scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updated code to latest version for open-sourcing.

Browse source
This commit is contained in:
Fletcher Haynes 2022-09-02 13:23:39 -07:00
parent 227632544d
commit eb06bb50ae
1338 changed files with 452272 additions and 49527 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
.circleci/config.yml
View file

@ -1,195 +0,0 @@
version: 2
defaults: &defaults
working_directory: /go/src/github.com/pilosa/pilosa
docker:
- image: circleci/golang:1.13
environment:
GO111MODULE: "on"
fast-checkout: &fast-checkout
attach_workspace:
at: .
jobs:
setup:
<<: *defaults
steps:
- checkout
- restore_cache:
keys:
- mod-cache-{{ checksum "go.sum" }}
- run: "go mod download"
- save_cache:
key: mod-cache-{{ checksum "go.sum" }}
paths:
- /go/pkg/mod/
- persist_to_workspace:
root: .
paths: "*"
check-license-headers:
<<: *defaults
steps:
- *fast-checkout
- run: make check-license-headers
linter:
<<: *defaults
steps:
- *fast-checkout
- run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
- run: sudo cp bin/golangci-lint /usr/local/bin/
- run: make golangci-lint
test-build-arm:
<<: *defaults
steps:
- *fast-checkout
- run: make build GOOS=linux GOARCH=arm GOARM=5
- run: make build GOOS=linux GOARCH=arm GOARM=6
- run: make build GOOS=linux GOARCH=arm GOARM=7
- run: make build GOOS=linux GOARCH=arm64
test-golang-1.13: &base-test
<<: *defaults
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run: make test
test-golang-1.13-shard22:
<<: *defaults
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run: make test SHARD_WIDTH=22
test-golang-1.13-race:
<<: *defaults
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run:
command: make test TESTFLAGS="-race -v -timeout=30m"
no_output_timeout: 30m
test-golang-1.13-386:
<<: *base-test
environment:
GO111MODULE: "on"
GOARCH: 386
test-golang-1.13-enterprise:
<<: *defaults
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run: make test ENTERPRISE=1
test-golang-1.12:
<<: *defaults
docker:
- image: circleci/golang:1.12
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run: make test
test-golang-1.11:
<<: *defaults
docker:
- image: circleci/golang:1.11
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run: make test
cluster-tests:
<<: *defaults
steps:
- *fast-checkout
- setup_remote_docker
- run: make clustertests-build
prerelease:
<<: *base-test
steps:
- *fast-checkout
- run: make prerelease
- store_artifacts:
path: build
- persist_to_workspace:
root: .
paths: build
release:
<<: *defaults
steps:
- *fast-checkout
- run: make release
- store_artifacts:
path: build
- persist_to_workspace:
root: .
paths: build
prerelease-upload:
docker:
- image: circleci/python:2.7-jessie
steps:
- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
- *fast-checkout
- run: sudo pip install awscli
- run: make prerelease-upload
dockerhub-upload:
<<: *defaults
steps:
- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
- *fast-checkout
- setup_remote_docker
- run: make docker
- run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
- run: docker login -u $DOCKER_USER -p $DOCKER_PASS
- run: docker push pilosa/pilosa:master
workflows:
version: 2
test:
jobs:
- setup
- linter:
requires:
- setup
- check-license-headers:
requires:
- setup
- test-build-arm:
requires:
- setup
- test-golang-1.13-enterprise:
requires:
- setup
- test-golang-1.13-race:
requires:
- setup
- test-golang-1.13-386:
requires:
- setup
- test-golang-1.13:
requires:
- setup
- test-golang-1.12:
requires:
- setup
- test-golang-1.11:
requires:
- setup
- cluster-tests:
requires:
- setup
- prerelease:
requires:
- linter
- check-license-headers
- test-golang-1.13
- release:
requires:
- linter
- check-license-headers
- test-golang-1.13
filters:
tags:
only: /^v.*/
branches:
ignore: /.*/
- prerelease-upload:
requires:
- prerelease
- dockerhub-upload:
requires:
- linter
- check-license-headers
- test-golang-1.13

16
.github/ISSUE_TEMPLATE.md vendored
View file

@ -1,16 +0,0 @@
For bugs, please provide the following:
### What's going wrong?
### What was expected?
### Steps to reproduce the behavior
### Information about your environment (OS/architecture, CPU, RAM, cluster/solo, configuration, etc.)
For feature requests, please provide the following:
### Description
### Success criteria (What criteria will consider this ticket closeable?)

28
.github/PULL_REQUEST_TEMPLATE.md vendored
View file

@ -1,28 +0,0 @@
## Overview
[Describe what this pull request addresses.]
Fixes #
## Pull request checklist
- [ ] I have read the [contributing guide](https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md).
- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa).
- [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs).
- [ ] I have resolved any merge conflicts.
- [ ] I have included tests that cover my changes.
- [ ] All new and existing tests pass.
- [ ] Make sure PR title conforms to convention in CHANGELOG.md.
- [ ] Add appropriate changelog label to PR (if applicable).
## Code review checklist
This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for.
- [ ] Ensure that any changes to external docs have been included in this pull request.
- [ ] If the changes require that minor/major versions need to be updated, tag the PR appropriately.
- [ ] Ensure the new code is [properly commented](https://github.com/golang/go/wiki/CodeReviewComments#doc-comments) and follows [Idiomatic Go](https://dmitri.shuralyov.com/idiomatic-go).
- [ ] Check that tests have been written and that they cover the new functionality.
- [ ] Run tests and ensure they pass.
- [ ] Build and run the code, performing any applicable integration testing.
- [ ] Make sure PR title conforms to convention in CHANGELOG.md.
- [ ] Make sure PR is tagged with appropriate changelog label.

715
CHANGELOG.md
View file

@ -1,715 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## [1.4.0] - 2019-09-17
This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions.
**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release.
### Added
- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028))
- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029))
- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016))
- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023))
- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004))
- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983))
- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992))
- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939))
- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950))
- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956))
### Changed
- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064))
- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001))
- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994))
- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993))
- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978))
- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902))
- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966))
### Fixed
- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050))
- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034))
- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033))
- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026))
- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021))
- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019))
- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017))
- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012))
- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000))
- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997))
- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975))
- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973))
- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970))
- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954))
- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951))
- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947))
### Performance
- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988))
- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048))
- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041))
- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024))
- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974))
### Removed
- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996))
## [1.3.1] - 2019-05-01
This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions.
### Fixed
- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964))
## [1.3.0] - 2019-04-16
This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions.
### Added
- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940))
- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921))
- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915))
- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859))
- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910))
- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906))
- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903))
- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881))
- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761))
- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832))
- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839))
- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831))
- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763))
- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820))
- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818))
- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811))
### Fixed
- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944))
- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943))
- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938))
- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936))
- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931))
- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876))
- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863))
- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887))
- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848))
- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846))
- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842))
- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834))
- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835))
- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836))
- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826))
- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827))
- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822))
- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802))
### Changed
- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924))
- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937))
- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909))
- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900))
- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616))
- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804))
- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851))
- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830))
### Performance
- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916))
- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901))
- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897))
- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892))
- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871))
- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865))
- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814))
- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803))
## [1.2.0] - 2018-12-20
This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions.
### Added
- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773))
- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774))
- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771))
- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753))
- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751))
- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756))
- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684))
- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733))
- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743))
- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717))
- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713))
- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707))
- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647))
- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699))
- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663))
- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683))
- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666))
- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671))
- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673))
- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645))
- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658))
- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653))
- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646))
- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738))
- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635))
- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631))
- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625))
- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628))
### Changed
- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780))
- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755))
- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748))
- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740))
- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725))
- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724))
- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677))
- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664))
### Performance
- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619))
- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741))
### Fixed
- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795))
- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787))
- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790))
- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785))
- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782))
- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777))
- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769))
- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764))
- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750))
- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749))
- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732))
- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744))
- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742))
- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737))
- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729))
- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719))
- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727))
- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712))
- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711))
- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674))
- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675))
- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696))
- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686))
- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679))
- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676))
- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665))
- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662))
- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655))
- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656))
- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651))
- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612))
- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621))
- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620))
- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614))
## [1.1.0] - 2018-08-21
This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions.
### Added
- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610))
- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608))
- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601))
- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582))
- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584))
- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527))
- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524))
- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532))
### Fixed
- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607))
- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586))
- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603))
- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602))
- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552))
- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558))
## [1.0.2] - 2018-08-01
This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions.
### Fixed
- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551))
- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547))
- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552))
- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541))
- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542))
- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533))
- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502))
- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555))
## [1.0.1] - 2018-07-11
This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions.
### Fixed
- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491))
- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485))
- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486))
- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483))
- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493))
## [1.0.0] - 2018-07-09
This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions.
### Added
- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337))
- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327))
### Changed
- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466))
- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297))
- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389))
- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452))
- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426))
- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424))
- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461))
- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451))
- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407))
- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463))
- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355))
- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365))
- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333))
- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326))
- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311))
- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454))
### Removed
- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397))
- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462))
- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396))
- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363))
- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347))
- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346))
- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341))
- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335))
- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332))
- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468))
### Fixed
- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423))
- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401))
- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415))
- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350))
## [0.10.0] - 2018-05-15
This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions.
### Added
- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285))
- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236))
### Changed
- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265))
### Removed
- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286))
- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284))
- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238))
- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255))
### Fixed
- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269))
- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279))
- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278))
- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268))
- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266))
- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242))
- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234))
- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253))
## [0.9.0] - 2018-05-04
This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions.
*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.*
### Added
- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908))
- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088))
- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133))
- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121))
- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138))
- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132))
- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109))
- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064))
- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063))
- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048))
- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245))
- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985))
- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963))
- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191))
- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246))
- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250))
### Changed
- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163))
- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166))
- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158))
- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148))
- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797))
- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115))
- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099))
- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077))
- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032))
- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027))
- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001))
- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220))
- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223))
- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216))
- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221))
### Removed
- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171))
- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142))
- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130))
- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226))
- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222))
- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219))
- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212))
- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209))
- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207))
### Fixed
- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143))
- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139))
- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110))
- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102))
- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105))
- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101))
- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084))
- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979))
- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980))
- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233))
- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229))
- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228))
### Deprecated
- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205))
### Performance
- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017))
## [0.8.8] - 2018-02-19
This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions.
### Fixed
- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118))
## [0.8.7] - 2018-02-12
This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions.
### Fixed
- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111))
## [0.8.6] - 2018-02-09
This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions.
### Fixed
- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106)
- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083)
## [0.8.5] - 2018-01-18
This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions.
### Fixed
- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061))
## [0.8.4] - 2018-01-10
This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions.
### Fixed
- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038))
- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033))
### Performance
- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046))
- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042))
## [0.8.3] - 2017-12-12
This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions.
### Fixed
- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000))
## [0.8.2] - 2017-12-05
This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions.
### Fixed
- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994))
## [0.8.1] - 2017-11-15
This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions.
### Fixed
- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969))
- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968))
## [0.8.0] - 2017-11-15
This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions.
### Added
- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895))
- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933))
- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938))
- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942))
- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867))
- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898))
- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889))
- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881))
- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955))
### Changed
- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943))
- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884))
- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892))
- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945))
- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
### Fixed
- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941))
- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934))
- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919))
- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887))
- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882))
- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949))
### Performance
- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874))
## [0.7.2] - 2017-11-15
This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion.
### Changed
- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920))
## [0.7.1] - 2017-10-09
This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions.
### Changed
- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
### Performance
- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874))
## [0.7.0] - 2017-10-03
This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions.
### Added
- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856))
- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785))
- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840))
- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791))
- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755))
- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778))
- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861))
- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847))
- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789))
- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837))
- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831))
- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830))
- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806))
- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783))
### Fixed
- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860))
- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814))
- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795))
- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794))
- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779))
- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773))
### Deprecated
- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839))
### Performance
- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859))
- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841))
- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842))
- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848))
- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846))
- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849))
- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820))
- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812))
## [0.6.0] - 2017-08-11
This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions.
### Added
- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758))
### Changed
- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750))
### Fixed
- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759))
- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674))
## [0.5.0] - 2017-08-02
This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions.
### Added
- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748))
- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646))
- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745))
- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741))
- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636))
- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635))
- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629))
- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600))
- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606))
- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721))
- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702))
- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670))
### Changed
- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747))
- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744))
- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699))
- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675))
### Fixed
- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729))
- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678))
## [0.4.0] - 2017-06-08
This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions.
*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).*
### Added
- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580))
- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586))
- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592))
- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571))
- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560))
- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537))
- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541))
- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563))
- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551))
- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540))
- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546))
- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535))
- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533))
- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510))
### Changed
- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520))
- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578))
- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573))
- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524))
- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525))
### Fixed
- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605))
- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594))
- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599))
- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597))
- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589))
- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565))
- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559))
- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555))
- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554))
- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548))
- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528))
- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503))
- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507))
- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462))
### Performance
- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590))
- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577))
- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562))
[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD
[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4
[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5
[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6
[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7
[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8
[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9
[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10
[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0
[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1
[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2

133
CODE_OF_CONDUCT.md Normal file
View file

@ -0,0 +1,133 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior include:
* The use of sexualized language or imagery, and sexual attention or advances of
any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email address,
without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
community@featurebase.com.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations

177
CONTRIBUTING.md
View file

@ -1,177 +0,0 @@
# Contributing to Pilosa
The workflow components of these instructions apply to all Pilosa repositories.
## Reporting a bug
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1].
## Submitting a feature request
Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4]
which starts by [opening a new issue][1] that describes the new feature proposal.
## Making code contributions
Before you start working on new features, you should [open a new issue][1] to let others know what
you're doing, otherwise you run the risk of duplicating effort. This also
gives others an opportunity to provide input for your feature.
If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6].
### Development Environment
- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information.
- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).
- Fork the [Pilosa repository][2] to your own account.
- It will be easier to follow these instructions if you:
```sh
export GH_USERNAME=<your github username>
```
- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa:
```sh
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
- `cd` to your pilosa directory:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
```
- Install Pilosa command line tools:
```sh
make install
```
Running `pilosa` should now run a Pilosa instance.
- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git
```
### Makefile
Pilosa includes a Makefile that automates several tasks:
- Install Pilosa:
```sh
make install
```
- Install build dependencies:
```sh
make install-build-deps
```
- Create the vendor directory:
```sh
make vendor
```
- Run the test suite:
```sh
make test
```
- View the coverage report:
```sh
make cover-viz
```
- Clear the `vendor/` and `build/` directories:
```sh
make clean
```
- Create release tarballs:
```sh
make release
```
- Regenerate protocol buffer files in `internal/`:
```sh
make generate-protoc
```
- Create tagged Docker image:
```sh
make docker
```
- Run tests inside Docker container:
```sh
make docker-test
```
Additional commands are available in the `Makefile`.
### Submitting code changes
- Before starting to work on a task, sync your branch with the upstream:
```sh
git checkout master
git pull
```
- Create a local feature branch:
```sh
git checkout -b something-amazing
```
- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/).
- Make sure that you've written tests for your new feature, and then run the tests:
```sh
make test
```
- Verify that your pull request is applied to the latest version of code on github:
```sh
git checkout master
git pull
git checkout something-amazing
git rebase master
```
- Push to your fork:
```sh
git push -u $GH_USERNAME something-amazing:something-amazing
```
- Submit a [pull request][3]
[1]: https://github.com/pilosa/pilosa/issues/new
[2]: https://github.com/pilosa/pilosa
[3]: https://github.com/pilosa/pilosa/compare/
[4]: https://github.com/pilosa/general/blob/master/proposal.md
[5]: https://github.com/pilosa/pilosa/issues
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer

51
Dockerfile
View file

@ -1,22 +1,55 @@
FROM golang:1.13.0 as builder
ARG GO_VERSION=latest
COPY . pilosa
#######################
### Lattice builder ###
#######################
RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
FROM moleculacorp/nodejs:latest as lattice-builder
WORKDIR /lattice
FROM alpine:3.9.4
COPY lattice/package.json ./
COPY lattice/yarn.lock ./
RUN yarn install
LABEL maintainer "dev@pilosa.com"
COPY lattice ./
RUN yarn build
######################
### Pilosa builder ###
######################
FROM golang:${GO_VERSION} as pilosa-builder
ARG MAKE_FLAGS
WORKDIR /pilosa
RUN go get github.com/rakyll/statik
COPY . ./
COPY --from=lattice-builder /lattice/build /lattice
RUN /go/bin/statik -src=/lattice -dest=/pilosa
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
#####################
### Pilosa runner ###
#####################
FROM alpine:3.13.2 as runner
LABEL maintainer "dev@molecula.com"
RUN apk add --no-cache curl jq
COPY --from=builder /go/bin/pilosa /pilosa
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
EXPOSE 10101
VOLUME /data
ENTRYPOINT ["/pilosa"]
CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
ENV PILOSA_DATA_DIR /data
ENV PILOSA_BIND 0.0.0.0:10101
ENV PILOSA_BIND_GRPC 0.0.0.0:20101
ENTRYPOINT ["/featurebase"]
CMD ["server"]

26
Dockerfile-clustertests
View file

@ -1,29 +1,35 @@
# This Dockerfile is used for cluster testing - it produces a much larger image
# and includes all of Go as well as some utilities.
FROM golang:1.11
FROM golang:1.19
LABEL maintainer "dev@pilosa.com"
COPY . /go/src/github.com/pilosa/pilosa/
COPY . /go/src/github.com/molecula/featurebase/
RUN cd /go/src/github.com/pilosa/pilosa \
&& GO111MODULE=on make vendor
RUN cd /go/src/github.com/pilosa/pilosa \
&& CGO_ENABLED=0 make install FLAGS="-a"
# download pumba for fault injection
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
RUN chmod +x /pumba
RUN cp /go/bin/pilosa /pilosa
# add docker client to pause/unpause nodes
RUN apt update
RUN apt install -y docker.io
# add docker-compose so tests can use it for stuff
ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
RUN chmod +x /usr/local/bin/docker-compose
# generate an instrumented binary to allow for calculating code coverage for clustertests
# the entrypoint for the binary is TestRunMain, which is wrapper for main
RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
EXPOSE 10101
VOLUME /data
ENTRYPOINT ["bash", "-c"]
CMD ["/pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

35
Dockerfile-clustertests-client Normal file
View file

@ -0,0 +1,35 @@
# This Dockerfile is used for cluster testing - it produces a much larger image
# and includes all of Go as well as some utilities.
FROM golang:1.19
LABEL maintainer "dev@pilosa.com"
COPY . /go/src/github.com/molecula/featurebase/
# download pumba for fault injection
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
RUN chmod +x /pumba
# add docker client to pause/unpause nodes
RUN apt update
RUN apt install -y docker.io
# add docker-compose so tests can use it for stuff
ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
RUN chmod +x /usr/local/bin/docker-compose
RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
COPY NOTICE /NOTICE
COPY ./internal/clustertests /go/src/github.com/molecula/featurebase/internal/clustertests
EXPOSE 10101
VOLUME /data
ENTRYPOINT ["bash", "-c"]
CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

0
LICENSE → LICENSE-2.0.txt
View file

337
Makefile
View file

@ -1,23 +1,34 @@
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
VARIANT = Molecula
GO=go
GOOS=$(shell $(GO) env GOOS)
GOARCH=$(shell $(GO) env GOARCH)
VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
SHARD_WIDTH = 20
LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
GO_VERSION=latest
ENTERPRISE ?= 0
ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE))
RELEASE ?= 0
RELEASE_ENABLED = $(subst 0,,$(RELEASE))
BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
LDFLAGS="-X github.com/molecula/featurebase/v3.Version=$(VERSION) -X github.com/molecula/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v3.Variant=$(VARIANT) -X github.com/molecula/featurebase/v3.Commit=$(COMMIT) -X github.com/molecula/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
GO_VERSION=1.19
DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ")
TEST_TAGS = roaringparanoia
UNAME := $(shell uname -s)
TEST_TIMEOUT=30m
RACE_TEST_TIMEOUT=90m
ifeq ($(UNAME), Darwin)
IS_MACOS:=1
else
IS_MACOS:=0
endif
export GO111MODULE=on
export GOPRIVATE=github.com/molecula
export CGO_ENABLED=0
# Run tests and compile Pilosa
default: test build
@ -25,17 +36,57 @@ default: test build
# Remove build directories
clean:
rm -rf vendor build
rm -f *.rpm *.deb
# Set up vendor directory using `go mod vendor`
vendor: go.mod
go mod vendor
$(GO) mod vendor
version:
@echo $(VERSION)
# We build a list of packages that omits the IDK packages because the IDK
# packages require fancy environment setup.
GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk")
# Run test suite
test:
go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS)
$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
# Run test suite with race flag
test-race:
CGO_ENABLED=1 $(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v
testv: testvsub
testv-race: testvsub-race
# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
# avoiding go -test ./... "package list mode" which doesn't give output
# until the test run finishes. Package list mode makes it hard to
# find which test is hung/deadlocked.
#
testvsub:
@set -e; for pkg in $(GOPACKAGES); do \
if [ $${pkg:0:38} == "github.com/molecula/featurebase/v3/idk" ]; then \
echo; echo "___ skipping subpkg $$pkg"; \
continue; \
fi; \
echo; echo "___ testing subpkg $$pkg"; \
$(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
echo; echo "999 done testing subpkg $$pkg"; \
done
testvsub-race:
@set -e; for pkg in $(GOPACKAGES); do \
echo; echo "___ testing subpkg $$pkg"; \
CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
echo; echo "999 done testing subpkg $$pkg"; \
done
bench:
go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
$(GO) test $(GOPACKAGES) -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
# Run test suite with coverage enabled
cover:
@ -44,19 +95,27 @@ cover:
# Run test suite with coverage enabled and view coverage results in browser
cover-viz: cover
go tool cover -html=build/coverage.out
$(GO) tool cover -html=build/coverage.out
# Compile Pilosa
build:
go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
# Create a single release build under the build directory
release-build:
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
cp NOTICE README.md build/pilosa-$(VERSION_ID)
$(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID))
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase"
cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID)
tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
@echo Created release build: build/featurebase-$(VERSION_ID).tar.gz
test-release-build: docker-build
mv build/featurebase-$(VERSION_ID).tar.gz install/
cd install && docker build -t featurebase:test_installation \
-f test_installation.Dockerfile \
--build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz .
mv install/featurebase-$(VERSION_ID).tar.gz build/
docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
featurebase:test_installation
# Error out if there are untracked changes in Git
check-clean:
@ -64,78 +123,196 @@ ifndef SKIP_CHECK_CLEAN
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
endif
# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
release: check-clean
# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release`
release: check-clean generate-statik-docker
$(MAKE) release-build GOOS=darwin GOARCH=amd64
$(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1
$(MAKE) release-build GOOS=darwin GOARCH=arm64
$(MAKE) release-build GOOS=linux GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1
$(MAKE) release-build GOOS=linux GOARCH=386
$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
$(MAKE) release-build GOOS=linux GOARCH=arm64
# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
release-sans-ui: check-clean
rm -f statik/statik.go
$(MAKE) release-build GOOS=darwin GOARCH=amd64
$(MAKE) release-build GOOS=darwin GOARCH=arm64
$(MAKE) release-build GOOS=linux GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=arm64
# try (e.g.) internal/clustertests/docker-compose-replication2.yml
DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
package:
go build -o featurebase ./cmd/featurebase
GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
# We allow setting a custom docker-compose "project". Multiple of the
# same docker-compose environment can exist simultaneously as long as
# they use different projects (the project name is prepended to
# container names and such). This is useful in a CI environment where
# we might be running multiple instances of the tests concurrently.
PROJECT ?= clustertests
DOCKER_COMPOSE = docker-compose -p $(PROJECT)
# Run cluster integration tests using docker. Requires docker daemon to be
# running. This will catch changes to internal/clustertests/*.go, but if you
# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
# pilosa image.
clustertests:
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) build client1
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
# running and docker-compose to be installed.
clustertests: vendor
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
# Like clustertests, but rebuilds all images.
clustertests-build:
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Create prerelease builds
prerelease:
$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
$(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release)
prerelease-upload:
aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read
# Run the cluster tests with authentication enabled
AUTH_ARGS="-c /go/src/github.com/molecula/featurebase/internal/clustertests/testdata/featurebase.conf"
authclustertests: vendor
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
# Install Pilosa
install:
go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
# Install the single-node PLG version of FeatureBase
plg:
$(GO) build -tags='plg $(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
install-bench:
$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
# Build the lattice assets
build-lattice:
docker build -t lattice:build ./lattice
export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE
# `go generate` protocol buffers
generate-protoc: require-protoc require-protoc-gen-gofast
go generate github.com/pilosa/pilosa/v2/internal
$(GO) generate github.com/molecula/featurebase/v3/pb
# `go generate` statik assets (lattice UI)
generate-statik: build-lattice require-statik
$(GO) generate github.com/molecula/featurebase/v3/statik
# `go generate` statik assets (lattice UI) in Docker
generate-statik-docker: build-lattice
docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f"
# `go generate` stringers
generate-stringer:
go generate github.com/pilosa/pilosa/v2
$(GO) generate github.com/molecula/featurebase/v3
generate-pql: require-peg
cd pql && peg -inline pql.peg && cd ..
generate-proto-grpc: require-protoc require-protoc-gen-go
protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto
# TODO: Modify above commands and remove the below mv if possible.
# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
# I couldn't get it to work during development - Cody
cp -r proto/github.com/molecula/featurebase/v3/proto/ proto/
rm -rf proto/github.com
# `go generate` all needed packages
generate: generate-protoc generate-stringer generate-pql
generate: generate-protoc generate-statik generate-stringer generate-pql
# Create release using Docker
docker-release:
$(MAKE) docker-build GOOS=linux GOARCH=amd64
$(MAKE) docker-build GOOS=linux GOARCH=arm64
$(MAKE) docker-build GOOS=darwin GOARCH=amd64
$(MAKE) docker-build GOOS=darwin GOARCH=arm64
# Build a release in Docker
docker-build: vendor
docker build \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
--target pilosa-builder \
--tag featurebase:build .
docker create --name featurebase-build featurebase:build
mkdir -p build/featurebase-$(VERSION_ID)
docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID)
cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID)
docker rm featurebase-build
tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
# Create Docker image from Dockerfile
docker:
docker build -t "pilosa:$(VERSION)" .
@echo Created docker image: pilosa:$(VERSION)
docker-image: vendor
docker build \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
--tag featurebase:$(VERSION) .
@echo Created docker image: featurebase:$(VERSION)
# Compile Pilosa inside Docker container
docker-build:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Create docker image (alias)
docker: docker-image # alias
# Tag and push a Docker image
docker-tag-push: vendor
docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET)
docker push $(DOCKER_TARGET)
@echo Pushed docker image: $(DOCKER_TARGET)
# These commands (docker-idk and docker-idk-tag-push)
# are designed to be used in CI.
# docker-idk builds idk docker images and tags them - intended for use in CI.
docker-idk: vendor
docker build \
-f idk/Dockerfile \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
# docker-idk-tag-push pushes tagged docker images to the GitLab container
# registry - intended for use in CI.
docker-idk-tag-push:
docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
pilosa-keydump:
$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
pilosa-chk:
$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
pilosa-fsck:
cd ./cmd/pilosa-fsck && make install && make release
# Run Pilosa tests inside Docker container
docker-test:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout $(TEST_TIMEOUT) $(GOPACKAGES)
# Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
# run top tests, not subdirs. print summary red/green after.
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
mv log.topt.roar log.topt.roar.prev || true
$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout $(RACE_TEST_TIMEOUT) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout $(RACE_TEST_TIMEOUT) -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
# Run golangci-lint
golangci-lint: require-golangci-lint
golangci-lint run
golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
# Alias
linter: golangci-lint
# Better alias
ocd: golangci-lint
# Run gometalinter with custom flags
# Note the "./..." in gometalinter is still allowed, because we do want
# linting to reach IDK pagkages.
gometalinter: require-gometalinter vendor
GO111MODULE=off gometalinter --vendor --disable-all \
--deadline=300s \
@ -158,13 +335,6 @@ gometalinter: require-gometalinter vendor
--exclude "^pql/pql.peg.go" \
./...
# Verify that all Go files have license header
check-license-headers: SHELL:=/bin/bash
check-license-headers:
@! find . -name '*.go' | grep -v '^./vendor' | while read fn;\
do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \
grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise
######################
# Build dependencies #
######################
@ -175,24 +345,37 @@ require-%:
$(info Verified build dependency "$*" is installed.),\
$(error Build dependency "$*" not installed. To install, try `make install-$*`))
install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
install-statik:
go install github.com/rakyll/statik@latest
install-stringer:
GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer
GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
install-protoc-gen-gofast:
GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast
GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
install-protoc-gen-go:
GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go
install-protoc:
@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
@echo On mac, brew install protobuf seems to work.
@echo As of the commit that added this line, protoc-gen-gofast was at 226206f39bd7, and the protoc version in use was:
@echo $$ protoc --version
@echo libprotoc 3.19.4
install-peg:
GO111MODULE=off go get github.com/pointlander/peg
GO111MODULE=off $(GO) get github.com/pointlander/peg
install-golangci-lint:
GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint
GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
install-gometalinter:
GO111MODULE=off go get -u github.com/alecthomas/gometalinter
GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter
GO111MODULE=off gometalinter --install
GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode
GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
test-external-lookup:
$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)

26
NOTES
View file

@ -1,26 +0,0 @@
Index Column
┌───────────▼────────────────────────────┐
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
Row──▶0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
F ▶│0000000000000000000000000000000000000000│
i ││0000000000000000000000000000000000000000│
e ││0000000000000000000000000000000000000000│
l ││0000000000000000000000000000000000000000│
d ▶│0000000000000000000000000000000000000000│
└────────────────────────────────────────┘
▲───────────▲
Shard
Fragment=intersection of field & shard

60
NOTICE
View file

@ -1,44 +1,12 @@
Software license
================
Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License").
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Enterprise Edition software license
===================================
Files contained under the directory `enterprise` are subject to the following
license notice (Full license included in the file `COPYING`):
Copyright (C) 2018 Pilosa Corp. All rights reserved.
Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Pilosa Enterprise Edition is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Pilosa Enterprise Edition. If not, see <http://www.gnu.org/licenses/>.
Copyright (C) 2017-2021 Molecula Corp. All rights reserved.
Third-party software licenses
=============================
The file /pilosa/lru/lru.go contains a redistribution of lru
The file /lru/lru.go contains a redistribution of lru
(github.com/golang/groupcache/lru); the license follows:
Copyright 2013 Google Inc.
@ -115,3 +83,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
The MIT License (MIT)
Copyright (c) 2015 Nick Galbreath
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

115
README.md
View file

@ -1,80 +1,85 @@
<p>
<a href="https://www.pilosa.com">
<img src="https://www.pilosa.com/img/logo.svg" width="50%">
</a>
</p>
# FeatureBase
[![CircleCI](https://circleci.com/gh/pilosa/pilosa/tree/master.svg?style=shield)](https://circleci.com/gh/pilosa/pilosa/tree/master)
[![GoDoc](https://godoc.org/github.com/pilosa/pilosa?status.svg)](https://godoc.org/github.com/pilosa/pilosa)
[![Go Report Card](https://goreportcard.com/badge/github.com/pilosa/pilosa)](https://goreportcard.com/report/github.com/pilosa/pilosa)
[![license](https://img.shields.io/github/license/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/blob/master/LICENSE)
[![CLA Assistant](https://cla-assistant.io/readme/badge/pilosa/pilosa)](https://cla-assistant.io/pilosa/pilosa)
[![GitHub release](https://img.shields.io/github/release/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/releases)
## Pilosa is now FeatureBase
## An open source, distributed bitmap index.
- [Docs](#docs)
- [Getting Started](#getting-started)
- [Data Model](#data-model)
- [Query Language](#query-language)
- [Client Libraries](#client-libraries)
- [Get Support](#get-support)
- [Contributing](#contributing)
As of September 7, 2022, the Pilosa project is now FeatureBase. The core of the project remains the same: FeatureBase is the first real-time distributed database built entirely on bitmaps. (More information about updated capabilities and improvements below.)
Want to contribute? One of the easiest ways is to [tell us how you're using (or want to use) Pilosa](https://github.com/pilosa/pilosa/issues/1074). We learn from every discussion!
## Docs
See our [Documentation](https://www.pilosa.com/docs/) for information about installing and working with Pilosa.
FeatureBase delivers low-latency query results, regardless of throughput or query volumes, on fresh data with extreme efficiency. It works because bitmaps are faster, simpler, and far more I/O efficient than traditional column-oriented data formats. With FeatureBase, you can ingest data from batch data sources (e.g. S3, CSV, Snowflake, BigQuery, etc.) and/or streaming data sources (e.g. Kafka/Confluent, Kinesis, Pulsar).
For more information about FeatureBase, please visit [www.featurebase.com][HomePage].
## Getting Started
1. [Install Pilosa](https://www.pilosa.com/docs/installation/).
### Build FeatureBase Server from source
2. [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
0. Install go. Ensure that your shell's search path includes the go/bin directory.
1. Clone the FeatureBase repository (or download as zip).
2. In the featurebase directory, run `make install` to compile the FeatureBase server binary. By default, it will be installed in the go/bin directory.
3. In the idk directory, run `make install` to compile the ingester binaries. By default, they will be installed in the go/bin directory.
4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
5. Run `curl localhost:10101/status` to verify the server is running and accessible.
```shell
pilosa server
```
and verify that it's running:
```shell
curl localhost:10101/nodes
```
### Ingest Data and Query
3. Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.
1. Run
```
molecula-consumer-csv \
--index repository \
--header "language__ID_F,project_id__ID_F" \
--id-field project_id \
--batch-size 1000 \
--files example.csv
```
This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about ingesting into FeatureBase: [https://docs.featurebase.com/data-ingestion/enterprise/ingesters][Ingest]
## Data Model
2. Query your data.
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'Row(example=5)'
```
Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL].
Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) works.
### Data Model
Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented.
Data Model Guide: [https://docs.featurebase.com/data-modeling-guide/data-modeling][DataModel]
## Query Language
### More Information
You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL).
Installation:[https://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase][Install]
Configuration: [https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration][Config]
## Client Libraries
## Community
There are supported libraries for the following languages:
- [Go](https://www.pilosa.com/docs/client-libraries/#go)
- [Java](https://www.pilosa.com/docs/client-libraries/#java)
- [Python](https://www.pilosa.com/docs/client-libraries/#python)
You can email us at comminuty@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community].
## Licenses
Chat with us: [https://discord.gg/bKAP5CEY][Discord]
The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0.
If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General
Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise)
directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and
[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses.
## What's Changed Since the Pilosa Days?
## Get Support
A lot has changed since the days of Pilosa. This list highlights some new capabilites included in FeatureBase. We have also made signficant improvements to the performance, scalability, and stability of the FeatureBase product.
There are [several channels](https://www.pilosa.com/community/#support) available for you to reach out to us for support. The Slack channel (#pilosa in the [Golang](https://invite.slack.golangbridge.org/) team) is the most active.
* Query Languages: FeatureBase supports Pilosa Query Language (PQL), as well as SQL
* Stream and Batch Ingest: Combine real-time data streams with batch historical data and act on it within milliseconds.
* Mutable: Perform inserts, updates, and deletes at scale, in real time and on-the-fly. This is key for meeting data compliance requirements, and for reflecting the constantly-changing nature of high-volume data.
* Multi-Valued Set Fields: Store multiple comma-delimited values within a single field while *increasing* query performance of counts, TopKs, etc.
* Time Quantums: Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to YMD, ranged Row queries down to the granularity of a day are supported.
* RBF storage backend: this is a new compressed bitmap format which improves performance in a number of ways: ACID support on a per shard basis, prevents issues with the number of open files, reduces memory allocation and lock contention for reads, provides more consistent garbage collection, and allows backups to run concurrently with writes. However, because of this change, Pilosa backup files cannot be restored into FeatureBase.
## Contributing
## License
Pilosa is an open source project. Please see our [Contributing Guide](CONTRIBUTING.md) for information about how to get involved.
FeatureBase is licensed under the [Apache License, Version 2.0][License]
[Community]: https://www.featurebase.com/community
[Config]: https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration
[DataModel]: https://docs.featurebase.com/data-modeling-guide/data-modeling
[Discord]: https://discord.gg/bKAP5CEY
[HomePage]: https://www.featurebase.com
[Ingest]: https://docs.featurebase.com/data-ingestion/enterprise/ingesters
[Install]: https://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase
[License]: http://www.apache.org/licenses/LICENSE-2.0
[PQL]: https://docs.featurebase.com/data-querying/pql/introduction
[SQL]: https://docs.featurebase.com/data-querying/sql

2708
api.go
View file

File diff suppressed because it is too large Load diff

203
api/client/grpc.go Normal file
View file

@ -0,0 +1,203 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package client
import (
"context"
"crypto/tls"
"sync"
"github.com/molecula/featurebase/v3/logger"
pb "github.com/molecula/featurebase/v3/proto"
"github.com/pkg/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials"
)
const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
// GRPCClient is a client for working with the gRPC server.
type GRPCClient struct {
dialTargets []string
tlsConfig *tls.Config
logger logger.Logger
mu sync.RWMutex
conn *grpc.ClientConn
targetIndex int
}
// NewGRPCClient returns a new instance of GRPCClient.
func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config, logger logger.Logger) (*GRPCClient, error) {
c := &GRPCClient{
dialTargets: dialTargets,
tlsConfig: tlsConfig,
logger: logger,
}
// resetConn sets GRPCClient.conn when it doesn't
// exist yet.
if err := c.resetConn(); err != nil {
return nil, errors.Wrap(err, "setting connection")
}
return c, nil
}
// resetConn resets the gRPC client connection. This method
// can also be used to initially set the client connection
// because it only tries to first close the connection if
// the connection already exists.
func (c *GRPCClient) resetConn() error {
c.mu.Lock()
defer c.mu.Unlock()
// If an existing connection exists, close it first.
if c.conn != nil {
if err := c.conn.Close(); err != nil {
return errors.Wrap(err, "closing existing connection")
}
}
var opts []grpc.DialOption
if c.tlsConfig != nil {
creds := credentials.NewTLS(c.tlsConfig)
opts = append(opts, grpc.WithTransportCredentials(creds))
} else {
opts = append(opts, grpc.WithInsecure())
}
opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
var err error
if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
return errors.Wrap(err, "creating new grpc client")
}
return nil
}
// getTargetIndex gets the current target index, then increments it for
// next time. Unprotected.
func (c *GRPCClient) getTargetIndex() int {
if len(c.dialTargets) == 0 {
return 0
}
ret := c.targetIndex
c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
return ret
}
// Close closes any connections the client has opened.
func (c *GRPCClient) Close() error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// Conn returns the gRPC client connection. If the connection
// has gone into state `TransientFailure`, this method tries
// to reset the connection and return that new connection.
func (c *GRPCClient) Conn() *grpc.ClientConn {
c.mu.RLock()
if c.conn == nil {
c.mu.RUnlock()
return nil
} else if c.conn.GetState() != connectivity.TransientFailure {
defer c.mu.RUnlock()
return c.conn
}
c.mu.RUnlock()
if err := c.resetConn(); err != nil {
c.logger.Errorf("error resetting connection: %s", err)
}
c.mu.RLock()
defer c.mu.RUnlock()
return c.conn
}
// Query returns a stream of RowResponse for the given index and PQL string.
func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
conn := c.Conn()
if conn == nil {
return nil, errors.New("client has not established a grpc connection")
}
grpcClient := pb.NewPilosaClient(conn)
stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
Index: index,
Pql: pql,
})
if err != nil {
return nil, errors.Wrap(err, "getting stream")
} else if stream == nil {
return nil, errors.New("could not create stream")
}
return stream, err
}
// QueryUnary returns a TableResponse for the given index and PQL string.
func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
conn := c.Conn()
if conn == nil {
return nil, errors.New("client has not established a grpc connection")
}
grpcClient := pb.NewPilosaClient(conn)
return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
Index: index,
Pql: pql,
})
}
// Inspect returns a stream of RowResponse for the given index, columns, and filters.
// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
conn := c.Conn()
if conn == nil {
return nil, errors.New("client has not established a grpc connection")
}
if len(columnIDs) > 0 && len(columnKeys) > 0 {
return nil, errors.New("only provide column ids or keys, not both")
}
// Convert columns to proto type IdsOrKeys.
idsOrKeys := &pb.IdsOrKeys{}
if len(columnKeys) > 0 {
idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
} else {
idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
}
grpcClient := pb.NewPilosaClient(conn)
stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
Index: index,
Columns: idsOrKeys,
FilterFields: fieldFilters,
Limit: limit,
Offset: offset,
Query: query,
})
if err != nil {
return nil, errors.Wrap(err, "getting stream")
} else if stream == nil {
return nil, errors.New("could not create stream")
}
return stream, err
}

1782
api_test.go
View file

File diff suppressed because it is too large Load diff

40
apimethod_string.go
View file

@ -19,25 +19,37 @@ func _() {
_ = x[apiFragmentBlockData-8]
_ = x[apiFragmentBlocks-9]
_ = x[apiFragmentData-10]
_ = x[apiField-11]
_ = x[apiFieldAttrDiff-12]
_ = x[apiImport-13]
_ = x[apiImportValue-14]
_ = x[apiIndex-15]
_ = x[apiIndexAttrDiff-16]
_ = x[apiTranslateData-11]
_ = x[apiFieldTranslateData-12]
_ = x[apiField-13]
_ = x[apiImport-14]
_ = x[apiImportValue-15]
_ = x[apiIndex-16]
_ = x[apiQuery-17]
_ = x[apiRecalculateCaches-18]
_ = x[apiRemoveNode-19]
_ = x[apiResizeAbort-20]
_ = x[apiSetCoordinator-21]
_ = x[apiShardNodes-22]
_ = x[apiViews-23]
_ = x[apiApplySchema-24]
_ = x[apiSchema-19]
_ = x[apiShardNodes-20]
_ = x[apiState-21]
_ = x[apiViews-22]
_ = x[apiApplySchema-23]
_ = x[apiStartTransaction-24]
_ = x[apiFinishTransaction-25]
_ = x[apiTransactions-26]
_ = x[apiGetTransaction-27]
_ = x[apiActiveQueries-28]
_ = x[apiPastQueries-29]
_ = x[apiIDReserve-30]
_ = x[apiIDCommit-31]
_ = x[apiIDReset-32]
_ = x[apiPartitionNodes-33]
_ = x[apiIngestOperations-34]
_ = x[apiIngestNodeOperations-35]
_ = x[apiMutexCheck-36]
}
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 499, 522, 535}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

205
attr.go
View file

@ -1,205 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"sort"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/internal"
)
// Attribute data type enum.
const (
attrTypeString = 1
attrTypeInt = 2
attrTypeBool = 3
attrTypeFloat = 4
)
// AttrStore represents an interface for handling row/column attributes.
type AttrStore interface {
Path() string
Open() error
Close() error
Attrs(id uint64) (m map[string]interface{}, err error)
SetAttrs(id uint64, m map[string]interface{}) error
SetBulkAttrs(m map[uint64]map[string]interface{}) error
Blocks() ([]AttrBlock, error)
BlockData(i uint64) (map[uint64]map[string]interface{}, error)
}
// nopStore represents an AttrStore that doesn't do anything.
var nopStore AttrStore = nopAttrStore{}
// newNopAttrStore returns an attr store which does nothing. It returns a global
// object to avoid unnecessary allocations.
func newNopAttrStore(string) AttrStore { return nopStore }
// nopAttrStore represents a no-op implementation of the AttrStore interface.
type nopAttrStore struct{}
// Path is a no-op implementation of AttrStore Path method.
func (s nopAttrStore) Path() string { return "" }
// Open is a no-op implementation of AttrStore Open method.
func (s nopAttrStore) Open() error { return nil }
// Close is a no-op implementation of AttrStore Close method.
func (s nopAttrStore) Close() error { return nil }
// Attrs is a no-op implementation of AttrStore Attrs method.
func (s nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return nil, nil }
// SetAttrs is a no-op implementation of AttrStore SetAttrs method.
func (s nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil }
// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method.
func (s nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { return nil }
// Blocks is a no-op implementation of AttrStore Blocks method.
func (s nopAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil }
// BlockData is a no-op implementation of AttrStore BlockData method.
func (s nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil }
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// attrBlocks represents a list of blocks.
type attrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a attrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
a, other = a[1:], other[1:]
}
}
}
func encodeAttrs(m map[string]interface{}) []*internal.Attr {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
a := make([]*internal.Attr, len(keys))
for i := range keys {
a[i] = encodeAttr(keys[i], m[keys[i]])
}
return a
}
func decodeAttrs(pb []*internal.Attr) map[string]interface{} {
m := make(map[string]interface{}, len(pb))
for i := range pb {
key, value := decodeAttr(pb[i])
m[key] = value
}
return m
}
// encodeAttr converts a key/value pair into an Attr internal representation.
func encodeAttr(key string, value interface{}) *internal.Attr {
pb := &internal.Attr{Key: key}
switch value := value.(type) {
case string:
pb.Type = attrTypeString
pb.StringValue = value
case float64:
pb.Type = attrTypeFloat
pb.FloatValue = value
case uint64:
pb.Type = attrTypeInt
pb.IntValue = int64(value)
case int64:
pb.Type = attrTypeInt
pb.IntValue = value
case bool:
pb.Type = attrTypeBool
pb.BoolValue = value
}
return pb
}
// decodeAttr converts from an Attr internal representation to a key/value pair.
func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
switch attr.Type {
case attrTypeString:
return attr.Key, attr.StringValue
case attrTypeInt:
return attr.Key, attr.IntValue
case attrTypeBool:
return attr.Key, attr.BoolValue
case attrTypeFloat:
return attr.Key, attr.FloatValue
default:
return attr.Key, nil
}
}
// cloneAttrs returns a shallow clone of m.
func cloneAttrs(m map[string]interface{}) map[string]interface{} {
other := make(map[string]interface{}, len(m))
for k, v := range m {
other[k] = v
}
return other
}
// EncodeAttrs encodes an attribute map into a byte slice.
func EncodeAttrs(attr map[string]interface{}) ([]byte, error) {
return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
}
// DecodeAttrs decodes a byte slice into an attribute map.
func DecodeAttrs(v []byte) (map[string]interface{}, error) {
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}

201
attr_test.go
View file

@ -1,201 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"io/ioutil"
"os"
"reflect"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
)
// Ensure database can set and retrieve column attributes.
func TestAttrStore_Attrs(t *testing.T) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
if err := s.SetAttrs(1, map[string]interface{}{"A": 100, "C": -27}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil {
t.Fatal(err)
}
// Retrieve attributes for column #1.
if m, err := s.Attrs(1); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) {
t.Fatalf("unexpected attrs(1): %#v", m)
}
// Retrieve attributes for column #2.
if m, err := s.Attrs(2); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) {
t.Fatalf("unexpected attrs(2): %#v", m)
}
}
// Ensure database returns a non-nil empty map if unset.
func TestAttrStore_Attrs_Empty(t *testing.T) {
s := MustOpenAttrStore()
defer s.Close()
if m, err := s.Attrs(100); err != nil {
t.Fatal(err)
} else if m == nil || len(m) > 0 {
t.Fatalf("unexpected attrs: %#v", m)
}
}
// Ensure database can unset attributes if explicitly set to nil.
func TestAttrStore_Attrs_Unset(t *testing.T) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
if err := s.SetAttrs(1, map[string]interface{}{"A": "X", "B": "Y"}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(1, map[string]interface{}{"B": nil}); err != nil {
t.Fatal(err)
}
// Verify attributes.
if m, err := s.Attrs(1); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": "X"}) {
t.Fatalf("unexpected attrs: %#v", m)
}
}
// Ensure attribute block checksums can be returned.
func TestAttrStore_Blocks(t *testing.T) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(100, map[string]interface{}{"B": "VALUE"}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(350, map[string]interface{}{"C": "FOO"}); err != nil {
t.Fatal(err)
}
// Retrieve blocks.
blks0, err := s.Blocks()
if err != nil {
t.Fatal(err)
} else if len(blks0) != 3 || blks0[0].ID != 0 || blks0[1].ID != 1 || blks0[2].ID != 3 {
t.Fatalf("unexpected blocks: %#v", blks0)
}
// Change second block.
if err := s.SetAttrs(100, map[string]interface{}{"X": 12}); err != nil {
t.Fatal(err)
}
// Ensure second block changed.
blks1, err := s.Blocks()
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(blks0[0], blks1[0]) {
t.Fatalf("block 0 mismatch: %#v != %#v", blks0[0], blks1[0])
} else if reflect.DeepEqual(blks0[1], blks1[1]) {
t.Fatalf("block 1 match: %#v ", blks0[0])
} else if !reflect.DeepEqual(blks0[2], blks1[2]) {
t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
}
}
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(string) pilosa.AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
errchan := make(chan error)
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
errchan <- err
}
}
}()
}
go func() {
wg.Wait()
close(errchan)
}()
if err := <-errchan; err != nil {
b.Fatal(err)
}
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() pilosa.AttrStore {
s := NewAttrStore("")
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

13
audit.go Normal file
View file

@ -0,0 +1,13 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"github.com/molecula/featurebase/v3/testhook"
)
var NewAuditor func() testhook.Auditor = NewNopAuditor
func NewNopAuditor() testhook.Auditor {
return testhook.NewNopAuditor()
}

40
audit_internal_test.go Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"fmt"
"reflect"
"github.com/molecula/featurebase/v3/testhook"
)
// These audit hooks are desireable during testing, but not in
// production.
type auditorViewHooks struct{}
type auditorFragmentHooks struct{}
// static type checks
var _ testhook.RegistryHookLive = &auditorViewHooks{}
var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
if entry != nil && entry.OpenCount != 0 {
return fmt.Errorf("view %s still open", o.(*view).name)
}
return nil
}
func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
if entry != nil && entry.OpenCount != 0 {
return fmt.Errorf("fragment %s still open", o.(*fragment).path())
}
return nil
}
func GetInternalTestHooks() testhook.RegistryHooks {
return map[reflect.Type]testhook.RegistryHook{
reflect.TypeOf((*view)(nil)): &auditorViewHooks{},
reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
}
}

95
audit_test.go Normal file
View file

@ -0,0 +1,95 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa_test
import (
"fmt"
"os"
"reflect"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/testhook"
)
// AuditLeaksOn is a global switch to turn on resource
// leak checking at the end of a test run.
var AuditLeaksOn = true
// for tests, we use a single shared auditor used by all of the holders.
var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
// These audit hooks are desireable during testing, but not in
// production.
type auditorIndexHooks struct{}
type auditorFieldHooks struct{}
type auditorHolderHooks struct{}
// static type checking
var _ testhook.RegistryHookLive = &auditorIndexHooks{}
var _ testhook.RegistryHookLive = &auditorFieldHooks{}
var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
var _ testhook.RegistryHookLive = &auditorHolderHooks{}
var testHooks = map[reflect.Type]testhook.RegistryHook{
reflect.TypeOf((*pilosa.Index)(nil)): &auditorIndexHooks{},
reflect.TypeOf((*pilosa.Field)(nil)): &auditorFieldHooks{},
reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
}
func init() {
if !AuditLeaksOn {
return
}
for k, v := range pilosa.GetInternalTestHooks() {
testHooks[k] = v
}
testhook.RegisterPreTestHook(func() error {
pilosa.NewAuditor = NewTestAuditor
return nil
})
testhook.RegisterPostTestHook(func() error {
err, errs := globalTestAuditor.FinalCheck()
if err != nil {
for i, e := range errs {
fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
}
}
return err
})
}
func NewTestAuditor() testhook.Auditor {
return globalTestAuditor
}
func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
if entry != nil && entry.OpenCount != 0 {
return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
}
return nil
}
func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
if entry != nil && entry.OpenCount != 0 {
return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
}
return nil
}
func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
path := o.(*pilosa.Holder).Path()
if path == "" {
fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
ent.Stack)
} else {
os.RemoveAll(o.(*pilosa.Holder).Path())
}
return err
}
func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
if entry != nil && entry.OpenCount != 0 {
return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
}
return nil
}

443
authn/authenticate.go Normal file
View file

@ -0,0 +1,443 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// Package authn handles authentication
package authn
import (
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/golang-jwt/jwt"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
"golang.org/x/oauth2"
)
const (
// AccessCookieName is the name of the cookie that holds the access token.
AccessCookieName = "molecula-chip"
// RefreshCookieName is the name of the cookie that holds the refresh token.
RefreshCookieName = "refresh-molecula-chip"
// RefreshHeaderName is the name of the header that holds the refresh token.
RefreshHeaderName = "X-Molecula-Refresh-Token"
// ContextValueAccessToken is the key used to set AccessTokens in a ctx.
ContextValueAccessToken = "Access"
// ContextValueRefreshToken is the key used to set RefreshTokens in a ctx.
ContextValueRefreshToken = "Refresh"
)
// cachedGroups is used to hold groups and when they were last cached
type cachedGroups struct {
cacheTime time.Time
groups []Group
}
// UserInfo holds the information about the user from the token
type UserInfo struct {
UserID string `json:"userid"`
UserName string `json:"username"`
Groups []Group `json:"groups"`
Expiry time.Time `json:"expiry"`
Token string `json:"token"`
RefreshToken string `json:"refreshtoken"`
}
// Group holds group information for an authenticated user
type Group struct {
GroupID string `json:"id"`
GroupName string `json:"displayName"`
}
// Groups holds a slice of Group for marshalling from JSON
type Groups struct {
NextLink string `json:"@odata.nextLink"`
Groups []Group `json:"value"`
}
// Auth holds state, configuration, and utilities needed for authentication.
type Auth struct {
logger logger.Logger
accessCookieName string
refreshCookieName string
secretKey []byte
groupEndpoint string
logoutEndpoint string
fbURL string // fbURL is the domain featurebase is hosted on, used for post logout redirection
oAuthConfig *oauth2.Config
cacheTTL time.Duration // cacheTTL is used to determine if a cached item should be refreshed or not
groupsCache map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
lastCacheClean time.Time // last cache clean is the time that the cache was last cleaned
allowedNetworks []net.IPNet // list of allowed networks for ingest
}
// NewAuth instantiates and returns a new Auth struct
func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string, configuredIPs []string) (auth *Auth, err error) {
auth = &Auth{
logger: logger,
accessCookieName: AccessCookieName,
refreshCookieName: RefreshCookieName,
groupEndpoint: groupEndpoint,
logoutEndpoint: logout,
fbURL: url,
oAuthConfig: &oauth2.Config{
RedirectURL: fmt.Sprintf("%s/redirect", url),
ClientID: clientID,
ClientSecret: clientSecret,
Scopes: scopes,
Endpoint: oauth2.Endpoint{
AuthURL: authURL,
TokenURL: tokenURL,
},
},
groupsCache: map[string]cachedGroups{},
cacheTTL: 10 * time.Minute,
lastCacheClean: time.Now(),
}
if auth.secretKey, err = decodeHex(secretKey); err != nil {
return nil, errors.Wrap(err, "decoding secret key")
}
// convert IPs and add them to allowed networks
err = auth.convertIP(configuredIPs)
if err != nil {
return nil, err
}
return auth, nil
}
// CleanOAuthConfig returns a's oauthConfig without the client secret
func (a Auth) CleanOAuthConfig() oauth2.Config {
b := *a.oAuthConfig
b.ClientSecret = ""
return b
}
// SecretKey is a convenient function to get the SecretKey from an Auth struct
func (a Auth) SecretKey() []byte {
return a.secretKey
}
// refreshToken refreshes a given access/refresh token pair
func (a *Auth) refreshToken(access, refresh string) (string, string, error) {
resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
url.Values{
"grant_type": {"refresh_token"},
"refresh_token": {refresh},
"client_id": {a.oAuthConfig.ClientID},
"client_secret": {a.oAuthConfig.ClientSecret},
},
)
if err != nil {
return "", "", errors.Wrap(err, "refreshing token")
}
if resp.StatusCode != http.StatusOK {
return "", "", fmt.Errorf("refreshing token: %s", resp.Status)
}
defer resp.Body.Close()
var t oauth2.Token
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
return "", "", errors.Wrap(err, "decoding refreshed token")
}
// remove the old groups from the groups cache
delete(a.groupsCache, access)
return t.AccessToken, t.RefreshToken, nil
}
// Authenticate takes in a auth token `access` and returns UserInfo from that token
// it is caller's responsibility to inform the user that the access token has been refreshed
func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) {
// clean up the cache every 30 minutes or so
if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute {
a.cleanCache()
}
if len(access) == 0 {
return nil, fmt.Errorf("auth token is empty")
}
// NOTE: we are using ParseUnverified here because the IDP validates the
// token's signature when we get the user's groups, we just need to make
// sure it's not expired and is well-formed
token, _, err := new(jwt.Parser).ParseUnverified(access, &jwt.MapClaims{})
// well-formed-ness check
if token == nil || token.Claims == nil || err != nil {
return nil, fmt.Errorf("parsing auth token: %v", err)
}
claims := *token.Claims.(*jwt.MapClaims)
// expiry check
if exp, ok := claims["exp"]; ok {
var expiry int64
switch v := exp.(type) {
case string:
expiry, err = strconv.ParseInt(v, 10, 64)
if err != nil {
return nil, fmt.Errorf("parsing exp string: %v", err)
}
case float64:
expiry = int64(v)
case int64:
expiry = v
}
if expiry < time.Now().UTC().Unix() {
access, refresh, err = a.refreshToken(access, refresh)
if err != nil {
return nil, fmt.Errorf("token is expired: %w", err)
}
}
}
userInfo := UserInfo{
Token: access,
RefreshToken: refresh,
Groups: []Group{},
}
if uid, ok := claims["oid"].(string); ok {
userInfo.UserID = uid
}
if name, ok := claims["name"].(string); ok {
userInfo.UserName = name
}
if userInfo.Groups, err = a.getGroups(access); err != nil {
return nil, errors.Wrap(err, "getting groups")
}
return &userInfo, nil
}
// cleanCache removes old items from our cache
func (a *Auth) cleanCache() {
for access, tkn := range a.groupsCache {
// if it's been more than 24 hours since the groups were cached
if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
// remove it from our cache
delete(a.groupsCache, access)
}
}
a.lastCacheClean = time.Now()
}
// Login redirects a user to login to their configured oAuth authorize endpoint
func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
}
// Logout clears out the user's cookie, removes the token from our cache, and
// redirects user to IdP's logout endpoint
func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
// remove the access token from a.groupsCache
if access, err := r.Cookie(a.accessCookieName); err == nil {
delete(a.groupsCache, access.Value)
}
// clear cookie
http.SetCookie(w, &http.Cookie{
Name: a.accessCookieName,
Value: "",
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(0, 0),
})
http.SetCookie(w, &http.Cookie{
Name: a.refreshCookieName,
Value: "",
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(0, 0),
})
http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect)
}
// Redirect handles the oAuth /redirect endpoint. It gets an access token and
// returns it to the user in the form of a cookie
func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline)
if err != nil {
a.logger.Warnf("getting token from IdP: %+v", err)
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
a.SetCookie(w, token.AccessToken, token.RefreshToken, token.Expiry)
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}
// getGroups gets the group membership for a given token from configured IdP
func (a *Auth) getGroups(token string) ([]Group, error) {
var groups Groups
gc, ok := a.groupsCache[token]
if ok && (time.Now().Sub(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 {
return gc.groups, nil
}
nextLink := a.groupEndpoint
for nextLink != "" {
req, err := http.NewRequest("GET", nextLink, nil)
if err != nil {
return nil, errors.Wrap(err, "creating new request to group endpoint")
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
response, err := http.DefaultClient.Do(req)
if err != nil {
return nil, errors.Wrap(err, "getting group membership info")
}
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("getting group membership info: %s", response.Status)
}
var g Groups
if err = json.NewDecoder(response.Body).Decode(&g); err != nil {
return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
}
response.Body.Close()
groups.Groups = append(groups.Groups, g.Groups...)
nextLink = g.NextLink
}
if len(groups.Groups) == 0 {
return nil, fmt.Errorf("no groups found")
}
a.groupsCache[token] = cachedGroups{
cacheTime: time.Now(),
groups: groups.Groups,
}
return groups.Groups, nil
}
func (a *Auth) SetCookie(w http.ResponseWriter, access, refresh string, expiry time.Time) error {
http.SetCookie(w, &http.Cookie{
Name: a.refreshCookieName,
Value: refresh,
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: expiry,
})
http.SetCookie(w, &http.Cookie{
Name: a.accessCookieName,
Value: access,
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: expiry,
})
return nil
}
func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) error {
mCookies := map[string]string{}
if c, ok := md["cookie"]; ok {
for _, cookie := range c {
name, val := parseCookie(cookie)
mCookies[name] = val
}
}
mCookies[a.accessCookieName] = access
mCookies[a.refreshCookieName] = refresh
cookies := []string{}
for name, val := range mCookies {
cookies = append(cookies, name+"="+val)
}
md["cookie"] = cookies
return grpc.SetHeader(ctx, md)
}
func decodeHex(hexstr string) ([]byte, error) {
data, err := hex.DecodeString(hexstr)
if err != nil {
return nil, errors.Wrap(err, "decoding hex string to byte slice")
}
if len(data) != 32 {
return nil, fmt.Errorf("invalid key length")
}
return data, nil
}
func (a *Auth) convertIP(configuredIPs []string) error {
sz := len(configuredIPs)
nets := make([]net.IPNet, sz)
for i, ip := range configuredIPs {
// skip empty strings
if ip == "" {
sz--
continue
}
// for IPs passed without a subnet, append /32 to only allow 1 IP
// this step is needed because ParseCIDR method assumes a CIDR address
if !strings.Contains(ip, "/") {
ip = ip + "/32"
}
_, subnet, err := net.ParseCIDR(ip)
if err != nil {
return errors.Wrapf(err, "parsing CIDR for %v", ip)
}
nets[i] = *subnet
}
a.allowedNetworks = nets[:sz]
return nil
}
// if IP is in allowed networks, then return true to grant admin permissions
func (a *Auth) CheckAllowedNetworks(clientIP string) bool {
clientIP = strings.Split(clientIP, ":")[0]
convertedIP := net.ParseIP(clientIP)
for _, network := range a.allowedNetworks {
if network.Contains(convertedIP) {
return true
}
}
return false
}
func parseCookie(cookie string) (name, data string) {
vals := strings.Split(cookie, "=")
if len(vals) == 0 {
vals = []string{"", ""}
} else if len(vals) < 2 {
vals = append(vals, "")
}
return vals[0], vals[1]
}

766
authn/authenticate_internal_test.go Normal file
View file

@ -0,0 +1,766 @@
package authn
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strconv"
"strings"
"testing"
"time"
"github.com/golang-jwt/jwt"
"github.com/molecula/featurebase/v3/logger"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
func NewTestAuth(t *testing.T) *Auth {
t.Helper()
var (
ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
LogoutURL = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
Scopes = []string{"https://graph.microsoft.com/.default", "offline_access"}
Key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
configuredIPs = []string{}
)
a, err := NewAuth(
logger.NewStandardLogger(os.Stdout),
"http://localhost:10101/",
Scopes,
AuthorizeURL,
TokenURL,
GroupEndpointURL,
LogoutURL,
ClientID,
ClientSecret,
Key,
configuredIPs,
)
if err != nil {
t.Fatalf("building auth object%s", err)
}
return a
}
func TestSetGRPCMetadata(t *testing.T) {
a := NewTestAuth(t)
for name, md := range map[string]metadata.MD{
"empty": {},
"something": {"cookie": []string{a.accessCookieName + "=something"}},
"somethingElse": {"cookie": []string{
a.accessCookieName + "=something",
a.refreshCookieName + "=something",
}},
"otherCookies": {"cookie": []string{a.accessCookieName + "=something", "blah=blah"}},
} {
t.Run(name, func(t *testing.T) {
ogCookies, _ := md["cookie"]
ctx := grpc.NewContextWithServerTransportStream(
metadata.NewIncomingContext(context.TODO(),
md,
),
NewServerTransportStream(),
)
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
if err := grpc.SendHeader(ctx, md); err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
md, ok = metadata.FromIncomingContext(ctx)
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
c, ok := md["cookie"]
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
var accessCookie, refreshCookie string
for _, cookie := range c {
if strings.HasPrefix(cookie, a.accessCookieName) {
accessCookie = cookie
} else if strings.HasPrefix(cookie, a.refreshCookieName) {
refreshCookie = cookie
}
if refreshCookie != "" && accessCookie != "" {
break
}
}
exp := a.accessCookieName + "=accesstoken!"
if accessCookie != exp {
t.Fatalf("expected '%v', got '%v'", exp, accessCookie)
}
exp = a.refreshCookieName + "=refreshtoken!"
if refreshCookie != exp {
t.Fatalf("expected '%v', got '%v'", exp, refreshCookie)
}
for _, cookie := range c {
if strings.HasPrefix(cookie, a.accessCookieName) || strings.HasPrefix(cookie, a.refreshCookieName) {
continue
}
found := false
for _, ogCookie := range ogCookies {
if cookie == ogCookie {
found = true
break
}
}
if !found {
t.Fatal("SetGRPCMetadata did not maintain the previous cookie list")
}
}
})
}
}
func TestAuth(t *testing.T) {
a := NewTestAuth(t)
t.Run("SetCookie", func(t *testing.T) {
w := httptest.NewRecorder()
err := a.SetCookie(w, "access", "refresh", time.Now().Add(time.Hour))
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
if w.Result().Cookies()[0].Value == "" {
t.Errorf("expected something, got empty string")
}
if got, want := w.Result().Cookies()[0].Path, "/"; got != want {
t.Fatalf("path=%s, want %s", got, want)
}
})
t.Run("KeyLength", func(t *testing.T) {
_, err := NewAuth(
logger.NewStandardLogger(os.Stdout),
"http://localhost:10101/",
[]string{"https://graph.microsoft.com/.default", "offline_access"},
"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
"https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
"https://login.microsoftonline.com/common/oauth2/v2.0/logout",
"e9088663-eb08-41d7-8f65-efb5f54bbb71",
"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
"DEADBEEFD",
[]string{},
)
if err == nil || !strings.Contains(err.Error(), "decoding secret key") {
t.Fatalf("expected error decoding secret key got: %v", err)
}
})
t.Run("GetSecretKey", func(t *testing.T) {
want, _ := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
if got := a.SecretKey(); !bytes.Equal(got, want) {
t.Fatalf("expected %v, got %v", got, want)
}
})
}
func TestAuthenticate(t *testing.T) {
cases := []struct {
name string
uid string
uname string
exp int64
refresh bool
refreshToken string
malformed bool
empty bool
groups []Group
err error
}{
{
name: "GoodToken",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
},
{
name: "Malformed",
malformed: true,
err: fmt.Errorf("parsing auth token: token contains an invalid number of segments"),
},
{
name: "Empty",
empty: true,
err: fmt.Errorf("auth token is empty"),
},
{
name: "ExpiredTokenNoRefresh",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
exp: -17764800,
err: fmt.Errorf("token is expired: refreshing token: 400 Bad Request"),
},
{
name: "ExpiredTokenYesRefresh",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
refresh: true,
refreshToken: "refreshToken",
exp: -17764800,
},
{
name: "ExpiredTokenYesRefreshButError",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
refresh: true,
refreshToken: "blah!!",
exp: -17764800,
err: fmt.Errorf("token is expired: refreshing token: 403 Forbidden"),
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
// setup the test
a := NewTestAuth(t)
token := ""
var err error
if !test.malformed && !test.empty {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = test.uid
claims["name"] = test.uname
if test.exp != 0 {
claims["exp"] = float64(test.exp)
}
token, err = tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
} else if !test.empty {
token = "asdfasdfasdfasdF"
}
if len(test.groups) > 0 {
a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
}
if test.refresh {
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
refresh := r.Form.Get("refresh_token")
if refresh != test.refreshToken {
t.Fatalf("refresh token not passed properly, expected %v, got %v", test.refreshToken, refresh)
return
}
if refresh != "refreshToken" {
http.Error(w, "bad token", http.StatusForbidden)
}
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = test.uid
claims["name"] = test.uname
expiry := float64(time.Now().Add(2 * time.Hour).Unix())
claims["exp"] = expiry
fresh, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
fmt.Fprintf(w, `{"access_token": "`+fresh+`", "refresh_token": "blah", "token_type": "bearer", "expires": `+strconv.FormatFloat(expiry, 'f', 0, 64)+` }`)
}))
defer srv.Close()
a.oAuthConfig.Endpoint.TokenURL = srv.URL
}
// do the actual testing
uinfo, err := a.Authenticate(token, test.refreshToken)
// okay this part kind of sucks bc we need to check errors and i
// dont want to write a whole new test for things that should have
// errors just to avoid this mess. errors.Is doesn't work either
if (test.err == nil && err != nil) || (test.err != nil && err == nil) {
t.Fatalf("expected %v, but got %v", test.err, err)
} else if test.err != nil && err != nil {
if test.err.Error() != err.Error() {
t.Fatalf("expected %v, but got %v", test.err, err)
} else {
return
}
}
if !reflect.DeepEqual(uinfo.Groups, test.groups) {
t.Fatalf("expected %v, got %v", test.groups, uinfo.Groups)
}
if !reflect.DeepEqual(uinfo.UserID, test.uid) {
t.Fatalf("expected %v, got %v", test.uid, uinfo.UserID)
}
if !reflect.DeepEqual(uinfo.UserName, test.uname) {
t.Fatalf("expected %v, got %v", test.uname, uinfo.UserName)
}
})
}
}
func TestAuthenticate_CleanCache(t *testing.T) {
// this deserves its own test bc it has gross setup required
t.Run("should clean", func(t *testing.T) {
a := NewTestAuth(t)
now := time.Now()
a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
a.lastCacheClean = now.Add(-45 * time.Minute)
_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
if a.lastCacheClean.Sub(now) <= time.Nanosecond {
t.Fatalf("cache should have been cleaned")
}
if _, ok := a.groupsCache["oldy"]; ok {
t.Errorf("oldy should have been deleted")
}
if _, ok := a.groupsCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
})
t.Run("shouldn't clean", func(t *testing.T) {
a := NewTestAuth(t)
now := time.Now()
a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
a.lastCacheClean = now
_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
if a.lastCacheClean.Sub(now) >= time.Nanosecond {
t.Fatalf("cache should not have been cleaned")
}
if _, ok := a.groupsCache["oldy"]; !ok {
t.Errorf("oldy should not have been deleted")
}
if _, ok := a.groupsCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
})
}
func TestGetGroups(t *testing.T) {
a := NewTestAuth(t)
a.groupsCache = map[string]cachedGroups{
"the world is changed": {
cacheTime: time.Now(),
groups: []Group{
{
GroupID: "a han noston ned wilith",
GroupName: "I smell it in the air",
},
},
},
}
srvNext := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := json.Marshal(
Groups{
Groups: []Group{
{
GroupID: "han mathon ne chae",
GroupName: "I feel it in the earth",
},
},
},
)
if err != nil {
t.Fatalf("unexpected error marshalling groups response: %v", err)
}
fmt.Fprintf(w, "%s", body)
}))
defer srvNext.Close()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := json.Marshal(
Groups{
NextLink: srvNext.URL,
Groups: []Group{
{
GroupID: "han mathon ne nen",
GroupName: "i feel it in the water",
},
},
},
)
if err != nil {
t.Fatalf("unexpected error marshalling groups response: %v", err)
}
fmt.Fprintf(w, "%s", body)
}))
defer srv.Close()
a.groupEndpoint = srv.URL
for name, test := range map[string]struct {
token string
groups []Group
}{
"InCache": {
token: "the world is changed",
groups: []Group{
{
GroupID: "a han noston ned wilith",
GroupName: "I smell it in the air",
},
},
},
"NotInCache": {
token: "i smell it in the air",
groups: []Group{
{
GroupID: "han mathon ne nen",
GroupName: "i feel it in the water",
},
{
GroupID: "han mathon ne chae",
GroupName: "I feel it in the earth",
},
},
},
} {
t.Run(name, func(t *testing.T) {
if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) {
t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err)
}
})
}
}
func TestDecodeHex(t *testing.T) {
t.Run("cantDecode", func(t *testing.T) {
_, err := decodeHex("gggg")
if err == nil {
t.Fatalf("expected err cannot decode slice, got nil")
}
})
t.Run("tooSmall", func(t *testing.T) {
_, err := decodeHex("DEADBEEF")
if err == nil {
t.Fatalf("expected err wrong length, got nil")
}
})
t.Run("tooBig", func(t *testing.T) {
_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
if err == nil {
t.Fatalf("expected err wrong length, got nil")
}
})
t.Run("justRight", func(t *testing.T) {
_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
if err != nil {
t.Fatalf("expected nil, got %v", err)
}
})
}
func TestHandlers(t *testing.T) {
a := NewTestAuth(t)
t.Run("login", func(t *testing.T) {
req := httptest.NewRequest("GET", "/login", nil)
w := httptest.NewRecorder()
a.Login(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
t.Fatalf("expected redirect, got %v", resp.StatusCode)
}
redirect := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
if got, err := resp.Location(); err != nil || got.String() != redirect {
t.Fatalf("expected %v, got %v", redirect, got.Path)
}
})
t.Run("logout", func(t *testing.T) {
req := httptest.NewRequest("GET", "/logout", nil)
w := httptest.NewRecorder()
req.AddCookie(
&http.Cookie{
Name: a.accessCookieName,
Value: "test",
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(3000000, 0),
},
)
req.AddCookie(
&http.Cookie{
Name: a.refreshCookieName,
Value: "test",
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(3000000, 0),
},
)
a.groupsCache["test"] = cachedGroups{}
a.Logout(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
t.Fatalf("expected redirect, got %v", resp.StatusCode)
}
redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL)
if got, err := resp.Location(); err != nil || got.String() != redirect {
t.Fatalf("expected %v, got %v", redirect, got.Path)
}
for _, c := range resp.Cookies() {
if c.Name == a.accessCookieName || c.Name == a.refreshCookieName {
if c.Value != "" {
t.Fatalf("cookie not set to empty value!")
}
want := time.Unix(0, 0).Unix()
got := c.Expires.Unix()
if want != got {
t.Fatalf("expected %v, got %v", want, got)
}
}
}
if _, ok := a.groupsCache["test"]; ok {
t.Fatalf("groups not deleted!")
}
})
t.Run("redirectGood", func(t *testing.T) {
req := httptest.NewRequest("GET", "/redirect", nil)
w := httptest.NewRecorder()
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = "user id"
claims["name"] = "user name"
expiresIn := 2 * time.Hour
exp := time.Now().Add(expiresIn)
expiry := float64(exp.Unix())
claims["exp"] = expiry
fresh, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write([]byte(body))
}))
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.Redirect(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
t.Fatalf("expected redirect, got %v", resp.StatusCode)
}
if got, err := resp.Location(); err != nil || got.String() != "/" {
t.Fatalf("expected %v, got %v", "/", got.Path)
}
cookies := resp.Cookies()
for _, c := range cookies {
if c.Name == a.accessCookieName && c.Value != fresh {
t.Fatalf("expected %v, got %v", exp, c.Value)
} else if c.Name == a.refreshCookieName && c.Value != "blah" {
t.Fatalf("expected %v, got %v", "blah", c.Value)
}
}
})
t.Run("redirectBad", func(t *testing.T) {
req := httptest.NewRequest("GET", "/redirect", nil)
w := httptest.NewRecorder()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Server Error", http.StatusInternalServerError)
}))
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.Redirect(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
}
})
}
// This type is used for mocking ServerTransportStreams in tests
type ServerTransportStream struct {
md metadata.MD
method string
}
func NewServerTransportStream() *ServerTransportStream {
return &ServerTransportStream{
md: metadata.MD{},
method: "test",
}
}
func (s *ServerTransportStream) Method() string {
return s.method
}
func (s *ServerTransportStream) SetHeader(md metadata.MD) error {
s.md = md
return nil
}
func (s *ServerTransportStream) SendHeader(md metadata.MD) error {
_ = md
return nil
}
func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
_ = md
return nil
}
func TestCleanOAuthConfig(t *testing.T) {
a := NewTestAuth(t)
res := a.CleanOAuthConfig()
assertEqual("", res.ClientSecret, t)
assertEqual(a.oAuthConfig.ClientID, res.ClientID, t)
assertEqual(a.oAuthConfig.RedirectURL, res.RedirectURL, t)
assertEqual(a.oAuthConfig.Scopes, res.Scopes, t)
assertEqual(a.oAuthConfig.Endpoint, res.Endpoint, t)
}
func assertEqual(exp, got interface{}, t *testing.T) {
if !reflect.DeepEqual(exp, got) {
t.Fatalf("expected %v, got %v", exp, got)
}
}
func TestCheckAllowedNetworks(t *testing.T) {
tests := []struct {
requestIP string
configuredIPs []string
isAdmin bool
}{
{
requestIP: "10.0.0.1",
configuredIPs: []string{"10.0.0.1"},
isAdmin: true,
},
{
requestIP: "10.0.0.3",
configuredIPs: []string{"10.0.0.1", "10.0.0.2"},
isAdmin: false,
},
{
requestIP: "10.0.0.2",
configuredIPs: []string{"10.0.0.1/30"},
isAdmin: true,
},
// it is possible for the client IP to have a port
{
requestIP: "10.0.0.2:22",
configuredIPs: []string{"10.0.0.1/30"},
isAdmin: true,
},
{
requestIP: "10.1.0.3",
configuredIPs: []string{"10.0.0.1/32"},
isAdmin: false,
},
{
requestIP: "10.0.0.254",
configuredIPs: []string{"10.0.0.1/24"},
isAdmin: true,
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
a := NewTestAuth(t)
if err := a.convertIP(test.configuredIPs); err != nil {
t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
}
got := a.CheckAllowedNetworks(test.requestIP)
if got != test.isAdmin {
t.Fatalf("expected %v, got %v", test.isAdmin, got)
}
})
}
}
func TestConvertIP(t *testing.T) {
tests := []struct {
configuredIPs []string
convertedIPs []net.IPNet
}{
{
configuredIPs: []string{"10.0.0.1"},
convertedIPs: []net.IPNet{
{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(32, 32)},
},
},
{
configuredIPs: []string{"10.0.0.1/30"},
convertedIPs: []net.IPNet{
{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(30, 32)},
},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
a := NewTestAuth(t)
if err := a.convertIP(test.configuredIPs); err != nil {
t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
}
if len(a.allowedNetworks) != len(test.convertedIPs) {
t.Fatalf("expected len of %v networks, got %v", len(test.convertedIPs), len(a.allowedNetworks))
}
for i := range a.allowedNetworks {
expected, got := test.convertedIPs[i], a.allowedNetworks[i]
if got.IP.String() != expected.IP.String() {
t.Fatalf("for IP, expected %v, got %v", expected.IP, got.IP)
}
if got.Mask.String() != expected.Mask.String() {
t.Fatalf("for mask, expected %v, got %v", expected.Mask.String(), got.Mask.String())
}
}
})
}
}

131
authz/authorization.go Normal file
View file

@ -0,0 +1,131 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package authz
import (
"fmt"
"io"
"io/ioutil"
"github.com/molecula/featurebase/v3/authn"
"gopkg.in/yaml.v2"
)
type GroupPermissions struct {
Permissions map[string]map[string]Permission `yaml:"user-groups"`
Admin string `yaml:"admin"`
}
type Permission string
const (
None Permission = ""
Read Permission = "read"
Write Permission = "write"
Admin Permission = "admin"
)
// Satisfies returns whether `p` satisfies the permissions required by `b`
func (p Permission) Satisfies(b Permission) bool {
switch p {
case "":
return b == ""
case "read":
return b == "" || b == "read"
case "write":
return b == "" || b == "read" || b == "write"
case "admin":
return b == "" || b == "read" || b == "write" || b == "admin"
}
return false
}
func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
permsData, err := ioutil.ReadAll(permsFile)
if err != nil {
return fmt.Errorf("reading permissions failed with error: %s", err)
}
err = yaml.UnmarshalStrict(permsData, &p)
if err != nil {
return fmt.Errorf("unmarshalling permissions failed with error: %s", err)
}
return
}
func (p *GroupPermissions) GetPermissions(user *authn.UserInfo, index string) (permission Permission, errors error) {
groups := user.Groups
if admin := p.IsAdmin(groups); admin {
return Admin, nil
}
allPermissions := map[Permission]bool{
Write: false,
Read: false,
}
if len(groups) == 0 {
return None, fmt.Errorf("user is not part of any groups in identity provider")
}
var groupsDenied []string
for _, group := range groups {
if _, ok := p.Permissions[group.GroupID]; ok {
if perm, ok := p.Permissions[group.GroupID][index]; ok {
allPermissions[perm] = true
} else {
return None, fmt.Errorf("user %s does not have permission to index %s", user.UserID, index)
}
} else {
groupsDenied = append(groupsDenied, group.GroupID)
}
}
if len(groupsDenied) == len(groups) {
return None, fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied)
}
if allPermissions[Write] {
return Write, nil
} else if allPermissions[Read] {
return Read, nil
} else {
return None, fmt.Errorf("no permissions found")
}
}
func (p *GroupPermissions) IsAdmin(groups []authn.Group) bool {
for _, group := range groups {
if p.Admin == group.GroupID {
return true
}
}
return false
}
func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission Permission) (indexList []string) {
// if user is admin, find all indexes in permissions file and return them
if p.IsAdmin(groups) {
for groupId := range p.Permissions {
for index := range p.Permissions[groupId] {
indexList = append(indexList, index)
}
}
return indexList
}
for _, group := range groups {
if _, ok := p.Permissions[group.GroupID]; ok {
for index, permission := range p.Permissions[group.GroupID] {
if permission.Satisfies(desiredPermission) {
indexList = append(indexList, index)
}
}
}
}
return indexList
}

305
authz/authorization_test.go Normal file
View file

@ -0,0 +1,305 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package authz_test
import (
"fmt"
"reflect"
"sort"
"strings"
"testing"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/authz"
)
func TestAuth_ReadPermissionsFile(t *testing.T) {
singleInput := `user-groups:
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "read"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
multiInput := `user-groups:
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "read"
"test2": "write"
"dca35310-ecda-4f23-86cd-876aee559900":
"test": "write"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
singlePermission := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
multiPermission := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read, "test2": authz.Write},
"dca35310-ecda-4f23-86cd-876aee559900": {"test": authz.Write}},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
tests := []struct {
input string
output authz.GroupPermissions
}{
{singleInput, singlePermission},
{multiInput, multiPermission},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
permFile := strings.NewReader(test.input)
var p authz.GroupPermissions
err := p.ReadPermissionsFile(permFile)
if err != nil {
t.Fatalf("readPermissionsFile error: %s", err)
}
if !reflect.DeepEqual(p, test.output) {
t.Fatalf("expected output %s, but got %s", test.output, p)
}
},
)
}
}
func TestAuth_GetPermissions(t *testing.T) {
// initializes different example of permissions file in yaml
permissions1 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "read"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
permissions2 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee559900":
"test": "write"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
permissions3 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "write"
"test2": "read"
"dca35310-ecda-4f23-86cd-876aee559900":
"test": "read"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
permissions4 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee559900":
"test": ""
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
// initializes groups that are returned from identity provider
groupName := "name"
groupsList1 := []authn.Group{}
groupsList2 := []authn.Group{{
GroupID: "fake-group",
GroupName: groupName}}
groupsList3 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: groupName},
{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: groupName},
}
groupsList4 := []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: groupName}}
tests := []struct {
yamlData string
groups []authn.Group
index string
userAccess authz.Permission
err string
}{
{
permissions1,
groupsList1,
"test",
authz.None,
"user is not part of any groups in identity provider",
},
{
permissions1,
groupsList3,
"test1",
authz.None,
"does not have permission to index",
},
{
permissions2,
groupsList2,
"test",
authz.None,
"does not have permission to FeatureBase",
},
{
permissions1,
groupsList3,
"test",
authz.Read,
"",
},
{
permissions2,
groupsList3,
"test",
authz.Write,
"",
},
{
permissions3,
groupsList4,
"test",
authz.Admin,
"",
},
{
permissions4,
groupsList3,
"test",
authz.None,
"no permissions found",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
permFile := strings.NewReader(test.yamlData)
var p authz.GroupPermissions
if err := p.ReadPermissionsFile(permFile); err != nil {
t.Errorf("Error: %s", err)
}
p1, err := p.GetPermissions(&authn.UserInfo{Groups: test.groups}, test.index)
if p1 != test.userAccess {
t.Errorf("expected permission to be %s, but got %s", test.userAccess, p1)
}
if err != nil {
if !strings.Contains(err.Error(), test.err) {
t.Errorf("expected error to contain %s, but got %s", test.err, err.Error())
}
}
})
}
}
func TestAuth_IsAdmin(t *testing.T) {
group1 := []authn.Group{
{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
}
group2 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
}
groupPermissions := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Write},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
tests := []struct {
groups []authn.Group
groupPermissions authz.GroupPermissions
output bool
}{
{
group1, groupPermissions, true,
},
{
group2, groupPermissions, false,
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
p := test.groupPermissions
resp := p.IsAdmin(test.groups)
if resp != test.output {
t.Errorf("expected %t, but got %t", test.output, resp)
}
})
}
}
func TestAuth_GetAuthorizedIndexList(t *testing.T) {
group1 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
}
group2 := []authn.Group{
{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
}
group3 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: "group-name"},
}
p := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {
"test1": authz.Read,
"test2": authz.Write,
},
"dca35310-ecda-4f23-86cd-876aee559900": {
"test3": authz.Read,
},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
tests := []struct {
groups []authn.Group
permission authz.Permission
output []string
}{
{
group1,
authz.Read,
[]string{"test1", "test2"},
},
{
group1,
authz.Write,
[]string{"test2"},
},
{
group3,
authz.Write,
nil,
},
{
group2,
authz.Read,
[]string{"test1", "test2", "test3"},
},
{
group2,
authz.Write,
[]string{"test1", "test2", "test3"},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
indexList := p.GetAuthorizedIndexList(test.groups, test.permission)
sort.Strings(indexList)
if !reflect.DeepEqual(indexList, test.output) {
t.Errorf("expected %s, but got %s", test.output, indexList)
}
})
}
}

423
boltdb/attrstore.go
View file

@ -1,423 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa/v2"
"github.com/pkg/errors"
)
// attrBlockSize is the size of attribute blocks for anti-entropy.
const attrBlockSize = 100
// attrCache represents a cache for attributes.
type attrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
}
// Get returns the cached attributes for a given id.
func (c *attrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
}
// Set updates the cached attributes for a given id.
func (c *attrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
}
// attrStore represents a storage layer for attributes.
type attrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *attrCache
}
// newAttrCache returns a new instance of AttrCache.
func newAttrCache() *attrCache {
return &attrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) pilosa.AttrStore {
return &attrStore{
path: path,
attrCache: newAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *attrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *attrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return errors.Wrap(err, "opening storage")
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte("attrs"))
return err
}); err != nil {
return errors.Wrap(err, "initializing")
}
return nil
}
// Close closes the store.
func (s *attrStore) Close() error {
if s.db != nil {
s.db.Close()
}
return nil
}
// Attrs returns a set of attributes by ID.
func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
return err
}); err != nil {
return nil, errors.Wrap(err, "finding attributes")
}
// Add to cache.
s.attrCache.Set(id, m)
return m, nil
}
// SetAttrs sets attribute values for a given ID.
func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return errors.Wrap(err, "checking attrs")
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return errors.Wrap(err, "updating store")
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
}
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *attrStore) Blocks() (blocks []pilosa.AttrBlock, err error) {
err = s.db.View(func(tx *bolt.Tx) error {
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize)
// Iterate over each block.
for cur.nextBlock() {
block := pilosa.AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
// hash function writes don't usually need to be checked
_, _ = h.Write(k)
_, _ = h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return nil
})
if err != nil {
return nil, errors.Wrap(err, "getting blocks")
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *attrStore) BlockData(i uint64) (m map[uint64]map[string]interface{}, err error) {
m = make(map[uint64]map[string]interface{})
// Start read-only transaction.
err = s.db.View(func(tx *bolt.Tx) error {
// Move to the start of the block.
min := u64tob(i * attrBlockSize)
max := u64tob((i + 1) * attrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
attrs, err := pilosa.DecodeAttrs(v)
if err != nil {
return errors.Wrap(err, "decoding attrs")
}
m[btou64(k)] = attrs
}
return nil
})
if err != nil {
return nil, errors.Wrap(err, "getting block data")
}
return m, nil
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
return pilosa.DecodeAttrs(v)
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
}
}
// Marshal and save new values.
buf, err := pilosa.EncodeAttrs(attr)
if err != nil {
return nil, errors.Wrap(err, "encoding attrs")
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, errors.Wrap(err, "saving attrs")
}
return attr, nil
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor { // nolint: unparam
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}

577
boltdb/translate.go
View file

@ -1,39 +1,53 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package boltdb
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"time"
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa/v2"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
"runtime/pprof"
)
var _ = pprof.StartCPUProfile
var (
// ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader
// and the underlying store is closed.
ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
// ErrTranslateKeyNotFound is returned when translating key
// and the underlying store returns an empty set
ErrTranslateKeyNotFound = errors.New("boltdb: translating key returned empty set")
bucketKeys = []byte("keys")
bucketIDs = []byte("ids")
bucketFree = []byte("free")
freeKey = []byte("free")
)
const (
// snapshotExt is the file extension used for an in-process snapshot.
snapshotExt = ".snapshotting"
errFmtTranslateBucketNotFound = "boltdb: translate bucket '%s' not found"
)
// OpenTranslateStore opens and initializes a boltdb translation store.
func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) {
s := NewTranslateStore(index, field)
func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) {
s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled)
s.Path = path
if err := s.Open(); err != nil {
return nil, err
@ -45,46 +59,71 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error
var _ pilosa.TranslateStore = &TranslateStore{}
// TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
// An empty string will be converted into the sentinel byte slice:
// var emptyKey = []byte{
// 0x00, 0x00, 0x00,
// 0x4d, 0x54, 0x4d, 0x54, // MTMT
// 0x00,
// 0xc2, 0xa0, // NO-BREAK SPACE
// 0x00,
// }
type TranslateStore struct {
mu sync.RWMutex
db *bolt.DB
index string
field string
index string
field string
partitionID int
partitionN int
once sync.Once
closing chan struct{}
readOnly bool
writeNotify chan struct{}
readOnly bool
fsyncEnabled bool
writeNotify chan struct{}
// File path to database file.
Path string
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore(index, field string) *TranslateStore {
func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore {
return &TranslateStore{
index: index,
field: field,
closing: make(chan struct{}),
writeNotify: make(chan struct{}),
index: index,
field: field,
partitionID: partitionID,
partitionN: partitionN,
closing: make(chan struct{}),
writeNotify: make(chan struct{}),
fsyncEnabled: fsyncEnabled,
}
}
// Open opens the translate file.
func (s *TranslateStore) Open() (err error) {
if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
// add the path to the problem database if we panic handling it.
defer func() {
r := recover()
if r != nil {
panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r))
}
}()
if err := os.MkdirAll(filepath.Dir(s.Path), 0750); err != nil {
return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
} else if s.db, err = bolt.Open(s.Path, 0600, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil {
return errors.Wrapf(err, "open file: %s", err)
}
// Initialize buckets.
if err := s.db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte("keys")); err != nil {
if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil {
return err
} else if _, err := tx.CreateBucketIfNotExists([]byte("ids")); err != nil {
} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
return err
} else if _, err := tx.CreateBucketIfNotExists(bucketFree); err != nil {
return err
}
return nil
@ -108,6 +147,11 @@ func (s *TranslateStore) Close() (err error) {
return nil
}
// PartitionID returns the partition id the store was initialized with.
func (s *TranslateStore) PartitionID() int {
return s.partitionID
}
// ReadOnly returns true if the store is in read-only mode.
func (s *TranslateStore) ReadOnly() bool {
s.mu.RLock()
@ -135,109 +179,139 @@ func (s *TranslateStore) Size() int64 {
return tx.Size()
}
// TranslateKeys converts a string key to an integer ID.
// If key does not have an associated id then one is created.
func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
// Find id by key under read lock.
if err := s.db.View(func(tx *bolt.Tx) error {
id = findIDByKey(tx.Bucket([]byte("keys")), key)
return nil
}); err != nil {
return 0, err
} else if id != 0 {
return id, nil
}
if s.ReadOnly() {
return 0, pilosa.ErrTranslateStoreReadOnly
}
// Find or create id under write lock.
var written bool
if err := s.db.Update(func(tx *bolt.Tx) (err error) {
bkt := tx.Bucket([]byte("keys"))
if id = findIDByKey(bkt, key); id != 0 {
return nil
} else if id, err = bkt.NextSequence(); err != nil {
return err
} else if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
return err
} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
return err
// FindKeys looks up the ID for each key.
// Keys are not created if they do not exist.
// Missing keys are not considered errors, so the length of the result may be less than that of the input.
func (s *TranslateStore) FindKeys(keys ...string) (map[string]uint64, error) {
result := make(map[string]uint64, len(keys))
err := s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(bucketKeys)
if bkt == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
}
for _, key := range keys {
id, _ := findIDByKey(bkt, key)
if id == 0 {
// The key does not exist.
continue
}
result[key] = id
}
written = true
return nil
}); err != nil {
return 0, err
})
if err != nil {
return nil, err
}
if written {
s.notifyWrite()
}
return id, nil
return result, nil
}
// TranslateKeys converts a string key to an integer ID.
// If key does not have an associated id then one is created.
func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
if len(keys) == 0 {
return nil, nil
}
// Allocate slice for ID mapping.
ids = make([]uint64, len(keys))
// Find ids by key under read lock.
var found int
if err := s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket([]byte("keys"))
for i, key := range keys {
if id := findIDByKey(bkt, key); id != 0 {
ids[i] = id
found++
}
}
return nil
}); err != nil {
return nil, err
} else if found == len(keys) {
return ids, nil
}
// translateTransactionSize governs the number of writes to a single
// boltDB bucket we will make in a single db.Update(), before starting
// a new Update. We do this because Put() is quadratic, but Commit is
// expensive enough that we want to do a fair number of updates before
// paying for it.
const translateTransactionSize = 16384
// CreateKeys maps all keys to IDs, creating the IDs if they do not exist.
// If the translator is read-only, this will return an error.
func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) {
if s.ReadOnly() {
return ids, pilosa.ErrTranslateStoreReadOnly
return nil, pilosa.ErrTranslateStoreReadOnly
}
// Find or create ids under write lock if any keys were not found.
var written bool
if err := s.db.Update(func(tx *bolt.Tx) (err error) {
bkt := tx.Bucket([]byte("keys"))
for i, key := range keys {
if ids[i] != 0 {
continue
written := false
result := make(map[string]uint64, len(keys))
idScratch := make([]byte, translateTransactionSize*8)
for len(keys) > 0 {
// boltdb performs badly if you write really large numbers of
// keys all at once...
err := s.db.Update(func(tx *bolt.Tx) error {
keyBucket := tx.Bucket(bucketKeys)
if keyBucket == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
}
idBucket := tx.Bucket(bucketIDs)
if idBucket == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs)
}
freeBucket := tx.Bucket(bucketFree)
if freeBucket == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketFree)
}
puts := 0
if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
continue
} else if ids[i], err = bkt.NextSequence(); err != nil {
return err
} else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
return err
} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
return err
// we create a freeIDGetter to reduce marshalling
getter := newFreeIDGetter(freeBucket)
defer getter.Close()
for idx, key := range keys {
id, boltKey := findIDByKey(keyBucket, key)
if id != 0 {
result[key] = id
continue
}
// see if we can re-use any IDs first
if id = getter.GetFreeID(); id == 0 {
id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
}
idBytes := idScratch[puts*8 : puts*8+8]
binary.BigEndian.PutUint64(idBytes, id)
puts++
if err := keyBucket.Put(boltKey, idBytes); err != nil {
return err
} else if err := idBucket.Put(idBytes, boltKey); err != nil {
return err
}
result[key] = id
written = true
if puts == translateTransactionSize {
keys = keys[idx+1:]
return nil
}
}
written = true
keys = keys[len(keys):]
return nil
})
if err != nil {
return nil, err
}
return nil
}); err != nil {
return nil, err
}
if written {
s.notifyWrite()
}
return ids, nil
return result, nil
}
// Match finds the IDs of all keys matching a filter.
func (s *TranslateStore) Match(filter func([]byte) bool) ([]uint64, error) {
var matches []uint64
err := s.db.View(func(tx *bolt.Tx) error {
// This uses the id bucket instead of the key bucket so that matches are produced in sorted order.
idBucket := tx.Bucket(bucketIDs)
if idBucket == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs)
}
return idBucket.ForEach(func(id, key []byte) error {
if bytes.Equal(key, emptyKey) {
key = nil
}
if filter(key) {
matches = append(matches, btou64(id))
}
return nil
})
})
if err != nil {
return nil, err
}
return matches, nil
}
// TranslateID converts an integer ID to a string key.
@ -248,7 +322,7 @@ func (s *TranslateStore) TranslateID(id uint64) (string, error) {
return "", err
}
defer func() { _ = tx.Rollback() }()
return findKeyByID(tx.Bucket([]byte("ids")), id), nil
return findKeyByID(tx.Bucket(bucketIDs), id), nil
}
// TranslateIDs converts a list of integer IDs to a list of string keys.
@ -263,9 +337,11 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
}
defer func() { _ = tx.Rollback() }()
bucket := tx.Bucket(bucketIDs)
keys := make([]string, len(ids))
for i, id := range ids {
keys[i] = findKeyByID(tx.Bucket([]byte("ids")), id)
keys[i] = findKeyByID(bucket, id)
}
return keys, nil
}
@ -273,9 +349,9 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
// ForceSet writes the id/key pair to the store even if read only. Used by replication.
func (s *TranslateStore) ForceSet(id uint64, key string) error {
if err := s.db.Update(func(tx *bolt.Tx) (err error) {
if err := tx.Bucket([]byte("keys")).Put([]byte(key), u64tob(id)); err != nil {
if err := tx.Bucket(bucketKeys).Put([]byte(key), u64tob(id)); err != nil {
return err
} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
} else if err := tx.Bucket(bucketIDs).Put(u64tob(id), []byte(key)); err != nil {
return err
}
return nil
@ -286,7 +362,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
return nil
}
// Reader returns a reader that streams the underlying data file.
// EntryReader returns a reader that streams the underlying data file.
func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
ctx, cancel := context.WithCancel(ctx)
return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@ -311,9 +387,7 @@ func (s *TranslateStore) notifyWrite() {
// MaxID returns the highest id in the store.
func (s *TranslateStore) MaxID() (max uint64, err error) {
if err := s.db.View(func(tx *bolt.Tx) error {
if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil {
max = btou64(key)
}
max = maxID(tx)
return nil
}); err != nil {
return 0, err
@ -321,6 +395,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) {
return max, nil
}
// WriteTo writes the contents of the store to the writer.
func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
tx, err := s.db.Begin(false)
if err != nil {
return 0, err
}
defer func() { _ = tx.Rollback() }()
return tx.WriteTo(w)
}
// ReadFrom reads the content and overwrites the existing store.
func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
// Close store.
if err := s.Close(); err != nil {
return 0, errors.Wrap(err, "closing store")
}
// Create a temporary file to snapshot to.
snapshotPath := s.Path + snapshotExt
file, err := os.Create(snapshotPath)
if err != nil {
return n, errors.Wrap(err, "creating snapshot file")
}
// Write payload to snapshot.
if n, err = io.Copy(file, r); err != nil {
file.Close()
return n, errors.Wrap(err, "snapshot write to")
}
// we close the file here so we don't still have it open when trying
// to open it in a moment.
file.Close()
// Move snapshot to data file location.
if err := os.Rename(snapshotPath, s.Path); err != nil {
return n, errors.Wrap(err, "renaming snapshot")
}
// Re-open the store.
if err := s.Open(); err != nil {
return n, errors.Wrap(err, "re-opening store")
}
return n, nil
}
// MaxID returns the highest id in the store.
func maxID(tx *bolt.Tx) uint64 {
if key, _ := tx.Bucket(bucketIDs).Cursor().Last(); key != nil {
return btou64(key)
}
return 0
}
type TranslateEntryReader struct {
ctx context.Context
store *TranslateStore
@ -353,7 +482,7 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
var found bool
if err := r.store.db.View(func(tx *bolt.Tx) error {
// Find ID/key lookup at offset or later.
cur := tx.Bucket([]byte("ids")).Cursor()
cur := tx.Bucket(bucketIDs).Cursor()
key, value := cur.Seek(u64tob(r.offset))
if key == nil {
return nil
@ -387,13 +516,203 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
}
}
func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
if value := bkt.Get([]byte(key)); value != nil {
return btou64(value)
type boltWrapper struct {
tx *bolt.Tx
db *bolt.DB
}
func (w *boltWrapper) Commit() error {
if w.tx != nil {
return w.tx.Commit()
}
return 0
return nil
}
func (w *boltWrapper) Rollback() {
if w.tx != nil {
w.tx.Rollback()
}
}
func (s *TranslateStore) FreeIDs() (*roaring.Bitmap, error) {
result := roaring.NewBitmap()
err := s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(bucketFree)
if bkt == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
}
b := bkt.Get(freeKey)
err := result.UnmarshalBinary(b)
if err != nil {
return err
}
return nil
})
return result, err
}
func (s *TranslateStore) MergeFree(tx *bolt.Tx, newIDs *roaring.Bitmap) error {
bkt := tx.Bucket(bucketFree)
b := bkt.Get(freeKey)
buf := new(bytes.Buffer)
if b != nil { //if existing combine with newIDs
before := roaring.NewBitmap()
err := before.UnmarshalBinary(b)
if err != nil {
return err
}
final := newIDs.Union(before)
_, err = final.WriteTo(buf)
if err != nil {
return err
}
} else {
newIDs.WriteTo(buf)
}
return bkt.Put(freeKey, buf.Bytes())
}
// Delete removes the lookeup pairs in order to make avialble for reuse but doesn't commit the
// transaction for that is tied to the associated rbf transaction being successful
func (s *TranslateStore) Delete(records *roaring.Bitmap) (pilosa.Commitor, error) {
tx, err := s.db.Begin(true)
if err != nil {
return nil, err
}
keyBucket := tx.Bucket(bucketKeys)
idBucket := tx.Bucket(bucketIDs)
ids := records.Slice()
for i := range ids {
id := u64tob(ids[i])
boltKey := idBucket.Get(id)
err = keyBucket.Delete(boltKey)
if err != nil {
tx.Rollback()
return &boltWrapper{}, err
}
err = idBucket.Delete(id)
if err != nil {
tx.Rollback()
return &boltWrapper{}, err
}
}
return &boltWrapper{tx: tx}, s.MergeFree(tx, records)
}
// emptyKey is a sentinel byte slice which stands for "" as a key.
var emptyKey = []byte{
0x00, 0x00, 0x00,
0x4d, 0x54, 0x4d, 0x54, // MTMT
0x00,
0xc2, 0xa0, // NO-BREAK SPACE
0x00,
}
func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
var boltKey []byte
if key == "" {
boltKey = emptyKey
} else {
boltKey = []byte(key)
}
if value := bkt.Get(boltKey); value != nil {
return btou64(value), boltKey
}
return 0, boltKey
}
// freeIDGetter reduces the amount of marshaling required to get multiple ids
type freeIDGetter struct {
freeBucket *bolt.Bucket
b *roaring.Bitmap
changed bool
}
// newFreeIDGetter initializes a new freeIDGetter. If at any point there is a
// failure, it returns an error.
//
// NOTE: For changes to be persisted to the bucket, you must call
// (*freeIDGetter).Close()
func newFreeIDGetter(freeBucket *bolt.Bucket) *freeIDGetter {
g := &freeIDGetter{
freeBucket: freeBucket,
}
// we ignore this value because it's okay if we dont have a bitmap just yet
_ = g.getBitmap()
return g
}
func (g *freeIDGetter) getBitmap() bool {
if g.b == nil {
// get the bitmap from freeBucket
value := g.freeBucket.Get(freeKey)
if value == nil {
return false
}
// turn the value into a bitmap
b := roaring.NewBitmap()
if err := b.UnmarshalBinary(value); err != nil {
return false
}
g.b = b
}
return true
}
// GetFreeID tries to get a free ID from the free id bucket. If at any point it
// fails to do so, it returns a 0. Otherwise, it returns the first free ID in the
// bucket
func (g *freeIDGetter) GetFreeID() (id uint64) {
if !g.getBitmap() {
return 0
}
// get the first free id
id, ok := g.b.Min()
if !ok {
return 0
}
// remove that id from the free id bitmap
if changed, err := g.b.RemoveN(id); changed == 0 || err != nil {
return 0
} else {
g.changed = true
}
return id
}
// Close persists any changes to the bitmap back to the bucket and then nils the
// references for safety.
func (g *freeIDGetter) Close() error {
if g.changed {
// convert bitmap to binary
buf, err := g.b.MarshalBinary()
if err != nil {
return errors.Wrap(err, "closing free ID Getter")
}
// put updated bitmap back into the freeBucket
if err := g.freeBucket.Put(freeKey, buf); err != nil {
return errors.Wrap(err, "closing free ID Getter")
}
}
g.b = nil
g.freeBucket = nil
return nil
}
func findKeyByID(bkt *bolt.Bucket, id uint64) string {
return string(bkt.Get(u64tob(id)))
boltKey := bkt.Get(u64tob(id))
if bytes.Equal(boltKey, emptyKey) {
return ""
}
return string(boltKey)
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }

107
boltdb/translate_internal_test.go Normal file
View file

@ -0,0 +1,107 @@
package boltdb
import (
"path/filepath"
"testing"
"github.com/molecula/featurebase/v3/roaring"
bolt "go.etcd.io/bbolt"
)
func TestGetFreeID(t *testing.T) {
boltDir := t.TempDir()
db, err := bolt.Open(filepath.Join(boltDir, "testDB"), 0600, nil)
if err != nil {
t.Fatalf("unexpected error opening test boltdb: %v", err)
}
defer db.Close()
makeTestBucket := func(tx *bolt.Tx, b *roaring.Bitmap) *bolt.Bucket {
if b == nil {
t.Fatalf("unexpected nil bitmap")
}
free, err := tx.CreateBucketIfNotExists(bucketFree)
if err != nil {
t.Fatalf("unexpected error making freeBucket: %v", err)
}
buf, err := b.MarshalBinary()
if err != nil {
t.Fatalf("unexpected error marshaling bitmap (%v) to binary: %v", b, err)
}
if err := free.Put(freeKey, buf); err != nil {
t.Fatalf("unexpected error adding data (%v) to freeBucket: %v", b, err)
}
return free
}
for name, test := range map[string]struct {
bits *roaring.Bitmap
want uint64
}{
"bucket is there, but nobody's home": {
bits: roaring.NewBitmap(),
want: 0,
},
"good bucket": {
bits: roaring.NewBitmap(1, 2, 34, 55, 9000),
want: 1,
},
} {
t.Run(name, func(t *testing.T) {
tx, err := db.Begin(true)
if err != nil {
t.Fatalf("unexpected error starting bolt transaction: %v", err)
}
defer tx.Rollback()
freeBucket := makeTestBucket(tx, test.bits)
getter := newFreeIDGetter(freeBucket)
defer getter.Close()
if got := getter.GetFreeID(); got != test.want {
t.Fatalf("expected %v got %v", test.want, got)
}
})
}
t.Run("CorrectOrdering", func(t *testing.T) {
tx, err := db.Begin(true)
if err != nil {
t.Fatalf("unexpected error starting bolt transaction: %v", err)
}
defer tx.Rollback()
bucket := makeTestBucket(tx, roaring.NewBitmap(1, 34, 2, 55, 9000))
getter := newFreeIDGetter(bucket)
defer getter.Close()
for _, want := range []uint64{1, 2, 34, 55, 9000} {
if got := getter.GetFreeID(); got != want {
t.Fatalf("expected %v got %v", want, got)
}
}
if got := getter.GetFreeID(); got != 0 {
t.Fatalf("expected 0 got %v", got)
}
})
t.Run("NotABitmap", func(t *testing.T) {
tx, err := db.Begin(true)
if err != nil {
t.Fatalf("unexpected error starting bolt transaction: %v", err)
}
defer tx.Rollback()
free, err := tx.CreateBucketIfNotExists(bucketFree)
if err != nil {
t.Fatalf("unexpected error making freeBucket: %v", err)
}
if err := free.Put(freeKey, []byte("this isn't right!")); err != nil {
t.Fatalf("unexpected error adding data to freeBucket: %v", err)
}
getter := newFreeIDGetter(free)
defer getter.Close()
if got := getter.GetFreeID(); got != 0 {
t.Fatalf("expected 0 got %v", got)
}
})
}

385
boltdb/translate_test.go
View file

@ -1,125 +1,99 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package boltdb_test
import (
"bytes"
"context"
"io/ioutil"
"os"
"fmt"
"reflect"
"strconv"
"testing"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/testhook"
)
func TestTranslateStore_TranslateKey(t *testing.T) {
s := MustOpenNewTranslateStore()
//var vv = pilosa.VV
func TestTranslateStore_CreateKeys(t *testing.T) {
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Ensure initial key translates to ID 1.
if id, err := s.TranslateKey("foo"); err != nil {
ids, err := s.CreateKeys("abc", "abc")
if err != nil {
t.Fatal(err)
} else if got, want := id, uint64(1); got != want {
t.Fatalf("TranslateKey()=%d, want %d", got, want)
} else if _, ok := ids["abc"]; !ok {
t.Fatalf(`missing "abc"; got %v`, ids)
} else if len(ids) > 1 {
t.Fatalf("expected one key, got %d in %v", len(ids), ids)
}
// Ensure next key autoincrements.
if id, err := s.TranslateKey("bar"); err != nil {
// Ensure different keys translate to different IDs.
ids1, err := s.CreateKeys("foo", "bar")
if err != nil {
t.Fatal(err)
} else if got, want := id, uint64(2); got != want {
t.Fatalf("TranslateKey()=%d, want %d", got, want)
}
// Ensure retranslating existing key returns original ID.
if id, err := s.TranslateKey("foo"); err != nil {
t.Fatal(err)
} else if got, want := id, uint64(1); got != want {
t.Fatalf("TranslateKey()=%d, want %d", got, want)
}
}
func TestTranslateStore_TranslateKeys(t *testing.T) {
s := MustOpenNewTranslateStore()
defer MustCloseTranslateStore(s)
// Ensure initial keys translate to incrementing IDs.
if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
t.Fatal(err)
} else if got, want := ids[0], uint64(1); got != want {
t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
} else if got, want := ids[1], uint64(2); got != want {
t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
} else if foo, bar := ids1["foo"], ids1["bar"]; foo == bar {
t.Fatalf(`"foo" and "bar" map back to the same ID %d`, foo)
}
// Ensure retranslation returns original IDs.
if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
if ids, err := s.CreateKeys("bar", "foo"); err != nil {
t.Fatal(err)
} else if got, want := ids[0], uint64(1); got != want {
t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
} else if got, want := ids[1], uint64(2); got != want {
t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
} else if !reflect.DeepEqual(ids, ids1) {
t.Fatalf("retranslation produced result %v which is different from original translation %v", ids, ids1)
}
// Ensure retranslating with existing and non-existing keys returns correctly.
if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
if ids, err := s.CreateKeys("foo", "baz", "bar"); err != nil {
t.Fatal(err)
} else if got, want := ids[0], uint64(1); got != want {
t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
} else if got, want := ids[1], uint64(3); got != want {
t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
} else if got, want := ids[2], uint64(2); got != want {
t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
} else if got, want := ids["foo"], ids1["foo"]; got != want {
t.Fatalf(`mismatched ID %d for "foo" (previously %d)`, got, want)
} else if _, ok := ids["baz"]; !ok {
t.Fatalf(`missing translation for "baz"; got %v`, ids)
} else if got, want := ids["bar"], ids1["bar"]; got != want {
t.Fatalf(`mismatched ID %d for "bar" (previously %d)`, got, want)
}
}
func TestTranslateStore_TranslateID(t *testing.T) {
s := MustOpenNewTranslateStore()
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Setup initial keys.
if _, err := s.TranslateKey("foo"); err != nil {
t.Fatal(err)
} else if _, err := s.TranslateKey("bar"); err != nil {
ids, err := s.CreateKeys("foo", "bar", "")
if err != nil {
t.Fatal(err)
}
// Ensure IDs can be translated back to keys.
if key, err := s.TranslateID(1); err != nil {
t.Fatal(err)
} else if got, want := key, "foo"; got != want {
t.Fatalf("TranslateID()=%s, want %s", got, want)
}
if key, err := s.TranslateID(2); err != nil {
t.Fatal(err)
} else if got, want := key, "bar"; got != want {
t.Fatalf("TranslateID()=%s, want %s", got, want)
for key, id := range ids {
k, err := s.TranslateID(id)
if err != nil {
t.Fatal(err)
}
if k != key {
t.Fatalf("TranslateID()=%s, want %s", k, key)
}
}
}
func TestTranslateStore_TranslateIDs(t *testing.T) {
s := MustOpenNewTranslateStore()
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Setup initial keys.
if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
ids, err := s.CreateKeys("foo", "bar")
if err != nil {
t.Fatal(err)
}
// Ensure IDs can be translated back to keys.
if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil {
if keys, err := s.TranslateIDs([]uint64{ids["foo"], ids["bar"], 1}); err != nil {
t.Fatal(err)
} else if got, want := keys[0], "foo"; got != want {
t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want)
@ -130,13 +104,114 @@ func TestTranslateStore_TranslateIDs(t *testing.T) {
}
}
func TestTranslateStore_FindKeys(t *testing.T) {
cases := []struct {
name string
data []string
lookup []string
}{
{
name: "All",
data: []string{"plugh", "xyzzy", "h"},
lookup: []string{"plugh", "xyzzy", "h"},
},
{
name: "Extra",
data: []string{"plugh", "xyzzy", "h"},
lookup: []string{"plugh", "xyzzy", "h", "65"},
},
{
name: "None",
data: []string{"a", "b", "c"},
lookup: []string{"d", "e"},
},
{
name: "Empty",
lookup: []string{"h"},
},
{
name: "LookupNothing",
},
}
for _, c := range cases {
c := c
t.Run(c.name, func(t *testing.T) {
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
var naiveMap map[string]uint64
if c.data != nil {
// Load in key data.
keys := c.data
ids, err := s.CreateKeys(keys...)
if err != nil {
t.Errorf("failed to import keys: %v", err)
return
}
if len(ids) != len(keys) {
t.Errorf("mapped %d keys to %d ids", len(keys), len(ids))
return
}
naiveMap = ids
}
// Compute expected lookup result.
result := map[string]uint64{}
for _, key := range c.lookup {
id, ok := naiveMap[key]
if !ok {
// The key is expected to be missing.
continue
}
result[key] = id
}
// Find the keys.
found, err := s.FindKeys(c.lookup...)
if err != nil {
t.Errorf("failed to find keys: %v", err)
} else if !reflect.DeepEqual(result, found) {
t.Errorf("expected %v but found %v", result, found)
}
})
}
}
func TestTranslateStore_MaxID(t *testing.T) {
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Generate a bunch of keys.
var lastk uint64
for i := 0; i < 1026; i++ {
key := strconv.Itoa(i)
ids, err := s.CreateKeys(key)
if err != nil {
t.Fatalf("translating %d: %v", i, err)
}
lastk = ids[key]
}
// Verify the max ID.
max, err := s.MaxID()
if err != nil {
t.Fatalf("checking max ID: %v", err)
}
if max != lastk {
t.Fatalf("last key is %d but max is %d", lastk, max)
}
}
func TestTranslateStore_EntryReader(t *testing.T) {
t.Run("OK", func(t *testing.T) {
s := MustOpenNewTranslateStore()
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Create multiple new keys.
if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
ids1, err := s.CreateKeys("foo", "bar")
if err != nil {
t.Fatal(err)
}
@ -151,7 +226,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Read first entry.
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(1); got != want {
} else if got, want := entry.ID, ids1["foo"]; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "foo"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@ -160,21 +235,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Read next entry.
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(2); got != want {
} else if got, want := entry.ID, ids1["bar"]; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "bar"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
}
// Insert next key while reader is open.
if _, err := s.TranslateKey("baz"); err != nil {
ids2, err := s.CreateKeys("baz")
if err != nil {
t.Fatal(err)
}
// Read newly created entry.
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(3); got != want {
} else if got, want := entry.ID, ids2["baz"]; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "baz"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@ -188,7 +264,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Ensure reader will read as soon as a new write comes in using WriteNotify().
t.Run("WriteNotify", func(t *testing.T) {
s := MustOpenNewTranslateStore()
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Start reader from initial position.
@ -198,20 +274,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
}
defer r.Close()
// cache holds the translated key id so we can check it later
cache := make(chan uint64)
// Insert key in separate goroutine.
// Sleep momentarily to reader hangs.
translateErr := make(chan error)
go func() {
time.Sleep(100 * time.Millisecond)
if _, err := s.TranslateKey("foo"); err != nil {
ids, err := s.CreateKeys("foo")
if err != nil {
translateErr <- err
}
cache <- ids["foo"]
}()
var entry pilosa.TranslateEntry
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(1); got != want {
} else if got, want := entry.ID, <-cache; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "foo"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@ -226,7 +307,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Ensure exits read on close.
t.Run("Close", func(t *testing.T) {
s := MustOpenNewTranslateStore()
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Start reader from initial position.
@ -260,7 +341,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Ensure exits read on store close.
t.Run("StoreClose", func(t *testing.T) {
s := MustOpenNewTranslateStore()
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Start reader from initial position.
@ -294,24 +375,138 @@ func TestTranslateStore_EntryReader(t *testing.T) {
}
// MustNewTranslateStore returns a new TranslateStore with a temporary path.
func MustNewTranslateStore() *boltdb.TranslateStore {
f, err := ioutil.TempFile("", "")
func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
f, err := testhook.TempFile(tb, "translate-store")
if err != nil {
panic(err)
} else if err := f.Close(); err != nil {
panic(err)
}
s := boltdb.NewTranslateStore("I", "F")
s := boltdb.NewTranslateStore("I", "F", 0, disco.DefaultPartitionN, false)
s.Path = f.Name()
return s
}
func TestTranslateStore_Delete(t *testing.T) {
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Setup initial keys.
ids, err := s.CreateKeys("foo", "bar", "deleteme")
if err != nil {
t.Fatal(err)
}
records := roaring.NewBitmap(ids["deleteme"])
c, err := s.Delete(records)
if err != nil {
t.Fatal(err)
}
if err = c.Commit(); err != nil {
t.Fatal(err)
}
r, e := s.FreeIDs()
if e != nil {
t.Fatal(err)
}
freeids := r.Slice()
if len(freeids) == 0 {
t.Fatalf("expected to have free id")
}
if freeids[0] != ids["deleteme"] {
t.Fatalf("expected [%v] and got %v", ids["deleteme"], freeids[0])
}
records2 := roaring.NewBitmap(ids["foo"])
c, err = s.Delete(records2)
if err != nil {
t.Fatal(err)
}
if err = c.Commit(); err != nil {
t.Fatal(err)
}
r, e = s.FreeIDs()
if e != nil {
t.Fatal(err)
}
freeids = r.Slice()
if len(freeids) != 2 {
t.Fatalf("expected to have 2 free ids")
}
}
func TestTranslateStore_ReadWrite(t *testing.T) {
t.Run("WriteTo_ReadFrom", func(t *testing.T) {
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
batch0 := []string{}
for i := 0; i < 100; i++ {
batch0 = append(batch0, fmt.Sprintf("key%d", i))
}
batch1 := []string{}
for i := 100; i < 200; i++ {
batch1 = append(batch1, fmt.Sprintf("key%d", i))
}
// Populate the store with the keys in batch0.
batch0IDs, err := s.CreateKeys(batch0...)
if err != nil {
t.Fatal(err)
}
// Put the contents of the store into a buffer.
buf := bytes.NewBuffer(nil)
expN := s.Size()
// After this, the buffer should contain batch0.
if n, err := s.WriteTo(buf); err != nil {
t.Fatalf("writing to buffer: %s", err)
} else if n != expN {
t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
}
// Populate the store with the keys in batch1.
batch1IDs, err := s.CreateKeys(batch1...)
if err != nil {
t.Fatal(err)
}
expIDs := map[string]uint64{
"key50": batch0IDs["key50"],
"key150": batch1IDs["key150"],
}
// Check the IDs for a key from each batch.
if ids, err := s.FindKeys("key50", "key150"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(expIDs, ids) {
t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids)
}
// Reset the contents of the store with the data in the buffer.
if n, err := s.ReadFrom(buf); err != nil {
t.Fatalf("reading from buffer: %s", err)
} else if n != expN {
t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
}
// This time, we expect the second key to be different because
// we overwrote the store, and then just set that key.
if ids, err := s.CreateKeys("key50", "key150"); err != nil {
t.Fatal(err)
} else if ids["key50"] != expIDs["key50"] {
t.Fatalf("last expected ids[key50]: %d, but got: %d", expIDs["key50"], ids["key50"])
} else if ids["key150"] == expIDs["key150"] {
t.Fatalf("last expected different ids[key150]: %d, but got: %d", expIDs["key150"], ids["key150"])
}
})
}
// MustOpenNewTranslateStore returns a new, opened TranslateStore.
func MustOpenNewTranslateStore() *boltdb.TranslateStore {
s := MustNewTranslateStore()
func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
s := MustNewTranslateStore(tb)
if err := s.Open(); err != nil {
panic(err)
tb.Fatalf("opening s: %v", err)
}
return s
}
@ -320,7 +515,5 @@ func MustOpenNewTranslateStore() *boltdb.TranslateStore {
func MustCloseTranslateStore(s *boltdb.TranslateStore) {
if err := s.Close(); err != nil {
panic(err)
} else if err := os.Remove(s.Path); err != nil {
panic(err)
}
}

71
broadcast.go
View file

@ -1,22 +1,11 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"fmt"
"github.com/molecula/featurebase/v3/disco"
"github.com/pkg/errors"
)
@ -26,11 +15,22 @@ type Serializer interface {
Unmarshal([]byte, Message) error
}
// NopSerializer represents a Serializer that doesn't do anything.
var NopSerializer Serializer = &nopSerializer{}
type nopSerializer struct{}
// Marshal is a no-op implementation of Serializer Marshal method.
func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil }
// Unmarshal is a no-op implementation of Serializer Unmarshal method.
func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
// broadcaster is an interface for broadcasting messages.
type broadcaster interface {
SendSync(Message) error
SendAsync(Message) error
SendTo(*Node, Message) error
SendTo(*disco.Node, Message) error
}
// Message is the interface implemented by all core pilosa types which can be serialized to messages.
@ -49,7 +49,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil }
func (nopBroadcaster) SendAsync(Message) error { return nil }
// SendTo is a no-op implementation of Broadcaster SendTo method.
func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
func (nopBroadcaster) SendTo(*disco.Node, Message) error { return nil }
// Broadcast message types.
const (
@ -61,14 +61,17 @@ const (
messageTypeCreateView
messageTypeDeleteView
messageTypeClusterStatus
messageTypeResizeInstruction
messageTypeResizeInstructionComplete
messageTypeSetCoordinator
messageTypeUpdateCoordinator
messageTypeUNUSED0 // used to be ResizeInstruction
messageTypeUNUSED1 // used to be ResizeInstructionComplete
messageTypeNodeState
messageTypeRecalculateCaches
messageTypeLoadSchemaMessage
messageTypeNodeEvent
messageTypeNodeStatus
messageTypeTransaction
messageTypeUNUSED2 // used to be ResizeNodeMessage
messageTypeUNUSED3 // used to be ResizeAbortMessage
messageTypeUpdateField
)
// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@ -100,22 +103,20 @@ func getMessage(typ byte) Message {
return &DeleteViewMessage{}
case messageTypeClusterStatus:
return &ClusterStatus{}
case messageTypeResizeInstruction:
return &ResizeInstruction{}
case messageTypeResizeInstructionComplete:
return &ResizeInstructionComplete{}
case messageTypeSetCoordinator:
return &SetCoordinatorMessage{}
case messageTypeUpdateCoordinator:
return &UpdateCoordinatorMessage{}
case messageTypeNodeState:
return &NodeStateMessage{}
case messageTypeRecalculateCaches:
return &RecalculateCaches{}
case messageTypeLoadSchemaMessage:
return &LoadSchemaMessage{}
case messageTypeNodeEvent:
return &NodeEvent{}
case messageTypeNodeStatus:
return &NodeStatus{}
case messageTypeTransaction:
return &TransactionMessage{}
case messageTypeUpdateField:
return &UpdateFieldMessage{}
default:
panic(fmt.Sprintf("unknown message type %d", typ))
}
@ -139,22 +140,20 @@ func getMessageType(m Message) byte {
return messageTypeDeleteView
case *ClusterStatus:
return messageTypeClusterStatus
case *ResizeInstruction:
return messageTypeResizeInstruction
case *ResizeInstructionComplete:
return messageTypeResizeInstructionComplete
case *SetCoordinatorMessage:
return messageTypeSetCoordinator
case *UpdateCoordinatorMessage:
return messageTypeUpdateCoordinator
case *NodeStateMessage:
return messageTypeNodeState
case *RecalculateCaches:
return messageTypeRecalculateCaches
case *LoadSchemaMessage:
return messageTypeLoadSchemaMessage
case *NodeEvent:
return messageTypeNodeEvent
case *NodeStatus:
return messageTypeNodeStatus
case *TransactionMessage:
return messageTypeTransaction
case *UpdateFieldMessage:
return messageTypeUpdateField
default:
panic(fmt.Sprintf("don't have type for message %#v", m))
}

284
bsi.go Normal file
View file

@ -0,0 +1,284 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"math/bits"
"github.com/molecula/featurebase/v3/roaring"
)
// bsiData contains BSI-structured data.
type bsiData []*Row
// pivotDescending loops over nonzero BSI values in descending order.
// For each value, the provided function is called with the value and a slice of the associated columns.
// If limit or offset are not-nil, they will be applied.
// Applying a limit or offset may modify the pointed-to value.
func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
// This "pivot" algorithm works by treating the BSI data as a tree.
// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
// Each range is subdivided into 2 ranges of half size, which form lower branches.
// Eventually, a range of width 1 cannot be subdivided and forms a leaf.
// At each branch and leaf, there is a bitmap of all columns within the corresponding range.
// The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range.
// This function uses a depth-first search over this virtual tree.
switch {
case !filter.Any():
// There are no remaining data.
case offset != nil && *offset >= filter.Count():
// Skip this entire branch.
*offset -= filter.Count()
case limit != nil && *limit == 0:
// The limit has been reached.
// No more data is necessary.
case len(bsi) == 0:
// This is a leaf node.
cols := filter.Columns()
if offset != nil {
cols = cols[*offset:]
*offset = 0
}
if limit != nil {
if *limit < uint64(len(cols)) {
cols = cols[:*limit]
}
*limit -= uint64(len(cols))
}
fn(branch, cols...)
default:
// Pivot over the highest bit.
upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
splitBit := bsi[len(bsi)-1]
lowerBits := bsi[:len(bsi)-1]
lowerBits.pivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
lowerBits.pivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
}
}
/*
// distribution generates a BSI histogram for the input.
// TODO: I forgot what I was going to use this for.
// Could probbably use this for:
// - quartile queries
// - TopN on int
func (bsi bsiData) distribution(filter *Row) bsiData {
var dist bsiData
bsi.pivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
dist.insert(count, uint64(len(values)))
})
return dist
}
*/
var placeholderBitmap = roaring.NewBitmap()
// addBSI adds two BSI bitmaps together.
// It does not handle sign and has no concept of overflow.
func addBSI(x, y bsiData) bsiData {
// Accumulate row segments.
segments := make([][]rowSegment, len(x)+len(y))
xsegs, ysegs := segments[:len(x)], segments[len(x):]
for i, r := range x {
xsegs[i] = r.segments
}
for i, r := range y {
ysegs[i] = r.segments
}
var dst bsiData
var xbitmaps, ybitmaps []*roaring.Bitmap
for {
// Find the next shard.
next := ^uint64(0)
for _, s := range segments {
if len(s) == 0 {
continue
}
shard := s[0].shard
if shard < next {
next = shard
}
}
if next == ^uint64(0) {
// There are no remaining shards.
break
}
// Accumulate bitmaps for this shard.
xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
for i, segs := range xsegs {
if len(segs) == 0 || segs[0].shard != next {
continue
}
xsegs[i] = segs[1:]
bm := segs[0].data
if !bm.Any() {
continue
}
for len(xbitmaps) < i {
xbitmaps = append(xbitmaps, placeholderBitmap)
}
xbitmaps = append(xbitmaps, bm)
}
for i, segs := range ysegs {
if len(segs) == 0 || segs[0].shard != next {
continue
}
ysegs[i] = segs[1:]
bm := segs[0].data
if !bm.Any() {
continue
}
for len(ybitmaps) < i {
ybitmaps = append(ybitmaps, placeholderBitmap)
}
ybitmaps = append(ybitmaps, bm)
}
// Add the shard values together.
var out []*roaring.Bitmap
switch {
case len(xbitmaps) == 0:
// There are no values in x.
out = ybitmaps
case len(ybitmaps) == 0:
// There are no values in y.
out = xbitmaps
default:
out = roaring.Add(xbitmaps, ybitmaps)
}
// Convert the bitmaps to output segments.
for i, b := range out {
if !b.Any() {
continue
}
for len(dst) <= i {
dst = append(dst, NewRow())
}
dst[i].segments = append(dst[i].segments, rowSegment{
shard: next,
writable: true,
data: b,
n: b.Count(),
})
}
}
return dst
}
// rowBuilder builds a row quickly from individual values.
// It is optimized for the case in which values are generated sequentially.
type rowBuilder struct {
bm *roaring.Bitmap
mask *[1024]uint64
array []uint16
key uint64
n int32
}
// flushKey flushes the data at the current key to the bitmap.
func (b *rowBuilder) flushKey() {
var c *roaring.Container
switch {
case b.mask != nil:
c = roaring.NewContainerBitmapN(b.mask[:], b.n)
b.mask = nil
case len(b.array) > 0:
c = roaring.NewContainerArrayCopy(b.array)
b.array = b.array[:0]
default:
return
}
if b.bm == nil {
b.bm = roaring.NewBitmap()
}
if old := b.bm.Containers.Get(b.key); old != nil {
c = roaring.Union(c, old)
}
b.bm.Containers.Put(b.key, c)
}
// Add a value to the bitmap.
// Values must be added sequentially.
func (b *rowBuilder) Add(v uint64) {
vkey := v / (1 << 16)
if b.key != vkey {
// This is a new key, so flush the old one.
b.flushKey()
b.key = vkey
}
if b.mask != nil {
// Add to the mask.
b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
b.mask[uint16(v)/64] |= 1 << (v % 64)
return
}
// Add to an array.
b.array = append(b.array, uint16(v))
if len(b.array) >= roaring.ArrayMaxSize {
// The array is too big.
// Convert it to a bitmask.
m := [1024]uint64{}
for _, v := range b.array {
m[v/64] |= 1 << (v % 64)
}
b.n = int32(len(b.array))
b.array = b.array[:0]
b.mask = &m
}
}
// Build a Row from stored data.
// This resets the builder.
func (b *rowBuilder) Build() *Row {
// Flush the active key to the bitmap.
b.flushKey()
// Remove the bitmap and convert it to a Row.
bm := b.bm
b.bm = nil
if bm == nil {
return NewRow()
}
return NewRowFromBitmap(bm)
}
// bsiBuilder assembles BSI data.
// It is optimized for the case in which values are generated sequentially.
type bsiBuilder []rowBuilder
// Insert a value into the BSI data.
// Columns must be inserted sequentially, and duplicates are not allowed.
func (b *bsiBuilder) Insert(col, val uint64) {
for val != 0 {
i := bits.TrailingZeros64(val)
val &^= 1 << i
for len(*b) <= i {
*b = append(*b, rowBuilder{})
}
(*b)[i].Add(col)
}
}
// Build BSI data.
// This resets the builder.
func (b *bsiBuilder) Build() bsiData {
builders := *b
*b = builders[:0]
rows := make(bsiData, len(builders))
for i := range builders {
rows[i] = builders[i].Build()
}
return rows
}

160
bsi_test.go Normal file
View file

@ -0,0 +1,160 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"fmt"
"math/rand"
"sort"
"testing"
)
// TestBSIAdd does a number of iterations. For each iteration, it
// generates a random number of ids, and two random values for each id
// to add together.
func TestBSIAdd(t *testing.T) {
// TODO wouldn't it be cool if our test suite had a randomized
// burn-in mode where you could run any test which supported it
// with a random seed and way more iterations?
rnd := rand.New(rand.NewSource(99))
//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
var builderA, builderB bsiBuilder
// a and b are generated slices of numbers to add together
var a, b []uint64
// idToIndex maps record ids to indexes in a and b
idToIndex := make(map[int]int)
// indexToID has the record id for each value in a and b
indexToID := []uint64{}
min := 999999999
max := 0
for iteration := 0; iteration < 1; iteration++ {
t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
// reset generated data
a, b = a[:0], b[:0]
indexToID = indexToID[:0]
for k := range idToIndex {
delete(idToIndex, k)
}
// z generates the values, they can be fairly large, but are usually small
z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
id := -1
for i := 0; true; i++ {
// get the next id, skipping a random amount
id = id + int(idZipf.Uint64()+1)
if id >= ShardWidth {
if i < min {
min = i
}
if max < i {
max = i
}
break
}
idToIndex[id] = int(i)
indexToID = append(indexToID, uint64(id))
// append a random value to each data slice
a = append(a, z.Uint64())
b = append(b, z.Uint64())
}
// build the BSIs based on the data slices and generated IDs
for index, id := range indexToID {
va, vb := a[index], b[index]
builderA.Insert(uint64(id), va)
builderB.Insert(uint64(id), vb)
}
dataA, dataB := builderA.Build(), builderB.Build()
dataC := addBSI(dataA, dataB)
// build results from added bsiData; results[i] should hold a[i]+b[i]
results := make([]uint64, len(a))
dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
for _, id := range ids {
results[idToIndex[int(id)]] = count
}
})
for i, res := range results {
if res != a[i]+b[i] {
t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
}
}
})
}
}
type bsiAddCase struct {
positions []uint64
a []uint64
b []uint64
}
func (b bsiAddCase) Len() int {
return len(b.positions)
}
// Less reports whether the element with
// index i should sort before the element with index j.
func (b bsiAddCase) Less(i, j int) bool {
return b.positions[i] < b.positions[j]
}
// Swap swaps the elements with indexes i and j.
func (b bsiAddCase) Swap(i, j int) {
b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
b.a[i], b.a[j] = b.a[j], b.a[i]
b.b[i], b.b[j] = b.b[j], b.b[i]
}
// TestBSIAddCases tests specific cases of bsiAdd (would generally be
// pulled from randomly generated ones from TestBSIAdd upon failure).
func TestBSIAddCases(t *testing.T) {
tests := []bsiAddCase{
{
positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
a: []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
b: []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
},
{
positions: []uint64{17570, 54607},
a: []uint64{1010, 71},
b: []uint64{25, 150},
},
}
var builderA, builderB bsiBuilder
for i, tst := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
}
sort.Sort(tst)
for i := 0; i < len(tst.a); i++ {
builderA.Insert(tst.positions[i], tst.a[i])
builderB.Insert(tst.positions[i], tst.b[i])
}
dataA, dataB := builderA.Build(), builderB.Build()
dataC := addBSI(dataA, dataB)
// maps id to count
results := make(map[uint64]uint64)
dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
for _, id := range ids {
results[id] = count
}
})
for i, id := range tst.positions {
if results[id] != tst.a[i]+tst.b[i] {
t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
}
}
})
}
}

344
cache.go
View file

@ -1,29 +1,20 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"bytes"
"encoding/json"
"fmt"
"io"
"sort"
"sync"
"time"
"github.com/pilosa/pilosa/v2/lru"
"github.com/pilosa/pilosa/v2/stats"
"github.com/molecula/featurebase/v3/lru"
pb "github.com/molecula/featurebase/v3/proto"
"github.com/molecula/featurebase/v3/stats"
"github.com/pkg/errors"
)
const (
@ -52,6 +43,9 @@ type cache interface {
// SetStats defines the stats client used in the cache.
SetStats(s stats.StatsClient)
// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
Clear()
}
// lruCache represents a least recently used Cache implementation.
@ -59,14 +53,17 @@ type lruCache struct {
cache *lru.Cache
counts map[uint64]uint64
stats stats.StatsClient
// maxEntries is saved to support Clear which recreates the cache.
maxEntries uint32
}
// newLRUCache returns a new instance of LRUCache.
func newLRUCache(maxEntries uint32) *lruCache {
c := &lruCache{
cache: lru.New(int(maxEntries)),
counts: make(map[uint64]uint64),
stats: stats.NopStatsClient,
cache: lru.New(int(maxEntries)),
counts: make(map[uint64]uint64),
stats: stats.NopStatsClient,
maxEntries: maxEntries,
}
c.cache.OnEvicted = c.onEvicted
return c
@ -118,7 +115,8 @@ func (c *lruCache) Top() []bitmapPair {
Count: n,
})
}
sort.Sort(bitmapPairs(a))
pairs := bitmapPairs(a)
sort.Sort(&pairs)
return a
}
@ -127,6 +125,13 @@ func (c *lruCache) SetStats(s stats.StatsClient) {
c.stats = s
}
func (c *lruCache) Clear() {
for k := range c.counts {
delete(c.counts, k)
}
c.cache = lru.New(int(c.maxEntries))
}
func (c *lruCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
// Ensure LRUCache implements Cache.
@ -134,9 +139,12 @@ var _ cache = &lruCache{}
// rankCache represents a cache with sorted entries.
type rankCache struct {
mu sync.Mutex
entries map[uint64]uint64
rankings []bitmapPair // cached, ordered list
// TODO why does this have a lock and lruCache doesn't?
mu sync.Mutex
entries map[uint64]uint64
rankings bitmapPairs // cached, ordered list
rankingsRead bool
dirty bool
updateN int
updateTime time.Time
@ -164,14 +172,35 @@ func NewRankCache(maxEntries uint32) *rankCache {
}
}
func (c *rankCache) Clear() {
c.mu.Lock()
defer c.mu.Unlock()
for k := range c.entries {
delete(c.entries, k)
}
c.rankings = c.rankings[:0]
c.rankingsRead = false
c.dirty = false
c.updateN = 0
c.updateTime = time.Time{}
c.thresholdValue = 0
}
// Add adds a count to the cache.
func (c *rankCache) Add(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
// Flag the cache as dirty.
// This forces recalculation if top is called before the cache is recalculated.
c.dirty = true
// Ignore if the column count is below the threshold,
// unless the count is 0, which is effectively used
// to clear the cache value.
if n < c.thresholdValue && n > 0 {
delete(c.entries, id)
return
}
@ -184,11 +213,25 @@ func (c *rankCache) Add(id uint64, n uint64) {
func (c *rankCache) BulkAdd(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
// Flag the cache as dirty.
// This forces recalculation if top is called before the cache is recalculated.
c.dirty = true
if n < c.thresholdValue {
delete(c.entries, id)
return
}
c.entries[id] = n
// FB-1206: Periodically invalidate the cache when we are bulk loading
// as this can take up an upbounded amount of memory. This is especially
// true when restoring shards as all rows will be added.
if len(c.entries) > int(2*c.maxEntries) {
c.stats.Count(MetricRecalculateCache, 1, 1.0)
c.recalculate()
}
}
// Get returns a count for a given id.
@ -209,12 +252,15 @@ func (c *rankCache) Len() int {
func (c *rankCache) IDs() []uint64 {
c.mu.Lock()
defer c.mu.Unlock()
a := make([]uint64, 0, len(c.entries))
for id := range c.entries {
a = append(a, id)
if len(c.entries) == 0 {
return nil
}
sort.Sort(uint64Slice(a))
return a
ids := make([]uint64, 0, len(c.entries))
for id := range c.entries {
ids = append(ids, id)
}
sort.Sort(uint64Slice(ids))
return ids
}
// Invalidate recalculates the entries by rank.
@ -228,7 +274,7 @@ func (c *rankCache) Invalidate() {
func (c *rankCache) Recalculate() {
c.mu.Lock()
defer c.mu.Unlock()
c.stats.Count("cache.recalculate", 1, 1.0)
c.stats.Count(MetricRecalculateCache, 1, 1.0)
c.recalculate()
}
@ -236,27 +282,42 @@ func (c *rankCache) invalidate() {
// Don't invalidate more than once every X seconds.
// TODO: consider making this configurable.
if time.Since(c.updateTime).Seconds() < 10 {
// Skipping recalculation means that the ranked cache's growth is unbounded.
// This is somewhat necessary for now since recalculation is not cheap.
// The cache will remain flagged as dirty and will be recalculated if Top is called.
// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
// Ensure that we're marked as dirty even if we weren't otherwise.
c.dirty = true
return
}
c.stats.Count("cache.invalidate", 1, 1.0)
c.stats.Count(MetricInvalidateCache, 1, 1.0)
c.recalculate()
}
func (c *rankCache) recalculate() {
if c.rankingsRead {
c.rankings = nil
c.rankingsRead = false
}
// Convert cache to a sorted list.
rankings := make([]bitmapPair, 0, len(c.entries))
rankings := c.rankings[:0]
if cap(rankings) < len(c.entries) {
rankings = make([]bitmapPair, 0, len(c.entries))
}
for id, cnt := range c.entries {
rankings = append(rankings, bitmapPair{
ID: id,
Count: cnt,
})
}
sort.Sort(bitmapPairs(rankings))
c.rankings = rankings
sort.Sort(&c.rankings)
// Store the count of the item at the threshold index.
c.rankings = rankings
length := len(c.rankings)
c.stats.Gauge("RankCache", float64(length), 1.0)
c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
var removeItems []bitmapPair // cached, ordered list
if length > int(c.maxEntries) {
@ -272,11 +333,14 @@ func (c *rankCache) recalculate() {
// If size is larger than the threshold then trim it.
if len(c.entries) > c.thresholdBuffer {
c.stats.Count("cache.threshold", 1, 1.0)
c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
for _, pair := range removeItems {
delete(c.entries, pair.ID)
}
}
// The cache is no longer dirty.
c.dirty = false
}
// SetStats defines the stats client used in the cache.
@ -285,7 +349,19 @@ func (c *rankCache) SetStats(s stats.StatsClient) {
}
// Top returns an ordered list of pairs.
func (c *rankCache) Top() []bitmapPair { return c.rankings }
func (c *rankCache) Top() []bitmapPair {
c.mu.Lock()
defer c.mu.Unlock()
if c.dirty {
// The cache is dirty, so we need to recalculate it to get a consistent view.
c.stats.Count(MetricReadDirtyCache, 1, 1.0)
c.recalculate()
}
c.rankingsRead = true
return c.rankings
}
// WriteTo writes the cache to w.
func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
@ -309,17 +385,68 @@ type bitmapPair struct {
// bitmapPairs is a sortable list of BitmapPair objects.
type bitmapPairs []bitmapPair
func (p bitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p bitmapPairs) Len() int { return len(p) }
func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
func (p *bitmapPairs) Swap(i, j int) { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
func (p *bitmapPairs) Len() int { return len(*p) }
func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
// Pair holds an id/count pair.
type Pair struct {
ID uint64 `json:"id"`
Key string `json:"key,omitempty"`
Key string `json:"key"`
Count uint64 `json:"count"`
}
// PairField is a Pair with its associated field.
type PairField struct {
Pair Pair
Field string
}
func (p PairField) Clone() (r PairField) {
return PairField{
Pair: p.Pair,
Field: p.Field,
}
}
// ToTable implements the ToTabler interface.
func (p PairField) ToTable() (*pb.TableResponse, error) {
return pb.RowsToTable(p, 1)
}
// ToRows implements the ToRowser interface.
func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
if p.Pair.Key != "" {
return callback(&pb.RowResponse{
Headers: []*pb.ColumnInfo{
{Name: p.Field, Datatype: "string"},
{Name: "count", Datatype: "uint64"},
},
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
},
})
} else {
return callback(&pb.RowResponse{
Headers: []*pb.ColumnInfo{
{Name: p.Field, Datatype: "uint64"},
{Name: "count", Datatype: "uint64"},
},
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
},
})
}
}
// MarshalJSON marshals PairField into a JSON-encoded byte slice,
// excluding `Field`.
func (p PairField) MarshalJSON() ([]byte, error) {
return json.Marshal(p.Pair)
}
// Pairs is a sortable slice of Pair objects.
type Pairs []Pair
@ -395,6 +522,82 @@ func (p Pairs) String() string {
return buf.String()
}
// PairsField is a Pairs object with its associated field.
type PairsField struct {
Pairs []Pair
Field string
}
func (p *PairsField) Clone() (r *PairsField) {
r = &PairsField{
Pairs: make([]Pair, len(p.Pairs)),
Field: p.Field,
}
copy(r.Pairs, p.Pairs)
return
}
// ToTable implements the ToTabler interface.
func (p *PairsField) ToTable() (*pb.TableResponse, error) {
return pb.RowsToTable(p, len(p.Pairs))
}
// ToRows implements the ToRowser interface.
func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
// Determine if the ID has string keys.
var stringKeys bool
if len(p.Pairs) > 0 {
if p.Pairs[0].Key != "" {
stringKeys = true
}
}
dtype := "uint64"
if stringKeys {
dtype = "string"
}
ci := []*pb.ColumnInfo{
{Name: p.Field, Datatype: dtype},
{Name: "count", Datatype: "uint64"},
}
for _, pair := range p.Pairs {
if stringKeys {
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
} else {
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
}
ci = nil //only send on the first
}
return nil
}
// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
// excluding `Field`.
func (p PairsField) MarshalJSON() ([]byte, error) {
return json.Marshal(p.Pairs)
}
// int64Slice represents a sortable slice of int64 numbers.
type int64Slice []int64
func (p int64Slice) Len() int { return len(p) }
func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64
@ -402,66 +605,6 @@ func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
ret := make([]uint64, 0, len(p))
i, j := 0, 0
for i < len(p) && j < len(other) {
a, b := p[i], other[j]
if a == b {
ret = append(ret, a)
i, j = i+1, j+1
} else if a < b {
ret = append(ret, a)
i++
} else {
ret = append(ret, b)
j++
}
}
if i < len(p) {
ret = append(ret, p[i:]...)
} else if j < len(other) {
ret = append(ret, other[j:]...)
}
return ret
}
// bitmapCache provides an interface for caching full bitmaps.
type bitmapCache interface {
Fetch(id uint64) (*Row, bool)
Add(id uint64, b *Row)
}
// simpleCache implements BitmapCache
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same row within a short time frame (i.e. good for write-heavy loads)
// A read-heavy use case would cause the cache to get bigger, potentially causing the
// node to run out of memory.
type simpleCache struct {
cache map[uint64]*Row
}
// Fetch retrieves the bitmap at the id in the cache.
func (s *simpleCache) Fetch(id uint64) (*Row, bool) {
m, ok := s.cache[id]
return m, ok
}
// Add adds the bitmap to the cache, keyed on the id. A nil row means
// deleting the row from the cache.
func (s *simpleCache) Add(id uint64, b *Row) {
if b != nil {
s.cache[id] = b
} else {
delete(s.cache, id)
}
}
// nopCache represents a no-op Cache implementation.
type nopCache struct {
stats stats.StatsClient
@ -481,6 +624,7 @@ func (c nopCache) Invalidate() {}
func (c nopCache) Len() int { return 0 }
func (c nopCache) Recalculate() {}
func (c nopCache) SetStats(stats.StatsClient) {}
func (c nopCache) Clear() {}
func (c nopCache) Top() []bitmapPair {
return []bitmapPair{}

86
cache_test.go
View file

@ -1,27 +1,16 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa_test
import (
"reflect"
"testing"
"github.com/pilosa/pilosa/v2"
pilosa "github.com/molecula/featurebase/v3"
)
// Ensure a bitmap query can be executed.
func TestCache_Rank(t *testing.T) {
// Ensure cache stays constrained to its configured size.
func TestCache_Rank_Size(t *testing.T) {
cacheSize := uint32(3)
cache := pilosa.NewRankCache(cacheSize)
for i := 1; i < int(2*cacheSize); i++ {
@ -31,5 +20,66 @@ func TestCache_Rank(t *testing.T) {
if cache.Len() != int(cacheSize) {
t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
}
}
// Ensure cache entries set below threshold are handled appropriately.
func TestCache_Rank_Threshold(t *testing.T) {
cacheSize := uint32(5)
cache := pilosa.NewRankCache(cacheSize)
for i := 1; i < int(2*cacheSize); i++ {
cache.Add(uint64(i), 3)
}
// Set the cache value for rows 4 and 5 to a number below the threshold
// value (which is 3), and ensure that they gets zeroed out.
cache.Add(4, 1)
cache.BulkAdd(5, 1)
cache.Recalculate()
if cache.Get(4) != 0 {
t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
}
if cache.Get(5) != 0 {
t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
}
}
// Test that consecutive writes show up in Top.
// On later writes, the cache skips recalculation to save CPU time.
// This used to mean that the later writes would not show up in Top.
// Now, the cache is flagged as dirty and recalculated during the call to Top.
func TestCache_Rank_Dirty(t *testing.T) {
cacheSize := uint32(5)
cache := pilosa.NewRankCache(cacheSize)
type pair struct{ ID, Count uint64 }
expect := []pair{
{5, 2},
{4, 1},
}
for _, v := range expect {
cache.Add(v.ID, v.Count)
}
var got []pair
for _, p := range cache.Top() {
got = append(got, pair(p))
}
if !reflect.DeepEqual(expect, got) {
t.Fatalf("wrote %v but got %v", expect, got)
}
}
func TestCache_Rank_BulkAdd(t *testing.T) {
const cacheSize = 10
cache := pilosa.NewRankCache(uint32(cacheSize))
for i := uint64(0); i < 1000; i++ {
cache.BulkAdd(i, i)
if n := cache.Len(); n > cacheSize*2 {
t.Fatalf("entry count exceed 2x cache size: %d", n)
}
}
}

248
catcher.go Normal file
View file

@ -0,0 +1,248 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/vprint"
)
// catcher is useful to report error locations with a
// Stack dump before the complexity
// of the executor_test swallows up
// the location of a PanicOn.
type catcherTx struct {
b Tx
}
func newCatcherTx(b Tx) *catcherTx {
return &catcherTx{b: b}
}
func init() {
// keep golangci-lint happy
_ = newCatcherTx
}
var _ Tx = (*catcherTx)(nil)
func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
return c.b.NewTxIterator(index, field, view, shard)
}
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
}
func (c *catcherTx) Rollback() {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
c.b.Rollback()
}
func (c *catcherTx) Commit() error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Commit()
}
func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.RoaringBitmap(index, field, view, shard)
}
func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Container(index, field, view, shard, key)
}
func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.PutContainer(index, field, view, shard, key, rc)
}
func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.RemoveContainer(index, field, view, shard, key)
}
func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Add(index, field, view, shard, a...)
}
func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Remove(index, field, view, shard, a...)
}
func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Contains(index, field, view, shard, key)
}
func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
}
func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ForEach(index, field, view, shard, fn)
}
func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.ForEachRange(index, field, view, shard, start, end, fn)
}
func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Count(index, field, view, shard)
}
func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Max(index, field, view, shard)
}
func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.Min(index, field, view, shard)
}
func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.CountRange(index, field, view, shard, start, end)
}
func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack())
vprint.PanicOn(r)
}
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *catcherTx) Type() string {
return c.b.Type()
}
func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
}
func (c *catcherTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) {
return c.b.ApplyRewriter(index, field, view, shard, ckey, filter)
}
func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
return c.b.GetSortedFieldViewList(idx, shard)
}
func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
return 0, nil
}

174
client.go
View file

@ -1,174 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"context"
"io"
)
// Bit represents the intersection of a row and a column. It can be specified by
// integer ids or string keys.
type Bit struct {
RowID uint64
ColumnID uint64
RowKey string
ColumnKey string
Timestamp int64
}
// FieldValue represents the value for a column within a
// range-encoded field.
type FieldValue struct {
ColumnID uint64
ColumnKey string
Value int64
}
// InternalClient should be implemented by any struct that enables any transport between nodes
// TODO: Refactor
// Note from Travis: Typically an interface containing more than two or three methods is an indication that
// something hasn't been architected correctly.
// While I understand that putting the entire Client behind an interface might require this many methods,
// I don't want to let it go unquestioned.
type InternalClient interface {
MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
Nodes(ctx context.Context) ([]*Node, error)
Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureField(ctx context.Context, indexName string, fieldName string) error
EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error
ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error
ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string) error
CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
}
//===============
// InternalQueryClient is the internal interface for querying a node.
type InternalQueryClient interface {
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
}
type nopInternalQueryClient struct{}
func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func newNopInternalQueryClient() *nopInternalQueryClient {
return &nopInternalQueryClient{}
}
var _ InternalQueryClient = newNopInternalQueryClient()
//===============
type nopInternalClient struct{}
func newNopInternalClient() nopInternalClient {
return nopInternalClient{}
}
var _ InternalClient = newNopInternalClient()
func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error {
return nil
}
func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
return nil
}
func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
return nil, nil
}
func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) {
return nil, nil
}
func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error {
return nil
}
func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
return nil
}
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
return nil
}
func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
return nil
}
func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
return nil
}
func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error {
return nil
}
func (n nopInternalClient) ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error {
return nil
}
func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
return nil
}
func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil }
func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
return nil, nil
}
func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
return nil, nil, nil
}
func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
return nil
}
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
return nil, nil
}

85
client/README.md Normal file
View file

@ -0,0 +1,85 @@
# Go Client for Pilosa
Go client for Pilosa high performance distributed index.
## Usage
If you have the pilosa repo in your `GOPATH`,
you can import the library in your code using:
```go
import "github.com/pilosa/pilosa/v2/client"
```
### Quick overview
Assuming [Pilosa](https://github.com/pilosa/pilosa) server is running at `localhost:10101` (the default):
```go
package main
import (
"fmt"
"github.com/pilosa/pilosa/v2/client"
)
func main() {
// Create the default client
cli := client.DefaultClient()
// Retrieve the schema
schema, err := cli.Schema()
// Create an Index object
myindex := schema.Index("myindex")
// Create a Field object
myfield := myindex.Field("myfield")
// make sure the index and the field exists on the server
err := cli.SyncSchema(schema)
// Send a Set query. If err is non-nil, response will be nil.
response, err := cli.Query(myfield.Set(5, 42))
// Send a Row query. If err is non-nil, response will be nil.
response, err = cli.Query(myfield.Row(5))
// Get the result
result := response.Result()
// Act on the result
if result != nil {
columns := result.Row().Columns
fmt.Println("Got columns: ", columns)
}
// You can batch queries to improve throughput
response, err = cli.Query(myindex.BatchQuery(
myfield.Row(5),
myfield.Row(10)))
if err != nil {
fmt.Println(err)
}
for _, result := range response.Results() {
// Act on the result
fmt.Println(result.Row().Columns)
}
}
```
## Documentation
### Data Model and Queries
See: [Data Model and Queries](docs/data-model-queries.md)
### Executing Queries
See: [Server Interaction](docs/server-interaction.md)
### Other Documentation
* [Tracing](docs/tracing.md)

1778
client/batch.go Normal file
View file

File diff suppressed because it is too large Load diff

1841
client/batch_test.go Normal file
View file

File diff suppressed because it is too large Load diff

1781
client/client.go Normal file
View file

File diff suppressed because it is too large Load diff

893
client/client_it_test.go Normal file
View file

@ -0,0 +1,893 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package client
import (
"bytes"
"fmt"
"io/ioutil"
"testing"
"time"
featurebase "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/disco"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/test"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
var (
testIndex *Index
testIndexWithKeys *Index
testIndexWithKeysNoTrack *Index
testIndexAtomicRecord *Index
testIndexKeyTranslation *Index
testField *Field
testFieldTimestamp *Field
testFieldInt *Field
testFieldTimeQuantum *Field
testFieldInt0 *Field
testFieldInt1 *Field
)
func setup(t *testing.T, cli *Client) {
t.Helper()
testSchema := NewSchema()
testIndex = testSchema.Index("test-index")
testIndexWithKeys = testSchema.Index("test-index-keys", OptIndexKeys(true))
testIndexWithKeysNoTrack = testSchema.Index("test-index-keys-notrack",
OptIndexKeys(true),
OptIndexTrackExistence(false),
)
testField = testIndex.Field("test-field")
testFieldTimeQuantum = testIndex.Field("test-field-timequantum", OptFieldTypeTime(TimeQuantumYear))
testFieldTimestamp = testIndex.Field("test-field-timestamp", OptFieldTypeTimestamp(time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC), "s"))
testFieldInt = testIndex.Field("test-field-int", OptFieldTypeInt(0, 100000))
testIndexKeyTranslation = testSchema.Index("test-index-key-translation", OptIndexKeys(true))
testIndexAtomicRecord = testSchema.Index("test-index-atomic-record")
testFieldInt0 = testIndexAtomicRecord.Field("test-field-int0", OptFieldTypeInt(-1000, 1000))
testFieldInt1 = testIndexAtomicRecord.Field("test-field-int1", OptFieldTypeInt(-1000, 1000))
require.NoErrorf(t, cli.SyncSchema(testSchema), "SyncSchema")
}
func tearDown(t *testing.T, cli *Client) {
t.Helper()
for _, i := range []*Index{testIndex, testIndexWithKeys, testIndexWithKeysNoTrack, testIndexAtomicRecord, testIndexKeyTranslation} {
require.NoErrorf(t, cli.DeleteIndex(i), "DeleteIndex(%s)", i.name)
}
}
func TestClientAgainstCluster(t *testing.T) {
for size, replicaN := 3, 1; replicaN <= 2; replicaN++ {
testName := fmt.Sprintf("%d.%d", size, replicaN)
t.Run(testName, func(t *testing.T) {
// Start size.replicaN cluster
c := test.MustNewCluster(t, size)
for _, n := range c.Nodes {
n.Config.Cluster.ReplicaN = replicaN
}
err := c.Start()
require.NoError(t, err, "Start cluster "+testName)
urls := make([]string, len(c.Nodes))
for i, n := range c.Nodes {
urls[i] = n.URL()
}
defer c.Close()
// Create a new client for the cluster
cli, err := newClientFromAddresses(urls, &ClientOptions{})
require.NoErrorf(t, err, "newClientFromAddresses(%v): %v", urls, err)
defer cli.Close()
t.Run("GetStatus", func(t *testing.T) {
status, err := cli.Status()
require.NoErrorf(t, err, "GET /status")
require.Equalf(t, disco.ClusterStateNormal, disco.ClusterState(status.State), "GET /status")
})
t.Run("QueryRow", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
resp, err := cli.Query(testField.Row(1))
require.NoErrorf(t, err, "Query Row")
require.NotNil(t, resp, "Response should not be nil")
})
t.Run("IntBase", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testIndex.Field("intbase", OptFieldTypeInt(-10, -5))
testIndex.Field("intbaseplus", OptFieldTypeInt(5, 10))
err = cli.SyncIndex(testIndex)
require.NoError(t, err)
schema, err := cli.Schema()
require.NoError(t, err)
if base := schema.Index("test-index").Field("intbase").Options().base; base != -5 {
t.Fatalf("unexpected base is not -5: %d", base)
}
if base := schema.Index("test-index").Field("intbaseplus").Options().base; base != 5 {
t.Fatalf("unexpected base is not 5: %d", base)
}
})
t.Run("QueryWithShards", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
shardWidth := uint64(1 << shardwidth.Exponent)
_, err := cli.Query(testField.Set(1, 1))
require.NoErrorf(t, err, "Set(1, %d)", 1)
_, err = cli.Query(testField.Set(1, shardWidth))
require.NoErrorf(t, err, "Set(1, %d)", shardWidth)
_, err = cli.Query(testField.Set(1, shardWidth*3))
require.NoErrorf(t, err, "Set(1, %d)", shardWidth*3)
resp, err := cli.Query(testField.Row(1), OptQueryShards(0, 3))
require.NoErrorf(t, err, "Row(1) OptQueryShards(0, 3)")
cols := resp.Result().Row().Columns
require.Equalf(t, []uint64{1, shardWidth * 3}, cols, "Unexpected results: %#v", cols)
})
t.Run("OrmCount", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldCount := testIndex.Field("test-field-count")
err := cli.EnsureField(testFieldCount)
require.NoError(t, err)
qry := testIndex.BatchQuery(
testFieldCount.Set(10, 20),
testFieldCount.Set(10, 21),
testFieldCount.Set(15, 25),
)
_, err = cli.Query(qry)
require.NoErrorf(t, err, "BatchQuery")
resp, err := cli.Query(testIndex.Count(testFieldCount.Row(10)))
require.NoErrorf(t, err, "Count")
require.Equalf(t, int64(2), resp.Result().Count(), "Count")
})
t.Run("DecimalField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldDec := testIndex.Field("test-field-dec", OptFieldTypeDecimal(3))
err := cli.EnsureField(testFieldDec)
require.NoError(t, err)
sch, err := cli.Schema()
require.NoErrorf(t, err, "Schema")
idx := sch.indexes[testIndex.name]
opts := idx.Field(testFieldDec.name).Options()
require.Equalf(t, int64(3), opts.scale, "%s scale", testFieldDec.name)
})
t.Run("IntersectReturns", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldSegments := testIndex.Field("test-field-segments")
err := cli.EnsureField(testFieldSegments)
require.NoError(t, err)
qry1 := testIndex.BatchQuery(
testFieldSegments.Set(2, 10),
testFieldSegments.Set(2, 15),
testFieldSegments.Set(3, 10),
testFieldSegments.Set(3, 20),
)
_, err = cli.Query(qry1)
require.NoErrorf(t, err, "BatchQuery")
qry2 := testIndex.Intersect(testFieldSegments.Row(2), testFieldSegments.Row(3))
resp, err := cli.Query(qry2)
require.NoErrorf(t, err, "Intersect")
require.Equalf(t, 1, len(resp.Results()), "Intersect number of results")
require.Equalf(t, []uint64{10}, resp.Result().Row().Columns, "Intersect columns results")
})
t.Run("TopNReturns", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldTopN := testIndex.Field("test-field-topn")
err := cli.EnsureField(testFieldTopN)
require.NoError(t, err)
qry := testIndex.BatchQuery(
testFieldTopN.Set(10, 5),
testFieldTopN.Set(10, 10),
testFieldTopN.Set(10, 15),
testFieldTopN.Set(20, 5),
testFieldTopN.Set(30, 5),
)
_, err = cli.Query(qry)
require.NoErrorf(t, err, "BatchQuery")
// XXX: The following is required to make this test pass. See: https://github.com/molecula/featurebase/issues/625
_, _, err = cli.HTTPRequest("POST", "/recalculate-caches", nil, nil)
require.NoErrorf(t, err, "POST /recalculate-caches")
resp, err := cli.Query(testFieldTopN.TopN(2))
require.NoErrorf(t, err, "TopN(2)")
items := resp.Result().CountItems()
require.Equalf(t, 2, len(items), "TopN result CountItems")
item := items[0]
require.Equalf(t, uint64(10), item.ID, "TopN result item[0].ID")
require.Equalf(t, uint64(3), item.Count, "TopN result item[0].Count")
})
t.Run("MinMaxRow", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldMinMax := testIndex.Field("test-field-minmax")
err := cli.EnsureField(testFieldMinMax)
require.NoError(t, err)
qry := testIndex.BatchQuery(
testFieldMinMax.Set(10, 5),
testFieldMinMax.Set(10, 10),
testFieldMinMax.Set(10, 15),
testFieldMinMax.Set(20, 5),
testFieldMinMax.Set(30, 5),
)
_, err = cli.Query(qry)
require.NoErrorf(t, err, "Setting bits")
resp, err := cli.Query(testFieldMinMax.MinRow())
require.NoErrorf(t, err, "MinRow")
min := resp.Result().CountItem().ID
require.Equalf(t, uint64(10), min, "Min")
resp, err = cli.Query(testFieldMinMax.MaxRow())
require.NoErrorf(t, err, "MaxRow")
max := resp.Result().CountItem().ID
require.Equalf(t, uint64(30), max, "Max")
})
t.Run("SetMutexField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldMutex := testIndex.Field("test-field-mutex", OptFieldTypeMutex(CacheTypeDefault, 0))
err := cli.EnsureField(testFieldMutex)
require.NoError(t, err)
// can set mutex
_, err = cli.Query(testFieldMutex.Set(1, 100))
require.NoErrorf(t, err, "Set(1, 100)")
resp, err := cli.Query(testFieldMutex.Row(1))
require.NoErrorf(t, err, "Row(1)")
target := []uint64{100}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
// setting another row removes the previous
_, err = cli.Query(testFieldMutex.Set(42, 100))
require.NoErrorf(t, err, "Set(42, 100)")
resp, err = cli.Query(testIndex.BatchQuery(
testFieldMutex.Row(1),
testFieldMutex.Row(42),
))
require.NoErrorf(t, err, "BatchQuery")
target1 := []uint64(nil)
target42 := []uint64{100}
require.Equalf(t, target1, resp.Results()[0].Row().Columns, "Row Results[0] Columns")
require.Equalf(t, target42, resp.Results()[1].Row().Columns, "Row Results[1] Columns")
})
t.Run("SetBoolField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldBool := testIndex.Field("test-field-bool", OptFieldTypeBool())
err := cli.EnsureField(testFieldBool)
require.NoError(t, err)
// can set bool
_, err = cli.Query(testFieldBool.Set(true, 100))
require.NoErrorf(t, err, "Set(true, 100)")
resp, err := cli.Query(testFieldBool.Row(true))
require.NoErrorf(t, err, "Row(true)")
target := []uint64{100}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("ClearRowQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldClear := testIndex.Field("test-field-clear")
err := cli.EnsureField(testFieldClear)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldClear.Set(1, 100),
testFieldClear.Set(1, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200)")
resp, err := cli.Query(testFieldClear.Row(1))
require.NoErrorf(t, err, "Row(1)")
target := []uint64{100, 200}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
_, err = cli.Query(testFieldClear.ClearRow(1))
require.NoErrorf(t, err, "ClearRow(1)")
resp, err = cli.Query(testFieldClear.Row(1))
require.NoErrorf(t, err, "Row(1)")
target = []uint64(nil)
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("RowsQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldRows := testIndex.Field("test-field-rows")
err := cli.EnsureField(testFieldRows)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldRows.Set(1, 100),
testFieldRows.Set(1, 200),
testFieldRows.Set(2, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
resp, err := cli.Query(testFieldRows.Rows())
require.NoErrorf(t, err, "Rows")
target := RowIdentifiersResult{
IDs: []uint64{1, 2},
}
require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
})
t.Run("UnionRowsQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldRows := testIndex.Field("test-field-rows")
err := cli.EnsureField(testFieldRows)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldRows.Set(1, 100),
testFieldRows.Set(1, 200),
testFieldRows.Set(2, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
resp, err := cli.Query(testFieldRows.Rows().Union())
require.NoErrorf(t, err, "Rows Union")
target := []uint64{100, 200}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("LikeQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldLike := testIndex.Field("test-field-like", OptFieldKeys(true))
err := cli.EnsureField(testFieldLike)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldLike.Set("a", 100),
testFieldLike.Set("b", 200),
testFieldLike.Set("bc", 200),
))
require.NoErrorf(t, err, "Set(a, 100) Set(b, 200) Set(bc, 200)")
resp, err := cli.Query(testFieldLike.Like("b%"))
require.NoErrorf(t, err, `Like(b%)`)
target := RowIdentifiersResult{
Keys: []string{"b", "bc"},
}
require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result")
})
t.Run("GroupByQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldGroupBy := testIndex.Field("test-field-group-by")
err := cli.EnsureField(testFieldGroupBy)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldGroupBy.Set(1, 100),
testFieldGroupBy.Set(1, 200),
testFieldGroupBy.Set(2, 200),
))
require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)")
resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
require.NoErrorf(t, err, `Like(b%)`)
target := []GroupCount{
{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 1}}, Count: 2},
{Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 2}}, Count: 1},
}
assertGroupBy(t, target, resp.Result().GroupCounts())
})
t.Run("GroupByQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldGroupBy := testIndex.Field("test-field-group-by-int", OptFieldTypeInt(-10, 10))
err := cli.EnsureField(testFieldGroupBy)
require.NoError(t, err)
_, err = cli.Query(testIndex.RawQuery(`
Set(0, test-field-group-by-int=1)
Set(1, test-field-group-by-int=2)
Set(2, test-field-group-by-int=-2)
Set(3, test-field-group-by-int=-1)
Set(4, test-field-group-by-int=4)
Set(10, test-field-group-by-int=0)
Set(100, test-field-group-by-int=0)
Set(1000, test-field-group-by-int=0)
Set(10000, test-field-group-by-int=0)
Set(100000, test-field-group-by-int=0)
`))
require.NoError(t, err, "Set(0..100000)")
resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows()))
require.NoErrorf(t, err, `GroupBy(Rows)`)
var a, b, c, d, e, f int64 = -2, -1, 0, 1, 2, 4
target := []GroupCount{
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &a}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &b}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &c}}, Count: 5},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &d}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &e}}, Count: 1},
{Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &f}}, Count: 1},
}
assertGroupBy(t, target, resp.Result().GroupCounts())
})
t.Run("CreateDeleteIndexField", func(t *testing.T) {
tmpIndex := NewIndex("tmp-index")
tmpField := tmpIndex.Field("tmp-field")
err := cli.CreateIndex(tmpIndex)
require.NoError(t, err)
err = cli.CreateField(tmpField)
require.NoError(t, err)
err = cli.DeleteField(tmpField)
require.NoError(t, err)
err = cli.DeleteIndex(tmpIndex)
require.NoError(t, err)
})
t.Run("ErrorCreatingIndexField", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
require.ErrorIs(t, cli.CreateIndex(testIndex), ErrIndexExists)
require.ErrorIs(t, cli.CreateField(testField), ErrFieldExists)
})
t.Run("Failover", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
uri, _ := pnet.NewURIFromAddress("does-not-resolve.foo.bar")
tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0))
_, err := tmpcli.Query(testIndex.All())
require.Error(t, err, ErrHTTPRequest)
})
t.Run("InvalidQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
_, _, err := cli.HTTPRequest("INVALID METHOD", "/foo", nil, nil)
require.Error(t, err)
_, err = cli.Query(testIndex.RawQuery("Invalid query"))
require.Error(t, err)
})
t.Run("Sync", func(t *testing.T) {
testIndexRemote := NewIndex("test-index-remote")
err := cli.EnsureIndex(testIndexRemote)
require.NoError(t, err)
testFieldRemote := testIndexRemote.Field("test-field-remote")
err = cli.EnsureField(testFieldRemote)
require.NoError(t, err)
schema := NewSchema()
idx1 := schema.Index("index-1")
idx1.Field("field-1-1")
idx1.Field("field-1-2")
idx2 := schema.Index("index-2")
idx2.Field("field-2-1")
schema.Index(testIndexRemote.Name())
err = cli.SyncSchema(schema)
require.NoError(t, err)
err = cli.DeleteIndex(testIndexRemote)
require.NoError(t, err)
err = cli.DeleteIndex(idx1)
require.NoError(t, err)
err = cli.DeleteIndex(idx2)
require.NoError(t, err)
})
t.Run("FetchFragmentNodes", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
nodes, err := cli.fetchFragmentNodes(testIndex.Name(), 0)
require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name)
require.Equalf(t, replicaN, len(nodes), "len(nodes)")
// running the same for coverage
nodes, err = cli.fetchFragmentNodes(testIndex.Name(), 0)
require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name)
require.Equalf(t, replicaN, len(nodes), "len(nodes)")
})
t.Run("RowRangeQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldRange := testIndex.Field("test-field-range", OptFieldTypeTime(TimeQuantumMonthDayHour))
err := cli.EnsureField(testFieldRange)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldRange.SetTimestamp(10, 100, time.Date(2017, time.January, 1, 0, 0, 0, 0, time.UTC)),
testFieldRange.SetTimestamp(10, 100, time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC)),
testFieldRange.SetTimestamp(10, 100, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)),
))
require.NoErrorf(t, err, "BatchQuery SetTimestamp")
start := time.Date(2017, time.January, 5, 0, 0, 0, 0, time.UTC)
end := time.Date(2018, time.January, 5, 0, 0, 0, 0, time.UTC)
resp, err := cli.Query(testFieldRange.RowRange(10, start, end))
require.NoErrorf(t, err, "RowRange(10, %v, %v)", start, end)
target := []uint64{100}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("StoreQuery", func(t *testing.T) {
schema := NewSchema()
testIndexStore := schema.Index("test-index-store")
testFieldFrom := testIndexStore.Field("test-field-from")
testFieldTo := testIndexStore.Field("test-field-to")
err := cli.SyncSchema(schema)
require.NoError(t, err)
defer func() {
cerr := cli.DeleteIndex(testIndexStore)
require.NoErrorf(t, cerr, "failed to delete index: %v", testIndexStore.name)
}()
_, err = cli.Query(testIndexStore.BatchQuery(
testFieldFrom.Set(10, 100),
testFieldFrom.Set(10, 200),
testFieldTo.Store(testFieldFrom.Row(10), 1),
))
require.NoErrorf(t, err, "Set(10, 100) Set(10, 200) Store(Row(10), 1)")
resp, err := cli.Query(testFieldTo.Row(1))
require.NoErrorf(t, err, "Row(1)")
target := []uint64{100, 200}
require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns")
})
t.Run("MultipleClientKeyQuery", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldMultiClient := testIndexWithKeys.Field("test-field-multiclient")
err := cli.EnsureField(testFieldMultiClient)
require.NoError(t, err)
eg := &errgroup.Group{}
for i := 0; i < 10; i++ {
rowID := uint64(i)
eg.Go(func() error {
_, e := cli.Query(testFieldMultiClient.Set(rowID, "col"))
return e
})
}
require.NoError(t, eg.Wait())
})
t.Run("ExportRowIDColumnID", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldExport := testIndex.Field("test-field-export")
err := cli.EnsureField(testFieldExport)
require.NoError(t, err)
_, err = cli.Query(testIndex.BatchQuery(
testFieldExport.Set(1, 1),
testFieldExport.Set(1, 10),
testFieldExport.Set(2, 1048577),
), nil)
require.NoErrorf(t, err, "Set(1, 1) Set(1, 10) Set(2, 1048577)")
r, err := cli.ExportField(testFieldExport)
require.NoErrorf(t, err, "ExportField")
b, err := ioutil.ReadAll(r)
require.NoError(t, err)
target := "1,1\n1,10\n2,1048577\n"
require.Equalf(t, target, string(b), "Export Field Response")
})
t.Run("ExportRowIDColumnKey", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldExport := testIndexWithKeys.Field("test-field-export")
err := cli.EnsureField(testFieldExport)
require.NoError(t, err)
_, err = cli.Query(testIndexWithKeys.BatchQuery(
testFieldExport.Set(1, "one"),
testFieldExport.Set(1, "ten"),
testFieldExport.Set(2, "big-number"),
), nil)
require.NoErrorf(t, err, "Set(1, one) Set(1, ten) Set(2, big-number)")
r, err := cli.ExportField(testFieldExport)
require.NoErrorf(t, err, "ExportField")
b, err := ioutil.ReadAll(r)
require.NoError(t, err)
target := "1,one\n1,ten\n2,big-number\n"
require.Equalf(t, target, string(b), "Export Field Response")
})
t.Run("TranslateRowKeys", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
testFieldTranslate := testIndexKeyTranslation.Field("test-field-translate", OptFieldKeys(true))
err := cli.EnsureField(testFieldTranslate)
require.NoError(t, err)
trans, err := cli.CreateFieldKeys(testFieldTranslate, "key1", "key2")
require.NoErrorf(t, err, "CreateFieldKeys")
target := map[string]uint64{"key1": 1, "key2": 2}
require.Equalf(t, target, trans, "CreateFieldKeys")
trans, err = cli.FindFieldKeys(testFieldTranslate, "key1", "key2", "key3")
require.NoErrorf(t, err, "FindFieldKeys")
require.Equalf(t, target, trans, "FindFieldKeys")
})
t.Run("TranslateColKeys", func(t *testing.T) {
setup(t, cli)
defer tearDown(t, cli)
created, err := cli.CreateIndexKeys(testIndexKeyTranslation, "key1", "key2")
require.NoErrorf(t, err, "CreateIndexKeys")
if _, ok := created["key1"]; !ok {
t.Error("key1 missing")
}
if _, ok := created["key2"]; !ok {
t.Error("key2 missing")
}
found, err := cli.FindIndexKeys(testIndexKeyTranslation, "key1", "key2", "key3")
require.NoErrorf(t, err, "FindIndexKeys")
require.Equalf(t, created, found, "IndexKeys")
})
t.Run("Transactions", func(t *testing.T) {
trns, err := cli.StartTransaction("blah", time.Minute, false, time.Minute)
require.NoErrorf(t, err, "StartTransaction(blah)")
require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
require.Truef(t, trns.Active, "TranslateColumnKeys Active")
trnsMap, err := cli.Transactions()
require.NoErrorf(t, err, "Transactions")
require.Equalf(t, 1, len(trnsMap), "Transactions len")
require.Truef(t, trnsMap["blah"].Active, "Transactions Active")
trns, err = cli.GetTransaction("blah")
require.NoErrorf(t, err, "GetTransaction(blah)")
require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
require.Truef(t, trns.Active, "TranslateColumnKeys Active")
trns, err = cli.FinishTransaction("blah")
require.NoErrorf(t, err, "FinishTransaction(blah)")
require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID")
require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout")
require.Truef(t, trns.Active, "TranslateColumnKeys Active")
})
t.Run("ImportRoaringShard", func(t *testing.T) {
setup(t, cli)
shardWidth := uint64(1 << shardwidth.Exponent)
bitmap := roaring.NewBitmap(1, shardWidth*2+1, shardWidth*3+1)
buf := &bytes.Buffer{}
_, err := bitmap.WriteTo(buf)
if err != nil {
t.Fatalf("serializing bitmap: %v", err)
}
request := &featurebase.ImportRoaringShardRequest{
Remote: true,
Views: []featurebase.RoaringUpdate{
{
Field: "test-field",
View: "standard",
Set: buf.Bytes(),
},
{
Field: "test-field-timestamp",
View: "bsig_test-field-timestamp",
Set: buf.Bytes(),
},
{
Field: "test-field-int",
View: "bsig_test-field-int",
Set: buf.Bytes(),
},
},
}
err = cli.ImportRoaringShard("test-index", 3, request)
if err != nil {
t.Fatalf("import-roaring-shard: %v", err)
}
if resp, err := cli.Query(testField.Row(2)); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldInt.NotNull()); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldTimestamp.NotNull()); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
if resp, err := cli.Query(testIndex.RawQuery("Row(test-field-timestamp>'1969-12-31T23:59:59Z')")); err != nil {
t.Fatalf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 {
t.Fatalf("unexpected result: %v", res)
}
// now write more data
bitmap = roaring.NewBitmap(1, 2, shardWidth*3+1, shardWidth*3+2)
buf = &bytes.Buffer{}
_, err = bitmap.WriteTo(buf)
if err != nil {
t.Fatalf("serializing bitmap: %v", err)
}
request = &featurebase.ImportRoaringShardRequest{
Remote: true,
Views: []featurebase.RoaringUpdate{
{
Field: "test-field",
View: "standard",
Set: buf.Bytes(),
},
{
Field: "test-field-timestamp",
View: "bsig_test-field-timestamp",
Set: buf.Bytes(),
},
{
Field: "test-field-int",
View: "bsig_test-field-int",
Set: buf.Bytes(),
},
},
}
err = cli.ImportRoaringShard("test-index", 3, request)
if err != nil {
t.Fatalf("import-roaring-shard: %v", err)
}
if resp, err := cli.Query(testField.Row(3)); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldInt.NotNull()); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
if resp, err := cli.Query(testFieldTimestamp.NotNull()); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
if resp, err := cli.Query(testIndex.RawQuery("Row(test-field-timestamp>'1969-12-31T23:59:59Z')")); err != nil {
t.Errorf("querying: %v", err)
} else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 {
t.Errorf("unexpected result: %v", res)
}
})
})
}
}
func assertGroupBy(t *testing.T, expected, results []GroupCount) {
t.Helper()
require.Equalf(t, len(expected), len(results), "number of groupings mismatch")
for i, result := range results {
require.Equalf(t, expected[i], result, "unexpected result at %d", i)
}
}

210
client/client_test.go Normal file
View file

@ -0,0 +1,210 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
"crypto/tls"
"errors"
"reflect"
"testing"
pnet "github.com/molecula/featurebase/v3/net"
)
func TestQueryWithError(t *testing.T) {
var err error
client := DefaultClient()
index := NewIndex("foo")
invalid := NewPQLRowQuery("", index, errors.New("invalid"))
_, err = client.Query(invalid, nil)
if err == nil {
t.Fatalf("Should have failed")
}
}
func TestClientOptions(t *testing.T) {
targets := []*ClientOptions{
{SocketTimeout: 10},
{ConnectTimeout: 5},
{PoolSizePerRoute: 7},
{TotalPoolSize: 17},
{TLSConfig: &tls.Config{InsecureSkipVerify: true}},
}
optionsList := [][]ClientOption{
{OptClientSocketTimeout(10)},
{OptClientConnectTimeout(5)},
{OptClientPoolSizePerRoute(7)},
{OptClientTotalPoolSize(17)},
{OptClientTLSConfig(&tls.Config{InsecureSkipVerify: true})},
}
for i := 0; i < len(targets); i++ {
options := &ClientOptions{}
err := options.addOptions(optionsList[i]...)
if err != nil {
t.Fatal(err)
}
target := targets[i]
if !reflect.DeepEqual(target, options) {
t.Fatalf("%v != %v", target, options)
}
}
}
func TestNewClientWithErrorredOption(t *testing.T) {
_, err := NewClient(":8888", ClientOptionErr(0))
if err == nil {
t.Fatalf("Should have failed")
}
}
func TestNewClient(t *testing.T) {
client, err := NewClient(":9999", OptClientManualServerAddress(true))
if err != nil {
t.Fatal(err)
}
targetURI, err := pnet.NewURIFromAddress(":9999")
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(targetURI, client.manualServerURI) {
t.Fatalf("%v != %v", targetURI, client.manualServerURI)
}
targetFragmentNode := &fragmentNode{
Scheme: "http",
Host: "localhost",
Port: 9999,
}
if !reflect.DeepEqual(targetFragmentNode, client.manualFragmentNode) {
t.Fatalf("%v != %v", targetFragmentNode, client.manualFragmentNode)
}
client, err = NewClient(":9999")
if err != nil {
t.Fatal(err)
}
targetURI, err = pnet.NewURIFromAddress(":9999")
if err != nil {
t.Fatal(err)
}
target := []*pnet.URI{targetURI}
if !reflect.DeepEqual(target, client.cluster.hosts) {
t.Fatalf("%v != %v", target, client.cluster.hosts)
}
client, err = NewClient([]string{":9999"})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(target, client.cluster.hosts) {
t.Fatalf("%v != %v", target, client.cluster.hosts)
}
targetURI1, err := pnet.NewURIFromAddress(":8888")
if err != nil {
t.Fatal(err)
}
targetURI2, err := pnet.NewURIFromAddress(":9999")
if err != nil {
t.Fatal(err)
}
client, err = NewClient([]*pnet.URI{targetURI1, targetURI2})
if err != nil {
t.Fatal(err)
}
target = []*pnet.URI{targetURI1, targetURI2}
if !reflect.DeepEqual(target, client.cluster.hosts) {
t.Fatalf("%v != %v", target, client.cluster.hosts)
}
client, err = NewClient([]*pnet.URI{targetURI})
if err != nil {
t.Fatal(err)
}
target = []*pnet.URI{targetURI}
if !reflect.DeepEqual(target, client.cluster.hosts) {
t.Fatalf("%v != %v", target, client.cluster.hosts)
}
client, err = NewClient(DefaultCluster())
if err != nil {
t.Fatal(err)
}
target = []*pnet.URI{}
if !reflect.DeepEqual(target, client.cluster.hosts) {
t.Fatalf("%v != %v", target, client.cluster.hosts)
}
}
func TestNewClientWithInvalidAddr(t *testing.T) {
_, err := NewClient(10)
if err != ErrAddrURIClusterExpected {
t.Fatalf("%v != %v", ErrAddrURIClusterExpected, err)
}
_, err = NewClient(":invalid")
if err == nil {
t.Fatalf("should have failed: %+v", err)
}
_, err = NewClient([]string{"valid:8000", ":invalid"})
if err != pnet.ErrInvalidAddress {
t.Fatalf("Should have failed '%v, got '%v'", pnet.ErrInvalidAddress, err)
}
}
func TestNewClientManualAddressWithNoURIs(t *testing.T) {
_, err := NewClient([]string{}, OptClientManualServerAddress(true))
if err != ErrSingleServerAddressRequired {
t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
}
_, err = NewClient([]*pnet.URI{}, OptClientManualServerAddress(true))
if err != ErrSingleServerAddressRequired {
t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
}
}
func TestNewClientManualAddressWithMultipleURIs(t *testing.T) {
_, err := NewClient([]string{":9000", ":5000"}, OptClientManualServerAddress(true))
if err != ErrSingleServerAddressRequired {
t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
}
targetURI1, err := pnet.NewURIFromAddress(":9000")
if err != nil {
t.Fatal(err)
}
targetURI2, err := pnet.NewURIFromAddress(":5000")
if err != nil {
t.Fatal(err)
}
_, err = NewClient([]*pnet.URI{targetURI1, targetURI2}, OptClientManualServerAddress(true))
if err != ErrSingleServerAddressRequired {
t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err)
}
}
func ClientOptionErr(int) ClientOption {
return func(*ClientOptions) error {
return errors.New("Some error")
}
}
func TestQueryOptionsError(t *testing.T) {
client := DefaultClient()
index := NewIndex("foo")
_, err := client.Query(index.RawQuery(""), QueryOptionErr(0))
if err == nil {
t.Fatalf("should have failed")
}
}
func QueryOptionErr(int) QueryOption {
return func(*QueryOptions) error {
return errors.New("Some error")
}
}

88
client/cluster.go Normal file
View file

@ -0,0 +1,88 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
"sync"
pnet "github.com/molecula/featurebase/v3/net"
)
// Cluster contains hosts in a Pilosa cluster.
type Cluster struct {
hosts []*pnet.URI
okList []bool
mutex *sync.RWMutex
lastHostIdx int
}
// DefaultCluster returns the default Cluster.
func DefaultCluster() *Cluster {
return &Cluster{
hosts: make([]*pnet.URI, 0),
okList: make([]bool, 0),
mutex: &sync.RWMutex{},
}
}
// NewClusterWithHost returns a cluster with the given URIs.
func NewClusterWithHost(hosts ...*pnet.URI) *Cluster {
cluster := DefaultCluster()
for _, host := range hosts {
cluster.AddHost(host)
}
return cluster
}
// AddHost adds a host to the cluster.
func (c *Cluster) AddHost(address *pnet.URI) {
c.mutex.Lock()
defer c.mutex.Unlock()
c.hosts = append(c.hosts, address)
c.okList = append(c.okList, true)
}
// Host returns a host in the cluster.
func (c *Cluster) Host() *pnet.URI {
c.mutex.Lock()
var host *pnet.URI
for i := range c.okList {
idx := (i + c.lastHostIdx) % len(c.okList)
ok := c.okList[idx]
if ok {
host = c.hosts[idx]
break
}
}
c.lastHostIdx++
c.mutex.Unlock()
if host != nil {
return host
}
c.reset()
return host
}
// Hosts returns all available hosts in the cluster.
func (c *Cluster) Hosts() []pnet.URI {
c.mutex.RLock()
defer c.mutex.RUnlock()
hosts := make([]pnet.URI, 0, len(c.hosts))
for i, host := range c.hosts {
if c.okList[i] {
hosts = append(hosts, *host)
}
}
return hosts
}
func (c *Cluster) reset() {
c.mutex.Lock()
defer c.mutex.Unlock()
for i := range c.okList {
c.okList[i] = true
}
}

52
client/cluster_test.go Normal file
View file

@ -0,0 +1,52 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
"testing"
pnet "github.com/molecula/featurebase/v3/net"
)
func TestNewClusterWithHost(t *testing.T) {
c := NewClusterWithHost(pnet.DefaultURI())
hosts := c.Hosts()
if len(hosts) != 1 || !hosts[0].Equals(pnet.DefaultURI()) {
t.Fail()
}
}
func TestAddHost(t *testing.T) {
const addr = "http://localhost:3000"
c := DefaultCluster()
if c.Hosts() == nil {
t.Fatalf("Hosts should not be nil")
}
uri, err := pnet.NewURIFromAddress(addr)
if err != nil {
t.Fatalf("Cannot parse address")
}
target, err := pnet.NewURIFromAddress(addr)
if err != nil {
t.Fatalf("Cannot parse address")
}
c.AddHost(uri)
hosts := c.Hosts()
if len(hosts) != 1 || !hosts[0].Equals(target) {
t.Fail()
}
}
func TestHosts(t *testing.T) {
c := DefaultCluster()
if c.Host() != nil {
t.Fatalf("Hosts with empty cluster should return nil")
}
c = NewClusterWithHost(pnet.DefaultURI())
if !c.Host().Equals(pnet.DefaultURI()) {
t.Fatalf("Host should return a value if there are hosts in the cluster")
}
}

182
client/csv/csv.go Normal file
View file

@ -0,0 +1,182 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package csv
import (
"bufio"
"errors"
"fmt"
"io"
"strconv"
"strings"
"time"
"github.com/molecula/featurebase/v3/client"
)
// Format is the format of the data in the CSV file.
type Format uint
const (
// RowIDColumnID formatted data is ROW_ID,COLUMN_ID.
RowIDColumnID Format = iota
// RowIDColumnKey formatted data is ROW_ID,COLUMN_KEY.
RowIDColumnKey
// RowKeyColumnID formatted data is ROW_KEY,COLUMN_ID.
RowKeyColumnID
// RowKeyColumnKey formatted data is ROW_KEY,COLUMN_ID.
RowKeyColumnKey
// ColumnID formatted data is COLUMN_ID. Valid only for value import.
ColumnID
// ColumnKey formatted data is COLUMN_KEY. Valud only for value import.
ColumnKey
)
// ColumnUnmarshaller creates a RecordUnmarshaller for importing columns with the given format.
func ColumnUnmarshaller(format Format) RecordUnmarshaller {
return ColumnUnmarshallerWithTimestamp(format, "")
}
// ColumnUnmarshallerWithTimestamp creates a RecordUnmarshaller for importing columns with the given format and timestamp format.
func ColumnUnmarshallerWithTimestamp(format Format, timestampFormat string) RecordUnmarshaller {
return func(text string) (client.Record, error) {
var err error
column := client.Column{}
parts := strings.Split(text, ",")
if len(parts) < 2 {
return nil, errors.New("invalid CSV line")
}
hasRowKey := format == RowKeyColumnID || format == RowKeyColumnKey
hasColumnKey := format == RowIDColumnKey || format == RowKeyColumnKey
if hasRowKey {
column.RowKey = parts[0]
} else {
column.RowID, err = strconv.ParseUint(parts[0], 10, 64)
if err != nil {
return nil, errors.New("invalid row ID")
}
}
if hasColumnKey {
column.ColumnKey = parts[1]
} else {
column.ColumnID, err = strconv.ParseUint(parts[1], 10, 64)
if err != nil {
return nil, errors.New("invalid column ID")
}
}
timestamp := int64(0)
if len(parts) == 3 {
if timestampFormat == "" {
if tsInt, err := strconv.Atoi(parts[2]); err != nil {
return nil, err
} else {
timestamp = int64(tsInt)
}
} else {
t, err := time.Parse(timestampFormat, parts[2])
if err != nil {
return nil, err
}
timestamp = t.Unix() * int64(time.Second) // Casting a duration to int64 gives the number of nanoseconds in that duration.
}
}
column.Timestamp = timestamp
return column, nil
}
}
// RecordUnmarshaller is a function which creates a Record from a CSV file line with column data.
type RecordUnmarshaller func(text string) (client.Record, error)
// Iterator reads records from a Reader.
// Each line should contain a single record in the following form:
// field1,field2,...
type Iterator struct {
reader io.Reader
line int
scanner *bufio.Scanner
unmarshaller RecordUnmarshaller
}
// NewIterator creates a CSVIterator from a Reader.
func NewIterator(reader io.Reader, unmarshaller RecordUnmarshaller) *Iterator {
return &Iterator{
reader: reader,
line: 0,
scanner: bufio.NewScanner(reader),
unmarshaller: unmarshaller,
}
}
// NewColumnIterator creates a new iterator for column data.
func NewColumnIterator(format Format, reader io.Reader) *Iterator {
return NewIterator(reader, ColumnUnmarshaller(format))
}
// NewColumnIteratorWithTimestampFormat creates a new iterator for column data with timestamp.
func NewColumnIteratorWithTimestampFormat(format Format, reader io.Reader, timestampFormat string) *Iterator {
return NewIterator(reader, ColumnUnmarshallerWithTimestamp(format, timestampFormat))
}
// NewValueIterator creates a new iterator for value data.
func NewValueIterator(format Format, reader io.Reader) *Iterator {
return NewIterator(reader, FieldValueUnmarshaller(format))
}
// NextRecord iterates on lines of a Reader.
// Returns io.EOF on end of iteration.
func (c *Iterator) NextRecord() (client.Record, error) {
if ok := c.scanner.Scan(); ok {
c.line++
text := strings.TrimSpace(c.scanner.Text())
if text != "" {
rc, err := c.unmarshaller(text)
if err != nil {
return nil, fmt.Errorf("%s at line: %d", err.Error(), c.line)
}
return rc, nil
}
}
err := c.scanner.Err()
if err != nil {
return nil, err
}
return nil, io.EOF
}
// FieldValueUnmarshaller is a function which creates a Record from a CSV file line with value data.
func FieldValueUnmarshaller(format Format) RecordUnmarshaller {
return func(text string) (client.Record, error) {
parts := strings.Split(text, ",")
if len(parts) < 2 {
return nil, errors.New("invalid CSV")
}
value, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return nil, errors.New("invalid value")
}
switch format {
case ColumnID:
columnID, err := strconv.ParseUint(parts[0], 10, 64)
if err != nil {
return nil, errors.New("invalid column ID at line: %d")
}
return client.FieldValue{
ColumnID: uint64(columnID),
Value: value,
}, nil
case ColumnKey:
return client.FieldValue{
ColumnKey: parts[0],
Value: value,
}, nil
default:
return nil, fmt.Errorf("invalid format: %d", format)
}
}
}

49
client/csv/csv_it_test.go Normal file
View file

@ -0,0 +1,49 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
//go:build integration
// +build integration
package csv_test
import (
"io"
"reflect"
"strings"
"testing"
"github.com/molecula/featurebase/v3/client"
"github.com/molecula/featurebase/v3/client/csv"
)
func TestCSVIterate(t *testing.T) {
text := `10,7
10,5
2,3
7,1`
iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text))
recs := consumeIterator(t, iterator)
target := []client.Record{
client.Column{RowID: 10, ColumnID: 7},
client.Column{RowID: 10, ColumnID: 5},
client.Column{RowID: 2, ColumnID: 3},
client.Column{RowID: 7, ColumnID: 1},
}
if !reflect.DeepEqual(target, recs) {
t.Fatalf("%v != %v", target, recs)
}
}
func consumeIterator(t *testing.T, it *csv.Iterator) []client.Record {
recs := []client.Record{}
for {
r, err := it.NextRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
recs = append(recs, r)
}
return recs
}

255
client/csv/csv_test.go Normal file
View file

@ -0,0 +1,255 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package csv_test
import (
"errors"
"io"
"reflect"
"strings"
"testing"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/client"
"github.com/molecula/featurebase/v3/client/csv"
)
func TestCSVColumnIterator(t *testing.T) {
reader := strings.NewReader(`1,10,683793200
5,20,683793300
3,41,683793385`)
iterator := csv.NewColumnIterator(csv.RowIDColumnID, reader)
columns := []client.Record{}
for {
column, err := iterator.NextRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
columns = append(columns, column)
}
if len(columns) != 3 {
t.Fatalf("There should be 3 columns")
}
target := []client.Column{
{RowID: 1, ColumnID: 10, Timestamp: 683793200},
{RowID: 5, ColumnID: 20, Timestamp: 683793300},
{RowID: 3, ColumnID: 41, Timestamp: 683793385},
}
for i := range target {
if !reflect.DeepEqual(target[i], columns[i]) {
t.Fatalf("%v != %v", target[i], columns[i])
}
}
}
func TestCSVColumnIteratorWithTimestampFormatRowIDColumnID(t *testing.T) {
format := "2006-01-02T03:04"
reader := strings.NewReader(`1,10,1991-09-02T09:33
5,20,1991-09-02T09:35
3,41,1991-09-02T09:36`)
iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format)
records := []client.Record{}
for {
record, err := iterator.NextRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
records = append(records, record)
}
target := []client.Column{
{RowID: 1, ColumnID: 10, Timestamp: 683803980000000000},
{RowID: 5, ColumnID: 20, Timestamp: 683804100000000000},
{RowID: 3, ColumnID: 41, Timestamp: 683804160000000000},
}
if len(records) != len(target) {
t.Fatalf("There should be %d columns", len(target))
}
for i := range target {
if !reflect.DeepEqual(target[i], records[i]) {
t.Fatalf("%v != %v", target[i], records[i])
}
}
}
func TestCSVColumnIteratorWithTimestampFormatRowKeyColumnKey(t *testing.T) {
format := "2006-01-02T03:04"
reader := strings.NewReader(`one,ten,1991-09-02T09:33
five,twenty,1991-09-02T09:35
three,forty-one,1991-09-02T09:36`)
iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowKeyColumnKey, reader, format)
records := []client.Record{}
for {
record, err := iterator.NextRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
records = append(records, record)
}
target := []client.Column{
{RowKey: "one", ColumnKey: "ten", Timestamp: 683803980000000000},
{RowKey: "five", ColumnKey: "twenty", Timestamp: 683804100000000000},
{RowKey: "three", ColumnKey: "forty-one", Timestamp: 683804160000000000},
}
if len(records) != len(target) {
t.Fatalf("There should be %d columns", len(target))
}
for i := range target {
if !reflect.DeepEqual(target[i], records[i]) {
t.Fatalf("%v != %v", target[i], records[i])
}
}
}
func TestCSVColumnIteratorWithTimestampFormatFail(t *testing.T) {
format := "2014-07-16"
reader := strings.NewReader(`1,10,X`)
iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format)
_, err := iterator.NextRecord()
if err == nil {
t.Fatalf("Should have failed")
}
}
func TestCSVValueIteratorWithColumnID(t *testing.T) {
reader := strings.NewReader(`1,10
5,-20
3,41
`)
iterator := csv.NewValueIterator(csv.ColumnID, reader)
values := []client.Record{}
for {
value, err := iterator.NextRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
values = append(values, value)
}
target := []pilosa.FieldValue{
{ColumnID: 1, Value: 10},
{ColumnID: 5, Value: -20},
{ColumnID: 3, Value: 41},
}
if len(values) != len(target) {
t.Fatalf("There should be %d values, got %d", len(target), len(values))
}
for i := range target {
v := values[i].(client.FieldValue)
if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) {
t.Fatalf("'%+v' != '%+v'", target[i], values[i])
}
}
}
func TestCSVValueIteratorWithColumnKey(t *testing.T) {
reader := strings.NewReader(`one,10
five,-20
three,41
`)
iterator := csv.NewValueIterator(csv.ColumnKey, reader)
values := []client.Record{}
for {
value, err := iterator.NextRecord()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
values = append(values, value)
}
target := []pilosa.FieldValue{
{ColumnKey: "one", Value: 10},
{ColumnKey: "five", Value: -20},
{ColumnKey: "three", Value: 41},
}
if len(values) != len(target) {
t.Fatalf("There should be %d values, got %d", len(target), len(values))
}
for i := range target {
v := values[i].(client.FieldValue)
if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) {
t.Fatalf("%v != %v", target[i], values[i])
}
}
}
func TestCSValueIteratorWithInvalidFormat(t *testing.T) {
reader := strings.NewReader("1,2")
iterator := csv.NewValueIterator(csv.RowIDColumnID, reader)
_, err := iterator.NextRecord()
if err == nil {
t.Fatalf("should have failed")
}
}
func TestCSVColumnIteratorInvalidInput(t *testing.T) {
invalidInputs := []string{
// less than 2 columns
"155",
// invalid row ID
"a5,155",
// invalid column ID
"155,a5",
// invalid timestamp
"155,255,a5",
}
for _, text := range invalidInputs {
iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text))
_, err := iterator.NextRecord()
if err == nil {
t.Fatalf("CSVColumnIterator input: %s should fail", text)
}
}
}
func TestCSVValueIteratorInvalidInput(t *testing.T) {
invalidInputs := []string{
// less than 2 columns
"155",
// invalid column ID
"a5,155",
// invalid value
"155,a5",
}
for _, text := range invalidInputs {
iterator := csv.NewValueIterator(csv.ColumnID, strings.NewReader(text))
_, err := iterator.NextRecord()
if err == nil {
t.Fatalf("CSVValueIterator input: %s should fail", text)
}
}
}
func TestCSVColumnIteratorError(t *testing.T) {
iterator := csv.NewColumnIterator(csv.RowIDColumnID, &BrokenReader{})
_, err := iterator.NextRecord()
if err == nil {
t.Fatal("CSVColumnIterator should fail with error")
}
}
func TestCSVValueIteratorError(t *testing.T) {
iterator := csv.NewValueIterator(csv.ColumnID, &BrokenReader{})
_, err := iterator.NextRecord()
if err == nil {
t.Fatal("CSVValueIterator should fail with error")
}
}
type BrokenReader struct{}
func (r BrokenReader) Read(p []byte) (n int, err error) {
return 0, errors.New("broken reader")
}

56
client/doc.go Normal file
View file

@ -0,0 +1,56 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
/*
Package client enables querying a Pilosa server.
This client uses Pilosa's http+protobuf API.
Usage:
import (
"fmt"
"github.com/molecula/featurebase/v3/client"
)
// Create a Client instance
cli := client.DefaultClient()
// Create a Schema instance
schema, err := cli.Schema()
if err != nil {
panic(err)
}
// Create an Index instance
index, err := schema.Index("repository")
if err != nil {
panic(err)
}
// Create a Field instance
stargazer, err := index.Field("stargazer")
if err != nil {
panic(err)
}
// Sync the schema with the server-side, so non-existing indexes/fields are created on the server-side.
err = cli.SyncSchema(schema)
if err != nil {
panic(err)
}
// Execute a query
response, err := cli.Query(stargazer.Row(5))
if err != nil {
panic(err)
}
// Act on the result
fmt.Println(response.Result())
See also https://www.pilosa.com/docs/api-reference/ and https://www.pilosa.com/docs/query-language/.
*/
package client

149
client/docs/data-model-queries.md Normal file
View file

@ -0,0 +1,149 @@
# Data Model and Queries
## Indexes and Fields
*Index* and *field*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs/latest/data-model/) for more detail about the data model.
The `schema.Index` function is used to create an index instance. Note that this does not create an index on the server; the index object simply defines the schema.
```go
schema := client.NewSchema()
repository := schema.Index("repository")
```
You can pass options while creating index instances:
```go
repository := schema.Index("repository", pilosa.OptIndexKeys(true))
```
Field definitions are created with a call to the `Field` function of an index:
```go
stargazer := repository.Field("stargazer")
```
You can pass options to fields:
```go
stargazer := repository.Field("stargazer", pilosa.OptFieldTypeTime(TimeQuantumYearMonthDay))
```
In case the schema already exists on the server, you can retrieve that instead of creating the schema:
```go
cli := client.DefaultClient()
schema, err := cli.Schema()
if err != nil {
// act on the error
}
repository := schema.Index("repository")
```
## Queries
Once you have indexes and field definitions, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows with related methods attached to fields.
For instance, `Row` queries work on rows; use a `Field` object to create those queries:
```go
rowQuery := stargazer.Row(1) // corresponds to PQL: Row(stargazer=1)
```
`Union` queries work on columns; use the index to create them:
```go
query := repository.Union(rowQuery1, rowQuery2)
```
In order to increase throughput, you may want to batch queries sent to the Pilosa server. The `index.BatchQuery` function is used for that purpose:
```go
query := repository.BatchQuery(
stargazer.Row(1),
repository.Union(stargazer.Row(100), stargazer.Row(5)))
```
The recommended way of creating query instances is using dedicated functions attached to index and field objects, but sometimes it would be desirable to send raw queries to Pilosa. You can use `index.RawQuery` method for that. Note that query string is not validated before sending to the server:
```go
query := repository.RawQuery("Row(stargazer=5)")
```
Raw queries are only sent to the coordinator node of a Pilosa cluster, so currently there's a possible performance hit using them instead of ORM functions attached to index or field instances.
This client supports [range queries using bit sliced indexes (BSI)](https://www.pilosa.com/docs/latest/query-language/#range-bsi). Read the [Range Encoded Bitmaps](https://www.pilosa.com/blog/range-encoded-bitmaps/) blog post for more information about the BSI implementation of range encoding in Pilosa.
In order to use BSI range queries, an integer field should be created. The field should have its minimum and maximum set. Here's how you would do that:
```go
index := schema.Index("animals")
captivity := index.Field("captivity", pilosa.OptFieldTypeInt(0, 956))
```
If the field with the necessary field already exists on the server, you don't need to create the field instance, `cli.SyncSchema(schema)` would load that to `schema`. You can then add some data:
```go
// Add the captivity values to the field.
data := []int{3, 392, 47, 956, 219, 14, 47, 504, 21, 0, 123, 318}
query := index.BatchQuery()
for i, x := range data {
column := uint64(i + 1)
query.Add(captivity.SetIntValue(column, x))
}
cli.Query(query)
```
Let's write a range query:
```go
// Query for all animals with more than 100 specimens
response, _ := cli.Query(captivity.GT(100))
fmt.Println(response.Result().Row().Columns)
// Query for the total number of animals in captivity
response, _ = cli.Query(captivity.Sum(nil))
fmt.Println(response.Result().Value())
```
If you pass a row query to `Sum` as a filter, then only the columns matching the filter will be considered in the `Sum` calculation:
```go
// Let's run a few set queries first
cli.Query(index.BatchQuery(
field.Set(42, 1),
field.Set(42, 6)))
// Query for the total number of animals in captivity where row 42 is set
response, _ = cli.Query(captivity.Sum(field.Row(42)))
fmt.Println(response.Result().Value())
```
See the functions further below for the list of functions that can be used with a `Field`.
Please check [Pilosa documentation](https://www.pilosa.com/docs) for PQL details. Here is a list of methods corresponding to PQL calls:
Index:
* `Union(rows *PQLRowQuery...) *PQLRowQuery`
* `Intersect(rows *PQLRowQuery...) *PQLRowQuery`
* `Difference(rows *PQLRowQuery...) *PQLRowQuery`
* `Xor(rows ...*PQLRowQuery) *PQLRowQuery`
* `Not(row) *PQLRowQuery`
* `Count(row *PQLRowQuery) *PQLBaseQuery`
* `Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery`
Field:
* `Row(rowID uint64) *PQLRowQuery`
* `Set(rowID uint64, columnID uint64) *PQLBaseQuery`
* `SetTimestamp(rowID uint64, columnID uint64, timestamp time.Time) *PQLBaseQuery`
* `Clear(rowID uint64, columnID uint64) *PQLBaseQuery`
* `TopN(n uint64) *PQLRowQuery`
* `RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery`
* `Range(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
* `RowRange(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
* `ClearRow(rowIDOrKey interface{}) *PQLBaseQuery`
* `Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery`
* `LT(n int) *PQLRowQuery`
* `LTE(n int) *PQLRowQuery`
* `GT(n int) *PQLRowQuery`
* `GTE(n int) *PQLRowQuery`
* `Between(a int, b int) *PQLRowQuery`
* `Sum(row *PQLRowQuery) *PQLBaseQuery`
* `Min(row *PQLRowQuery) *PQLBaseQuery`
* `Max(row *PQLRowQuery) *PQLBaseQuery`
* `SetIntValue(columnID uint64, value int) *PQLBaseQuery`

160
client/docs/server-interaction.md Normal file
View file

@ -0,0 +1,160 @@
# Server Interaction
## Pilosa URI
A Pilosa URI has the `${SCHEME}://${HOST}:${PORT}` format:
* **Scheme**: Protocol of the URI. Default: `http`.
* **Host**: Hostname or ipv4/ipv6 IP address. Default: localhost.
* **Port**: Port number. Default: `10101`.
All parts of the URI are optional, but at least one of them must be specified. The following are equivalent:
* `http://localhost:10101`
* `http://localhost`
* `http://:10101`
* `localhost:10101`
* `localhost`
* `:10101`
A Pilosa URI is represented by the `github.com/pilosa/pilosa/v2/net URI` struct. Below are a few ways to create `URI` objects:
```go
import pnet "github.com/pilosa/pilosa/v2/net"
// create the default URI: http://localhost:10101
uri1 := pnet.DefaultURI()
// create a URI from string address
uri2, err := pnet.NewURIFromAddress("index1.pilosa.com:20202");
// create a URI with the given host and port
uri3, err := pnet.NewURIFromHostPort("index1.pilosa.com", 20202);
```
## Pilosa Client
In order to interact with a Pilosa server, an instance of `client.Client` should be created. The client is thread-safe and uses a pool of connections to the server, so we recommend creating a single instance of the client and sharing it when necessary.
If the Pilosa server is running at the default address (`http://localhost:10101`) you can create the client with default options using:
```go
import "github.com/pilosa/pilosa/v2/client"
cli := client.DefaultClient()
```
To use a custom server address, use the `NewClient` function:
```go
uri, err := pnet.NewURIFromAddress("http://index1.pilosa.com:15000")
if err != nil {
// Act on the error
}
cli, err := client.NewClient(uri)
```
Equivalently:
```go
cli, err := client.NewClient("http://index1.pilosa.com:15000")
```
If you are running a cluster of Pilosa servers, you can create a `Cluster` struct that keeps addresses of those servers:
```go
uri1, err := pnet.NewURIFromAddress(":10101")
uri2, err := pnet.NewURIFromAddress(":10110")
uri3, err := pnet.NewURIFromAddress(":10111")
cluster := client.NewClusterWithHost(uri1, uri2, uri3)
// Create a client with the cluster
cli, err := client.NewClient(cluster)
```
That is equivalent to:
```go
cli, err := client.NewClient([]string{":10101", ":10110", ":10111"})
```
It is possible to customize the behaviour of the underlying HTTP client by passing `ClientOption` structs to the `NewClient` function:
```go
cli, err := client.NewClient(cluster,
client.OptClientConnectTimeout(1000), // if can't connect in a second, close the connection
client.OptClientSocketTimeout(10000), // if no response received in 10 seconds, close the connection
client.OptClientPoolSizePerRoute(3), // number of connections in the pool per host
client.OptClientTotalPoolSize(10)) // number of total connections in the pool
```
Once you create a client, you can create indexes, fields or start sending queries.
Here is how you would create a index and field:
```go
// materialize repository index definition and stargazer field definition initialized before
err := cli.SyncSchema(schema)
```
You can send queries to a Pilosa server using the `Query` function of the `Client` struct:
```go
response, err := cli.Query(field.Row(5));
```
## Server Response
When a query is sent to a Pilosa server, the server either fulfills the query or sends an error message. In the case of an error, a `pilosa.Error` struct is returned, otherwise a `QueryResponse` struct is returned.
A `QueryResponse` struct may contain zero or more results of `QueryResult` type. You can access all results using the `Results` function of `QueryResponse` (which returns a list of `QueryResult` objects), or you can use the `Result` method (which returns either the first result or `nil` if there are no results):
```go
response, err := cli.Query(field.Row(5))
if err != nil {
// Act on the error
}
// check that there's a result and act on it
result := response.Result()
if result != nil {
// Act on the result
}
// iterate over all results
for _, result := range response.Results() {
// Act on the result
}
```
`QueryResult` objects contain:
* `Row()` function to retrieve a row result,
* `CountItems()` function to retrieve column count per row ID entries returned from `TopN` queries,
* `Count()` function to retrieve the number of rows per the given row ID returned from `Count` queries.
* `Value()` function to retrieve the result of `Min`, `Max` or `Sum` queries.
* `Changed()` function returns whether a `Set` or `Clear` query changed a column.
```go
row := result.Row()
columns := row.Columns
countItems := result.CountItems()
count := result.Count()
value := result.Value()
changed := result.Changed()
```
## SSL/TLS
Make sure the Pilosa server runs on a TLS address. [How To Set Up a Secure Cluster](https://www.pilosa.com/docs/latest/tutorials/#how-to-set-up-a-secure-cluster) tutorial explains how to do that.
In order to enable TLS support on the client side, the scheme of the address should be explicitly specified as `https`, e.g.: `https://01.pilosa.local:10501`
This client library uses the `net/http` module of Go standard library. You can pass a [tls.Config](https://golang.org/pkg/crypto/tls/#Config) struct in a `pilosa.TLSConfig` option to the client. If the Pilosa server is using a certificate from a recognized authority, you can use the defaults.
If you are using a self signed certificate, just pass `pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true})` to `pilosa.NewClient` function:
```go
client, _ := pilosa.NewClient("https://01.pilosa.local:10501", pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true}))
```

111
client/docs/tracing.md Normal file
View file

@ -0,0 +1,111 @@
# Tracing
Pilosa client supports distributed tracing via the [OpenTracing](https://opentracing.io/) API.
In order to use a tracer with Go-Pilosa, you should:
1. Create the tracer,
2. Pass the `OptClientOption(tracer)` to `NewClient`.
In this document, we will be using the [Jaeger](https://www.jaegertracing.io) tracer, but OpenTracing has support for [other tracing systems](https://opentracing.io/docs/supported-tracers/).
## Running the Pilosa Server
Let's run a temporary Pilosa container:
$ docker run -it --rm -p 10101:10101 pilosa/pilosa:v1.2.0
Check that you can access Pilosa:
$ curl localhost:10101
Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.
## Running the Jaeger Server
Let's run a Jaeger Server container:
$ docker run -it --rm -p 5775:5775/udp -p 16686:16686 jaegertracing/all-in-one:latest
...<title>Jaeger UI</title>...
## Writing the Sample Code
The sample code depdends on the Jaeger Go client, so let's install it first:
$ go get -u github.com/uber/jaeger-client-go/
Save the following sample code as `gopilosa-tracing.go`:
```go
package main
import (
"log"
"time"
"github.com/pilosa/pilosa/v2/client"
"github.com/uber/jaeger-client-go"
"github.com/uber/jaeger-client-go/config"
)
func main() {
// Create the tracer.
cfg := config.Configuration{
Sampler: &config.SamplerConfig{
Type: "const",
Param: 1,
},
Reporter: &config.ReporterConfig{
LogSpans: true,
BufferFlushInterval: 1 * time.Second,
// Jaeger Server address
LocalAgentHostPort: "127.0.0.1:5775",
},
}
tracer, closer, err := cfg.New(
"go_pilosa_test",
config.Logger(jaeger.StdLogger),
)
// Don't forget to close the tracer.
defer closer.Close()
// Create the client, and pass the tracer.
cli, err := client.NewClient(":10101", pilosa.OptClientTracer(tracer))
if err != nil {
log.Fatal(err)
}
// Read the schema from the server.
// This should create a trace on the Jaeger server.
schema, err := cli.Schema()
if err != nil {
log.Fatal(err)
}
// Create and sync the sample schema.
// This should create a trace on the Jaeger server.
myIndex := schema.Index("my-index")
myField := myIndex.Field("my-field")
err = cli.SyncSchema(schema)
if err != nil {
log.Fatal(err)
}
// Run a query on Pilosa.
// This should create a trace on the Jaeger server.
_, err = cli.Query(myField.Set(1, 1000))
if err != nil {
log.Fatal(err)
}
}
```
## Checking the Tracing Data
Run the sample code:
$ go run gopilosa-tracing.go
* Open http://localhost:16686 in your web browser to visit Jaeger UI.
* Click on the *Search* tab and select `go_pilosa_test` in the *Service* dropdown on the right.
* Click on *Find Traces* button at the bottom left.
* You should see a couple of traces, such as: `Client.Query`, `Client.CreateField`, `Client.Schema`, etc.

111
client/egpool/egpool.go Normal file
View file

@ -0,0 +1,111 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package egpool
import (
"errors"
"fmt"
"sync"
)
type Group struct {
PoolSize int
jobs chan func() error
sema chan struct{}
errMu sync.Mutex
firstErr error
errs []error
}
func (eg *Group) Go(f func() error) {
if eg.PoolSize <= 0 {
eg.PoolSize = 1
}
if eg.jobs == nil {
eg.jobs = make(chan func() error)
eg.sema = make(chan struct{}, eg.PoolSize)
}
// Start the job in an idle worker if possible.
select {
case eg.jobs <- f:
return
default:
}
// Start a new worker if necessary.
select {
case eg.jobs <- f:
// A worker finished its previous job and took this one over.
return
case eg.sema <- struct{}{}:
// Start a new worker.
go eg.processJobs()
eg.jobs <- f
}
}
func (eg *Group) err(err error) {
eg.errMu.Lock()
defer eg.errMu.Unlock()
if eg.firstErr == nil {
eg.firstErr = err
}
eg.errs = append(eg.errs, err)
}
type ErrPanic struct {
Value interface{}
}
func (p ErrPanic) Error() string {
return fmt.Sprintf("panic: %v", p.Value)
}
var ErrGoexit = errors.New("runtime.Goexit used in job function")
func (eg *Group) processJobs() {
// Notify pool of shutdown.
defer func() { <-eg.sema }()
// Handle panic and Goexit.
var finished bool
defer func() {
if !finished {
if p := recover(); p != nil {
eg.err(ErrPanic{p})
} else {
eg.err(ErrGoexit)
}
}
}()
// Run jobs from queue.
for jobFn := range eg.jobs {
err := jobFn()
if err != nil {
eg.err(err)
}
}
finished = true
}
func (eg *Group) Wait() error {
if eg.jobs == nil {
return nil
}
close(eg.jobs)
for i := 0; i < eg.PoolSize; i++ {
eg.sema <- struct{}{}
}
return eg.firstErr
}
func (eg *Group) Errors() []error {
return eg.errs
}

38
client/egpool/egpool_test.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package egpool_test
import (
"errors"
"testing"
"github.com/molecula/featurebase/v3/client/egpool"
)
func TestEGPool(t *testing.T) {
eg := egpool.Group{}
a := make([]int, 10)
for i := 0; i < 10; i++ {
i := i
eg.Go(func() error {
a[i] = i
if i == 7 {
return errors.New("blah")
}
return nil
})
}
err := eg.Wait()
if err == nil || err.Error() != "blah" {
t.Errorf("expected err blah, got: %v", err)
}
for i := 0; i < 10; i++ {
if a[i] != i {
t.Errorf("expected a[%d] to be %d, but is %d", i, i, a[i])
}
}
}

26
client/error.go Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package client
import "github.com/pkg/errors"
// Predefined Pilosa errors.
var (
ErrEmptyCluster = errors.New("No usable addresses in the cluster")
ErrIndexExists = errors.New("Index exists")
ErrFieldExists = errors.New("Field exists")
ErrInvalidIndexName = errors.New("Invalid index name")
ErrInvalidFieldName = errors.New("Invalid field name")
ErrInvalidLabel = errors.New("Invalid label")
ErrInvalidKey = errors.New("Invalid key")
ErrHTTPRequest = errors.New("Failed all HTTP retries")
ErrAddrURIClusterExpected = errors.New("Addresses, URIs or a cluster is expected")
ErrInvalidQueryOption = errors.New("Invalid query option")
ErrInvalidIndexOption = errors.New("Invalid index option")
ErrInvalidFieldOption = errors.New("Invalid field option")
ErrNoFragmentNodes = errors.New("No fragment nodes")
ErrNoShard = errors.New("Index has no shards")
ErrUnknownType = errors.New("Unknown type")
ErrSingleServerAddressRequired = errors.New("OptClientManualServerAddress requires a single URI or address")
ErrPreconditionFailed = errors.New("Precondition failed")
)

145
client/ingest_api_batch.go Normal file
View file

@ -0,0 +1,145 @@
package client
import (
"time"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
)
// NewIngestAPIBatch creates an alternate implementation of
// RecordBatch which exists to aid in testing the new Ingest API and
// is likely far slower than the Batch.
func NewIngestAPIBatch(client *Client, size int, logger logger.Logger, fields []*Field) *ingestAPIBatch {
if len(fields) == 0 {
return nil
}
return &ingestAPIBatch{
client: client,
log: logger,
fields: fields,
keyed: fields[0].index.Opts().Keys(),
index: fields[0].index.Name(),
batchSize: size,
recordsK: make(map[string]map[string]interface{}),
records: make(map[uint64]map[string]interface{}),
}
}
type ingestAPIBatch struct {
client *Client
log logger.Logger
batchSize int
fields []*Field
keyed bool
index string
// map[recordKey][fieldName]value
recordsK map[string]map[string]interface{}
records map[uint64]map[string]interface{}
}
func (b *ingestAPIBatch) Add(row Row) error {
if len(row.Clears) > 0 {
return errors.New("ingest api batch does not support clears")
}
values := make(map[string]interface{})
for i, val := range row.Values {
field := b.fields[i]
// val can be string, uint64, int64, []string, []uint64, nil
// TODO timestamp field might need special handling
// TODO check that the Row.Clears field is only used for packed bools, and then issue a warning/error (in IDK) if the ingest API mode is used in conjunction w/ packed bools.
if val == nil {
continue
}
zero := QuantizedTime{}
if field.Options().Type() == FieldTypeTime && row.Time != zero {
timeq, err := row.Time.Time()
if err != nil {
return errors.Wrap(err, "parsing row time")
}
values[field.Name()] = map[string]interface{}{"time": timeq.Format(time.RFC3339), "values": val}
} else {
values[field.Name()] = val
}
}
if b.keyed {
switch rowID := row.ID.(type) {
case string:
b.recordsK[rowID] = values
case []byte:
b.recordsK[string(rowID)] = values
default:
return errors.Errorf("unsupported rowID %v of type %[1]T, must be string, or []byte for keyed index", rowID)
}
if len(b.recordsK) >= b.batchSize {
return ErrBatchNowFull
}
} else {
rowID, ok := row.ID.(uint64)
if !ok {
return errors.Errorf("unsupported rowID %v of type %[1]T, must be uint64 for unkeyed index", row.ID)
}
b.records[rowID] = values
if len(b.records) >= b.batchSize {
return ErrBatchNowFull
}
}
return nil
}
func (b *ingestAPIBatch) Import() error {
if b.keyed {
return b.importKeyed()
}
return b.importUnkeyed()
}
func (b *ingestAPIBatch) importKeyed() error {
req := []map[string]interface{}{
{
"action": "set",
"records": b.recordsK,
},
}
bod, err := b.client.IngestData(b.index, req)
if err != nil {
return errors.Wrapf(err, "importKeyed, body: %s", bod)
}
for k := range b.recordsK {
delete(b.recordsK, k)
}
return nil
}
func (b *ingestAPIBatch) importUnkeyed() error {
req := []map[string]interface{}{
{
"action": "set",
"records": b.records,
},
}
bod, err := b.client.IngestData(b.index, req)
if err != nil {
return errors.Wrapf(err, "importKeyed, body: %s", bod)
}
for v := range b.records {
delete(b.records, v)
}
return nil
}
func (b *ingestAPIBatch) Len() int {
if b.keyed {
return len(b.recordsK)
}
return len(b.records)
}
func (b *ingestAPIBatch) Flush() error { return nil }

305
client/ingest_api_batch_test.go Normal file
View file

@ -0,0 +1,305 @@
package client
import (
"strings"
"testing"
"time"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/test"
)
func TestIngestAPIBatchAdd(t *testing.T) {
t.Run("unkeyed", func(t *testing.T) {
batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
{
name: "a",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeSet,
},
},
{
name: "b",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeSet,
keys: true,
},
},
{
name: "c",
index: &Index{name: "idxname", options: &IndexOptions{}},
options: &FieldOptions{
fieldType: FieldTypeTime,
keys: true,
},
},
})
qt := QuantizedTime{}
qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
err := batch.Add(Row{
ID: uint64(1),
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
if err != nil {
t.Fatalf("adding row to batch: %v", err)
}
if batch.records[1]["a"] != uint64(2) {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["b"] != "bkey" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
if batch.records[1]["c"].(map[string]interface{})["values"] != "ckey" {
t.Fatalf("unexpected batch.records: %+v", batch.records)
}
})
t.Run("keyed", func(t *testing.T) {
batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{
{
name: "a",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeSet,
},
},
{
name: "b",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeSet,
keys: true,
},
},
{
name: "c",
index: &Index{name: "idxname", options: &IndexOptions{keys: true}},
options: &FieldOptions{
fieldType: FieldTypeTime,
keys: true,
},
},
})
qt := QuantizedTime{}
qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC))
err := batch.Add(Row{
ID: "1",
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
checkResult := func(batch *ingestAPIBatch, id string, err error) {
if err != nil {
t.Fatalf("adding row to batch: %v", err)
}
if batch.recordsK[id]["a"] != uint64(2) {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["b"] != "bkey" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
if batch.recordsK[id]["c"].(map[string]interface{})["values"] != "ckey" {
t.Fatalf("unexpected batch.records: %+v", batch.recordsK)
}
}
checkResult(batch, "1", err)
// test wrong type row ID
if err := batch.Add(Row{ID: 64.5}); !strings.Contains(err.Error(), "unsupported rowID") {
t.Fatalf("unexpected error w/ floating point rowID: %v", err)
}
// test that byte slice ID works same as string
err = batch.Add(Row{
ID: []byte("2"),
Values: []interface{}{uint64(2), "bkey", "ckey"},
Time: qt,
})
checkResult(batch, "2", err)
})
}
func TestIngestAPIBatch(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
urls := make([]string, len(c.Nodes))
for i, n := range c.Nodes {
urls[i] = n.URL()
}
// Create a new client for the cluster
cli, err := newClientFromAddresses(urls, &ClientOptions{})
if err != nil {
t.Fatalf("getting new client: %v", err)
}
defer cli.Close()
cli.IngestSchema(map[string]interface{}{
"index-name": "test-1",
"index-action": "create",
"primary-key-type": "uint",
"field-action": "create",
"fields": []map[string]interface{}{
{
"field-name": "astr",
"field-type": "string",
"field-options": map[string]interface{}{},
},
{
"field-name": "bint",
"field-type": "int",
"field-options": map[string]interface{}{},
},
{
"field-name": "cid",
"field-type": "id",
"field-options": map[string]interface{}{},
},
{
"field-name": "dtimestamp",
"field-type": "timestamp",
"field-options": map[string]interface{}{
"unit": "s",
},
},
{
"field-name": "etime",
"field-type": "string",
"field-options": map[string]interface{}{
"time-quantum": "YMD",
},
},
{
"field-name": "fdecimal",
"field-type": "decimal",
"field-options": map[string]interface{}{
"scale": 3,
},
},
{
"field-name": "gbool",
"field-type": "bool",
"field-options": map[string]interface{}{},
},
},
})
schema, err := cli.Schema()
if err != nil {
t.Fatalf("getting schema: %v", err)
}
index := schema.Index("test-1")
defer cli.DeleteIndex(index)
batch := NewIngestAPIBatch(cli, 10, logger.NopLogger, []*Field{
{
name: "astr",
index: &Index{name: "test-1", options: &IndexOptions{}},
options: &FieldOptions{fieldType: FieldTypeSet, keys: true},
},
{
name: "bint",
options: &FieldOptions{fieldType: FieldTypeInt},
},
{
name: "cid",
options: &FieldOptions{fieldType: FieldTypeSet, keys: false},
},
{
name: "dtimestamp",
options: &FieldOptions{fieldType: FieldTypeTimestamp},
},
{
name: "etime",
options: &FieldOptions{fieldType: FieldTypeTime, keys: true, timeQuantum: TimeQuantumYearMonthDay},
},
{
name: "fdecimal",
options: &FieldOptions{fieldType: FieldTypeDecimal, scale: 3},
},
{
name: "gbool",
options: &FieldOptions{fieldType: FieldTypeBool},
},
})
qt0 := &QuantizedTime{}
qt0.Set(time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC))
if err := batch.Add(Row{
ID: uint64(7),
Values: []interface{}{"a", -2, 9, 1287367623, "e", 1.2345, true},
Time: *qt0,
}); err != nil {
t.Fatalf("adding row: %v", err)
}
// test nil value case
if err := batch.Add(Row{
ID: uint64(8),
Values: []interface{}{nil, nil, nil, nil, nil, nil, nil},
Time: QuantizedTime{},
}); err != nil {
t.Fatalf("error adding all nil batch which should affect nothing: %v", err)
}
if err := batch.Import(); err != nil {
t.Fatalf("importing row: %v", err)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(astr=a)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(bint==-2)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(bint==-2) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(cid=9)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(cid=9) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(dtimestamp=='2010-10-18T02:07:03Z')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(dtimestamp=='2010-10-18T02:07:03Z') result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(etime=e, from='2010-01-01', to='2010-01-02')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(etime=e, from='2010-01-01', to='2010-01-02') result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(fdecimal==1.234)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(fdecimal==1.234) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(gbool=true)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(gbool=true) result: %+v", resp.Result().Row().Columns)
}
}

33
client/logimport.go Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package client
import (
"encoding/gob"
"io"
)
type importLog struct {
Index string
Path string
Shard uint64
IsRoaring bool
Timestamp int64 // Unix Nanoseconds
Data []byte
}
type encoder interface {
Encode(thing interface{}) error
}
func newImportLogEncoder(w io.Writer) encoder {
return gob.NewEncoder(w)
}
type decoder interface {
Decode(thing interface{}) error
}
func newImportLogDecoder(r io.Reader) decoder {
return gob.NewDecoder(r)
}

143
client/logimport_test.go Normal file
View file

@ -0,0 +1,143 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package client
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
)
func TestEncodeDecode(t *testing.T) {
tests := []importLog{
{
Index: "go-testindex",
Path: "/index/go-testindex/field/importfield-batchsize/import?clear=false",
Shard: 0,
Data: make([]byte, 3918),
},
{
Index: "go-testindex",
Path: "/index/go-testindex/field/importfield-batchsize/import?clear=false",
Shard: 0,
Data: make([]byte, 3918),
},
{
Index: "eheh",
Path: "blah",
Shard: 9,
Data: []byte("something"),
},
{
Index: "",
Path: "",
Shard: 0,
Data: nil,
},
{
Index: "eheh",
Path: "blah",
Shard: 10,
Data: []byte("blahaslkdjfeoiwujf"),
},
{
Index: "eheh",
Path: "blah",
Shard: 10,
Data: make([]byte, 10000),
},
{
Index: "zoop",
Path: "blah",
Shard: 8923734,
Data: []byte("blahaslkdjfeoiwujf"),
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
nl := importLog{
Index: test.Index,
Path: test.Path,
Shard: test.Shard,
Data: make([]byte, len(test.Data)),
}
copy(nl.Data, test.Data)
buf := &bytes.Buffer{}
enc := newImportLogEncoder(buf)
err := enc.Encode(nl)
if err != nil {
t.Fatalf("writing to buf: %v", err)
}
dec := newImportLogDecoder(buf)
l2 := &importLog{}
err = dec.Decode(l2)
if err != nil {
t.Fatalf("reading from buf: %v", err)
}
if l2.Index != test.Index {
t.Errorf("indexes not equal:\n%s\n%s", test.Index, l2.Index)
}
if l2.Path != test.Path {
t.Errorf("paths not equal:\n%s\n%s", test.Path, l2.Path)
}
if l2.Shard != test.Shard {
t.Errorf("shards not equal exp: %d got %d", test.Shard, l2.Shard)
}
if !reflect.DeepEqual(test.Data, l2.Data) {
t.Errorf("data not equal \n%v\n%v", test.Data, l2.Data)
}
})
}
buf, err := ioutil.TempFile("", "")
if err != nil {
t.Fatalf("getting temp file: %v", err)
}
enc := newImportLogEncoder(buf)
for _, test := range tests {
a := &test
err := enc.Encode(a)
if err != nil {
t.Errorf("encoding to buf: %v", err)
}
}
name := buf.Name()
err = buf.Close()
if err != nil {
t.Fatalf("closing temp file: %v", err)
}
buf, err = os.Open(name)
if err != nil {
t.Fatalf("reopening: %v", err)
}
dec := newImportLogDecoder(buf)
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
l := &importLog{}
err := dec.Decode(l)
// err := l.ReadFrom(buf)
if err != nil {
t.Errorf("reading from buf: %v", err)
}
if l.Index != test.Index {
t.Errorf("indexes not equal:\n%s\n%s", test.Index, l.Index)
}
if l.Path != test.Path {
t.Errorf("paths not equal:\n%s\n%s", test.Path, l.Path)
}
if l.Shard != test.Shard {
t.Errorf("shards not equal exp: %d got %d", test.Shard, l.Shard)
}
if !reflect.DeepEqual(test.Data, l.Data) {
t.Errorf("data not equal \n%v\n%v", test.Data, l.Data)
}
})
}
}

28
client/metrics.go Normal file
View file

@ -0,0 +1,28 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package client
const (
// MetricBatchImportDurationSeconds records the full time of the
// RecordBatch.Import call. This includes starting and finishing a
// transaction, doing key translation, building fragments locally,
// importing all data, and resetting internal structures.
MetricBatchImportDurationSeconds = "batch_import_duration_seconds"
// MetricBatchFlushDurationSeconds records the full time for
// RecordBatch.Flush (if splitBatchMode is in use). This includes
// starting and finishing a transaction, importing all data, and
// resetting internal structures.
MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds"
// MetricBatchShardImportBuildRequestsSeconds is the time it takes
// after making fragments to build the shard-transactional request
// objects (but not actually import them or do any network activity).
MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds"
// MetricBatchShardImportDurationSeconds is the time it takes to
// import all data for all shards in the batch using the
// shard-transactional endpoint. This does not include the time it
// takes to build the requests locally.
MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds"
)

1586
client/orm.go Normal file
View file

File diff suppressed because it is too large Load diff

1235
client/orm_test.go Normal file
View file

File diff suppressed because it is too large Load diff

63
client/record.go Normal file
View file

@ -0,0 +1,63 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
// Record is a Column or a FieldValue.
type Record interface {
Shard(shardWidth uint64) uint64
Less(other Record) bool
}
// RecordIterator is an iterator for a record.
type RecordIterator interface {
NextRecord() (Record, error)
}
// Column defines a single Pilosa column.
type Column struct {
RowID uint64
ColumnID uint64
RowKey string
ColumnKey string
Timestamp int64
}
// Shard returns the shard for this column.
func (b Column) Shard(shardWidth uint64) uint64 {
return b.ColumnID / shardWidth
}
// Less returns true if this column sorts before the given Record.
func (b Column) Less(other Record) bool {
if ob, ok := other.(Column); ok {
if b.RowID == ob.RowID {
return b.ColumnID < ob.ColumnID
}
return b.RowID < ob.RowID
}
return false
}
// FieldValue represents the value for a column within a
// range-encoded field.
type FieldValue struct {
ColumnID uint64
ColumnKey string
Value int64
}
// Shard returns the shard for this field value.
func (v FieldValue) Shard(shardWidth uint64) uint64 {
return v.ColumnID / shardWidth
}
// Less returns true if this field value sorts before the given Record.
func (v FieldValue) Less(other Record) bool {
if ov, ok := other.(FieldValue); ok {
return v.ColumnID < ov.ColumnID
}
return false
}

71
client/record_test.go Normal file
View file

@ -0,0 +1,71 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client_test
import (
"testing"
"github.com/molecula/featurebase/v3/client"
)
func TestColumnShard(t *testing.T) {
a := client.Column{RowID: 15, ColumnID: 55, Timestamp: 100101}
target := uint64(0)
if a.Shard(100) != target {
t.Fatalf("shard %d != %d", target, a.Shard(100))
}
target = 5
if a.Shard(10) != target {
t.Fatalf("shard %d != %d", target, a.Shard(10))
}
}
func TestColumnLess(t *testing.T) {
a := client.Column{RowID: 10, ColumnID: 200}
a2 := client.Column{RowID: 10, ColumnID: 1000}
b := client.Column{RowID: 200, ColumnID: 10}
c := client.FieldValue{ColumnID: 1}
if !a.Less(a2) {
t.Fatalf("%v should be less than %v", a, a2)
}
if !a.Less(b) {
t.Fatalf("%v should be less than %v", a, b)
}
if b.Less(a) {
t.Fatalf("%v should not be less than %v", b, a)
}
if c.Less(a) {
t.Fatalf("%v should not be less than %v", c, a)
}
}
func TestFieldValueShard(t *testing.T) {
a := client.FieldValue{ColumnID: 55, Value: 125}
target := uint64(0)
if a.Shard(100) != target {
t.Fatalf("shard %d != %d", target, a.Shard(100))
}
target = 5
if a.Shard(10) != target {
t.Fatalf("shard %d != %d", target, a.Shard(10))
}
}
func TestFieldValueLess(t *testing.T) {
a := client.FieldValue{ColumnID: 55, Value: 125}
b := client.FieldValue{ColumnID: 100, Value: 125}
c := client.Column{ColumnID: 1, RowID: 2}
if !a.Less(b) {
t.Fatalf("%v should be less than %v", a, b)
}
if b.Less(a) {
t.Fatalf("%v should not be less than %v", b, a)
}
if c.Less(a) {
t.Fatalf("%v should not be less than %v", c, a)
}
}

496
client/response.go Normal file
View file

@ -0,0 +1,496 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
"encoding/json"
"fmt"
"github.com/molecula/featurebase/v3/pb"
)
// QueryResponse types.
const (
QueryResultTypeNil uint32 = iota
QueryResultTypeRow
QueryResultTypePairs
QueryResultTypePairsField
QueryResultTypeValCount
QueryResultTypeUint64
QueryResultTypeBool
QueryResultTypeRowIDs // this is not used by the client
QueryResultTypeGroupCounts
QueryResultTypeRowIdentifiers
QueryResultTypePair
QueryResultTypePairField
QueryResultTypeSignedRow
)
// QueryResponse represents the response from a Pilosa query.
type QueryResponse struct {
ResultList []QueryResult `json:"results,omitempty"`
ErrorMessage string `json:"error-message,omitempty"`
Success bool `json:"success,omitempty"`
}
func newQueryResponseFromInternal(response *pb.QueryResponse) (*QueryResponse, error) {
if response.Err != "" {
return &QueryResponse{
ErrorMessage: response.Err,
Success: false,
}, nil
}
results := make([]QueryResult, 0, len(response.Results))
for _, r := range response.Results {
result, err := newQueryResultFromInternal(r)
if err != nil {
return nil, err
}
results = append(results, result)
}
return &QueryResponse{
ResultList: results,
Success: true,
}, nil
}
// Results returns all results in the response.
func (qr *QueryResponse) Results() []QueryResult {
return qr.ResultList
}
// Result returns the first result or nil.
func (qr *QueryResponse) Result() QueryResult {
if len(qr.ResultList) == 0 {
return nil
}
return qr.ResultList[0]
}
// QueryResult represents one of the results in the response.
type QueryResult interface {
Type() uint32
Row() RowResult
CountItems() []CountResultItem
CountItem() CountResultItem
Count() int64
Value() int64
Changed() bool
GroupCounts() []GroupCount
RowIdentifiers() RowIdentifiersResult
}
func newQueryResultFromInternal(result *pb.QueryResult) (QueryResult, error) {
switch result.Type {
case QueryResultTypeNil:
return NilResult{}, nil
case QueryResultTypeRow:
return newRowResultFromInternal(result.Row)
case QueryResultTypePairs:
return countItemsFromInternal(result.Pairs), nil
case QueryResultTypePairsField:
return countItemsFromInternal(result.PairsField.Pairs), nil
case QueryResultTypeValCount:
return &ValCountResult{
Val: result.ValCount.Val,
Cnt: result.ValCount.Count,
}, nil
case QueryResultTypeUint64:
return IntResult(result.N), nil
case QueryResultTypeBool:
return BoolResult(result.Changed), nil
case QueryResultTypeRowIdentifiers:
return &RowIdentifiersResult{
IDs: result.RowIdentifiers.Rows,
Keys: result.RowIdentifiers.Keys,
}, nil
case QueryResultTypeGroupCounts:
return groupCountsFromInternal(result.GroupCounts), nil
case QueryResultTypePair:
return CountItem{CountResultItem: countItemFromInternal(result.Pairs[0])}, nil
case QueryResultTypePairField:
return CountItem{CountResultItem: countItemFromInternal(result.PairField.Pair)}, nil
}
return nil, ErrUnknownType
}
// CountResultItem represents a result from TopN call.
type CountResultItem struct {
ID uint64 `json:"id"`
Key string `json:"key,omitempty"`
Count uint64 `json:"count"`
}
func (c *CountResultItem) String() string {
if c.Key != "" {
return fmt.Sprintf("%s:%d", c.Key, c.Count)
}
return fmt.Sprintf("%d:%d", c.ID, c.Count)
}
type CountItem struct {
CountResultItem
}
// Type is the type of this result.
func (CountItem) Type() uint32 { return QueryResultTypePairField }
// Row returns a RowResult.
func (CountItem) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (t CountItem) CountItems() []CountResultItem { return []CountResultItem{t.CountResultItem} }
// CountItem returns a CountResultItem
func (t CountItem) CountItem() CountResultItem { return t.CountResultItem }
// Count returns the result of a Count call.
func (CountItem) Count() int64 { return 0 }
// Value returns the result of a Min, Max or Sum call.
func (CountItem) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (CountItem) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (CountItem) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (CountItem) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
func countItemFromInternal(item *pb.Pair) CountResultItem {
return CountResultItem{ID: item.ID, Key: item.Key, Count: item.Count}
}
func countItemsFromInternal(items []*pb.Pair) TopNResult {
result := make([]CountResultItem, 0, len(items))
for _, v := range items {
result = append(result, countItemFromInternal(v))
}
return TopNResult(result)
}
// TopNResult is returned from TopN call.
type TopNResult []CountResultItem
// Type is the type of this result.
func (TopNResult) Type() uint32 { return QueryResultTypePairsField }
// Row returns a RowResult.
func (TopNResult) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (t TopNResult) CountItems() []CountResultItem { return t }
// CountItem returns a CountResultItem
func (t TopNResult) CountItem() CountResultItem {
if len(t) >= 1 {
return t[0]
}
return CountResultItem{}
}
// Count returns the result of a Count call.
func (TopNResult) Count() int64 { return 0 }
// Value returns the result of a Min, Max or Sum call.
func (TopNResult) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (TopNResult) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (TopNResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (TopNResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// RowResult represents a result from Row, Union, Intersect, Difference and Range PQL calls.
type RowResult struct {
Columns []uint64 `json:"columns"`
Keys []string `json:"keys"`
}
func newRowResultFromInternal(row *pb.Row) (*RowResult, error) {
return &RowResult{
Columns: row.Columns,
Keys: row.Keys,
}, nil
}
// Type is the type of this result.
func (RowResult) Type() uint32 { return QueryResultTypeRow }
// Row returns a RowResult.
func (b RowResult) Row() RowResult { return b }
// CountItems returns a CountResultItem slice.
func (RowResult) CountItems() []CountResultItem { return nil }
// CountItem returns a CountResultItem
func (RowResult) CountItem() CountResultItem { return CountResultItem{} }
// Count returns the result of a Count call.
func (RowResult) Count() int64 { return 0 }
// Value returns the result of a Min, Max or Sum call.
func (RowResult) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (RowResult) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (RowResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (RowResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// MarshalJSON serializes this row result.
func (b RowResult) MarshalJSON() ([]byte, error) {
columns := b.Columns
if columns == nil {
columns = []uint64{}
}
keys := b.Keys
if keys == nil {
keys = []string{}
}
return json.Marshal(struct {
Columns []uint64 `json:"columns"`
Keys []string `json:"keys"`
}{
Columns: columns,
Keys: keys,
})
}
// ValCountResult is returned from Min, Max and Sum calls.
type ValCountResult struct {
Val int64 `json:"val"`
Cnt int64 `json:"count"`
}
// Type is the type of this result.
func (ValCountResult) Type() uint32 { return QueryResultTypeValCount }
// Row returns a RowResult.
func (ValCountResult) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (ValCountResult) CountItems() []CountResultItem { return nil }
// CountItem returns a CountResultItem
func (ValCountResult) CountItem() CountResultItem { return CountResultItem{} }
// Count returns the result of a Count call.
func (c ValCountResult) Count() int64 { return c.Cnt }
// Value returns the result of a Min, Max or Sum call.
func (c ValCountResult) Value() int64 { return c.Val }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (ValCountResult) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (ValCountResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (ValCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// IntResult is returned from Count call.
type IntResult int64
// Type is the type of this result.
func (IntResult) Type() uint32 { return QueryResultTypeUint64 }
// Row returns a RowResult.
func (IntResult) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (IntResult) CountItems() []CountResultItem { return nil }
// CountItem returns a CountResultItem
func (IntResult) CountItem() CountResultItem { return CountResultItem{} }
// Count returns the result of a Count call.
func (i IntResult) Count() int64 { return int64(i) }
// Value returns the result of a Min, Max or Sum call.
func (IntResult) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (IntResult) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (IntResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (IntResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// BoolResult is returned from Set and Clear calls.
type BoolResult bool
// Type is the type of this result.
func (BoolResult) Type() uint32 { return QueryResultTypeBool }
// Row returns a RowResult.
func (BoolResult) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (BoolResult) CountItems() []CountResultItem { return nil }
// CountItem returns a CountResultItem
func (BoolResult) CountItem() CountResultItem { return CountResultItem{} }
// Count returns the result of a Count call.
func (BoolResult) Count() int64 { return 0 }
// Value returns the result of a Min, Max or Sum call.
func (BoolResult) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (b BoolResult) Changed() bool { return bool(b) }
// GroupCounts returns the result of a GroupBy call.
func (BoolResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (BoolResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// NilResult is returned from calls which don't return a value.
type NilResult struct{}
// Type is the type of this result.
func (NilResult) Type() uint32 { return QueryResultTypeNil }
// Row returns a RowResult.
func (NilResult) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (NilResult) CountItems() []CountResultItem { return nil }
// CountItem returns a CountResultItem
func (NilResult) CountItem() CountResultItem { return CountResultItem{} }
// Count returns the result of a Count call.
func (NilResult) Count() int64 { return 0 }
// Value returns the result of a Min, Max or Sum call.
func (NilResult) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (NilResult) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (NilResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (NilResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// FieldRow represents a Group in a GroupBy call result.
type FieldRow struct {
FieldName string `json:"field"`
RowID uint64 `json:"rowID"`
RowKey string `json:"rowKey"`
Value *int64 `json:"value,omitempty"`
}
// GroupCount contains groups and their count in a GroupBy call result.
type GroupCount struct {
Groups []FieldRow `json:"groups"`
Count int64 `json:"count"`
Agg int64 `json:"agg"`
}
// GroupCountResult is returned from GroupBy call.
type GroupCountResult []GroupCount
// Type is the type of this result.
func (GroupCountResult) Type() uint32 { return QueryResultTypeGroupCounts }
// Row returns a RowResult.
func (GroupCountResult) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (GroupCountResult) CountItems() []CountResultItem { return nil }
// CountItem returns a CountResultItem
func (GroupCountResult) CountItem() CountResultItem { return CountResultItem{} }
// Count returns the result of a Count call.
func (GroupCountResult) Count() int64 { return 0 }
// Value returns the result of a Min, Max or Sum call.
func (GroupCountResult) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (GroupCountResult) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (r GroupCountResult) GroupCounts() []GroupCount { return r }
// RowIdentifiers returns the result of a Rows call.
func (GroupCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// RowIdentifiersResult is returned from a Rows call.
type RowIdentifiersResult struct {
IDs []uint64 `json:"ids"`
Keys []string `json:"keys,omitempty"`
}
// Type is the type of this result.
func (RowIdentifiersResult) Type() uint32 { return QueryResultTypeRowIdentifiers }
// Row returns a RowResult.
func (RowIdentifiersResult) Row() RowResult { return RowResult{} }
// CountItems returns a CountResultItem slice.
func (RowIdentifiersResult) CountItems() []CountResultItem { return nil }
// CountItem returns a CountResultItem
func (RowIdentifiersResult) CountItem() CountResultItem { return CountResultItem{} }
// Count returns the result of a Count call.
func (RowIdentifiersResult) Count() int64 { return 0 }
// Value returns the result of a Min, Max or Sum call.
func (RowIdentifiersResult) Value() int64 { return 0 }
// Changed returns whether the corresponding Set or Clear call changed the value of a bit.
func (RowIdentifiersResult) Changed() bool { return false }
// GroupCounts returns the result of a GroupBy call.
func (RowIdentifiersResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (r RowIdentifiersResult) RowIdentifiers() RowIdentifiersResult { return r }
func groupCountsFromInternal(items *pb.GroupCounts) GroupCountResult {
result := make([]GroupCount, 0, len(items.Groups))
for _, g := range items.Groups {
groups := make([]FieldRow, 0, len(g.Group))
for _, f := range g.Group {
fr := FieldRow{
FieldName: f.Field,
RowID: f.RowID,
RowKey: f.RowKey,
}
if f.Value != nil {
fr.Value = &f.Value.Value
}
groups = append(groups, fr)
}
result = append(result, GroupCount{
Groups: groups,
Count: int64(g.Count),
Agg: int64(g.Agg),
})
}
return GroupCountResult(result)
}

273
client/response_test.go Normal file
View file

@ -0,0 +1,273 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
"encoding/json"
"fmt"
"log"
"reflect"
"testing"
"github.com/molecula/featurebase/v3/pb"
)
func TestNewRowResultFromInternal(t *testing.T) {
targetColumns := []uint64{5, 10}
row := &pb.Row{
Columns: []uint64{5, 10},
}
result, err := newRowResultFromInternal(row)
if err != nil {
t.Fatalf("Failed with error: %s", err)
}
if !reflect.DeepEqual(targetColumns, result.Columns) {
t.Fatal()
}
}
func TestNewQueryResponseFromInternal(t *testing.T) {
targetColumns := []uint64{5, 10}
targetCountItems := []CountResultItem{
{ID: 10, Count: 100},
}
row := &pb.Row{
Columns: []uint64{5, 10},
}
pairs := []*pb.Pair{
{ID: 10, Count: 100},
}
response := &pb.QueryResponse{
Results: []*pb.QueryResult{
{Type: QueryResultTypeRow, Row: row},
{Type: QueryResultTypePairs, Pairs: pairs},
},
Err: "",
}
qr, err := newQueryResponseFromInternal(response)
if err != nil {
t.Fatalf("Failed with error: %s", err)
}
if qr.ErrorMessage != "" {
t.Fatalf("ErrorMessage should be empty")
}
if !qr.Success {
t.Fatalf("IsSuccess should be true")
}
results := qr.Results()
if len(results) != 2 {
t.Fatalf("Number of results should be 2")
}
if results[0] != qr.Result() {
t.Fatalf("Result() should return the first result")
}
if !reflect.DeepEqual(targetColumns, results[0].Row().Columns) {
t.Fatalf("The row result should contain the columns")
}
if !reflect.DeepEqual(targetCountItems, results[1].CountItems()) {
t.Fatalf("The response should include count items")
}
}
func TestNewQueryResponseWithErrorFromInternal(t *testing.T) {
response := &pb.QueryResponse{
Err: "some error",
}
qr, err := newQueryResponseFromInternal(response)
if err != nil {
t.Fatalf("Failed with error: %s", err)
}
if qr.ErrorMessage != "some error" {
t.Fatalf("The response should include the error message")
}
if qr.Success {
t.Fatalf("IsSuccess should be false")
}
if qr.Result() != nil {
t.Fatalf("If there are no results, Result should return nil")
}
}
func TestCountResultItemToString(t *testing.T) {
tests := []struct {
item *CountResultItem
expected string
}{
{item: &CountResultItem{ID: 100, Count: 50}, expected: "100:50"},
{item: &CountResultItem{Key: "blah", Count: 50}, expected: "blah:50"},
{item: &CountResultItem{Key: "blah", ID: 22, Count: 50}, expected: "blah:50"},
{item: &CountResultItem{Key: "blah", ID: 22}, expected: "blah:0"},
{item: &CountResultItem{}, expected: "0:0"},
}
for i, tst := range tests {
t.Run(fmt.Sprintf("%d: ", i), func(t *testing.T) {
if tst.expected != tst.item.String() {
t.Fatalf("%s != %s", tst.expected, tst.item.String())
}
})
}
}
func TestMarshalResults(t *testing.T) {
row := &pb.Row{
Columns: []uint64{5, 10},
}
pairs := []*pb.Pair{
{ID: 10, Count: 100},
}
pbufResults := []*pb.QueryResult{
{Type: QueryResultTypeRow, Row: row},
{Type: QueryResultTypePairs, Pairs: pairs},
}
resultJSONStrings := make([]string, len(pbufResults))
for i, pr := range pbufResults {
r, err := newQueryResultFromInternal(pr)
if err != nil {
t.Fatal(err)
}
b, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
resultJSONStrings[i] = string(b)
}
targetJSON := []string{
`{"columns":[5,10],"keys":[]}`,
`[{"id":10,"count":100}]`,
}
for i := range targetJSON {
if sortedString(targetJSON[i]) != sortedString(resultJSONStrings[i]) {
t.Fatalf("%v != %v ", targetJSON[i], resultJSONStrings[i])
}
}
}
func TestUnknownQueryResultType(t *testing.T) {
result := &pb.QueryResult{
Type: 999,
}
_, err := newQueryResultFromInternal(result)
if err != ErrUnknownType {
t.Fatalf("Should have failed with ErrUnknownType")
}
}
func TestTopNResult(t *testing.T) {
result := TopNResult{
CountResultItem{ID: 100, Count: 10},
}
expectResult(t, result, QueryResultTypePairsField, RowResult{}, []CountResultItem{{100, "", 10}}, 0, 0, false, nil, RowIdentifiersResult{})
}
func TestRowResult(t *testing.T) {
result := RowResult{
Columns: []uint64{1, 2, 3},
}
targetBmp := RowResult{
Columns: []uint64{1, 2, 3},
}
expectResult(t, result, QueryResultTypeRow, targetBmp, nil, 0, 0, false, nil, RowIdentifiersResult{})
}
func TestRowResultNilColumns(t *testing.T) {
result := RowResult{
Columns: nil,
}
_, err := result.MarshalJSON()
if err != nil {
t.Fatal(err)
}
}
func TestSumCountResult(t *testing.T) {
result := ValCountResult{
Val: 100,
Cnt: 50,
}
expectResult(t, result, QueryResultTypeValCount, RowResult{}, nil, 100, 50, false, nil, RowIdentifiersResult{})
}
func TestIntResult(t *testing.T) {
result := IntResult(11)
expectResult(t, result, QueryResultTypeUint64, RowResult{}, nil, 0, 11, false, nil, RowIdentifiersResult{})
}
func TestBoolResult(t *testing.T) {
result := BoolResult(true)
expectResult(t, result, QueryResultTypeBool, RowResult{}, nil, 0, 0, true, nil, RowIdentifiersResult{})
}
func TestNilResult(t *testing.T) {
result := NilResult{}
expectResult(t, result, QueryResultTypeNil, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{})
}
func TestGroupCountResult(t *testing.T) {
result := GroupCountResult{
{Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2},
{Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1},
}
expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{
{Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2},
{Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1},
}, RowIdentifiersResult{})
}
func TestGroupCountWithValueResult(t *testing.T) {
var a, b int64 = -1, 1
result := GroupCountResult{
{Groups: []FieldRow{{FieldName: "f1", Value: &a}}, Count: 1},
{Groups: []FieldRow{{FieldName: "f1", Value: &b}}, Count: 1},
}
var aa, bb int64 = -1, 1
expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{
{Groups: []FieldRow{{FieldName: "f1", Value: &aa}}, Count: 1},
{Groups: []FieldRow{{FieldName: "f1", Value: &bb}}, Count: 1},
}, RowIdentifiersResult{})
}
func TestRowIdentifiersResult(t *testing.T) {
result := RowIdentifiersResult{
IDs: []uint64{1, 2, 3, 4},
}
expectResult(t, result, QueryResultTypeRowIdentifiers, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{
IDs: []uint64{1, 2, 3, 4},
})
}
func expectResult(t *testing.T, r QueryResult, resultType uint32, bmp RowResult,
countItems []CountResultItem, sum int64, count int64, changed bool,
groupCounts []GroupCount, rowIdentifiers RowIdentifiersResult) {
if resultType != r.Type() {
log.Fatalf("Result type: %d != %d", resultType, r.Type())
}
if !reflect.DeepEqual(bmp, r.Row()) {
log.Fatalf("Row: %v != %v", bmp, r.Row())
}
if !reflect.DeepEqual(countItems, r.CountItems()) {
log.Fatalf("Count items: %v != %v", countItems, r.CountItems())
}
if count != r.Count() {
log.Fatalf("Count: %d != %d", count, r.Count())
}
if sum != r.Value() {
log.Fatalf("Sum: %d != %d", sum, r.Value())
}
if changed != r.Changed() {
log.Fatalf("Changed: %v != %v", changed, r.Changed())
}
if !reflect.DeepEqual(groupCounts, r.GroupCounts()) {
log.Fatalf("Group counts: %v != %v", groupCounts, r.GroupCounts())
}
if !reflect.DeepEqual(rowIdentifiers, r.RowIdentifiers()) {
log.Fatalf("Row identifiers: %v != %v", rowIdentifiers, r.RowIdentifiers())
}
}

54
client/shardnodes.go Normal file
View file

@ -0,0 +1,54 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
"sync"
pnet "github.com/molecula/featurebase/v3/net"
)
type shardNodes struct {
data map[string]map[uint64][]*pnet.URI
mu *sync.RWMutex
}
func newShardNodes() shardNodes {
return shardNodes{
data: make(map[string]map[uint64][]*pnet.URI),
mu: &sync.RWMutex{},
}
}
func (s shardNodes) Get(index string, shard uint64) ([]*pnet.URI, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
if idx, ok := s.data[index]; ok {
if uris, ok := idx[shard]; ok {
return uris, true
}
}
return nil, false
}
func (s shardNodes) Put(index string, shard uint64, uris []*pnet.URI) {
s.mu.Lock()
defer s.mu.Unlock()
idx, ok := s.data[index]
if !ok {
idx = make(map[uint64][]*pnet.URI)
}
idx[shard] = uris
s.data[index] = idx
}
func (s shardNodes) Invalidate() {
s.mu.Lock()
defer s.mu.Unlock()
for k := range s.data {
delete(s.data, k)
}
}

77
client/tracer.go Normal file
View file

@ -0,0 +1,77 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
opentracing "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/log"
)
type NoopTracer struct{}
type NoopSpan struct{}
func (s NoopSpan) Finish() {
// pass
}
func (s NoopSpan) FinishWithOptions(opts opentracing.FinishOptions) {
// pass
}
func (s NoopSpan) Context() opentracing.SpanContext {
return nil
}
func (s NoopSpan) SetOperationName(operationName string) opentracing.Span {
return s
}
func (s NoopSpan) SetTag(key string, value interface{}) opentracing.Span {
return s
}
func (s NoopSpan) LogFields(fields ...log.Field) {
// pass
}
func (s NoopSpan) LogKV(alternatingKeyValues ...interface{}) {
// pass
}
func (s NoopSpan) SetBaggageItem(restrictedKey, value string) opentracing.Span {
return s
}
func (s NoopSpan) BaggageItem(restrictedKey string) string {
return ""
}
func (s NoopSpan) Tracer() opentracing.Tracer {
return nil
}
func (s NoopSpan) LogEvent(event string) {
// pass
}
func (s NoopSpan) LogEventWithPayload(event string, payload interface{}) {
// pass
}
func (s NoopSpan) Log(data opentracing.LogData) {
// pass
}
func (t NoopTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
return NoopSpan{}
}
func (t NoopTracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error {
return nil
}
func (t NoopTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
return nil, nil
}

42
client/validate.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import (
"regexp"
)
const (
maxLabel = 64
maxKey = 64
)
var labelRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9_-]*$")
var keyRegex = regexp.MustCompile("^[A-Za-z0-9_{}+/=.~%:-]*$")
// ValidLabel returns true if the given label is valid, otherwise false.
func ValidLabel(label string) bool {
return len(label) <= maxLabel && labelRegex.Match([]byte(label))
}
// ValidKey returns true if the given key is valid, otherwise false.
func ValidKey(key string) bool {
return len(key) <= maxKey && keyRegex.Match([]byte(key))
}
func validateLabel(label string) error {
if ValidLabel(label) {
return nil
}
return ErrInvalidLabel
}
func validateKey(key string) error {
if ValidKey(key) {
return nil
}
return ErrInvalidKey
}

61
client/validate_test.go Normal file
View file

@ -0,0 +1,61 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
import "testing"
func TestValidateLabel(t *testing.T) {
labels := []string{
"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, label := range labels {
if validateLabel(label) != nil {
t.Fatalf("Should be valid label: %s", label)
}
}
}
func TestValidateLabelInvalid(t *testing.T) {
labels := []string{
"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, label := range labels {
if validateLabel(label) == nil {
t.Fatalf("Should be invalid label: %s", label)
}
}
}
func TestValidateKey(t *testing.T) {
keys := []string{
"", "1", "ab", "ab1", "b-c", "d_e", "pilosa.com",
"bbf8d41c-7dba-40c4-94dc-94677b43bcf3", // UUID
"{bbf8d41c-7dba-40c4-94dc-94677b43bcf3}", // Windows GUID
"https%3A//www.pilosa.com/about/%23contact", // escaped URL
"aHR0cHM6Ly93d3cucGlsb3NhLmNvbS9hYm91dC8jY29udGFjdA==", // base64
"urn:isbn:1234567",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, key := range keys {
if validateKey(key) != nil {
t.Fatalf("Should be valid key: %s", key)
}
}
}
func TestValidateKeyInvalid(t *testing.T) {
keys := []string{
"\"", "'", "slice\\dice", "valid?no", "yüce", "*xyz", "with space", "<script>",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, key := range keys {
if validateKey(key) == nil {
t.Fatalf("Should be invalid key: %s", key)
}
}
}

9
client/version.go Normal file
View file

@ -0,0 +1,9 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
// package ctl contains all pilosa subcommands other than 'server'. These are
// generally administration, testing, and debugging tools.
package client
// Version is the client version.
const Version = "v1.3.0"

2738
cluster.go
View file

File diff suppressed because it is too large Load diff

879
cluster_internal_test.go
View file

@ -1,346 +1,84 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"bytes"
"fmt"
"io/ioutil"
"math/rand"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"reflect"
"strconv"
"strings"
"testing"
"testing/quick"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/gorilla/mux"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
"github.com/molecula/featurebase/v3/disco"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/testhook"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
// Ensure that fragCombos creates the correct fragment mapping.
func TestFragCombos(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
c := newCluster()
c.addNodeBasicSorted(node0)
c.addNodeBasicSorted(node1)
tests := []struct {
idx string
availableShards *roaring.Bitmap
fieldViews viewsByField
expected fragsByHost
}{
{
idx: "i",
availableShards: roaring.NewBitmap(0, 1, 2),
fieldViews: viewsByField{"f": []string{"v1", "v2"}},
expected: fragsByHost{
"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
"node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}},
},
},
{
idx: "foo",
availableShards: roaring.NewBitmap(0, 1, 2, 3),
fieldViews: viewsByField{"f": []string{"v0"}},
expected: fragsByHost{
"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
"node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}},
},
},
}
for _, test := range tests {
actual := c.fragCombos(test.idx, test.availableShards, test.fieldViews)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
// newIndexWithTempPath returns a new instance of Index.
func newIndexWithTempPath(name string) *Index {
path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
// newHolderWithTempPath returns a new instance of Holder.
func newHolderWithTempPath(tb testing.TB, backend string) *Holder {
path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-holder-")
if err != nil {
panic(err)
}
index, err := NewIndex(path, name)
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = backend
h := NewHolder(path, cfg)
PanicOn(h.Open())
testhook.Cleanup(tb, func() {
h.Close()
})
return h
}
// newIndexWithTempPath returns a new instance of Index.
func newIndexWithTempPath(tb testing.TB, name string) *Index {
path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
if err != nil {
panic(err)
}
cfg := DefaultHolderConfig()
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
h := NewHolder(path, cfg)
PanicOn(h.Open())
index, err := h.CreateIndex(name, IndexOptions{})
testhook.Cleanup(tb, func() {
h.Close()
})
if err != nil {
panic(err)
}
return index
}
// Ensure that fragSources creates the correct fragment mapping.
func TestFragSources(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
uri3, err := NewURIFromAddress("host3")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
node3 := &Node{ID: "node3", URI: *uri3}
c1 := newCluster()
c1.ReplicaN = 1
c1.addNodeBasicSorted(node0)
c1.addNodeBasicSorted(node1)
c2 := newCluster()
c2.ReplicaN = 1
c2.addNodeBasicSorted(node0)
c2.addNodeBasicSorted(node1)
c2.addNodeBasicSorted(node2)
c3 := newCluster()
c3.ReplicaN = 2
c3.addNodeBasicSorted(node0)
c3.addNodeBasicSorted(node1)
c4 := newCluster()
c4.ReplicaN = 2
c4.addNodeBasicSorted(node0)
c4.addNodeBasicSorted(node1)
c4.addNodeBasicSorted(node2)
c5 := newCluster()
c5.ReplicaN = 2
c5.addNodeBasicSorted(node0)
c5.addNodeBasicSorted(node1)
c5.addNodeBasicSorted(node2)
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
_, err = field.SetBit(1, 101, nil)
if err != nil {
t.Fatal(err)
}
_, err = field.SetBit(1, ShardWidth+1, nil)
if err != nil {
t.Fatal(err)
}
_, err = field.SetBit(1, ShardWidth*2+1, nil)
if err != nil {
t.Fatal(err)
}
_, err = field.SetBit(1, ShardWidth*3+1, nil)
if err != nil {
t.Fatal(err)
}
tests := []struct {
from *cluster
to *cluster
idx *Index
expected map[string][]*ResizeSource
err string
}{
{
from: c1,
to: c2,
idx: idx,
expected: map[string][]*ResizeSource{
"node0": {},
"node1": {},
"node2": {
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c4,
to: c3,
idx: idx,
expected: map[string][]*ResizeSource{
"node0": {
{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)},
},
"node1": {
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c5,
to: c4,
idx: idx,
expected: map[string][]*ResizeSource{
"node0": {
{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
},
"node1": {
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)},
},
"node2": {},
},
err: "",
},
{
from: c2,
to: c4,
idx: idx,
expected: nil,
err: "clusters are the same size",
},
{
from: c1,
to: c5,
idx: idx,
expected: nil,
err: "adding more than one node at a time is not supported",
},
{
from: c5,
to: c1,
idx: idx,
expected: nil,
err: "removing more than one node at a time is not supported",
},
}
for _, test := range tests {
actual, err := (test.from).fragSources(test.to, test.idx)
if test.err != "" {
if !strings.Contains(err.Error(), test.err) {
t.Fatalf("expected error: %s, got: %s", test.err, err.Error())
}
} else {
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
}
// Ensure that fragSources creates the correct fragment mapping.
func TestResizeJob(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
tests := []struct {
existingNodes []*Node
node *Node
action string
expectedIDs map[string]bool
}{
{
existingNodes: []*Node{node0, node1},
node: node2,
action: resizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
node: node2,
action: resizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
},
}
for _, test := range tests {
actual := newResizeJob(test.existingNodes, test.node, test.action)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual.IDs, test.expectedIDs) {
t.Errorf("expected: %v, but got: %v", test.expectedIDs, actual.IDs)
}
}
}
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := cluster{
nodes: []*Node{
noder: disco.NewLocalNoder([]*disco.Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
}),
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
cNodes := c.noder.Nodes()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := c.NewSnapshot()
// Verify nodes are distributed.
if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.nodes[0], c.nodes[1]}) {
if a := snap.PartitionNodes(0); !reflect.DeepEqual(a, []*disco.Node{cNodes[0], cNodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.nodes[2], c.nodes[0]}) {
if a := snap.PartitionNodes(2); !reflect.DeepEqual(a, []*disco.Node{cNodes[2], cNodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
@ -351,7 +89,7 @@ func TestCluster_Partition(t *testing.T) {
c := newCluster()
c.partitionN = partitionN
partitionID := c.partition(index, shard)
partitionID := disco.ShardToShardPartition(index, shard, partitionN)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
}
@ -381,7 +119,7 @@ func TestHasher(t *testing.T) {
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
hasher := &jmphasher{}
hasher := &disco.Jmphasher{}
if got := hasher.Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
@ -391,9 +129,14 @@ func TestHasher(t *testing.T) {
// Ensure ContainsShards can find the actual shard list for node and index.
func TestCluster_ContainsShards(t *testing.T) {
c := NewTestCluster(5)
c := NewTestCluster(t, 5)
c.ReplicaN = 3
shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
cNodes := c.noder.Nodes()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := c.NewSnapshot()
shards := snap.ContainsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), cNodes[2])
if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected shars for node's index: %v", shards)
@ -401,20 +144,22 @@ func TestCluster_ContainsShards(t *testing.T) {
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
const urisCount = 4
var uris []pnet.URI
arbitraryPorts := []int{17384, 17385, 17386, 17387}
for i := 0; i < urisCount; i++ {
uris = append(uris, NewTestURIFromHostPort(fmt.Sprintf("node%d", i), uint16(arbitraryPorts[i])))
}
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
node0 := &disco.Node{ID: "node0", URI: uris[0]}
node1 := &disco.Node{ID: "node1", URI: uris[1]}
node2 := &disco.Node{ID: "node2", URI: uris[2]}
node3 := &disco.Node{ID: "node3", URI: uris[3]}
nodes := []*Node{node0, node1, node2}
nodes := []*disco.Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
actual := disco.Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
@ -422,24 +167,24 @@ func TestCluster_Nodes(t *testing.T) {
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
actual := disco.Nodes(disco.Nodes(nodes).Filter(nodes[1])).URIs()
expected := []pnet.URI{uris[0], uris[2]}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
actual := disco.Nodes(disco.Nodes(nodes).FilterURI(uris[1])).URIs()
expected := []pnet.URI{uris[0], uris[2]}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
actualTrue := disco.Nodes(nodes).Contains(node1)
actualFalse := disco.Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
@ -449,395 +194,15 @@ func TestCluster_Nodes(t *testing.T) {
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
clone := disco.Nodes(nodes).Clone()
actual := disco.Nodes(clone).URIs()
expected := []pnet.URI{uris[0], uris[1], uris[2]}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
func TestCluster_PreviousNode(t *testing.T) {
node0 := &Node{ID: "node0"}
node1 := &Node{ID: "node1"}
node2 := &Node{ID: "node2"}
t.Run("OneNode", func(t *testing.T) {
c := newCluster()
c.addNodeBasicSorted(node0)
c.Node = node0
if prev := c.unprotectedPreviousNode(); prev != nil {
t.Errorf("expected: nil, but got: %v", prev)
}
})
t.Run("TwoNode", func(t *testing.T) {
c := newCluster()
c.addNodeBasicSorted(node0)
c.addNodeBasicSorted(node1)
c.Node = node0
if prev := c.unprotectedPreviousNode(); prev != node1 {
t.Errorf("expected: node1, but got: %v", prev)
}
c.Node = node1
if prev := c.unprotectedPreviousNode(); prev != node0 {
t.Errorf("expected: node0, but got: %v", prev)
}
})
t.Run("ThreeNode", func(t *testing.T) {
c := newCluster()
c.addNodeBasicSorted(node0)
c.addNodeBasicSorted(node1)
c.addNodeBasicSorted(node2)
c.Node = node0
if prev := c.unprotectedPreviousNode(); prev != node2 {
t.Errorf("expected: node2, but got: %v", prev)
}
c.Node = node1
if prev := c.unprotectedPreviousNode(); prev != node0 {
t.Errorf("expected: node0, but got: %v", prev)
}
c.Node = node2
if prev := c.unprotectedPreviousNode(); prev != node1 {
t.Errorf("expected: node1, but got: %v", prev)
}
})
}
// NEXT: move this test to internal and unexport IsCoordinator
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *newCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *newCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.isCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.isCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.addNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.nodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
nodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
nodeIDs: []string{node.Node.ID},
}
if err := tc.WriteTopology(node.Path, top); err != nil {
t.Fatalf("writing topology: %v", err)
}
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
nodeIDs: []string{"some-other-host"},
}
if err := tc.WriteTopology(node.Path, top); err != nil {
t.Fatalf("writing topology: %v", err)
}
// Open TestCluster.
expected := "coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || errors.Cause(err).Error() != expected {
t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatalf("opening cluster: %v", err)
}
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
nodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
nodeIDs: []string{"node0", "node2"},
}
if err := tc.WriteTopology(node0.Path, top); err != nil {
t.Fatalf("writing topology: %v", err)
}
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatalf("opening cluster: %v", err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
if err := tc.addNode(); err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node2 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
t.Fatalf("creating field: %v", err)
}
if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
if err := tc.SetBit("i", "f", 1, ShardWidth+1, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.holder.Field("i", "f")
node0View := node0Field.view("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
// addNode needs to block until the resize process has completed.
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
nodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.holder.Field("i", "f")
node1View := node1Field.view("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
func TestAE(t *testing.T) {
t.Run("AbortDoesn'tBlockUninitialized", func(t *testing.T) {
c := newCluster()
@ -846,6 +211,7 @@ func TestAE(t *testing.T) {
c.abortAntiEntropy()
close(ch)
}()
defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
select {
case <-ch:
return
@ -857,11 +223,13 @@ func TestAE(t *testing.T) {
t.Run("AbortBlocksInitialized", func(t *testing.T) {
c := newCluster()
c.initializeAntiEntropy()
ch := make(chan struct{})
go func() {
c.abortAntiEntropy()
close(ch)
}()
defer c.abortAntiEntropyQ() // avoid leak of goroutine.
select {
case <-ch:
t.Fatalf("aborting anti entropy on an initialized cluster didn't block")
@ -893,97 +261,4 @@ func TestAE(t *testing.T) {
t.Fatalf("abort should not have blocked this long")
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.nodes[0]
newNode := c.nodes[1]
// Update coordinator to the same value.
if c.updateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.updateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}
func TestCluster_confirmNodeDownUp(t *testing.T) {
r := mux.NewRouter()
r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "ignored")
}))
server := httptest.NewServer(r)
// Close the server when test finishes
defer server.Close()
u, err := url.Parse(server.URL)
if err != nil {
t.Error("bad test setup")
}
uri := URI{}
host, port, _ := net.SplitHostPort(u.Host)
uri.Scheme = u.Scheme
uri.Host = host
iport, err := strconv.ParseUint(port, 0, 16)
if err != nil {
t.Error(err)
}
uri.Port = uint16(iport)
if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
t.Errorf("expected node to be up")
}
}
func TestCluster_confirmNodeDownTimeout(t *testing.T) {
r := mux.NewRouter()
r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
fmt.Fprintln(w, "ignored")
}))
server := httptest.NewServer(r)
// Close the server when test finishes
defer server.Close()
u, err := url.Parse(server.URL)
if err != nil {
t.Error("bad test setup")
}
uri := URI{}
host, port, _ := net.SplitHostPort(u.Host)
uri.Scheme = u.Scheme
uri.Host = host
iport, err := strconv.ParseUint(port, 0, 16)
if err != nil {
t.Error(err)
}
uri.Port = uint16(iport)
if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
t.Errorf("expected node to be down")
}
}
func TestCluster_confirmNodeDownDown(t *testing.T) {
uri := URI{}
uri.Scheme = "http"
uri.Host = "DoesntMatter"
uri.Port = 6666
if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
t.Errorf("expected node to be down")
}
}

25
cmd.go
View file

@ -1,22 +1,11 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package pilosa
import (
"io"
"log"
"github.com/molecula/featurebase/v3/logger"
)
// CmdIO holds standard unix inputs and outputs.
@ -24,7 +13,7 @@ type CmdIO struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
logger *log.Logger
logger logger.Logger
}
// NewCmdIO returns a new instance of CmdIO with inputs and outputs set to the
@ -34,10 +23,10 @@ func NewCmdIO(stdin io.Reader, stdout, stderr io.Writer) *CmdIO {
Stdin: stdin,
Stdout: stdout,
Stderr: stderr,
logger: log.New(stderr, "", log.LstdFlags),
logger: logger.NewStandardLogger(stderr),
}
}
func (c *CmdIO) Logger() *log.Logger {
func (c *CmdIO) Logger() logger.Logger {
return c.logger
}

30
cmd/auth_token.go Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"context"
"io"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
func newAuthTokenCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
cmd := ctl.NewAuthTokenCommand(stdin, stdout, stderr)
ccmd := &cobra.Command{
Use: "auth-token",
Short: "Get an auth-token",
Long: `
Retrieves an auth-token for use in authenticating with FeatureBase from the configured identity provider.
`,
RunE: func(c *cobra.Command, args []string) error {
return cmd.Run(context.Background())
},
}
flags := ccmd.Flags()
flags.StringVar(&cmd.Host, "host", "https://localhost:10101", "The address (host:port) of FeatureBase (HTTPs).")
ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
return ccmd
}

38
cmd/backup.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"context"
"io"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
func newBackupCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
cmd := ctl.NewBackupCommand(stdin, stdout, stderr)
ccmd := &cobra.Command{
Use: "backup",
Short: "Back up FeatureBase server",
Long: `
Backs up a FeatureBase server to a local, tar-formatted snapshot file.
`,
RunE: func(c *cobra.Command, args []string) error {
return cmd.Run(context.Background())
},
}
flags := ccmd.Flags()
flags.StringVarP(&cmd.OutputDir, "output", "o", "", "Output directory to write to.")
flags.BoolVar(&cmd.NoSync, "no-sync", false, "Disable file sync")
flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "Number of concurrent backup goroutines.")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "The address (host:port) of FeatureBase (HTTP).")
flags.StringVar(&cmd.Index, "index", "", "Index to backup, default backs up all indexes. ")
flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
flags.StringVar(&cmd.HeaderTimeoutStr, "header-timeout", cmd.HeaderTimeoutStr, "Length of time to wait for initial HTTP response before giving up.")
return ccmd
}

138
cmd/badloader/badloader.go Normal file
View file

@ -0,0 +1,138 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package main
import (
"archive/tar"
"compress/gzip"
"context"
"time"
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/encoding/proto"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/vprint"
"os"
"strconv"
"strings"
)
func UploadTar(srcFile string, client *pilosa.InternalClient) error {
t0 := time.Now()
f, err := os.Open(srcFile)
if err != nil {
return (err)
}
defer f.Close()
var tarReader *tar.Reader
if strings.HasSuffix(srcFile, "gz") {
gzf, err := gzip.NewReader(f)
if err != nil {
return err
}
tarReader = tar.NewReader(gzf)
} else {
tarReader = tar.NewReader(f)
}
viewData := make(map[string][]byte)
//given ordered by index/field/view
//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
lastIndex := ""
lastField := ""
lastShard := uint64(0)
n := 0
for {
header, err := tarReader.Next()
if err == io.EOF {
if header != nil {
vprint.PanicOn("header should not be nil on err io.EOF")
}
//submit any stuff we have left
if len(viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
}
// Submit(lastIndex, lastField, lastShard, request)
uri := GetImportRoaringURI(lastIndex, lastShard)
err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
vprint.PanicOn(err)
}
return nil
}
n++
if n%500 == 0 {
vprint.VV("n = %v, progress, elapsed '%v'", n, time.Since(t0))
}
parts := strings.Split(header.Name, "/")
//vv("parts = '%#v'", parts)
index := parts[1]
field := parts[2]
view := parts[4]
shard, err := strconv.ParseUint(parts[6], 10, 64)
if err != nil {
return err
}
// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
if index != lastIndex || field != lastField || shard != lastShard {
if len(viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
}
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
vprint.PanicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
viewData = make(map[string][]byte)
//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
}
}
roaringData, err := ioutil.ReadAll(tarReader)
if err != nil {
return err
}
if _, already := viewData[view]; already {
vprint.PanicOn(fmt.Sprintf("view '%v' already present!", view))
}
viewData[view] = roaringData
lastIndex = index
lastField = field
//lastShard = shard
//vv("bottom of loop")
}
}
// badloader reproduce a union in place issue for us. slurp is
// the new "good" loader, and should always be preferred now
// when not trying to repro that bug. pulled from 85fa67e8
func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{}))
vprint.PanicOn(err)
tarSrcPath := "q2.tar.gz"
t0 := time.Now()
vprint.PanicOn(UploadTar(tarSrcPath, c))
vprint.VV("total elapsed '%v'", time.Since(t0))
}
var globURI *pnet.URI
func init() {
var err error
globURI, err = pnet.NewURIFromHostPort("127.0.0.1", 10101)
vprint.PanicOn(err)
}
// get correct node to go to.
func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
return globURI
}

46
cmd/check.go
View file

@ -1,46 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"fmt"
"io"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/v2/ctl"
)
var checker *ctl.CheckCommand
func newCheckCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
checker = ctl.NewCheckCommand(stdin, stdout, stderr)
checkCmd := &cobra.Command{
Use: "check <path> [path2]...",
Short: "Do a consistency check on a pilosa data file.",
Long: `
Performs a consistency check on data files.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("path required")
}
checker.Paths = args
return checker.Run(context.Background())
},
}
return checkCmd
}

36
cmd/check_test.go
View file

@ -1,36 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
)
func TestCheckHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "check", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa check") || err != nil {
t.Fatalf("Command 'check --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestCheckNoPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "check")
if !strings.Contains(err.Error(), "path required") {
t.Fatalf("Command 'check' without args should error but: err: '%v', output: '%v'", err, output)
}
}

31
cmd/chksum.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"context"
"io"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
func newChkSumCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
cmd := ctl.NewChkSumCommand(stdin, stdout, stderr)
ccmd := &cobra.Command{
Use: "chksum",
Short: "Digital signature of FeatureBase data",
Long: `
Generates a digital signature of all the data associated with a provided FeatureBase server
WARNING: could be slow if high cardinality fields exist
`,
RunE: func(c *cobra.Command, args []string) error {
return cmd.Run(context.Background())
},
}
flags := ccmd.Flags()
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
return ccmd
}

20
cmd/config.go
View file

@ -1,17 +1,5 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
@ -20,8 +8,8 @@ import (
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/v2/ctl"
"github.com/pilosa/pilosa/v2/server"
"github.com/molecula/featurebase/v3/ctl"
"github.com/molecula/featurebase/v3/server"
)
var conf *ctl.ConfigCommand

16
cmd/doc.go
View file

@ -1,17 +1,5 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
/*
Package cmd contains all the pilosa subcommand definitions (1 per file).

26
cmd/export.go
View file

@ -1,17 +1,5 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
@ -20,7 +8,7 @@ import (
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
)
var Exporter *ctl.ExportCommand
@ -29,7 +17,7 @@ func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
Exporter = ctl.NewExportCommand(stdin, stdout, stderr)
exportCmd := &cobra.Command{
Use: "export",
Short: "Export data from pilosa.",
Short: "Export data from FeatureBase.",
Long: `
Bulk exports a fragment to a CSV file. If the OUTFILE is not specified then
the output is written to STDOUT.
@ -46,11 +34,11 @@ The file does not contain any headers.
}
flags := exportCmd.Flags()
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of FeatureBase.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "FeatureBase index to export")
flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
ctl.SetTLSConfig(flags, "", &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.CACertPath, &Exporter.TLS.SkipVerify, &Exporter.TLS.EnableClientVerification)
return exportCmd
}

20
cmd/export_test.go
View file

@ -1,31 +1,19 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/v2/cmd"
"github.com/molecula/featurebase/v3/cmd"
)
func TestExportHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "export", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa export") || err != nil {
!strings.Contains(output, "featurebase export") || err != nil {
t.Fatalf("Command 'export --help' not working, err: '%v', output: '%s'", err, output)
}
}

48
cmd/featurebase-parse-sql/main.go Normal file
View file

@ -0,0 +1,48 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"strings"
"github.com/molecula/featurebase/v3/sql2"
)
func main() {
if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
os.Exit(1)
} else if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(ctx context.Context, args []string) error {
fs := flag.NewFlagSet("featurebase-parse-sql", flag.ContinueOnError)
if err := fs.Parse(args); err != nil {
return err
}
q := fs.Arg(0)
if q == "" {
return fmt.Errorf("query required")
}
stmt, err := sql2.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
return err
}
buf, err := json.MarshalIndent(stmt, "", " ")
if err != nil {
return err
}
fmt.Println(string(buf))
return nil
}

23
cmd/featurebase/main.go Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
/*
This is the entrypoint for the Pilosa binary.
*/
package main
import (
"fmt"
"os"
"github.com/molecula/featurebase/v3/cmd"
"github.com/molecula/featurebase/v3/monitor"
)
func main() {
defer monitor.CaptureMessage("Session:Ended")
rootCmd := cmd.NewRootCommand(os.Stdin, os.Stdout, os.Stderr)
if err := rootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}

13
cmd/featurebase/main_test.go Normal file
View file

@ -0,0 +1,13 @@
//go:build testrunmain
// +build testrunmain
package main
import (
"testing"
)
// Wrapper test for main function used to get code coverage for end2end tests
func TestRunMain(t *testing.T) {
main()
}

18
cmd/generate_config.go
View file

@ -1,17 +1,5 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
@ -20,7 +8,7 @@ import (
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
)
var generateConf *ctl.GenerateConfigCommand

71
cmd/import.go
View file

@ -1,36 +1,54 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"context"
"fmt"
"io"
"strconv"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/ctl"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/ctl"
"github.com/molecula/featurebase/v3/pql"
"github.com/spf13/cobra"
)
var Importer *ctl.ImportCommand
// newImportCommand runs the Pilosa import subcommand for ingesting bulk data.
// DecimalFlagValue is used to set the unexported value field in a decimal. It also
// fulfills the flag.Value interface.
type DecimalFlagValue struct {
dec *pql.Decimal
}
func (dfv *DecimalFlagValue) String() string {
return fmt.Sprintf("%v", dfv.dec.Value())
}
func (dfv *DecimalFlagValue) Set(s string) error {
i, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return err
}
if dfv.dec == nil {
d := pql.NewDecimal(0, 0)
dfv.dec = &d
}
dfv.dec.SetValue(i)
return nil
}
func (dfv *DecimalFlagValue) Type() string {
return fmt.Sprintf("%T", int64(0))
}
// newImportCommand runs the FeatureBase import subcommand for ingesting bulk data.
func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Importer = ctl.NewImportCommand(stdin, stdout, stderr)
importCmd := &cobra.Command{
Use: "import",
Short: "Bulk load data into pilosa.",
Short: "Bulk load data into FeatureBase.",
Long: `Bulk imports one or more CSV files to a host's index and field. The data
of the CSV file are grouped by shard for the most efficient import.
@ -47,23 +65,28 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
},
}
fieldMin := DecimalFlagValue{dec: &Importer.FieldOptions.Min}
fieldMax := DecimalFlagValue{dec: &Importer.FieldOptions.Max}
flags := importCmd.Flags()
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of FeatureBase.")
flags.StringVarP(&Importer.Index, "index", "i", "", "FeatureBase index to import into.")
flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
flags.BoolVar(&Importer.RowColMode, "row-col-mode", false, "Specify row-col-mode=true to read csv files as <row id>,<col id>")
flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, decimal, time, bool, mutex")
flags.Var(&fieldMin, "field-min", "Specify the minimum for an int field on creation") // TODO: noting that decimal field min/max are not supported here.
flags.Var(&fieldMax, "field-max", "Specify the maximum for an int field on creation")
flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
flags.DurationVarP(&Importer.FieldOptions.TTL, "time-to-live", "t", 0, "Specify the time to live for views created by time quantum. Supported time unit: \"s\", \"m\", \"h\"") // \"ns\", \"us\" (or \"µs\"), \"ms\" also supported but ommitted for simplicity
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
flags.StringVar(&Importer.AuthToken, "auth-token", "", "Authentication token")
return importCmd
}

27
cmd/import_test.go
View file

@ -1,33 +1,22 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/v2"
pilosa "github.com/molecula/featurebase/v3"
"github.com/pilosa/pilosa/v2/cmd"
"github.com/molecula/featurebase/v3/cmd"
"github.com/molecula/featurebase/v3/pql"
)
func TestImportHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "import", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa import") || err != nil {
!strings.Contains(output, "featurebase import") || err != nil {
t.Fatalf("Command 'import --help' not working, err: '%v', output: '%s'", err, output)
}
}
@ -58,8 +47,8 @@ field = "f1"
v.Check(cmd.Importer.Field, "f1")
v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{
Keys: true,
Max: 100,
Min: -10,
Max: pql.NewDecimal(100, 0),
Min: pql.NewDecimal(-10, 0),
CacheType: pilosa.CacheTypeRanked,
CacheSize: 50000,
})

49
cmd/inspect.go
View file

@ -1,49 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"fmt"
"io"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/v2/ctl"
)
var inspector *ctl.InspectCommand
func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
inspector = ctl.NewInspectCommand(stdin, stdout, stderr)
inspectCmd := &cobra.Command{
Use: "inspect",
Short: "Get stats on a pilosa data file.",
Long: `
Inspects a data file and provides stats.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("path required")
} else if len(args) > 1 {
return fmt.Errorf("only one path allowed")
}
inspector.Path = args[0]
return inspector.Run(context.Background())
},
}
return inspectCmd
}

42
cmd/inspect_test.go
View file

@ -1,42 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
)
func TestInspectHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "pilosa inspect") || err != nil {
t.Fatalf("Command 'inspect --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestInspectNoPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect")
if !strings.Contains(err.Error(), "path required") {
t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
}
}
func TestInspectMultiPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect", "one", "two")
if !strings.Contains(err.Error(), "only one path") {
t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
}
}

29
cmd/keygen.go Normal file
View file

@ -0,0 +1,29 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"context"
"io"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
func newKeygenCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
cmd := ctl.NewKeygenCommand(stdin, stdout, stderr)
ccmd := &cobra.Command{
Use: "keygen",
Short: "Generate secret key for authentication.",
Long: `
Generate secret key to configure FeatureBase for Authentication.
`,
RunE: func(c *cobra.Command, args []string) error {
return cmd.Run(context.Background())
},
}
flags := ccmd.Flags()
flags.IntVarP(&cmd.KeyLength, "length", "l", 32, "length of the key to produce")
return ccmd
}

346
cmd/pilosa-bench/main.go Normal file
View file

@ -0,0 +1,346 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package main
import (
"context"
"expvar"
"flag"
"fmt"
"io/ioutil"
"log"
"math/rand"
"net/http"
_ "net/http/pprof"
"os"
"sort"
"strings"
"time"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/encoding/proto"
"golang.org/x/sync/errgroup"
)
var (
requestCountVar = expvar.NewInt("request_count")
requestCurrentLatencyVar = expvar.NewFloat("request_current_latency") // seconds
requestAvgLatencyVar = expvar.NewFloat("request_avg_latency") // seconds
requestTotalLatencyVar = expvar.NewFloat("request_total_latency") // seconds
requestPerSecVar = expvar.NewFloat("request_per_sec")
)
func main() {
if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp {
os.Exit(1)
} else if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(ctx context.Context, args []string) (err error) {
fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError)
hostport := fs.String("hostport", "localhost:10101", "")
typ := fs.String("type", "row", "query type (row)")
n := fs.Int("n", 1000, "number of queries")
rate := fs.Int("rate", 1, "number of queries per second")
verbose := fs.Bool("v", false, "verbose logging")
from := fs.String("from", "", "from time for row-range queries (ISO 8601)")
to := fs.String("to", "", "to time for row-range queries (ISO 8601)")
if err := fs.Parse(args); err != nil {
return err
}
// Parse from/to time.
var opt queryOptions
if *from != "" {
if opt.from, err = time.Parse(time.RFC3339, *from); err != nil {
return fmt.Errorf("cannot parse -from time")
}
}
if *to != "" {
if opt.to, err = time.Parse(time.RFC3339, *to); err != nil {
return fmt.Errorf("cannot parse -to time")
}
}
if (*typ == "row-range" || *typ == "topk") && (opt.from.IsZero() || opt.to.IsZero()) {
return fmt.Errorf("-from and -to flags must be specified for topk & row-range queries")
}
// Clear time prefix on log.
log.SetFlags(0)
if !*verbose {
log.SetOutput(ioutil.Discard)
}
// Setup PRNG to have consistent values for the same set of data.
rand.Seed(0)
// Setup connection to pilosa.
client, err := pilosa.NewInternalClient(*hostport, http.DefaultClient, pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
return err
}
// Set up HTTP endpoint to provide /debug endpoints.
fmt.Println("Serving debug endpoint at http://localhost:7070/debug")
go func() { _ = http.ListenAndServe(":7070", nil) }()
// Run separate goroutine to calculate the current req/sec & latency.
go monitor()
// Load all id/keys for each field.
log.Printf("loading field identifiers")
fieldIDMap, err := loadFields(ctx, client)
if err != nil {
return fmt.Errorf("cannot load field identifiers: %w", err)
} else if len(fieldIDMap) == 0 {
return fmt.Errorf("no field identifiers available, please verify data exists")
}
// Generate list of sorted keys.
fieldKeys := make([]fieldKey, 0, len(fieldIDMap))
for k, f := range fieldIDMap {
switch *typ {
case "row-bsi":
if f.info.Options.Type != "int" {
continue
}
case "row-range", "topk":
if f.info.Options.Type != "time" {
continue
}
default:
if f.info.Options.Type == "int" || f.info.Options.Type == "time" {
continue
}
}
fieldKeys = append(fieldKeys, k)
}
sort.Slice(fieldKeys, func(i, j int) bool {
return compareFieldKeys(fieldKeys[i], fieldKeys[j]) == -1
})
// Ensure we have appropriate fields for our query type.
if len(fieldKeys) == 0 {
return fmt.Errorf("no available fields are appropriate for %q queries", *typ)
}
log.Printf("issuing %d queries at %d query/sec", *n, *rate)
// Repeatedly issue queries based on available row data.
ticker := time.NewTicker(time.Second / time.Duration(*rate))
var g errgroup.Group
for i := 0; i < *n; i++ {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
key := fieldKeys[rand.Intn(len(fieldKeys))]
q, err := generateQuery(*typ, key.index, key.field, fieldIDMap[key].info, fieldIDMap[key].identifiers, opt)
if err != nil {
return fmt.Errorf("cannot generate query: %w", err)
}
log.Printf("[query] %s", q)
g.Go(func() error {
t := time.Now()
_, err = client.Query(ctx, key.index, &pilosa.QueryRequest{Index: key.index, Query: q})
if err != nil {
return err
}
elapsed := time.Since(t).Seconds()
requestCountVar.Add(1)
requestTotalLatencyVar.Add(elapsed)
requestAvgLatencyVar.Set(requestTotalLatencyVar.Value() / float64(requestCountVar.Value()))
return nil
})
}
return g.Wait()
}
// monitor runs in a separate goroutine and updates metrics.
func monitor() {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
var lastTime time.Time
var lastN int64
var lastLatency float64
for range ticker.C {
now, n := time.Now(), requestCountVar.Value()
latency := requestTotalLatencyVar.Value()
if !lastTime.IsZero() {
elapsed := lastTime.Sub(now).Seconds()
if n > 0 {
requestCurrentLatencyVar.Set((lastLatency - latency) / float64(n))
}
requestPerSecVar.Set(float64(lastN-n) / elapsed)
}
lastTime, lastN, lastLatency = now, n, latency
}
}
func generateQuery(typ, index, field string, info *pilosa.FieldInfo, identifiers *pilosa.RowIdentifiers, opt queryOptions) (string, error) {
switch typ {
case "row":
return generateRowQuery(index, field, identifiers), nil
case "row-bsi":
return generateRowBSIQuery(index, field), nil
case "row-range":
return generateRowRangeQuery(index, field, identifiers, opt.from, opt.to), nil
case "count":
return generateCountQuery(index, field, identifiers), nil
case "intersect":
return generateIntersectQuery(index, field, identifiers), nil
case "union":
return generateUnionQuery(index, field, identifiers), nil
case "difference":
return generateDifferenceQuery(index, field, identifiers), nil
case "xor":
return generateXorQuery(index, field, identifiers), nil
case "groupby":
return generateGroupByQuery(index, field), nil
case "topk":
return generateTopKQuery(index, field, opt.from, opt.to), nil
default:
return "", fmt.Errorf("invalid query type: %q", typ)
}
}
func generateRowQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
if len(identifiers.Rows) > 0 {
return fmt.Sprintf("Row(%s=%d)", field, chooseRowID(identifiers))
}
return fmt.Sprintf("Row(%s=%q)", field, chooseRowKey(identifiers))
}
func generateRowBSIQuery(index, field string) string {
return fmt.Sprintf("Row(%s > 0)", field)
}
func generateRowRangeQuery(index, field string, identifiers *pilosa.RowIdentifiers, from, to time.Time) string {
if len(identifiers.Rows) > 0 {
return fmt.Sprintf("Row(%s=%d, from='%s', to='%s')", field, chooseRowID(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
}
return fmt.Sprintf("Row(%s=%q, from='%s', to='%s')", field, chooseRowKey(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
}
func generateRowQueries(index, field string, identifiers *pilosa.RowIdentifiers) string {
a := make([]string, rand.Intn(9)+1)
for i := range a {
a[i] = generateRowQuery(index, field, identifiers)
}
return strings.Join(a, ", ")
}
func generateCountQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Count(%s)", generateRowQuery(index, field, identifiers))
}
func generateIntersectQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Intersect(%s)", generateRowQueries(index, field, identifiers))
}
func generateUnionQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Union(%s)", generateRowQueries(index, field, identifiers))
}
func generateDifferenceQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Difference(%s)", generateRowQueries(index, field, identifiers))
}
func generateXorQuery(index, field string, identifiers *pilosa.RowIdentifiers) string {
return fmt.Sprintf("Xor(%s)", generateRowQueries(index, field, identifiers))
}
func generateGroupByQuery(index, field string) string {
return fmt.Sprintf("GroupBy(Rows(%s))", field)
}
func generateTopKQuery(index, field string, from, to time.Time) string {
return fmt.Sprintf("TopK(%s, from='%s', to='%s')", field, from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04"))
}
// loadFields returns a mapping of index/field names to field info & identifiers.
func loadFields(ctx context.Context, client *pilosa.InternalClient) (map[fieldKey]*fieldInfo, error) {
indexes, err := client.Schema(ctx)
if err != nil {
return nil, err
}
m := make(map[fieldKey]*fieldInfo)
for _, ii := range indexes {
for _, f := range ii.Fields {
log.Printf("field: index=%s name=%s type=%s", ii.Name, f.Name, f.Options.Type)
switch f.Options.Type {
case "set", "mutex", "time":
identifiers, err := fetchFieldIDs(ctx, client, ii.Name, f.Name)
if err != nil {
return nil, fmt.Errorf("fetch fields: %w", err)
} else if len(identifiers.Rows) > 0 || len(identifiers.Keys) > 0 {
m[fieldKey{ii.Name, f.Name}] = &fieldInfo{f, identifiers}
}
case "int":
m[fieldKey{ii.Name, f.Name}] = &fieldInfo{info: f}
}
}
}
return m, nil
}
// fetchFieldIDs returns a list of field IDs or keys.
func fetchFieldIDs(ctx context.Context, client *pilosa.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
if err != nil {
return nil, err
}
switch result := resp.Results[0].(type) {
case *pilosa.RowIdentifiers:
return result, nil
case pilosa.RowIdentifiers:
return &result, nil
default:
return nil, fmt.Errorf("unexpected result type: %T", result)
}
}
func chooseRowID(identifiers *pilosa.RowIdentifiers) uint64 {
return identifiers.Rows[rand.Intn(len(identifiers.Rows))]
}
func chooseRowKey(identifiers *pilosa.RowIdentifiers) string {
return identifiers.Keys[rand.Intn(len(identifiers.Keys))]
}
type fieldKey struct {
index string
field string
}
type fieldInfo struct {
info *pilosa.FieldInfo
identifiers *pilosa.RowIdentifiers
}
func compareFieldKeys(x, y fieldKey) int {
if cmp := strings.Compare(x.index, y.index); cmp != 0 {
return cmp
}
return strings.Compare(x.field, y.field)
}
type queryOptions struct {
from, to time.Time
}

33
cmd/pilosa/main.go
View file

@ -1,33 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
This is the entrypoint for the Pilosa binary.
*/
package main
import (
"fmt"
"os"
"github.com/pilosa/pilosa/v2/cmd"
)
func main() {
rootCmd := cmd.NewRootCommand(os.Stdin, os.Stdout, os.Stderr)
if err := rootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}

148
cmd/rbf.go Normal file
View file

@ -0,0 +1,148 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"context"
"errors"
"fmt"
"io"
"strconv"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
func newRBFCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
cmd := &cobra.Command{
Use: "rbf",
Short: "Inspect RBF data files.",
Long: `
Provides a set of commands for inspecting RBF data files.
`,
}
cmd.AddCommand(newRBFCheckCommand(stdin, stdout, stderr))
cmd.AddCommand(newRBFDumpCommand(stdin, stdout, stderr))
cmd.AddCommand(newRBFPagesCommand(stdin, stdout, stderr))
cmd.AddCommand(newRBFPageCommand(stdin, stdout, stderr))
return cmd
}
func newRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
c := ctl.NewRBFCheckCommand(stdin, stdout, stderr)
cmd := &cobra.Command{
Use: "check [flags] PATH",
Short: "Run consistency check on RBF data.",
Long: `
Executes a consistency check on an RBF data directory.
`,
Args: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("data directory path required")
} else if len(args) > 1 {
return fmt.Errorf("too many command line arguments")
}
c.Path = args[0]
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
return c.Run(context.Background())
},
}
return cmd
}
func newRBFDumpCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
c := ctl.NewRBFDumpCommand(stdin, stdout, stderr)
cmd := &cobra.Command{
Use: "dump [flags] PATH PGNO [PGNO...]",
Short: "Prints RBF raw page data",
Long: `
Dumps the raw hex data for one or more RBF pages.
`,
Args: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("data directory path required")
} else if len(args) == 1 {
return fmt.Errorf("page number required")
}
c.Path = args[0]
for _, arg := range args[1:] {
pgno, err := strconv.Atoi(arg)
if err != nil {
return errors.New("invalid page number")
}
c.Pgnos = append(c.Pgnos, uint32(pgno))
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
return c.Run(context.Background())
},
}
return cmd
}
func newRBFPagesCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
c := ctl.NewRBFPagesCommand(stdin, stdout, stderr)
cmd := &cobra.Command{
Use: "pages [flags] PATH",
Short: "Prints metadata for the list of all pages",
Long: `
Prints a line for every page in the database with its type/status.
`,
Args: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("data directory path required")
} else if len(args) > 1 {
return fmt.Errorf("too many command line arguments")
}
c.Path = args[0]
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
return c.Run(context.Background())
},
}
flags := cmd.Flags()
flags.BoolVar(&c.WithTree, "with-tree", false, "Display b-tree name for each row")
return cmd
}
func newRBFPageCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
c := ctl.NewRBFPageCommand(stdin, stdout, stderr)
cmd := &cobra.Command{
Use: "page [flags] PATH PGNO [PGNO...]",
Short: "Prints data for a page(s)",
Long: `
Prints the header & cell data for one or more pages.
`,
Args: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("data directory path required")
} else if len(args) == 1 {
return fmt.Errorf("page number required")
}
c.Path = args[0]
for _, arg := range args[1:] {
pgno, err := strconv.Atoi(arg)
if err != nil {
return errors.New("invalid page number")
}
c.Pgnos = append(c.Pgnos, uint32(pgno))
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
return c.Run(context.Background())
},
}
return cmd
}

42
cmd/restore.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"context"
"io"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
func newRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
cmd := ctl.NewRestoreCommand(stdin, stdout, stderr)
restoreCmd := &cobra.Command{
Use: "restore",
Short: "Restore from a backup",
Long: `
The Restore command will take a backup archive and restore it to a new, clean cluster.
`,
RunE: func(c *cobra.Command, args []string) error {
return cmd.Run(context.Background())
},
}
flags := restoreCmd.Flags()
flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.IntVar(&cmd.Concurrency, "concurrency", 1, "number of concurrent uploads")
flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
ctl.SetTLSConfig(
flags, "",
&cmd.TLS.CertificatePath,
&cmd.TLS.CertificateKeyPath,
&cmd.TLS.CACertPath,
&cmd.TLS.SkipVerify,
&cmd.TLS.EnableClientVerification,
)
return restoreCmd
}

16
cmd/roaring-migrate/ctim_darwin.go Normal file
View file

@ -0,0 +1,16 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
//go:build darwin
// +build darwin
package main
import (
"syscall"
)
func CTimeNano(stat *syscall.Stat_t) int64 {
NANOS := int64(1e9) // number of nanosecs in 1 sec
ts := stat.Ctimespec
return ts.Sec*NANOS + ts.Nsec
}

14
cmd/roaring-migrate/ctim_linux.go Normal file
View file

@ -0,0 +1,14 @@
// Copyright 2022 Molecula Corp. (DBA FeatureBase).
// SPDX-License-Identifier: Apache-2.0
//go:build linux
// +build linux
package main
import (
"syscall"
)
func CTimeNano(stat *syscall.Stat_t) int64 {
return stat.Ctim.Nano()
}

Some files were not shown because too many files have changed in this diff Show more