diff --git a/.circleci/config.yml b/.circleci/config.yml index 30b2ef5e4..498c56ea3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,13 +63,6 @@ jobs: - checkout-plus - run: go mod download - save-mod-cache - check-license-headers: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - run: make check-license-headers linter: executor: name: golang @@ -239,10 +232,6 @@ workflows: context: molecula requires: - setup - - check-license-headers: - context: molecula - requires: - - setup - go-mod-tidy: context: molecula requires: @@ -263,7 +252,7 @@ workflows: - setup matrix: parameters: - golang_version: ["1.14.15", "1.15.8", "1.16.3"] + golang_version: ["1.15.8", "1.16.10"] - test: name: << matrix.test_make_target >> resource_class: xlarge diff --git a/.cloudbuild/Dockerfile b/.cloudbuild/Dockerfile deleted file mode 100644 index 407e5b994..000000000 --- a/.cloudbuild/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM alpine:3.14.2 - -LABEL maintainer "dev@molecula.com" -LABEL org.opencontainers.image.authors="dev@molecula.com" - -RUN apk add --no-cache curl jq - -COPY LICENSE /LICENSE -COPY NOTICE /NOTICE -ARG SHORT_SHA - -COPY .cloudbuild/featurebase_linux_amd64 / - -RUN mv /featurebase_linux_amd64 /pilosa -RUN chmod ugo+x /pilosa -EXPOSE 10101 -VOLUME /data - -ENV PILOSA_DATA_DIR /data -ENV PILOSA_BIND 0.0.0.0:10101 -ENV PILOSA_BIND_GRPC 0.0.0.0:20101 - -ENTRYPOINT ["/pilosa"] -CMD ["server"] diff --git a/.cloudbuild/LICENSE b/.cloudbuild/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/.cloudbuild/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.cloudbuild/NOTICE b/.cloudbuild/NOTICE deleted file mode 100644 index 594272a27..000000000 --- a/.cloudbuild/NOTICE +++ /dev/null @@ -1,120 +0,0 @@ -Software license -================ - -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Third-party software licenses -============================= - -The file /pilosa/lru/lru.go contains a redistribution of lru -(github.com/golang/groupcache/lru); the license follows: - - Copyright 2013 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -The file /roaring/btree.go contains a modified redistribution of b -(https://github.com/cznic/b); the license follows: - - Copyright (c) 2014 The b Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the names of the authors nor the names of the - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The file /server/tlsconfig.go contains a modified redistribution of bridge -(https://github.com/robustirc/bridge); the license follows: - - Copyright © 2014-2015 The RobustIRC Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of RobustIRC nor the names of contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows: - - The MIT License (MIT) - - Copyright (c) 2015 Nick Galbreath - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. diff --git a/.cloudbuild/cloudbuild.yml b/.cloudbuild/cloudbuild.yml deleted file mode 100644 index 4ec21187a..000000000 --- a/.cloudbuild/cloudbuild.yml +++ /dev/null @@ -1,202 +0,0 @@ -steps: - - name: gcr.io/cloud-builders/yarn - args: - - install - dir: lattice/ - id: Install Lattice - waitFor: - - '-' - - name: gcr.io/cloud-builders/yarn - args: - - build - dir: lattice/ - id: Build Lattice - waitFor: - - Install Lattice - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - tar - - '-c' - - '-z' - - '-v' - - '-f' - - lattice.tar.gz - - lattice/build - id: Package Lattice Build - waitFor: - - Build Lattice - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - mv - - lattice.tar.gz - - /cache/ - waitFor: - - Package Lattice Build - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - go - - test - - '-covermode=atomic' - - '-coverprofile=/cache/coverage.out' - id: Generate code coverage report - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - '-c' - - | - go test -json > /cache/test-report.out - id: Generate test report - waitFor: - - '-' - entrypoint: bash - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - go - - tool - - cover - - '-html' - - /cache/coverage.out - - '-o' - - /cache/coverage.html - id: Calculate Coverage - waitFor: - - Generate code coverage report - - name: 'sonarsource/sonar-scanner-cli:4.6' - args: - - '-Dsonar.projectKey=molecula_featurebase' - - '-Dsonar.organization=molecula' - - '-Dsonar.sources=.' - - '-Dsonar.host.url=https://sonarcloud.io' - - '-Dsonar.go.coverage.reportPaths=/cache/coverage.out' - - '-Dsonar.go.tests.reportPaths=/cache/test-report.out' - id: SonarCloud - waitFor: - - Calculate Coverage - secretEnv: - - SONAR_TOKEN - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - jfrog - - config - - add - - molecula - - '--interactive=false' - - '--artifactory-url=https://molecula.jfrog.io/' - - '--access-token=$$JFROG_API_TOKEN' - secretEnv: - - JFROG_API_TOKEN - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=linux - - GOARCH=amd64 - args: - - go - - build - - '-o' - - /cache/featurebase_linux_amd64 - - ./cmd/featurebase - id: Build for Linux amd64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=darwin - - GOARCH=amd64 - args: - - go - - build - - '-o' - - /cache/featurebase_darwin_amd64 - - ./cmd/featurebase - id: Build for Darwin amd64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=linux - - GOARCH=arm64 - args: - - go - - build - - '-o' - - /cache/featurebase_linux_arm64 - - ./cmd/featurebase - id: Build for Linux arm64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=darwin - - GOARCH=arm64 - args: - - go - - build - - '-o' - - /cache/featurebase_darwin_arm64 - - ./cmd/featurebase - id: Build for Darwin arm64 - waitFor: - - '-' - - name: gcr.io/cloud-builders/gsutil - args: - - '-c' - - > - gsutil -m cp -r '/cache/*' - "gs://molecula-pipeline/featurebase/dev/$SHORT_SHA/" - id: Upload Binaries by Short SHA - waitFor: - - Package Lattice Build - - Calculate Coverage - - Build for Linux amd64 - - Build for Darwin amd64 - - Build for Linux arm64 - - Build for Darwin arm64 - entrypoint: bash - - name: gcr.io/cloud-builders/gsutil - args: - - '-c' - - > - gsutil -m cp -r '/cache/*' - "gs://molecula-pipeline/featurebase/dev/$BRANCH_NAME/" - id: Upload Binaries by Branch Name - waitFor: - - Package Lattice Build - - Calculate Coverage - - Build for Linux amd64 - - Build for Darwin amd64 - - Build for Linux arm64 - - Build for Darwin arm64 - entrypoint: bash - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - > - cp -v /cache/* .cloudbuild/ && docker build --build-arg - SHORT_SHA=$SHORT_SHA -t - northamerica-northeast1-docker.pkg.dev/featurebase-pipeline-dev/featurebase/main:$SHORT_SHA - -f .cloudbuild/Dockerfile . - id: Build Linux amd Container - entrypoint: bash - - name: gcr.io/cloud-builders/docker - args: - - push - - >- - northamerica-northeast1-docker.pkg.dev/featurebase-pipeline-dev/featurebase/main:$SHORT_SHA - waitFor: - - Build Linux amd Container -timeout: 1200s -logsBucket: 'gs://molecula-pipeline/logs' -options: - volumes: - - name: data - path: /cache - pool: - name: >- - projects/featurebase-pipeline-dev/locations/us-central1/workerPools/test-pool-00 -availableSecrets: - secretManager: - - versionName: projects/903994035405/secrets/jfrog-admin-token/versions/1 - env: JFROG_API_TOKEN - - versionName: projects/903994035405/secrets/sonarcloud-featurebase-token/versions/1 - env: SONAR_TOKEN diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2c1f4979d..f579dc42f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,13 +6,10 @@ Fixes # ## Pull request checklist -- [ ] I have read the [contributing guide](https://github.com/molecula/featurebase/blob/master/CONTRIBUTING.md). -- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). - [ ] I have updated the [documentation](https://github.com/molecula/docs). - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. - [ ] All new and existing tests pass. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Add appropriate changelog label to PR (if applicable). ## Code review checklist @@ -24,5 +21,4 @@ This is the checklist that the reviewer will follow while reviewing your pull re - [ ] Check that tests have been written and that they cover the new functionality. - [ ] Run tests and ensure they pass. - [ ] Build and run the code, performing any applicable integration testing. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Make sure PR is tagged with appropriate changelog label. diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3afbafbdc..f0a523e1a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -17,6 +17,7 @@ stages: - lint - test - build + - integration #before_script: #- echo "before_script" @@ -35,30 +36,23 @@ golangci-lint: - echo "Checking for issues in new code" - golangci-lint run -v -install lattice: - stage: test - image: node:14 - script: - - echo "Installing lattice dependencies..." - - cd lattice - - yarn install - build lattice: stage: test image: node:14 variables: - CI: "true" + CI: "false" script: - - echo "Building lattice..." - cd lattice + - yarn install - yarn build - - tar -czvf lattice.tar.gz lattice/build + - mv build ../ + - cd ../ + - rm -r lattice + - mv build lattice + - tar -czvf lattice.tar.gz lattice artifacts: paths: - - lattice/lattice.tar.gz - needs: - - job: install lattice - allow_failure: true + - lattice.tar.gz run jest tests: stage: test @@ -76,7 +70,7 @@ run jest tests: run go tests: stage: test - image: golang:1.16.9 + image: golang:1.16.10 extends: .go-cache script: - echo "Running featurebase unit tests..." @@ -85,9 +79,21 @@ run go tests: paths: - coverage.out +run go tests future: + stage: test + image: golang:1.17.3 + extends: .go-cache + script: + - echo "Running featurebase unit tests..." + - ./cover-everything.sh + artifacts: + paths: + - coverage.out + + run go tests with output: stage: test - image: golang:1.16.9 + image: golang:1.16.10 script: - echo "Running featurebase unit tests to capture JSON output..." - go test -json > test-report.out @@ -109,59 +115,59 @@ upload to sonarcloud: build for linux amd64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - extends: .go-cache - variables: - GOOS: "linux" - GOARCH: "amd64" + image: golang:1.16.10 script: - - go build -o featurebase_linux_amd64 ./cmd/featurebase + - rm -r lattice + - tar -xvf lattice.tar.gz + - go get -v -u github.com/rakyll/statik + - /go/bin/statik -src=lattice + - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64" artifacts: paths: - featurebase_linux_amd64 build for linux arm64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - extends: .go-cache - variables: - GOOS: "linux" - GOARCH: "arm64" + image: golang:1.16.10 script: - - go build -o featurebase_linux_arm64 ./cmd/featurebase + - rm -r lattice + - tar -xvf lattice.tar.gz + - go get -v -u github.com/rakyll/statik + - /go/bin/statik -src=lattice + - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64" artifacts: paths: - featurebase_linux_arm64 build for darwin amd64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - extends: .go-cache - variables: - GOOS: "darwin" - GOARCH: "amd64" + image: golang:1.16.10 script: - - go build -o featurebase_darwin_amd64 ./cmd/featurebase + - rm -r lattice + - tar -xvf lattice.tar.gz + - go get -v -u github.com/rakyll/statik + - /go/bin/statik -src=lattice + - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: - featurebase_darwin_amd64 build for darwin arm64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - extends: .go-cache - variables: - GOOS: "darwin" - GOARCH: "arm64" + image: golang:1.16.10 script: - - go build -o featurebase_darwin_arm64 ./cmd/featurebase + - rm -r lattice + - tar -xvf lattice.tar.gz + - go get -v -u github.com/rakyll/statik + - /go/bin/statik -src=lattice + - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64" artifacts: paths: - featurebase_darwin_arm64 package for linux amd64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + image: golang:1.16.10 variables: GOOS: "linux" GOARCH: "amd64" @@ -188,4 +194,30 @@ build container fb: - tag=${CI_REGISTRY_IMAGE}/server:${CI_COMMIT_REF_SLUG} - docker build --build-arg GO_VERSION=$GOVERSION -t $tag -f .gitlab/Dockerfile . - docker push $tag - - echo Created docker featurebase image with tag "$tag" \ No newline at end of file + - echo Created docker featurebase image with tag "$tag" + +# deploy EC2 instance, configure and run featurebase +deploy node for linux amd64: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "default" + AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY + before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem + - chmod 400 gitlab-featurebase-dev.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval `ssh-agent -s` + - mkdir -p ~/.ssh + - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq + script: + - ./qa/scripts/deployNode.sh $PROFILE + needs: + - job: build for linux amd64 diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index b22392ce1..b041f58ce 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -7,7 +7,6 @@ WORKDIR /featurebase RUN apk add --no-cache curl jq -COPY LICENSE . COPY NOTICE . COPY featurebase_linux_amd64 . diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 57de1f746..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,715 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) -and this project adheres to [Semantic Versioning](http://semver.org/). - -## [1.4.0] - 2019-09-17 - -This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions. - -**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release. - -### Added -- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028)) -- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029)) -- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016)) -- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023)) -- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004)) -- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983)) -- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992)) -- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939)) -- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950)) -- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956)) - -### Changed -- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064)) -- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001)) -- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994)) -- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993)) -- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978)) -- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902)) -- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966)) - -### Fixed -- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050)) -- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034)) -- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033)) -- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026)) -- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021)) -- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019)) -- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017)) -- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012)) -- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000)) -- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997)) -- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975)) -- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973)) -- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970)) -- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954)) -- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951)) -- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947)) - -### Performance -- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988)) -- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048)) -- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041)) -- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024)) -- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974)) - -### Removed -- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996)) - -## [1.3.1] - 2019-05-01 - -This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions. - -### Fixed -- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964)) - -## [1.3.0] - 2019-04-16 - -This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions. - -### Added -- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940)) -- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921)) -- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915)) -- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859)) -- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910)) -- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906)) -- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903)) -- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881)) -- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761)) -- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832)) -- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839)) -- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831)) -- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763)) -- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820)) -- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818)) -- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811)) - -### Fixed -- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944)) -- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943)) -- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938)) -- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936)) -- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931)) -- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876)) -- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863)) -- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887)) -- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848)) -- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846)) -- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842)) -- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834)) -- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835)) -- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836)) -- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826)) -- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827)) -- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822)) -- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802)) - -### Changed -- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924)) -- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937)) -- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909)) -- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900)) -- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616)) -- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804)) -- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851)) -- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830)) - -### Performance -- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916)) -- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901)) -- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897)) -- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892)) -- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871)) -- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865)) -- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814)) -- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803)) - -## [1.2.0] - 2018-12-20 - -This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. - -### Added - -- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) -- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774)) -- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) -- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) -- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) -- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) -- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) -- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) -- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) -- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717)) -- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) -- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) -- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) -- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) -- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) -- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) -- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666)) -- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) -- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) -- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) -- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) -- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) -- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) -- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738)) -- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) -- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) -- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) -- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628)) - -### Changed - -- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) -- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) -- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) -- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) -- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) -- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) -- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) -- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) - -### Performance - -- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) -- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741)) - -### Fixed - -- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795)) -- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) -- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) -- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) -- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782)) -- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) -- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) -- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) -- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) -- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) -- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) -- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) -- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) -- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) -- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729)) -- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) -- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) -- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) -- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711)) -- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674)) -- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675)) -- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) -- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) -- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) -- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) -- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) -- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) -- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) -- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) -- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) -- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) -- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) -- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620)) -- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) - - -## [1.1.0] - 2018-08-21 - -This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. - -### Added - -- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610)) -- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608)) -- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601)) -- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582)) -- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584)) -- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527)) -- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524)) -- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532)) - -### Fixed - -- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607)) -- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586)) -- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603)) -- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602)) -- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) - -## [1.0.2] - 2018-08-01 - -This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions. - -### Fixed - -- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551)) -- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547)) -- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541)) -- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542)) -- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533)) -- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502)) -- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555)) - -## [1.0.1] - 2018-07-11 - -This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions. - -### Fixed - -- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491)) -- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485)) -- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486)) -- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483)) -- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493)) - -## [1.0.0] - 2018-07-09 - -This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions. - -### Added - -- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337)) -- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327)) - -### Changed - -- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466)) -- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297)) -- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389)) -- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452)) -- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426)) -- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424)) -- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461)) -- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451)) -- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407)) -- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463)) -- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355)) -- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365)) -- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333)) -- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326)) -- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311)) -- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454)) - -### Removed - -- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397)) -- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462)) -- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396)) -- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363)) -- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347)) -- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346)) -- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341)) -- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335)) -- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332)) -- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468)) - -### Fixed - -- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423)) -- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401)) -- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) -- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) - -## [0.10.0] - 2018-05-15 - -This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions. - -### Added - -- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285)) -- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236)) - -### Changed - -- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265)) - -### Removed - -- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286)) -- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284)) -- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238)) -- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255)) - -### Fixed - -- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269)) -- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279)) -- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278)) -- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268)) -- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266)) -- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242)) -- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234)) -- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253)) - -## [0.9.0] - 2018-05-04 - -This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions. - -*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* - -### Added - -- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) -- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) -- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) -- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) -- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138)) -- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132)) -- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109)) -- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) -- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) -- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) -- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) -- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) -- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) -- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) -- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) - -### Changed - -- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163)) -- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166)) -- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158)) -- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148)) -- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797)) -- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115)) -- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099)) -- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077)) -- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032)) -- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) -- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) -- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) -- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) -- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) -- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) -- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) -- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) - -### Removed - -- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) -- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) -- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) -- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) -- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) -- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) -- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) -- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) -- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) - -### Fixed - -- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143)) -- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139)) -- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110)) -- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102)) -- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105)) -- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101)) -- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) -- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) -- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) -- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) -- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) -- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) - -### Deprecated - -- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) - -### Performance - -- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) - -## [0.8.8] - 2018-02-19 - -This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions. - -### Fixed - -- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118)) - -## [0.8.7] - 2018-02-12 - -This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions. - -### Fixed - -- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111)) - -## [0.8.6] - 2018-02-09 - -This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions. - -### Fixed - -- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106) -- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083) - -## [0.8.5] - 2018-01-18 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions. - -### Fixed - -- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061)) - -## [0.8.4] - 2018-01-10 - -This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions. - -### Fixed - -- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038)) -- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033)) - -### Performance - -- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046)) -- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042)) - -## [0.8.3] - 2017-12-12 - -This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions. - -### Fixed - -- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000)) - -## [0.8.2] - 2017-12-05 - -This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions. - -### Fixed - -- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994)) - -## [0.8.1] - 2017-11-15 - -This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions. - -### Fixed - -- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969)) -- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968)) - -## [0.8.0] - 2017-11-15 - -This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions. - -### Added - -- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895)) -- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933)) -- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938)) -- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942)) -- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867)) -- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898)) -- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889)) -- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881)) -- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955)) - -### Changed - -- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943)) -- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884)) -- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892)) -- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945)) -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Fixed - -- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941)) -- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934)) -- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919)) -- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887)) -- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882)) -- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949)) - -### Performance - -- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.2] - 2017-11-15 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion. - -### Changed - -- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920)) - -## [0.7.1] - 2017-10-09 - -This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions. - -### Changed - -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Performance - -- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.0] - 2017-10-03 - -This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions. - -### Added - -- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856)) -- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785)) -- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840)) -- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791)) -- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755)) -- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778)) -- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861)) -- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847)) -- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789)) -- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837)) -- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831)) -- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830)) -- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806)) -- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783)) - -### Fixed - -- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860)) -- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814)) -- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795)) -- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794)) -- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779)) -- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773)) - -### Deprecated - -- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839)) - -### Performance - -- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859)) -- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841)) -- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842)) -- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848)) -- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846)) -- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849)) -- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820)) -- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812)) - -## [0.6.0] - 2017-08-11 - -This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions. - -### Added - -- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758)) - -### Changed - -- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750)) - -### Fixed - -- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759)) -- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674)) - -## [0.5.0] - 2017-08-02 - -This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions. - -### Added - -- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748)) -- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646)) -- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745)) -- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741)) -- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636)) -- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635)) -- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629)) -- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600)) -- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606)) -- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721)) -- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702)) -- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670)) - -### Changed - -- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747)) -- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744)) -- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699)) -- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675)) - -### Fixed - -- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729)) -- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678)) - -## [0.4.0] - 2017-06-08 - -This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions. - -*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).* - -### Added -- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580)) -- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586)) -- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592)) -- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571)) -- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560)) -- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537)) -- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541)) -- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563)) -- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551)) -- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540)) -- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546)) -- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535)) -- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533)) -- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510)) - -### Changed -- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520)) -- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578)) -- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573)) -- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524)) -- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525)) - -### Fixed -- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605)) -- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594)) -- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599)) -- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597)) -- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589)) -- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565)) -- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559)) -- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555)) -- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554)) -- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548)) -- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528)) -- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503)) -- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507)) -- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462)) - -### Performance -- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590)) -- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577)) -- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562)) - -[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD -[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 -[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 -[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 -[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 -[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 -[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9 -[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 -[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 -[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1 -[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 8e4b45e4e..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,177 +0,0 @@ -# Contributing to Pilosa - -The workflow components of these instructions apply to all Pilosa repositories. - -## Reporting a bug - -If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]. - -## Submitting a feature request - -Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4] -which starts by [opening a new issue][1] that describes the new feature proposal. - -## Making code contributions - -Before you start working on new features, you should [open a new issue][1] to let others know what -you're doing, otherwise you run the risk of duplicating effort. This also -gives others an opportunity to provide input for your feature. - -If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6]. - - -### Development Environment - -- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information. - -- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). - -- Fork the [Pilosa repository][2] to your own account. - -- It will be easier to follow these instructions if you: - - ```sh - export GH_USERNAME= - ``` - -- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa: - - ```sh - mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ - git clone https://github.com/pilosa/pilosa.git - ``` - -- `cd` to your pilosa directory: - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - ``` - -- Install Pilosa command line tools: - - ```sh - make install - ``` - - Running `pilosa` should now run a Pilosa instance. - -- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git - ``` - -### Makefile - -Pilosa includes a Makefile that automates several tasks: - -- Install Pilosa: - - ```sh - make install - ``` - -- Install build dependencies: - - ```sh - make install-build-deps - ``` - -- Create the vendor directory: - - ```sh - make vendor - ``` - -- Run the test suite: - - ```sh - make test - ``` - -- View the coverage report: - - ```sh - make cover-viz - ``` - -- Clear the `vendor/` and `build/` directories: - - ```sh - make clean - ``` - -- Create release tarballs: - - ```sh - make release - ``` - -- Regenerate protocol buffer files in `internal/`: - - ```sh - make generate-protoc - ``` - -- Create tagged Docker image: - - ```sh - make docker - ``` - -- Run tests inside Docker container: - - ```sh - make docker-test - ``` - -Additional commands are available in the `Makefile`. - -### Submitting code changes - -- Before starting to work on a task, sync your branch with the upstream: - - ```sh - git checkout master - git pull - ``` - -- Create a local feature branch: - - ```sh - git checkout -b something-amazing - ``` - -- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/). - -- Make sure that you've written tests for your new feature, and then run the tests: - - ```sh - make test - ``` - -- Verify that your pull request is applied to the latest version of code on github: - - ```sh - git checkout master - git pull - git checkout something-amazing - git rebase master - ``` - -- Push to your fork: - - ```sh - git push -u $GH_USERNAME something-amazing:something-amazing - ``` - -- Submit a [pull request][3] - - -[1]: https://github.com/pilosa/pilosa/issues/new -[2]: https://github.com/pilosa/pilosa -[3]: https://github.com/pilosa/pilosa/compare/ -[4]: https://github.com/pilosa/general/blob/master/proposal.md -[5]: https://github.com/pilosa/pilosa/issues -[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer diff --git a/Dockerfile b/Dockerfile index 115855034..25ea9c3f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 8c60f11bf..4a3a0c196 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -20,7 +20,6 @@ RUN apt install -y docker.io RUN cp /go/bin/featurebase /featurebase -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa index a24c70ec7..6ee2e0bbf 100644 --- a/Dockerfile.pilosa +++ b/Dockerfile.pilosa @@ -24,7 +24,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/Makefile b/Makefile index 9878b09d8..f72517684 100644 --- a/Makefile +++ b/Makefile @@ -14,14 +14,10 @@ BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) LDFLAGS="-X github.com/molecula/featurebase/v2.Version=$(VERSION) -X github.com/molecula/featurebase/v2.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v2.Variant=$(VARIANT) -X github.com/molecula/featurebase/v2.Commit=$(COMMIT) -X github.com/molecula/featurebase/v2.TrialDeadline=$(TRIAL_DEADLINE)" -GO_VERSION=1.16.3 +GO_VERSION=1.16.10 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia -define LICENSE_HASH_CODE - head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/' -e 's/Pilosa Corp\./Molecula Corp./' | shasum | cut -f 1 -d " " -endef -LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) UNAME := $(shell uname -s) ifeq ($(UNAME), Darwin) IS_MACOS:=1 @@ -104,10 +100,19 @@ build: # Create a single release build under the build directory release-build: $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase" - cp NOTICE README.md LICENSE build/featurebase$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID) tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ @echo Created release build: build/featurebase-$(VERSION_ID).tar.gz +test-release-build: docker-build + mv build/featurebase-$(VERSION_ID).tar.gz install/ + cd install && docker build -t featurebase:test_installation \ + -f test_installation.Dockerfile \ + --build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz . + mv install/featurebase-$(VERSION_ID).tar.gz build/ + docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + featurebase:test_installation + # Error out if there are untracked changes in Git check-clean: ifndef SKIP_CHECK_CLEAN @@ -220,7 +225,7 @@ docker-build: vendor docker create --name featurebase-build featurebase:build mkdir -p build/featurebase-$(VERSION_ID) docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID) - cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID) docker rm featurebase-build tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ @@ -304,12 +309,6 @@ gometalinter: require-gometalinter vendor --exclude "^pql/pql.peg.go" \ ./... -# Verify that all Go files have license header -check-license-headers: SHELL:=/bin/bash -check-license-headers: - @! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\ - do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.' - ###################### # Build dependencies # ###################### diff --git a/NOTES b/NOTES deleted file mode 100644 index 8da55fe11..000000000 --- a/NOTES +++ /dev/null @@ -1,26 +0,0 @@ - - Index Column - ┌───────────▼────────────────────────────┐ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - Row──▶0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - F ▶│0000000000000000000000000000000000000000│ - i ││0000000000000000000000000000000000000000│ - e ││0000000000000000000000000000000000000000│ - l ││0000000000000000000000000000000000000000│ - d ▶│0000000000000000000000000000000000000000│ - └────────────────────────────────────────┘ - ▲───────────▲ - Shard - - -Fragment=intersection of field & shard diff --git a/NOTICE b/NOTICE index 594272a27..cb62dfdca 100644 --- a/NOTICE +++ b/NOTICE @@ -1,23 +1,12 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= -The file /pilosa/lru/lru.go contains a redistribution of lru +The file /lru/lru.go contains a redistribution of lru (github.com/golang/groupcache/lru); the license follows: Copyright 2013 Google Inc. diff --git a/client/client_it_test.go b/client/client_it_test.go index 1b5ca2884..b9137bee9 100644 --- a/client/client_it_test.go +++ b/client/client_it_test.go @@ -592,7 +592,6 @@ func TestClientAgainstCluster(t *testing.T) { target := []uint64{100} require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") }) - t.Run("StoreQuery", func(t *testing.T) { schema := NewSchema() testIndexStore := schema.Index("test-index-store") diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 32c0b6552..ab9adaf91 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -18,34 +18,46 @@ import ( "context" "flag" "fmt" + "io/ioutil" "math" "math/rand" nethttp "net/http" + "os" "strconv" "strings" "time" + "github.com/gogo/protobuf/proto" pilosa "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/client" "github.com/molecula/featurebase/v2/http" + "github.com/molecula/featurebase/v2/pb" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/vprint" + "github.com/pkg/errors" + vegeta "github.com/tsenart/vegeta/v12/lib" ) // RandomQueryConfig type RandomQueryConfig struct { - - // user facing flags - HostPort string // -hostport - TreeDepth int // -d - QueryCount int // -n - Verbose bool // -v - VeryVerbose bool // -V - TimeFromArg string // --time.from - TimeToArg string // --time.to - TimeFrom time.Time // parsed time - TimeTo time.Time // parsed time - TimeRange int64 // hours between parsed times + HostPort string + TreeDepth int + QueryCount int + Verbose bool + GenerateOnly bool + NumRuns int + TimeFromArg string + TimeToArg string + TimeFrom time.Time + TimeTo time.Time + TimeRange int64 + Index string + QPM int + Seed int + SrcFile string + Duration time.Duration + Target vegeta.Target IndexMap map[string]*Features @@ -93,12 +105,18 @@ var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour) // call DefineFlags before myflags.Parse() func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.") - fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") - fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") + fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") + fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") + fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") + fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to current time") + fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") + fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") - fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results") + fs.BoolVar(&cfg.GenerateOnly, "generate-only", false, "only generate do not run package") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") + fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating") } // call c.ValidateConfig() after myflags.Parse() @@ -123,6 +141,9 @@ func (c *RandomQueryConfig) ValidateConfig() error { return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)", c.TimeToArg, c.TimeFromArg) } + if c.QPM <= 0 { + return fmt.Errorf("-qpm must be positive") + } return nil } @@ -144,7 +165,6 @@ func main() { fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) os.Exit(1) } - err = cfg.Run() if err != nil { @@ -159,76 +179,24 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - ctx := context.Background() - totalQ := 0 - loops := 0 - t0 := time.Now() - - report := func() { - dur := time.Since(t0) - if dur > 0 { - qps := 1e9 * float64(totalQ) / float64(dur) - vprint.AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) - } else { - vprint.AlwaysPrintf("totalQueries run: %v elapsed: %v qps: N/A", totalQ, dur) - } - } - defer report() - -NewSetup: err = cfg.Setup(cli) if err != nil { return err } + rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Minute} + duration := cfg.Duration + targeter := vegeta.NewStaticTargeter(cfg.Target) + attacker := vegeta.NewAttacker() - if len(cfg.IndexMap) == 0 { - return fmt.Errorf("no rows to query") - } - - var indexes []string - for index := range cfg.IndexMap { - indexes = append(indexes, index) - } - - for j := 0; ; j++ { - if cfg.QueryCount > 0 { - if j >= cfg.QueryCount { - break - } - } else { - // else keep doing queries forever... - if loops > 0 && loops%500 == 0 { - // ...but account for any new data arrived by getting - // the schema and rows again every so often. - loops++ - goto NewSetup - } + for i := 0; i < cfg.NumRuns; i++ { + vprint.VV("================") + var metrics vegeta.Metrics + for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") { + metrics.Add(res) } - if totalQ > 0 && totalQ%100 == 0 { - report() - } - - index := indexes[cfg.Rnd.Intn(len(indexes))] - - pql, err := cfg.GenQuery(index) - vprint.PanicOn(err) - - if cfg.Verbose { - fmt.Printf("pql = '%v'\n", pql) - } - - // Query node0. - res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) - if err != nil { - vprint.AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) - return err - } - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) - } - totalQ++ - loops++ - + metrics.Close() + rpt := vegeta.NewTextReporter(&metrics) + rpt(os.Stdout) } return nil @@ -238,6 +206,7 @@ type Features struct { Slc []IndexFieldRow Ranges []IndexFieldRange Distinctables []IndexFieldRange + Stores []IndexFieldRow SlcWeight int RangeWeight int } @@ -283,7 +252,7 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree { endTime.Format(pilosaTimeFmt)) } if fea.IsRowKey { - return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)} + return &Tree{S: fmt.Sprintf(`Row(%v="%v"%s)`, fea.Field, fea.RowKey, fromTo)} } return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)} } @@ -341,65 +310,131 @@ func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree { // and spits back a PQL query // func (cfg *RandomQueryConfig) Setup(api API) (err error) { + if cfg.SrcFile != "" { + return cfg.buildPayload() + } ctx := context.Background() cfg.Info, err = api.Schema(ctx) if err != nil { return err } foundIntField := false + any := false for i, ii := range cfg.Info { - _ = i - for k, fld := range ii.Fields { - _ = k - switch fld.Options.Type { - case "set", "mutex", "time": - pql := fmt.Sprintf("Rows(%v)", fld.Name) + if ii.Name == cfg.Index { + any = true + _ = i + for k, fld := range ii.Fields { + _ = k + switch fld.Options.Type { + case "set", "mutex", "time": + pql := fmt.Sprintf("Rows(%v)", fld.Name) - res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) - vprint.PanicOn(err) - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) + res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) + vprint.PanicOn(err) + switch x := res.Results[0].(type) { + case *pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time", fld.Options.Type == "set") + case pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time", fld.Options.Type == "set") + } + case "int": + foundIntField = true + fallthrough + case "decimal": + cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") + default: + vprint.AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } - // if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers. - // e.g. - // success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"} - // success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}' - - switch x := res.Results[0].(type) { - case *pilosa.RowIdentifiers: - // internalClient gets this - cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time") - case pilosa.RowIdentifiers: - // test gets this - cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time") - } - case "int": - foundIntField = true - fallthrough // I bet you thought you'd never see this used - case "decimal": - cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") - default: - vprint.AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } } } - + if !any { + return errors.New(fmt.Sprintf("index %v not found", cfg.Index)) + } cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") } - seed := int64(42) - cfg.Rnd = rand.New(rand.NewSource(seed)) - - return nil + cfg.Rnd = rand.New(rand.NewSource(int64(cfg.Seed))) + return cfg.buildPayload() } -func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) { +func (cfg *RandomQueryConfig) buildPayload() error { + var request strings.Builder + if cfg.SrcFile != "" { + b, err := ioutil.ReadFile(cfg.SrcFile) // just pass the file name + if err != nil { + return err + } + request.WriteString(string(b)) + } else { + + for i := 0; i < cfg.QueryCount; i++ { + pql, err := cfg.GenQuery(cfg.Index) + if err != nil { + return err + } + request.WriteString(pql) + } + } + //TODO (twg) tls support + path := fmt.Sprintf("http://%s/index/%s/query", cfg.HostPort, cfg.Index) + header := nethttp.Header{ + "Content-Type": []string{"application/x-protobuf"}, + "Accept": []string{"application/x-protobuf"}, + "PQL-Version": []string{client.PQLVersion}, + } + + req := &pb.QueryRequest{ + Query: request.String(), + } + vprint.VV("%v", request.String()) + if cfg.GenerateOnly { + // just output the PQL and exit + os.Exit(0) + } + payload, err := proto.Marshal(req) + if err != nil { + return errors.Wrap(err, "marshaling request to protobuf") + } + cfg.Target = vegeta.Target{ + Method: "POST", + URL: path, + Body: payload, + Header: header, + } + return nil +} +func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool, isSet bool) { + storeID := uint64(0) for _, rowID := range x.Rows { cfg.AddFeature(index, field, rowID, "", false, hasTime) + storeID = rowID } + storeKey := "" for _, rowKey := range x.Keys { cfg.AddFeature(index, field, 0, rowKey, true, hasTime) + storeKey = rowKey + } + idx := cfg.IndexMap[index] + if isSet { + if storeID > 0 { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowID: storeID + 1, + }) + } + if storeKey != "" { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowKey: storeKey + "_1", + IsRowKey: true, + }) + + } } } @@ -411,10 +446,6 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } - if min.Scale != scale || max.Scale != scale { - vprint.PanicOn(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal", - min.Scale, max.Scale, scale)) - } effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which @@ -449,11 +480,32 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci } func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { - tree := cfg.GenTree(index, cfg.TreeDepth) pql = tree.ToPQL() - // avoid using too much bandwidth, just count the final bitmap. + dice := cfg.Rnd.Intn(9) + if dice == 3 { // 1 in 9 of getting a store + idx := cfg.IndexMap[index] + if len(idx.Stores) > 0 { + i := cfg.Rnd.Intn(len(idx.Stores)) + fr := idx.Stores[i] + var key string + if fr.IsRowKey { + key = fmt.Sprintf(`%v="%v"`, fr.Field, fr.RowKey) + c := strings.LastIndex(fr.RowKey, "_") + n, err := strconv.Atoi(fr.RowKey[c+1:]) + vprint.PanicOn(err) + n += 1 + fr.RowKey = fmt.Sprintf("%v%v", fr.RowKey[:c+1], n) + } else { + key = fmt.Sprintf("%v=%v", fr.Field, fr.RowID) + fr.RowID = fr.RowID + 1 + } + idx.Stores[i] = fr + pql = fmt.Sprintf("Store(%v,%v)Count(Row(%v))", pql, key, key) + return + } + } pql = fmt.Sprintf("Count(%v)", pql) return } @@ -522,7 +574,6 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { } func (tr *Tree) ToPQL() (s string) { - if len(tr.Chd) == 0 { // leaf return tr.S diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go index eb6cc9040..7ad6936fa 100644 --- a/cmd/random-query/main_test.go +++ b/cmd/random-query/main_test.go @@ -20,7 +20,7 @@ import ( "strconv" "testing" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" @@ -65,8 +65,8 @@ func Test_RandomQuery(t *testing.T) { ctx := context.Background() - indexes := []string{"rick", "morty"} - fieldName := []string{"f", "flying_car"} + indexes := []string{"rick"} + fieldName := []string{"f"} idx := make([]*pilosa.Index, len(indexes)) field := make([]*pilosa.Field, len(indexes)) @@ -144,7 +144,7 @@ func Test_RandomQuery(t *testing.T) { //qcx.Reset() } // end of setup. - + cfg.Index = indexes[0] PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API))) for j := 0; j < 4; j++ { diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 627d56ef2..e12348e56 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -243,6 +243,9 @@ func copyFile(src, dest string) error { } func Migrate(dataDir, backupPath string) error { + if strings.HasSuffix(dataDir, "/") { + dataDir = dataDir[:len(dataDir)-1] + } err := os.MkdirAll(backupPath, 0777) if err != nil { return err diff --git a/cover-everything.sh b/cover-everything.sh index f34640cfb..808fc9997 100755 --- a/cover-everything.sh +++ b/cover-everything.sh @@ -2,7 +2,8 @@ # actually get test coverage for every single package and subpackage # very slow but oh well what are you gonna do, not test things? +# note it skips the roaring migrate echo "mode: atomic" > coverage.out -for pkg in $(go list all | grep featurebase); do +for pkg in $(go list all | grep featurebase ); do go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg; tail -n +2 pkgcoverage.out >> coverage.out; done diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index c5672e563..b121de978 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -19,7 +19,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" @@ -1637,11 +1637,17 @@ func (s Serializer) decodePairField(pb *pb.PairField) pilosa.PairField { } func (s Serializer) decodeValCount(pb *pb.ValCount) pilosa.ValCount { + var t time.Time + t, err := time.Parse(time.RFC3339Nano, pb.TimestampVal) + if err != nil { + t = time.Time{} + } return pilosa.ValCount{ - Val: pb.Val, - FloatVal: pb.FloatVal, - DecimalVal: s.decodeDecimalStruct(pb.DecimalVal), - Count: pb.Count, + Val: pb.Val, + FloatVal: pb.FloatVal, + DecimalVal: s.decodeDecimalStruct(pb.DecimalVal), + TimestampVal: t, + Count: pb.Count, } } @@ -1849,10 +1855,11 @@ func (s Serializer) encodePairField(p pilosa.PairField) *pb.PairField { func (s Serializer) encodeValCount(vc pilosa.ValCount) *pb.ValCount { return &pb.ValCount{ - Val: vc.Val, - FloatVal: vc.FloatVal, - DecimalVal: s.encodeDecimal(vc.DecimalVal), - Count: vc.Count, + Val: vc.Val, + FloatVal: vc.FloatVal, + DecimalVal: s.encodeDecimal(vc.DecimalVal), + Count: vc.Count, + TimestampVal: vc.TimestampVal.Format(time.RFC3339Nano), } } diff --git a/executor.go b/executor.go index f99a0f667..009f55c7b 100644 --- a/executor.go +++ b/executor.go @@ -254,7 +254,7 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } } // Must copy out of Tx data before Commiting, because it will become invalid afterwards. - respSafeNoTxData := e.safeCopy(resp) + respSafeNoTxData := safeCopy(resp) // Commit transactions if writing; else let the defer grp.Abort do the rollbacks. if needWriteTxn { @@ -267,7 +267,7 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar // safeCopy copies everything in resp that has Bitmap material, // to avoid anything coming from the mmap-ed Tx storage. -func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { +func safeCopy(resp QueryResponse) (out QueryResponse) { out = QueryResponse{ Err: resp.Err, // error Profile: resp.Profile, // *tracing.Profile @@ -322,6 +322,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { safe[i] = v.Clone() } out.Results = append(out.Results, safe) + case DistinctTimestamp: + out.Results = append(out.Results, x) default: panic(fmt.Sprintf("handle %T here", v)) } @@ -1498,6 +1500,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if field == nil { return nil, ErrFieldNotFound } + bsig := field.bsiGroup(fieldName) if bsig == nil { result = &Row{ @@ -1534,9 +1537,25 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if bsig == nil { return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } + if field.Options().Type == FieldTypeTimestamp { + r, err := executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) + if err != nil { + return nil, err + } + results := make([]string, len(r.Pos.Columns())) + for i, val := range r.Pos.Columns() { + results[i] = FormatTimestampNano(int64(val), bsig.Base, field.options.TimeUnit) + } + return DistinctTimestamp{Name: fieldName, Values: results}, nil + } return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) } +type DistinctTimestamp struct { + Values []string + Name string +} + func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { index := idx.Name() tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) @@ -3045,10 +3064,11 @@ func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]Gr // FieldRow is used to distinguish rows in a group by result. type FieldRow struct { - Field string `json:"field"` - RowID uint64 `json:"rowID"` - RowKey string `json:"rowKey,omitempty"` - Value *int64 `json:"value,omitempty"` + Field string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey,omitempty"` + Value *int64 `json:"value,omitempty"` + FieldOptions *FieldOptions `json:"-"` } func (fr *FieldRow) Clone() (clone *FieldRow) { @@ -3062,6 +3082,11 @@ func (fr *FieldRow) Clone() (clone *FieldRow) { v := *fr.Value clone.Value = &v } + if fr.FieldOptions != nil { + // deep copy, for Extra Safety + v := *fr.FieldOptions + clone.FieldOptions = &v + } return } @@ -7550,8 +7575,11 @@ func (vc *ValCount) smaller(other ValCount) ValCount { extra += other.Count } return ValCount{ - Val: vc.Val, - Count: vc.Count + extra, + Val: vc.Val, + Count: vc.Count + extra, + DecimalVal: vc.DecimalVal, + FloatVal: vc.FloatVal, + TimestampVal: vc.TimestampVal, } } @@ -7601,8 +7629,11 @@ func (vc *ValCount) larger(other ValCount) ValCount { extra += other.Count } return ValCount{ - Val: vc.Val, - Count: vc.Count + extra, + Val: vc.Val, + Count: vc.Count + extra, + DecimalVal: vc.DecimalVal, + FloatVal: vc.FloatVal, + TimestampVal: vc.TimestampVal, } } @@ -7719,6 +7750,8 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children return nil, newNotFoundError(ErrFieldNotFound, fieldName) } gbi.fields[i].Field = fieldName + options := field.Options() + gbi.fields[i].FieldOptions = &options switch field.Type() { case FieldTypeSet, FieldTypeMutex, FieldTypeBool: @@ -7952,7 +7985,6 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool ret.Group = make([]FieldRow, len(gbi.rows)) copy(ret.Group, gbi.fields) for i, r := range gbi.rows { - ret.Group[i].RowID = r.id ret.Group[i].Value = r.value } diff --git a/executor_internal_test.go b/executor_internal_test.go index b752957ef..8337bb296 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -21,6 +21,7 @@ import ( "reflect" "strconv" "testing" + "time" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/testhook" @@ -301,6 +302,20 @@ func TestValCountComparisons(t *testing.T) { expLarger: ValCount{FloatVal: 10.7, Count: 3}, expSmaller: ValCount{FloatVal: 10.7, Count: 3}, }, + { + name: "timestampEquality", + vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + other: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + expLarger: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2}, + expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2}, + }, + { + name: "timestamp", + vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + other: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1}, + expLarger: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1}, + expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + }, } for i, test := range tests { @@ -475,3 +490,15 @@ func TestGetSorter(t *testing.T) { }) } } + +func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) { + result := DistinctTimestamp{Values: []string{"test", "test"}, Name: "test"} + results := make([]interface{}, 1) + results[0] = result + + response := QueryResponse{Results: results, Err: nil, Profile: nil} + copied := safeCopy(response) + if !reflect.DeepEqual(copied.Results, response.Results) { + t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results) + } +} diff --git a/executor_test.go b/executor_test.go index b6c4b993a..e4924887e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -8292,3 +8292,43 @@ func TestToRows(t *testing.T) { } } + +// TestMinMaxTimestampVariableNode tests Min() and Max() queries on +// timestamp values on a clusters of varying size. +func TestMinMaxTimestampVariableNode(t *testing.T) { + for i := 1; i < 4; i++ { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + // separate function so we can defer closing the cluster + // and so it looks prettier + MinMaxTimestampNodeTester(t, i) + }) + } +} + +// MinMaxTimestampNodeTester tests Min() and Max() queries on +// timestamp values on a cluster with `numNodes` nodes. +// fails if the min or max values are not correct +func MinMaxTimestampNodeTester(t *testing.T, numNodes int) { + index := "test_index" + field := "ts" + c := test.MustRunCluster(t, numNodes) + defer c.Close() + + // create an index and timestamp field + c.CreateField(t, index, pilosa.IndexOptions{}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s")) + + // add some data + expected := "2010-01-02T12:32:00Z" + c.Query(t, index, "Set(10, ts=\""+expected+"\")") + + // run a query on the cluster + min := c.Query(t, index, "Min("+field+")").Results[0].(pilosa.ValCount).TimestampVal.Format(time.RFC3339Nano) + if min != expected { + t.Fatalf("incorrect min timestamp val. expected: %v, got %v\n", expected, min) + } + + max := c.Query(t, index, "Max("+field+")").Results[0].(pilosa.ValCount).TimestampVal.Format(time.RFC3339Nano) + if max != expected { + t.Fatalf("incorrect max timestamp val. expected: %v, got %v\n", expected, min) + } +} diff --git a/fragment.go b/fragment.go index 613f4bc27..bc6bc33e1 100644 --- a/fragment.go +++ b/fragment.go @@ -2485,7 +2485,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 err = doFunc() } - if err != nil { + if err != nil && f.storage != nil { // we got an error. it's possible that the error indicates that something went wrong. mappedIn, mappedOut, unmappedIn, errs, e2 := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to) if errs != 0 { diff --git a/go.mod b/go.mod index 53a0860d3..add6082e3 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 + github.com/tsenart/vegeta/v12 v12.8.4 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/zeebo/blake3 v0.1.1 diff --git a/go.sum b/go.sum index 0b9624b38..4224f965d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/jsonschema v0.0.0-20180308105923-f2c93856175a/go.mod h1:qpebaTNSsyUn5rPSJMsfqEtDw71TTggXM6stUDI16HA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -42,8 +43,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -75,6 +79,9 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051 h1:ByJUvQYyTtNNCVfYNM48q6uYUT4fAlN0wNmd3th4BSo= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E= +github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77/go.mod h1:Va5MyIzkU0rAM92tn3hb3Anb7oz7KcnixF49+2wOMe4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -120,6 +127,24 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71 h1:BE6g8oinc3Ek2elIHq+uDOiZgX3/ODi+EerJ48yrrKc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71/go.mod h1:22dM4PLscQl+Nzf64qNBurVJvfyvZELT0iRW2l/NN70= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 h1:EvokxLQsaaQjcWVWSV38221VAK7qc2zhaO17bKys/18= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 h1:GUSkTcIe1SlregbHNUKbYDhBsS8lNgYfIp4S4cToUyU= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2/go.mod h1:pDgmNM6seYpwvPos3q+zxlXMsbve6mOIPucUnUOrI7Y= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029 h1:8jtTdc+Nfj9AR+0soOeia9UZSvYBvETVHZrugUowJ7M= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 h1:7qnwS9+oeSiOIsiUMajT+0R7HR6hw5NegnKPmn/94oI= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55 h1:Ajwn2ENgC/pKtVat0LEHEWNa4a4VGyYJ1feGSccOzFU= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55/go.mod h1:fmo8aiSEWkJeiGXUJf+sPvuDgEFgqIoZSs843ePKrGg= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 h1:V2IgdyerlBa/MxaEFRbV5juy/C3MGdj4ePi+g6ePIp4= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b h1:fbskpz/cPqWH8VqkQ7LJghFkl2KPAiIFUHrTJ2O3RGk= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b/go.mod h1:Z4GIJBJO3Wa4gD4vbwQxXXZ+WHmW6E9ixmNrwvs0iZs= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -184,6 +209,8 @@ github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10y github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a h1:vMqgISSVkIqWxCIZs8m1L4096temR7IbYyNdMiBxSPA= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -210,6 +237,8 @@ github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -217,6 +246,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -321,6 +351,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -338,6 +370,9 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo= +github.com/tsenart/vegeta/v12 v12.8.4 h1:UQ7tG7WkDorKj0wjx78Z4/vsMBP8RJQMGJqRVrkvngg= +github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= @@ -367,6 +402,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -416,6 +452,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= @@ -446,6 +483,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -486,6 +524,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -559,6 +598,8 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +pgregory.net/rapid v0.3.3 h1:jCjBsY4ln4Atz78QoBWxUEvAHaFyNDQg9+WU62aCn1U= +pgregory.net/rapid v0.3.3/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/install/featurebase.conf b/install/featurebase.conf new file mode 100644 index 000000000..73895ed6d --- /dev/null +++ b/install/featurebase.conf @@ -0,0 +1,373 @@ +# FEATUREBASE HOST CONFIGURATION +# +# Uncomment when/where appropriate + +# ============================================================================== +# Use advertise to specify the address advertised by the server to other nodes +# in the cluster and to clients via /status endpoint. Host defaults to IP +# address represented by bind parameter with network port. +# +# advertise = :10101 +# advertise-grpc = :20101 + + + +# "long-query-time" represents duration of time that will trigger log and stat +# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds +# +# long-query-time = "10s" + + + +# Unique name for node in cluster. This is just a human-readable label for +# convenience and not used by any underlying logic. +# +# name = "featurebase1" + + + +# Host:Port where Featurebase server listens for HTTP requests. +# Default is localhost:10101 +# +# bind = "localhost:10101" + + + +# The address and port featurebase will listen to for all GRPC connections +# Ex. python-molecula, grafana for queries, etc. +# +# bind-grpc = "0.0.0.0:20101" + + + +# Directory to store Featurebase data files +data-dir = "/var/lib/molecula" + + +# ============================================================================== +# CORS (Cross-Origin Resource Sharing) Allowed Origins +# List of allowed origin URIs for CORS +# +# [handler] +# allowed-origins = ["https://myapp.com", "https://myapp.org"] + + + +# Path to the log file +log-path = "/var/log/molecula/featurebase.log" + + + +# Verbose - Enable verbose logging. Valid options are true or false. +# Set to true only when debugging as directed by Molecula engineers. +# +# verbose = true + + + +# Soft limit on max number of files featurebase will keep open simultaneously. +# When past this limit, featurebase will only keep files open for as long as is +# needed to write updates. +# +# max-file-count = 900000 + + + +# Maximum number of active memory maps featurebase will use for fragment files. +# Actual total usage may be slightly higher. +# Best practice is to set this to ~10% lower than your system's max map count. +# See sysctl vm.max_map_count in Linux. +# +# max-map-count = 900000 + + + +# Max Writes Per Request - Max number of mutating commands allowed per request. +# This includes Set, Clear, ClearRow, and Store +# +# max-writes-per-request = 5000 + + + +# The following option sets the maximum number of queries that are maintained +# for the /query-history endpoint. +# This parameter is per-node, and the result combines the history from all nodes. +# +# query-history-length = 100 + + + +# External database to connect to for `ExternalLookup` queries. +# lookup-db-dsn = "postgres://localhost:5432/db" + + + +# ============================================================================== +# For cluster stanza, "name" represents name for cluster. Must be same on all +# nodes in cluster. "replicas" represents number of hosts each piece of data +# should be stored on. Must be greater than or equal to 1 & less than or equal +# to number of nodes in cluster. +# [cluster] +# name = "cluster1" +# replicas = 1 + + + +# ============================================================================== +# [etcd] +# etcd is the tool Featurebase uses for node-to-node, intra-cluster +# communication. etcd is embedded in the featurebase cluster rather than +# running as a separate instance. +# It's important to configure this correctly for your network and nodes, and +# that it is consistent across all nodes. +# +# The easiest setup can be used when all nodes can reach all other nodes via a +# local subnet: +# listen-peer-address = advertise-peer-address +# = (what's in the initial-cluster-list) +# = the nodes ip address (which can be reached by every +# other node +# (localhost:10401 would not work for this, as each node can't reach that) +# +# If each node is separated by a proxy, or must be reached via url / dns, you +# will need to use a more complicated setup: +# listen-peer-address = the nodes local ip address +# (specific ip, localhost, or 0.0.0.0 for all +# local ip's) +# advertise-peer-address = the nodes ip address, reachable by all other nodes +# (This address should also be included in +# initital-cluster-list) +# in this case, you specify a different url/ip for listen-peer and +# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case) +# you use a url for advertise. In each of these cases, you should set listen +# to the local ip, and you set advertise = to how each other node connects to +# this node, and you also use this same address in the initial cluster. +# The key here is that initial-cluster has to include the same node name and +# advertise-peer address as the node it's on (edited) + + + +# for additional assistance, and for help with config issues, +# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster +# that a new node should join when adding nodes to cluster. +# +# cluster-url = "http://localhost:10401" + + + +# Address and port to bind to for client communication +# listen-client-address = "http://localhost:10401" + + + +# Address and port to bind to for peer communication +# listen-peer-address = "http://localhost:10301" + + + +# Comma-separated list of node=address pairs that makes up initial cluster when +# first started. In each pair, "node" value (left side of = ) should match +# name of node specified by "name" configuration parameter +# +# initial-cluster = "featurebase1=http://localhost:10301" + + + +# ============================================================================== +# Profile Block Rate - Block Rate is passed directly to Go's +# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 +# per rate nanoseconds. A value of "1" samples every event, and 0 disables +# profiling. +# +# block-rate = 10000000 + +# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's +# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled. +# +# mutex-fraction = 100 + + + +# ============================================================================== +# PostgreSQL Section +# [postgres] +# +# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint. +# No PostgreSQL endpoint will be exposed unless a bind address is specified. +# Requires Molecula v3.0 or newer. +# +# bind = "localhost:55432" + + + +# The PostgreSQL endpoint has support for a connection limit. +# This is generally not necessary, so it is disabled by default. +# +# connection-limit = 10000 + + + +# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint +# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should +# typically be sufficient, but may be exceeded if a client sends an unusually +# large amount of configuration data. Oversized startup packets are typically +# caused by connecting with a different protocol, e.g. HTTP. +# +# max-startup-size = 10000000 + + + +# PostgreSQL Timeouts +# In order to detect stalled clients, the PostgreSQL endpoint has connection +# read and write timeouts. There is also a startup timeout, which is used for +# connection setup. The read timeout does not impact idle connections. Idle +# connections will only be closed by the server if TCP keepalive reports a +# break in the connection. TCP keepalives use the default configuration +# provided by the host. +# Caution: Due to a limitation of the PostgreSQL wire protocol, +# raising the write timeout may delay the shutdown of a featurebase node. +# +# startup-timeout = "20s" +# read-timeout = "20s" +# write-timeout - "20s" + + + +# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is +# structured the same as the TLS configuration for Featurebase's other endpoints, +# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint, +# Featurebase will reject unsecured connections. +# [postgres.tls] +# certificate = "/srv/pilosa/certs/server.crt" +# key = "/srv/pilosa/certs/server.key" +# ca-certificate = "/srv/pilosa/certs/ca.crt" +# enable-client-verification = true + + + +# ============================================================================== +# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is +# calculated periodically in the background and accessed by the UI/usage +# endpoint. Since this disk scan can take a long and unpredictable amount of +# time, its timing behavior is specified in a relative, rather than absolute +# sense. That is, the duty cycle sets the percentage of time that is spent +# recalculating this cache. This setting affects the results received from +# the "/ui/usage" http endpoint, as well as all data file and memory usage +# values and graphs on the webui "tables" page + +# Special considerations: +# * If disk usage can be calculated quickly (less than 5 seconds), fresh +# results will be calculated when accessed +# * When disk usage takes longer to calculate, there is a minimum of one +# hour wait between cache recalculations +# Setting this value to 0 will completely disable the calculation of disk usage +# +# usage-duty-cycle = 20 + + + +# ============================================================================== +# Use [metric] stanza to define attributes for monitoring. +# [metric] +# Specify which service to use for collecting metrics. Valid options are: +# "statsd", "expvar", "prometheus", "none" +# +# service = "prometheus" + + + +# Remote host to send statsd metrics to. +# host = "localhost:8125" + + + +# The interval to send statsd metrics. +# poll-interval = "10s" + + + +# Debugging flag to enable to send diagnostic information to Featurebase +# developers. +# +# diagnostics = false + + + +# ============================================================================== +# TLS Certificate Section - Path to TLC certificate used for service HTTPS. +# Suffix should contain .crt or .pem +# +# [tls] +# certificate = "/srv/pilosa/certs/server.crt" +# TLS Certificate Key - Path to TLS certificate key for HTTPS. Suffix should +# be .key +# +# key = "/srv/pilosa/certs/server.key" + + + +# ============================================================================== +# Tracing Section +# [tracing] +# +# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting", +# or "remote". Set to 'off' to disable tracing completely. +# +# sampler-type = "remote" + + +# Jaeger sampler parameter (number) +# sampler-param = 0.001 + + + +# Tracing Agent Host:Port +# agent-host-port = "localhost:6831" + + + +# ============================================================================== +# Configuration for the RBF storage format. +# [rbf] +# Maximum size for each RBF database file. +# Allocates virtual memory but does not preallocate physical disk space. +# If you get into the range where you have 16000 shards on a single node +# (across all indexes), you will need to lower this in order to not run out of +# virtual address space. +# +# max-db-size = 4294967296 + + + +# Maximum size for each RBF WAL file. +# Allocates virtual memory but does not preallocate physical disk space. +# This is the same as max-db-size, but for the write-ahead log. If you set it +# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely +# condense these options in the future). +# +# max-wal-size = 4294967296 + + + +# Minimum WAL size before WAL pages can be copied to the main database file. +# min-wal-checkpoint-size = 1048576 + + + +# Maximum WAL size before transactions are halted to copy WAL pages to the +# main database file. +# +# max-wal-checkpoint-size = 2147483648 + + +# ============================================================================== +# [storage] +# Sync all changes to the file system. +# Should not be changed in production systems unless you know what you are +# doing - Should always be on unless testing or possibly while performing a +# bulk import and you are not worried about data loss +# +# fsync = true + + +# ============================================================================== diff --git a/install/featurebase.debian.service b/install/featurebase.debian.service new file mode 100644 index 000000000..162264106 --- /dev/null +++ b/install/featurebase.debian.service @@ -0,0 +1,13 @@ +[Unit] +Description="Service for FeatureBase" +After=network.target + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/featurebase.redhat.service b/install/featurebase.redhat.service new file mode 100644 index 000000000..cf507d8b0 --- /dev/null +++ b/install/featurebase.redhat.service @@ -0,0 +1,12 @@ +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/test_installation.Dockerfile b/install/test_installation.Dockerfile new file mode 100644 index 000000000..5d9a93253 --- /dev/null +++ b/install/test_installation.Dockerfile @@ -0,0 +1,18 @@ +FROM fedora:rawhide + +RUN yum -y install systemd procps + +ARG release_tarball + +COPY $release_tarball . + +COPY test_installation.sh test_installation.sh + +RUN mkdir install && \ + tar -xf $release_tarball -C install --strip-components 1 && \ + cd install && \ + cp featurebase /usr/local/bin/featurebase && \ + cp featurebase.redhat.service /etc/systemd/system/featurebase.service && \ + cp featurebase.conf /etc/featurebase.conf + +CMD ["/bin/bash", "test_installation.sh"] diff --git a/install/test_installation.sh b/install/test_installation.sh new file mode 100644 index 000000000..4a301ea48 --- /dev/null +++ b/install/test_installation.sh @@ -0,0 +1,29 @@ +#!/bin/bash +tests_failed=0 + +# test validity of featurebase.service +systemd-analyze verify /etc/systemd/system/featurebase.service +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo 'featurebase.redhat.service invalid' + tests_failed=1 +fi + +# test validity of featurebase.conf +mkdir /var/log/molecula +featurebase_bin='/usr/local/bin/featurebase' +config_file='/etc/featurebase.conf' +if ! $featurebase_bin -c $config_file holder /dev/null 2>&1; then + echo 'featurebase.conf is invalid' + tests_failed=1 +fi + +# print success if both passed +if [ $tests_failed -eq 0 ]; then + echo 'featurebase.redhat.service is valid' + echo 'featurebase.conf is valid' +fi + +# tests_failed set to 0 if none of the tests failed +# otherwise set to non-zero +exit $tests_failed diff --git a/pb/public.pb.go b/pb/public.pb.go index ca1b84852..34c78e3d5 100644 --- a/pb/public.pb.go +++ b/pb/public.pb.go @@ -1168,6 +1168,7 @@ type ValCount struct { Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` FloatVal float64 `protobuf:"fixed64,3,opt,name=FloatVal,proto3" json:"FloatVal,omitempty"` DecimalVal *Decimal `protobuf:"bytes,4,opt,name=DecimalVal,proto3" json:"DecimalVal,omitempty"` + TimestampVal string `protobuf:"bytes,5,opt,name=TimestampVal,proto3" json:"TimestampVal,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1234,6 +1235,13 @@ func (m *ValCount) GetDecimalVal() *Decimal { return nil } +func (m *ValCount) GetTimestampVal() string { + if m != nil { + return m.TimestampVal + } + return "" +} + type Decimal struct { Value int64 `protobuf:"varint,1,opt,name=Value,proto3" json:"Value,omitempty"` Scale int64 `protobuf:"varint,2,opt,name=Scale,proto3" json:"Scale,omitempty"` @@ -2449,106 +2457,106 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1569 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0xd5, - 0x16, 0xce, 0xfc, 0xf8, 0x6f, 0xd9, 0x71, 0xd2, 0xdd, 0xb4, 0x67, 0x4e, 0x4f, 0xea, 0xe3, 0x8e, - 0x8e, 0x2a, 0xf7, 0x04, 0xa5, 0xc2, 0x40, 0x85, 0x2a, 0x01, 0x8a, 0xe3, 0x94, 0x8c, 0xda, 0xa4, - 0x65, 0x27, 0x04, 0x2e, 0xb8, 0x99, 0xd8, 0x1b, 0x77, 0xc4, 0xd8, 0x63, 0xc6, 0xe3, 0x3a, 0x16, - 0x0f, 0x00, 0x8f, 0xc0, 0x0b, 0xf0, 0x28, 0x08, 0xee, 0xe0, 0x92, 0x4b, 0x54, 0x5e, 0x04, 0xad, - 0xb5, 0xf7, 0xfc, 0xda, 0xad, 0xaa, 0x8a, 0xbb, 0x59, 0x3f, 0x7b, 0xfd, 0x7c, 0x6b, 0xed, 0xb5, - 0xb6, 0x0d, 0x8d, 0xe9, 0xfc, 0xd2, 0xf7, 0x06, 0xfb, 0xd3, 0x30, 0x88, 0x02, 0xa6, 0x4f, 0x2f, - 0xed, 0x25, 0x18, 0x3c, 0x58, 0x30, 0x0b, 0x2a, 0x87, 0x81, 0x3f, 0x1f, 0x4f, 0x66, 0x96, 0xd6, - 0x36, 0x3a, 0x26, 0x8f, 0x49, 0xc6, 0xc0, 0x7c, 0x2c, 0x96, 0x33, 0xcb, 0x68, 0x1b, 0x9d, 0x1a, - 0xa7, 0x6f, 0xd4, 0xe6, 0x81, 0x1b, 0x7a, 0x93, 0x91, 0x65, 0xb6, 0xb5, 0x4e, 0x83, 0xc7, 0x24, - 0xdb, 0x81, 0x92, 0x33, 0x19, 0x8a, 0x2b, 0xab, 0xd4, 0xd6, 0x3a, 0x35, 0x2e, 0x09, 0xe4, 0x3e, - 0xf2, 0x84, 0x3f, 0xb4, 0xca, 0x92, 0x4b, 0x84, 0xdd, 0x81, 0x1a, 0x0f, 0x16, 0x27, 0x6e, 0x14, - 0x7a, 0x57, 0xec, 0x3f, 0x60, 0xf2, 0x60, 0x21, 0xbd, 0xd7, 0xbb, 0x95, 0xfd, 0xe9, 0xe5, 0x3e, - 0x0f, 0x16, 0x9c, 0x98, 0xf6, 0x01, 0xd4, 0xce, 0xbc, 0xd1, 0x44, 0x0c, 0x31, 0xd4, 0x7f, 0x83, - 0xf1, 0x2c, 0x40, 0x45, 0x2d, 0xab, 0x88, 0x3c, 0x14, 0x9d, 0x8a, 0x91, 0xa5, 0x17, 0x44, 0xa7, - 0x62, 0x64, 0x7f, 0x08, 0x4d, 0x1e, 0x2c, 0x9c, 0xa1, 0x98, 0x44, 0xde, 0xd7, 0x9e, 0x08, 0x29, - 0xb1, 0xc4, 0xa3, 0x29, 0x1d, 0x25, 0xc9, 0xea, 0x69, 0xb2, 0xf6, 0x2d, 0x28, 0x3b, 0xfd, 0x27, - 0xde, 0x2c, 0x62, 0xdb, 0x60, 0x38, 0xfd, 0xf8, 0x00, 0x7e, 0xda, 0x87, 0x70, 0xed, 0xe8, 0x2a, - 0x0a, 0xdd, 0x41, 0x24, 0x86, 0x4e, 0x5f, 0x42, 0xc6, 0x9a, 0xa0, 0x3b, 0x7d, 0x8a, 0xcf, 0xe4, - 0xba, 0xd3, 0x67, 0x2d, 0x30, 0x2f, 0x5c, 0x5f, 0x1a, 0xad, 0x77, 0x01, 0xc3, 0x92, 0x06, 0x39, - 0xf1, 0xed, 0xaf, 0x72, 0x46, 0x14, 0x1e, 0x37, 0xa1, 0x4c, 0x28, 0x49, 0x77, 0x35, 0xae, 0x28, - 0x76, 0x3f, 0x2d, 0x94, 0xb4, 0x77, 0x03, 0xed, 0xad, 0x04, 0x91, 0xd4, 0xcf, 0xbe, 0x0d, 0x95, - 0xc7, 0x62, 0x49, 0xf1, 0xc7, 0xd9, 0x69, 0x99, 0xec, 0x7e, 0xd3, 0xe0, 0x7a, 0x72, 0xfa, 0xdc, - 0xbd, 0xf4, 0xc5, 0x85, 0xeb, 0xcf, 0x05, 0x6b, 0xc5, 0xb9, 0x6a, 0xf9, 0x98, 0x8f, 0x37, 0x28, - 0x73, 0x76, 0x27, 0x41, 0x0a, 0x15, 0xea, 0xa8, 0xa0, 0xdc, 0x1c, 0x6f, 0xa8, 0x2e, 0xd9, 0x85, - 0x6a, 0xef, 0xcc, 0x21, 0x73, 0x96, 0xd1, 0xd6, 0x3a, 0xc6, 0xf1, 0x06, 0x4f, 0x38, 0xec, 0x16, - 0x54, 0x4e, 0xe6, 0x91, 0xb8, 0x72, 0xfa, 0xd4, 0x43, 0xe6, 0xf1, 0x06, 0x8f, 0x19, 0x78, 0x92, - 0x3e, 0x1f, 0x8b, 0xa5, 0x6c, 0x24, 0x3c, 0x19, 0x73, 0xd8, 0x0e, 0x98, 0xbd, 0x20, 0xf0, 0xa9, - 0x99, 0xaa, 0xe8, 0x0d, 0xa9, 0x5e, 0x05, 0x4a, 0x64, 0xd8, 0xbe, 0x82, 0x9d, 0x7c, 0x42, 0xaa, - 0x2c, 0x0c, 0x0c, 0xb4, 0xa7, 0x29, 0x7b, 0x48, 0xb0, 0x6d, 0x2a, 0x95, 0xae, 0xfc, 0x63, 0xb1, - 0xee, 0x43, 0x99, 0xcc, 0xc8, 0x86, 0xaf, 0x77, 0xff, 0x95, 0x83, 0x37, 0x05, 0x88, 0x2b, 0xb5, - 0x5e, 0x8d, 0xf0, 0x7d, 0x1a, 0x3a, 0x7d, 0xfb, 0xa3, 0x22, 0x94, 0x54, 0x33, 0x84, 0xfd, 0xd4, - 0x1d, 0x0b, 0xe9, 0x99, 0xd3, 0x37, 0xf2, 0xce, 0x97, 0x53, 0x41, 0xae, 0x6b, 0x9c, 0xbe, 0xed, - 0x39, 0x34, 0xf3, 0xc7, 0x31, 0x98, 0x4c, 0x13, 0xac, 0x0d, 0x86, 0xe4, 0x49, 0x77, 0x74, 0x8b, - 0xdd, 0x61, 0xad, 0x9e, 0x28, 0x36, 0xc8, 0xc7, 0x60, 0x3e, 0x73, 0xbd, 0x70, 0xa5, 0x6d, 0xb7, - 0x25, 0x5e, 0x06, 0x45, 0x68, 0x48, 0xe0, 0x4b, 0x87, 0xc1, 0x7c, 0x12, 0x49, 0xc0, 0xb8, 0x24, - 0xec, 0x4f, 0xa0, 0x86, 0xe7, 0x65, 0xae, 0xbb, 0xd2, 0x98, 0xea, 0x9b, 0x2a, 0x7a, 0x47, 0x9a, - 0x4b, 0x17, 0xc9, 0x1c, 0xd0, 0xb3, 0x73, 0xa0, 0x07, 0x80, 0xd2, 0x99, 0xb4, 0xd0, 0x82, 0x12, - 0x51, 0x2a, 0xe5, 0xd4, 0x84, 0x64, 0xbf, 0xc2, 0xc6, 0x6d, 0x9c, 0x3b, 0xd1, 0x83, 0xf7, 0x51, - 0x2c, 0x3b, 0x0e, 0x23, 0x30, 0xb8, 0xea, 0x89, 0x00, 0xaa, 0x12, 0xa8, 0x60, 0x91, 0x1a, 0xd0, - 0x32, 0x06, 0x90, 0x8b, 0xf3, 0xa1, 0x1f, 0xe7, 0x46, 0x04, 0xde, 0x42, 0x1e, 0x2c, 0x52, 0x18, - 0x14, 0xc5, 0xfe, 0x1b, 0x7b, 0x31, 0x29, 0xcf, 0x1a, 0xdd, 0x0f, 0xf4, 0x1f, 0x3b, 0xfc, 0x12, - 0xe0, 0xd3, 0x30, 0x98, 0x4f, 0x09, 0x22, 0x66, 0x43, 0x89, 0x28, 0x95, 0x53, 0x03, 0xd5, 0xe3, - 0x78, 0xb8, 0x14, 0xad, 0x07, 0x17, 0x8b, 0x70, 0x30, 0x1a, 0xc9, 0xeb, 0xc3, 0xf1, 0xd3, 0xfe, - 0x0e, 0xaa, 0x17, 0xae, 0x9f, 0x48, 0x2f, 0x5c, 0x5f, 0xa5, 0x8a, 0x9f, 0x79, 0x2b, 0x46, 0x6c, - 0xe5, 0x16, 0x54, 0x1f, 0xf9, 0x81, 0x1b, 0xa1, 0x32, 0x9a, 0xd2, 0x78, 0x42, 0xb3, 0x3d, 0x80, - 0xbe, 0x18, 0x78, 0x63, 0xd7, 0x47, 0xa9, 0x99, 0x5e, 0x67, 0xc5, 0xe5, 0x19, 0xb1, 0xfd, 0x01, - 0x54, 0x14, 0xb5, 0x1e, 0x68, 0xe4, 0x9e, 0x0d, 0x5c, 0x5f, 0xc4, 0xfe, 0x89, 0xb0, 0x7f, 0xd6, - 0xa0, 0xf1, 0xd9, 0x5c, 0x84, 0x4b, 0x2e, 0xbe, 0x9d, 0x8b, 0x59, 0x84, 0x6a, 0x44, 0xc7, 0x35, - 0x20, 0x02, 0xd1, 0x3e, 0x7b, 0xee, 0x86, 0x43, 0xd9, 0xbc, 0x26, 0x57, 0x14, 0x55, 0x41, 0x8c, - 0x83, 0x48, 0xd0, 0x30, 0xa8, 0x72, 0x45, 0xb1, 0x3d, 0x68, 0x1c, 0x8d, 0x2f, 0xc5, 0x70, 0x28, - 0x86, 0x7d, 0x37, 0x72, 0xad, 0x6a, 0x7e, 0x77, 0xe4, 0x84, 0xec, 0x7f, 0xb0, 0xf9, 0x2c, 0x14, - 0xe7, 0xa1, 0x3b, 0x99, 0xf9, 0x6e, 0x24, 0x86, 0x56, 0x8d, 0x6c, 0xe5, 0x99, 0x6c, 0x17, 0x6a, - 0x27, 0xee, 0xd5, 0x89, 0x18, 0x07, 0xe1, 0xd2, 0x02, 0xca, 0x21, 0x65, 0xd8, 0x4f, 0x60, 0x53, - 0xa5, 0x31, 0x9b, 0x06, 0x93, 0x99, 0xc0, 0x02, 0x1c, 0x85, 0xa1, 0xca, 0x02, 0x3f, 0xd9, 0x3d, - 0xa8, 0x70, 0x31, 0x9b, 0xfb, 0x51, 0x7c, 0x03, 0xb7, 0x30, 0x9c, 0xf8, 0xd4, 0xdc, 0x8f, 0x78, - 0x2c, 0xb7, 0x7f, 0x2a, 0x41, 0x3d, 0x23, 0x48, 0x66, 0x02, 0xce, 0xb5, 0x4d, 0x39, 0x13, 0x70, - 0xa3, 0xf1, 0x60, 0xb1, 0xb2, 0xec, 0xb0, 0x8f, 0x1b, 0xa0, 0x9d, 0xaa, 0x66, 0xd1, 0x4e, 0xd3, - 0x6b, 0x63, 0xac, 0xbf, 0x36, 0xb8, 0xe0, 0x9f, 0xbb, 0x93, 0x91, 0x18, 0x52, 0x8d, 0xab, 0x3c, - 0x26, 0x59, 0x27, 0x6d, 0x28, 0xc2, 0x57, 0xf5, 0x67, 0xcc, 0xe3, 0x69, 0xbb, 0xc9, 0xdb, 0x80, - 0x6b, 0xa1, 0x22, 0xeb, 0x23, 0x29, 0xf6, 0x00, 0x9a, 0x4f, 0xfd, 0x61, 0xda, 0xef, 0x33, 0x55, - 0x89, 0x26, 0xda, 0x49, 0xd9, 0xbc, 0xa0, 0xc5, 0x1e, 0x16, 0x77, 0x32, 0xd5, 0xa4, 0xde, 0x65, - 0x2a, 0xcf, 0x8c, 0x84, 0x17, 0xb7, 0xf7, 0x5e, 0xe6, 0x49, 0x40, 0x85, 0xaa, 0x77, 0x37, 0xf1, - 0x58, 0xc2, 0xe4, 0x99, 0x27, 0xc3, 0x7e, 0x76, 0xc2, 0x58, 0x75, 0xd2, 0x6e, 0xc6, 0x08, 0x49, - 0x2e, 0xcf, 0xce, 0xa0, 0xbd, 0xcc, 0x48, 0xb3, 0x1a, 0xa9, 0xf1, 0x84, 0xc9, 0x33, 0x23, 0xef, - 0x70, 0xcd, 0xfa, 0xb6, 0x36, 0xe9, 0x50, 0x71, 0x37, 0x4b, 0x21, 0x5f, 0xb3, 0xee, 0x1f, 0x16, - 0x67, 0xbf, 0xd5, 0x4c, 0xa1, 0xc8, 0x4b, 0x78, 0x71, 0x4b, 0xec, 0x65, 0xde, 0x51, 0xd6, 0x56, - 0x1a, 0x6d, 0xc2, 0xe4, 0x99, 0x77, 0xd6, 0xbb, 0x50, 0xcf, 0x16, 0x6a, 0x9b, 0xd4, 0xb7, 0xf2, - 0x85, 0x9a, 0xf1, 0xac, 0x8e, 0xfd, 0x8b, 0x0e, 0x9b, 0xce, 0x78, 0x1a, 0x84, 0x51, 0xe6, 0xfa, - 0xca, 0x57, 0x9e, 0xb6, 0xf6, 0x95, 0xa7, 0x17, 0x06, 0x2b, 0x5d, 0x63, 0x1a, 0x3c, 0x26, 0x97, - 0x44, 0xa6, 0x95, 0xcc, 0x5c, 0x2b, 0xed, 0x42, 0x4d, 0xee, 0x25, 0x14, 0x95, 0x48, 0x94, 0x32, - 0xe4, 0xbb, 0x73, 0x41, 0xef, 0x8e, 0x0a, 0xbd, 0x61, 0x62, 0x92, 0xb5, 0x00, 0xa4, 0x1a, 0x09, - 0xab, 0x24, 0xcc, 0x70, 0x50, 0x7e, 0xee, 0x8d, 0xc5, 0x2c, 0x72, 0xc7, 0xd3, 0x99, 0x55, 0x6e, - 0x1b, 0x1d, 0x83, 0x67, 0x38, 0xec, 0x2e, 0x34, 0x29, 0x89, 0xc3, 0x50, 0xe0, 0x1c, 0x38, 0x88, - 0xa8, 0x15, 0x0d, 0x5e, 0xe0, 0xa2, 0x1e, 0xa5, 0x95, 0xea, 0xc9, 0x21, 0x51, 0xe0, 0xd2, 0x1c, - 0xf6, 0x85, 0x1b, 0x52, 0xb3, 0x55, 0xb9, 0x24, 0xec, 0x3f, 0x74, 0x60, 0x12, 0x49, 0xf9, 0x86, - 0xf8, 0xc7, 0xe0, 0x7c, 0x3d, 0x6c, 0x79, 0x70, 0x2a, 0x2b, 0xe0, 0xdc, 0x4c, 0xde, 0x3c, 0x12, - 0x18, 0x45, 0xb1, 0x36, 0xd4, 0xe3, 0x35, 0x81, 0x42, 0x44, 0x55, 0xe3, 0x59, 0x16, 0xb3, 0xa1, - 0x71, 0x16, 0xe1, 0xc3, 0x5f, 0xa9, 0xd4, 0xc8, 0x76, 0x8e, 0xb7, 0x06, 0x5a, 0x78, 0x43, 0x68, - 0xeb, 0xaf, 0x87, 0xb6, 0x91, 0x85, 0xf6, 0x7b, 0x0d, 0x1a, 0x07, 0x51, 0x30, 0xf6, 0x06, 0x5c, - 0x0c, 0x82, 0x70, 0xf8, 0x6a, 0x50, 0x25, 0x7c, 0x7a, 0x16, 0xbe, 0x0e, 0x18, 0xce, 0x8b, 0x50, - 0x8d, 0xce, 0x9b, 0xb4, 0xcc, 0x57, 0xaa, 0xc4, 0x51, 0x85, 0xdd, 0x01, 0xdd, 0x09, 0xa9, 0x67, - 0xeb, 0xdd, 0x6b, 0xa9, 0x62, 0xac, 0xa3, 0x3b, 0xa1, 0xfd, 0x0e, 0xec, 0xc8, 0x40, 0x62, 0x91, - 0xda, 0x15, 0x3b, 0x50, 0x3a, 0x0a, 0xc3, 0x20, 0xde, 0x16, 0x92, 0xc0, 0xd7, 0x6a, 0xb2, 0x7e, - 0xb0, 0x18, 0x6f, 0xd3, 0x13, 0xeb, 0x7e, 0xa2, 0xb5, 0xa1, 0x7e, 0x1a, 0x44, 0x5f, 0x84, 0x5e, - 0x44, 0xd3, 0x44, 0xce, 0xfc, 0x2c, 0xcb, 0xbe, 0x07, 0x37, 0x0a, 0x9e, 0xd3, 0xa5, 0x86, 0x6d, - 0x64, 0xa4, 0x3f, 0x73, 0xce, 0xe0, 0x7a, 0xa2, 0xea, 0xf4, 0xdf, 0x2a, 0xc6, 0x55, 0xa3, 0xff, - 0xcf, 0x64, 0x4e, 0x46, 0x95, 0xfb, 0x35, 0xd9, 0xd8, 0x3d, 0xb0, 0x14, 0x9a, 0xf2, 0x77, 0xa6, - 0x8a, 0xe0, 0xc2, 0x13, 0x8b, 0x57, 0x3d, 0xaf, 0xe9, 0x45, 0xa0, 0xd3, 0xaf, 0x53, 0xfa, 0xb6, - 0x7f, 0xd0, 0x61, 0x67, 0x9d, 0x91, 0xb4, 0xa1, 0xb4, 0x4c, 0x43, 0xb1, 0x2e, 0x94, 0x5e, 0x78, - 0x62, 0x11, 0xaf, 0xf1, 0xdd, 0x4c, 0xb1, 0x57, 0x62, 0xe0, 0x52, 0x15, 0x2f, 0xd2, 0xc1, 0x20, - 0xf2, 0x82, 0x49, 0xfc, 0x5c, 0x94, 0x14, 0x7a, 0xe8, 0xf9, 0xc1, 0xe0, 0x1b, 0xf9, 0x4b, 0x87, - 0x4b, 0x62, 0xcd, 0xc5, 0x28, 0xbd, 0xe1, 0xc5, 0x28, 0xaf, 0xbd, 0x18, 0x1d, 0xd8, 0xfa, 0x7c, - 0x3a, 0x74, 0x23, 0x71, 0x74, 0xe5, 0xcd, 0x22, 0x31, 0x19, 0x08, 0xab, 0x42, 0x19, 0x15, 0xd9, - 0xf6, 0x59, 0x6e, 0x09, 0xe0, 0xf4, 0x38, 0x18, 0x8d, 0x42, 0x31, 0x72, 0xa3, 0x18, 0xc6, 0x94, - 0xc1, 0xee, 0x42, 0x99, 0x94, 0x63, 0x24, 0x8a, 0x5b, 0x5d, 0x49, 0x7b, 0xdb, 0xbf, 0xbe, 0x6c, - 0x69, 0xbf, 0xbf, 0x6c, 0x69, 0x7f, 0xbe, 0x6c, 0x69, 0x3f, 0xfe, 0xd5, 0xda, 0xb8, 0x2c, 0xd3, - 0xdf, 0x0c, 0xef, 0xfd, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x70, 0xbe, 0xda, 0x79, 0x76, 0x10, 0x00, - 0x00, + // 1582 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5f, 0x6f, 0x1b, 0xd5, + 0x12, 0xcf, 0xfe, 0xf1, 0xbf, 0xb1, 0xe3, 0xa4, 0xa7, 0x69, 0xef, 0xde, 0xde, 0xd4, 0xd7, 0x5d, + 0x5d, 0x55, 0xee, 0x0d, 0x4a, 0x85, 0x81, 0x0a, 0x55, 0x02, 0x14, 0xc7, 0x29, 0x59, 0xb5, 0x49, + 0xcb, 0x49, 0x08, 0x3c, 0xf0, 0xb2, 0xb1, 0x0f, 0xee, 0x8a, 0xb5, 0xd7, 0xac, 0xd7, 0x75, 0xfc, + 0x09, 0xe0, 0x23, 0xf0, 0xc6, 0x13, 0x1f, 0x05, 0xc1, 0x1b, 0x3c, 0xf2, 0x88, 0xca, 0x17, 0x41, + 0x33, 0xe7, 0xec, 0x5f, 0xbb, 0x55, 0x55, 0xf1, 0xb6, 0xf3, 0xe7, 0xcc, 0x99, 0xf9, 0xcd, 0x9c, + 0x99, 0xb1, 0xa1, 0x31, 0x9d, 0x5f, 0xfa, 0xde, 0x60, 0x7f, 0x1a, 0x06, 0x51, 0xc0, 0xf4, 0xe9, + 0xa5, 0xbd, 0x04, 0x83, 0x07, 0x0b, 0x66, 0x41, 0xe5, 0x30, 0xf0, 0xe7, 0xe3, 0xc9, 0xcc, 0xd2, + 0xda, 0x46, 0xc7, 0xe4, 0x31, 0xc9, 0x18, 0x98, 0x8f, 0xc5, 0x72, 0x66, 0x19, 0x6d, 0xa3, 0x53, + 0xe3, 0xf4, 0x8d, 0xda, 0x3c, 0x70, 0x43, 0x6f, 0x32, 0xb2, 0xcc, 0xb6, 0xd6, 0x69, 0xf0, 0x98, + 0x64, 0x3b, 0x50, 0x72, 0x26, 0x43, 0x71, 0x65, 0x95, 0xda, 0x5a, 0xa7, 0xc6, 0x25, 0x81, 0xdc, + 0x47, 0x9e, 0xf0, 0x87, 0x56, 0x59, 0x72, 0x89, 0xb0, 0x3b, 0x50, 0xe3, 0xc1, 0xe2, 0xc4, 0x8d, + 0x42, 0xef, 0x8a, 0xfd, 0x07, 0x4c, 0x1e, 0x2c, 0xe4, 0xed, 0xf5, 0x6e, 0x65, 0x7f, 0x7a, 0xb9, + 0xcf, 0x83, 0x05, 0x27, 0xa6, 0x7d, 0x00, 0xb5, 0x33, 0x6f, 0x34, 0x11, 0x43, 0x74, 0xf5, 0xdf, + 0x60, 0x3c, 0x0b, 0x50, 0x51, 0xcb, 0x2a, 0x22, 0x0f, 0x45, 0xa7, 0x62, 0x64, 0xe9, 0x05, 0xd1, + 0xa9, 0x18, 0xd9, 0x1f, 0x42, 0x93, 0x07, 0x0b, 0x67, 0x28, 0x26, 0x91, 0xf7, 0xb5, 0x27, 0x42, + 0x0a, 0x2c, 0xb9, 0xd1, 0x94, 0x17, 0x25, 0xc1, 0xea, 0x69, 0xb0, 0xf6, 0x2d, 0x28, 0x3b, 0xfd, + 0x27, 0xde, 0x2c, 0x62, 0xdb, 0x60, 0x38, 0xfd, 0xf8, 0x00, 0x7e, 0xda, 0x87, 0x70, 0xed, 0xe8, + 0x2a, 0x0a, 0xdd, 0x41, 0x24, 0x86, 0x4e, 0x5f, 0x42, 0xc6, 0x9a, 0xa0, 0x3b, 0x7d, 0xf2, 0xcf, + 0xe4, 0xba, 0xd3, 0x67, 0x2d, 0x30, 0x2f, 0x5c, 0x5f, 0x1a, 0xad, 0x77, 0x01, 0xdd, 0x92, 0x06, + 0x39, 0xf1, 0xed, 0xaf, 0x72, 0x46, 0x14, 0x1e, 0x37, 0xa1, 0x4c, 0x28, 0xc9, 0xeb, 0x6a, 0x5c, + 0x51, 0xec, 0x7e, 0x9a, 0x28, 0x69, 0xef, 0x06, 0xda, 0x5b, 0x71, 0x22, 0xc9, 0x9f, 0x7d, 0x1b, + 0x2a, 0x8f, 0xc5, 0x92, 0xfc, 0x8f, 0xa3, 0xd3, 0x32, 0xd1, 0xfd, 0xa6, 0xc1, 0xf5, 0xe4, 0xf4, + 0xb9, 0x7b, 0xe9, 0x8b, 0x0b, 0xd7, 0x9f, 0x0b, 0xd6, 0x8a, 0x63, 0xd5, 0xf2, 0x3e, 0x1f, 0x6f, + 0x50, 0xe4, 0xec, 0x4e, 0x82, 0x14, 0x2a, 0xd4, 0x51, 0x41, 0x5d, 0x73, 0xbc, 0xa1, 0xaa, 0x64, + 0x17, 0xaa, 0xbd, 0x33, 0x87, 0xcc, 0x59, 0x46, 0x5b, 0xeb, 0x18, 0xc7, 0x1b, 0x3c, 0xe1, 0xb0, + 0x5b, 0x50, 0x39, 0x99, 0x47, 0xe2, 0xca, 0xe9, 0x53, 0x0d, 0x99, 0xc7, 0x1b, 0x3c, 0x66, 0xe0, + 0x49, 0xfa, 0x7c, 0x2c, 0x96, 0xb2, 0x90, 0xf0, 0x64, 0xcc, 0x61, 0x3b, 0x60, 0xf6, 0x82, 0xc0, + 0xa7, 0x62, 0xaa, 0xe2, 0x6d, 0x48, 0xf5, 0x2a, 0x50, 0x22, 0xc3, 0xf6, 0x15, 0xec, 0xe4, 0x03, + 0x52, 0x69, 0x61, 0x60, 0xa0, 0x3d, 0x4d, 0xd9, 0x43, 0x82, 0x6d, 0x53, 0xaa, 0x74, 0x75, 0x3f, + 0x26, 0xeb, 0x3e, 0x94, 0xc9, 0x8c, 0x2c, 0xf8, 0x7a, 0xf7, 0x5f, 0x39, 0x78, 0x53, 0x80, 0xb8, + 0x52, 0xeb, 0xd5, 0x08, 0xdf, 0xa7, 0xa1, 0xd3, 0xb7, 0x3f, 0x2a, 0x42, 0x49, 0x39, 0x43, 0xd8, + 0x4f, 0xdd, 0xb1, 0x90, 0x37, 0x73, 0xfa, 0x46, 0xde, 0xf9, 0x72, 0x2a, 0xe8, 0xea, 0x1a, 0xa7, + 0x6f, 0x7b, 0x0e, 0xcd, 0xfc, 0x71, 0x74, 0x26, 0x53, 0x04, 0x6b, 0x9d, 0x21, 0x79, 0x52, 0x1d, + 0xdd, 0x62, 0x75, 0x58, 0xab, 0x27, 0x8a, 0x05, 0xf2, 0x31, 0x98, 0xcf, 0x5c, 0x2f, 0x5c, 0x29, + 0xdb, 0x6d, 0x89, 0x97, 0x41, 0x1e, 0x1a, 0x12, 0xf8, 0xd2, 0x61, 0x30, 0x9f, 0x44, 0x12, 0x30, + 0x2e, 0x09, 0xfb, 0x13, 0xa8, 0xe1, 0x79, 0x19, 0xeb, 0xae, 0x34, 0xa6, 0xea, 0xa6, 0x8a, 0xb7, + 0x23, 0xcd, 0xe5, 0x15, 0x49, 0x1f, 0xd0, 0xb3, 0x7d, 0xa0, 0x07, 0x80, 0xd2, 0x99, 0xb4, 0xd0, + 0x82, 0x12, 0x51, 0x2a, 0xe4, 0xd4, 0x84, 0x64, 0xbf, 0xc2, 0xc6, 0x6d, 0xec, 0x3b, 0xd1, 0x83, + 0xf7, 0x51, 0x2c, 0x2b, 0x0e, 0x3d, 0x30, 0xb8, 0xaa, 0x89, 0x00, 0xaa, 0x12, 0xa8, 0x60, 0x91, + 0x1a, 0xd0, 0x32, 0x06, 0x90, 0x8b, 0xfd, 0xa1, 0x1f, 0xc7, 0x46, 0x04, 0xbe, 0x42, 0x1e, 0x2c, + 0x52, 0x18, 0x14, 0xc5, 0xfe, 0x1b, 0xdf, 0x62, 0x52, 0x9c, 0x35, 0x7a, 0x1f, 0x78, 0x7f, 0x7c, + 0xe1, 0x97, 0x00, 0x9f, 0x86, 0xc1, 0x7c, 0x4a, 0x10, 0x31, 0x1b, 0x4a, 0x44, 0xa9, 0x98, 0x1a, + 0xa8, 0x1e, 0xfb, 0xc3, 0xa5, 0x68, 0x3d, 0xb8, 0x98, 0x84, 0x83, 0xd1, 0x48, 0x3e, 0x1f, 0x8e, + 0x9f, 0xf6, 0x8f, 0x1a, 0x54, 0x2f, 0x5c, 0x3f, 0x11, 0x5f, 0xb8, 0xbe, 0x8a, 0x15, 0x3f, 0xf3, + 0x66, 0x8c, 0xd8, 0xcc, 0x2d, 0xa8, 0x3e, 0xf2, 0x03, 0x37, 0x42, 0x65, 0xb4, 0xa5, 0xf1, 0x84, + 0x66, 0x7b, 0x00, 0x7d, 0x31, 0xf0, 0xc6, 0xae, 0x8f, 0x52, 0x33, 0x7d, 0xcf, 0x8a, 0xcb, 0x33, + 0x62, 0x66, 0x43, 0xe3, 0xdc, 0x1b, 0x8b, 0x59, 0xe4, 0x8e, 0xa7, 0xa8, 0x2e, 0xdb, 0x7c, 0x8e, + 0x67, 0x7f, 0x00, 0x15, 0x75, 0x62, 0x7d, 0x36, 0x90, 0x7b, 0x36, 0x70, 0x7d, 0x11, 0xfb, 0x48, + 0x84, 0xfd, 0xb3, 0x06, 0x8d, 0xcf, 0xe6, 0x22, 0x5c, 0x72, 0xf1, 0xed, 0x5c, 0xcc, 0x22, 0x54, + 0x23, 0x3a, 0x4e, 0x14, 0x11, 0x98, 0x92, 0xb3, 0xe7, 0x6e, 0x38, 0x94, 0x15, 0x6e, 0x72, 0x45, + 0x51, 0xaa, 0xc4, 0x38, 0x88, 0x04, 0xf9, 0x54, 0xe5, 0x8a, 0x62, 0x7b, 0xd0, 0x38, 0x1a, 0x5f, + 0x8a, 0xe1, 0x50, 0x0c, 0xfb, 0x6e, 0xe4, 0x5a, 0xd5, 0xfc, 0x80, 0xc9, 0x09, 0xd9, 0xff, 0x60, + 0xf3, 0x59, 0x28, 0xce, 0x43, 0x77, 0x32, 0xf3, 0xdd, 0x48, 0x0c, 0xad, 0x1a, 0xd9, 0xca, 0x33, + 0xd9, 0x2e, 0xd4, 0x4e, 0xdc, 0xab, 0x13, 0x31, 0x0e, 0xc2, 0xa5, 0x05, 0x14, 0x43, 0xca, 0xb0, + 0x9f, 0xc0, 0xa6, 0x0a, 0x63, 0x36, 0x0d, 0x26, 0x33, 0x81, 0x49, 0x3a, 0x0a, 0x43, 0x15, 0x05, + 0x7e, 0xb2, 0x7b, 0x50, 0xe1, 0x62, 0x36, 0xf7, 0xa3, 0xf8, 0x99, 0x6e, 0xa1, 0x3b, 0xf1, 0xa9, + 0xb9, 0x1f, 0xf1, 0x58, 0x6e, 0xff, 0x54, 0x82, 0x7a, 0x46, 0x90, 0x34, 0x0e, 0x6c, 0x7e, 0x9b, + 0xb2, 0x71, 0xe0, 0xd8, 0xe3, 0xc1, 0x62, 0x65, 0x22, 0x62, 0xb1, 0x37, 0x40, 0x3b, 0x55, 0x15, + 0xa5, 0x9d, 0xa6, 0x6f, 0xcb, 0x58, 0xff, 0xb6, 0x70, 0x0b, 0x78, 0xee, 0x4e, 0x46, 0x62, 0x48, + 0x75, 0x50, 0xe5, 0x31, 0xc9, 0x3a, 0x69, 0xd1, 0x11, 0xbe, 0xaa, 0x88, 0x63, 0x1e, 0x4f, 0x4b, + 0x52, 0x3e, 0x19, 0x9c, 0x1d, 0x15, 0x99, 0x1f, 0x49, 0xb1, 0x07, 0xd0, 0x7c, 0xea, 0x0f, 0xd3, + 0x47, 0x31, 0x53, 0x99, 0x68, 0xa2, 0x9d, 0x94, 0xcd, 0x0b, 0x5a, 0xec, 0x61, 0x71, 0x70, 0x53, + 0x4e, 0xea, 0x5d, 0xa6, 0xe2, 0xcc, 0x48, 0x78, 0x71, 0xc4, 0xef, 0x65, 0xf6, 0x06, 0x4a, 0x54, + 0xbd, 0xbb, 0x89, 0xc7, 0x12, 0x26, 0xcf, 0xec, 0x15, 0xfb, 0xd9, 0x36, 0x64, 0xd5, 0x49, 0xbb, + 0x19, 0x23, 0x24, 0xb9, 0x3c, 0xdb, 0xa8, 0xf6, 0x32, 0x7d, 0xcf, 0x6a, 0xa4, 0xc6, 0x13, 0x26, + 0xcf, 0xf4, 0xc5, 0xc3, 0x35, 0x33, 0xde, 0xda, 0xa4, 0x43, 0xc5, 0x01, 0x2e, 0x85, 0x7c, 0xcd, + 0x4e, 0xf0, 0xb0, 0x38, 0x20, 0xac, 0x66, 0x0a, 0x45, 0x5e, 0xc2, 0x8b, 0xa3, 0x64, 0x2f, 0xb3, + 0x6c, 0x59, 0x5b, 0xa9, 0xb7, 0x09, 0x93, 0x67, 0x96, 0xb1, 0x77, 0xa1, 0x9e, 0x4d, 0xd4, 0x36, + 0xa9, 0x6f, 0xe5, 0x13, 0x35, 0xe3, 0x59, 0x1d, 0xfb, 0x17, 0x1d, 0x36, 0x9d, 0xf1, 0x34, 0x08, + 0xa3, 0xcc, 0xf3, 0x95, 0xab, 0xa0, 0xb6, 0x76, 0x15, 0xd4, 0x0b, 0xdd, 0x97, 0x9e, 0x31, 0x35, + 0x27, 0x93, 0x4b, 0x22, 0x53, 0x4a, 0x66, 0xae, 0x94, 0x76, 0xa1, 0x26, 0x87, 0x17, 0x8a, 0x4a, + 0x24, 0x4a, 0x19, 0x72, 0x39, 0x5d, 0xd0, 0x72, 0x52, 0xa1, 0x45, 0x27, 0x26, 0x59, 0x0b, 0x40, + 0xaa, 0x91, 0xb0, 0x4a, 0xc2, 0x0c, 0x07, 0xe5, 0x49, 0x23, 0x9b, 0x59, 0xe5, 0xb6, 0xd1, 0x31, + 0x78, 0x86, 0xc3, 0xee, 0x42, 0x93, 0x82, 0x38, 0x0c, 0x05, 0xf6, 0x81, 0x83, 0x88, 0x4a, 0xd1, + 0xe0, 0x05, 0x2e, 0xea, 0x51, 0x58, 0xa9, 0x9e, 0x6c, 0x12, 0x05, 0x2e, 0xf5, 0x6a, 0x5f, 0xb8, + 0x21, 0x15, 0x5b, 0x95, 0x4b, 0xc2, 0xfe, 0x43, 0x07, 0x26, 0x91, 0x94, 0x8b, 0xc6, 0x3f, 0x06, + 0xe7, 0xeb, 0x61, 0xcb, 0x83, 0x53, 0x59, 0x01, 0xe7, 0x66, 0xb2, 0x18, 0x49, 0x60, 0x14, 0xc5, + 0xda, 0x50, 0x8f, 0x47, 0x09, 0x0a, 0x11, 0x55, 0x8d, 0x67, 0x59, 0x38, 0x33, 0xce, 0x22, 0xfc, + 0x75, 0xa0, 0x54, 0x6a, 0x64, 0x3b, 0xc7, 0x5b, 0x03, 0x2d, 0xbc, 0x21, 0xb4, 0xf5, 0xd7, 0x43, + 0xdb, 0xc8, 0x42, 0xfb, 0x9d, 0x06, 0x8d, 0x83, 0x28, 0x18, 0x7b, 0x03, 0x2e, 0x06, 0x41, 0x38, + 0x7c, 0x35, 0xa8, 0x12, 0x3e, 0x3d, 0x0b, 0x5f, 0x07, 0x0c, 0xe7, 0x45, 0xa8, 0x5a, 0xe7, 0x4d, + 0x9a, 0xf8, 0x2b, 0x59, 0xe2, 0xa8, 0xc2, 0xee, 0x80, 0xee, 0x84, 0x54, 0xb3, 0xf5, 0xee, 0xb5, + 0x54, 0x31, 0xd6, 0xd1, 0x9d, 0xd0, 0x7e, 0x07, 0x76, 0xa4, 0x23, 0xb1, 0x48, 0xcd, 0x8a, 0x1d, + 0x28, 0x1d, 0x85, 0x61, 0x10, 0x4f, 0x0b, 0x49, 0xe0, 0x4a, 0x9b, 0x8c, 0x1f, 0x4c, 0xc6, 0xdb, + 0xd4, 0xc4, 0xba, 0xdf, 0x71, 0x6d, 0xa8, 0x9f, 0x06, 0xd1, 0x17, 0xa1, 0x17, 0x51, 0x37, 0x91, + 0x3d, 0x3f, 0xcb, 0xb2, 0xef, 0xc1, 0x8d, 0xc2, 0xcd, 0xe9, 0x50, 0xc3, 0x32, 0x32, 0xd2, 0xdf, + 0x42, 0x67, 0x70, 0x3d, 0x51, 0x75, 0xfa, 0x6f, 0xe5, 0xe3, 0xaa, 0xd1, 0xff, 0x67, 0x22, 0x27, + 0xa3, 0xea, 0xfa, 0x35, 0xd1, 0xd8, 0x3d, 0xb0, 0x14, 0x9a, 0xf2, 0xc7, 0xa8, 0xf2, 0xe0, 0xc2, + 0x13, 0x8b, 0x57, 0xed, 0xe0, 0xb4, 0x11, 0xe8, 0xf4, 0x13, 0x96, 0xbe, 0xed, 0xef, 0x75, 0xd8, + 0x59, 0x67, 0x24, 0x2d, 0x28, 0x2d, 0x53, 0x50, 0xac, 0x0b, 0xa5, 0x17, 0x9e, 0x58, 0xc4, 0x63, + 0x7c, 0x37, 0x93, 0xec, 0x15, 0x1f, 0xb8, 0x54, 0xc5, 0x87, 0x74, 0x30, 0x88, 0xbc, 0x60, 0x12, + 0xef, 0x94, 0x92, 0xc2, 0x1b, 0x7a, 0x7e, 0x30, 0xf8, 0x46, 0xfe, 0x1c, 0xe2, 0x92, 0x58, 0xf3, + 0x30, 0x4a, 0x6f, 0xf8, 0x30, 0xca, 0x6b, 0x1f, 0x46, 0x07, 0xb6, 0x3e, 0x9f, 0x0e, 0xdd, 0x48, + 0x1c, 0x5d, 0x79, 0xb3, 0x48, 0x4c, 0x06, 0xc2, 0xaa, 0x50, 0x44, 0x45, 0xb6, 0x7d, 0x96, 0x1b, + 0x02, 0xd8, 0x3d, 0x0e, 0x46, 0xa3, 0x50, 0x8c, 0xdc, 0x28, 0x86, 0x31, 0x65, 0xb0, 0xbb, 0x50, + 0x26, 0xe5, 0x18, 0x89, 0xe2, 0x54, 0x57, 0xd2, 0xde, 0xf6, 0xaf, 0x2f, 0x5b, 0xda, 0xef, 0x2f, + 0x5b, 0xda, 0x9f, 0x2f, 0x5b, 0xda, 0x0f, 0x7f, 0xb5, 0x36, 0x2e, 0xcb, 0xf4, 0x5f, 0xc4, 0x7b, + 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x88, 0x20, 0x3d, 0x60, 0x9b, 0x10, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -3557,6 +3565,13 @@ func (m *ValCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.TimestampVal) > 0 { + i -= len(m.TimestampVal) + copy(dAtA[i:], m.TimestampVal) + i = encodeVarintPublic(dAtA, i, uint64(len(m.TimestampVal))) + i-- + dAtA[i] = 0x2a + } if m.DecimalVal != nil { { size, err := m.DecimalVal.MarshalToSizedBuffer(dAtA[:i]) @@ -5224,6 +5239,10 @@ func (m *ValCount) Size() (n int) { l = m.DecimalVal.Size() n += 1 + l + sovPublic(uint64(l)) } + l = len(m.TimestampVal) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8213,6 +8232,38 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimestampVal", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimestampVal = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/pb/public.proto b/pb/public.proto index 70935c647..d86a80d5d 100644 --- a/pb/public.proto +++ b/pb/public.proto @@ -109,6 +109,7 @@ message ValCount { int64 Count = 2; double FloatVal = 3; Decimal DecimalVal = 4; + string TimestampVal = 5; } message Decimal { diff --git a/qa/scripts/cloud-init.sh b/qa/scripts/cloud-init.sh new file mode 100755 index 000000000..8d710d0b2 --- /dev/null +++ b/qa/scripts/cloud-init.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ex +# generate log +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + +# Install packages +yum update -y +yum install postgresql -y + +# Configure host system +echo 'cat /proc/sys/fs/file-max' +sysctl -w fs.file-max=262144 +sysctl -p +echo 'cat /proc/sys/fs/file-max' + +# yum install golang -y # latest verion in ec2 is 1.15.14 +# install go 1.16.9 manually +curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz +tar xvf go1.16.10.linux-amd64.tar.gz +chown -R root:root ./go +mv go /usr/local +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null +source /etc/profile + +# install aws session manager pluggin +curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" +yum install -y session-manager-plugin.rpm diff --git a/qa/scripts/configureFeatureBase.json b/qa/scripts/configureFeatureBase.json new file mode 100644 index 000000000..6afd135d9 --- /dev/null +++ b/qa/scripts/configureFeatureBase.json @@ -0,0 +1,20 @@ +{ + "Parameters": { + "commands": [ + "#!/bin/bash", + "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase", + "mv /home/ec2-user/featurebase.conf /etc/", + "mv /home/ec2-user/featurebase.service /etc/systemd/system/", + "adduser molecula", + "sudo mkdir /var/log/molecula", + "sudo chown molecula /var/log/molecula", + "sudo mkdir -p /opt/molecula/featurebase", + "sudo chown molecula /opt/molecula/featurebase", + "systemctl daemon-reload", + "sudo systemctl start featurebase", + "sudo systemctl enable featurebase", + "sudo systemctl status featurebase", + "curl localhost:10101" + ] + } +} \ No newline at end of file diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh new file mode 100755 index 000000000..b928a31e7 --- /dev/null +++ b/qa/scripts/deployNode.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# To run script: ./deployNode.sh $PROFILE + +function deploy_node() { + # get AMI, security group and subnet ID + AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) + if [[ $? > 0 ]]; then + echo "aws session manager failed to find AMI" + exit 1 + fi + + SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + if [[ $? > 0 ]]; then + echo "aws session manager failed to find security group" + exit 1 + fi + + SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) + if [[ $? > 0 ]]; then + echo "aws session manager failed to find subnet ID" + exit 1 + fi + + # launch EC2 instance and get instance ID + aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json + if [[ $? > 0 ]]; then + echo "aws run-instances failed to launch a new EC2 instance" + exit 1 + fi + + INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') + echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID +} + +function initialize_featurebase() { + # get IP for node + for i in {0..24} + do + IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + if [ -n "$IP" ]; then + echo "Public IP for EC2 instance found: " $IP + break + fi + + if [[ $? > 0 ]]; then + echo "aws cli describe-instances command failed to find public IP" + terminate_node + exit 1 + fi + + sleep 5 + done + + sleep 60 # to allow enough time for node to be ready for use + + # copy featurebase binary and files to ec2 instance + scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. + if [[ $? > 0 ]]; then + echo "scp of featurebase binary, service and config files to EC2 instance failed" + terminate_node + exit 1 + fi + + # execute script to configure featurebase on the EC2 node + aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION + if [[ $? > 0 ]]; then + echo "aws cli session manager send-command failed" + terminate_node + exit 1 + fi +} + +function terminate_node() { + aws ec2 terminate-instances --instance-ids $INSTANCE_ID --profile $PROFILE +} + +# Pass variables to shell script +PROFILE=$1 +shift + +# set some variables +INSTANCE="t3a.large" +REGION="us-east-2" + +# get AMI, security group and subnet for EC2 instance, +# launch instance, save instance Id and run cloud-init to set up node env +deploy_node + +# Get IP for instance, scp featurebase binary, config and service files; +# set up featurebase config in node +initialize_featurebase + +# terminate node +terminate_node diff --git a/qa/scripts/featurebase.conf b/qa/scripts/featurebase.conf new file mode 100644 index 000000000..7716d0f6b --- /dev/null +++ b/qa/scripts/featurebase.conf @@ -0,0 +1,30 @@ +name = "pilosa1" +bind = "0.0.0.0:10101" +bind-grpc = "0.0.0.0:20101" + +data-dir = "/opt/molecula/featurebase" +log-path = "/var/log/molecula/featurebase.log" + +max-file-count=900000 +max-map-count=900000 + +long-query-time = "10s" + +[postgres] + + bind = "localhost:55432" + +[cluster] + + name = "cluster1" + replicas = 1 + +[etcd] + + listen-client-address = "http://localhost:10401" + listen-peer-address = "http://localhost:10301" + initial-cluster = "pilosa1=http://localhost:10301" + +[metric] + + service = "prometheus" \ No newline at end of file diff --git a/qa/scripts/featurebase.service b/qa/scripts/featurebase.service new file mode 100644 index 000000000..a543b92af --- /dev/null +++ b/qa/scripts/featurebase.service @@ -0,0 +1,13 @@ +# Not Ansible managed + +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] \ No newline at end of file diff --git a/qa/scripts/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh new file mode 100755 index 000000000..23d11dd32 --- /dev/null +++ b/qa/scripts/ingestWorkload.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# To run: +# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag} + +function delete_field { + if (($INITIALIZE == 0)); + then + curl -XDELETE $HOST/index/$INDEX/field/$FIELD + fi +} + +# Script to replicate samsung workload of deleting and re-ingesting fields every night +# outline delete and re-ingest workload +function ingest_int_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_time_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_set_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +# path for featurebase binary +FEATUREBASE_PATH=$1 +shift + +# featurebase host & port +HOST=$1 +shift + +# path for directory with csv directory files for all fields to be ingested +CSV_DIR_PATH=$1 +shift + +# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing +INITIALIZE=$1 +shift + +# get a list of csv files in the directory +CSV_FILES=`ls $CSV_DIR_PATH/*.csv` + +# assign index name +INDEX="samsung" +if (($INITIALIZE == 1)); +then + curl -XPOST $HOST/index/$INDEX +fi + +# perform delete and re-ingest for all fields +for CSV_FILE in ${CSV_FILES[@]} + do + # get field name from csv file path + FIELD="$(basename $CSV_FILE .csv)" + if [[ "$FIELD" == *"age"* ]]; + then + MAX=100 + ingest_int_field + elif [[ "$FIELD" == *"identifier"* ]]; + then + MAX=$((2**63 - 1)) # compute max value for 64bit + ingest_int_field + elif [[ "$FIELD" == *"ip"* ]]; + then + MAX=$((2**31 - 1)) # compute max value for 32bit + ingest_int_field + elif [[ "$FIELD" == *"time"* ]]; + then + ingest_time_field + else + ingest_set_field + fi + done + + + diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go new file mode 100644 index 000000000..e94c619d8 --- /dev/null +++ b/qa/simulacraData/simulacra_data.go @@ -0,0 +1,357 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 +// INPUT: none +// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) + +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "strconv" + "time" + + "github.com/pkg/errors" +) + +var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", + "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", + "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", + "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", + "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", + "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", + "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", + "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", + "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", + "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", + "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", + "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", + "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", + "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", + "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", + "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", +} + +const totalRecords int = 200000000 + +func main() { + if err := GenerateAgeField(totalRecords); err != nil { + log.Fatalf("unable to generate age field: %v", err) + } + + if err := GenerateIPField(totalRecords); err != nil { + log.Fatalf("unable to generate IP field: %v", err) + } + + if err := GenerateArbIdField(totalRecords); err != nil { + log.Fatalf("unable to generate indentifier field: %v", err) + } + + if err := GenerateOptInField(totalRecords); err != nil { + log.Fatalf("unable to generate opt in field: %v", err) + } + + if err := GenerateCountryField(totalRecords); err != nil { + log.Fatalf("unable to generate country field: %v", err) + } + + if err := GenerateTimeField(totalRecords); err != nil { + log.Fatalf("unable to generate time field: %v", err) + } +} + +func GenerateAgeField(requestedRecords int) error { + csvFile, err0 := os.Create("age.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create age.csv") + } + + writer := bufio.NewWriter(csvFile) + + for i := 0; i < requestedRecords; i++ { + if rand.Intn(100) <= 20 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to age.csv") + } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to age.csv") + + } + } + + } + } + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close age.csv") + } + return nil +} + +func GenerateIPField(requestedRecords int) error { + csvFile, err0 := os.Create("ip.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create ip.csv") + } + + writer := bufio.NewWriter(csvFile) + + for i := 0; i < requestedRecords; i++ { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to ip.csv") + } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to ip.csv") + } + } + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close ip.csv") + } + return nil + +} + +func GenerateArbIdField(requestedRecords int) error { + csvFile, err0 := os.Create("identifier.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create identifier.csv") + } + + writer := bufio.NewWriter(csvFile) + + for i := 0; i < requestedRecords; i++ { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to identifier.csv") + } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to identifier.csv") + } + } + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close identifier.csv") + } + return nil + +} + +func GenerateTimeField(requestedRecords int) error { + csvFile, err0 := os.Create("time.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create time.csv") + } + + writer := bufio.NewWriter(csvFile) + + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 4000000 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < requestedRecords; i++ { + value := distro.Uint64() + urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) + + date := generateDate() + + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to time.csv") + } + + if rand.Intn(5) == 1 { + date = generateDate() + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to time.csv") + } + } + + if rand.Intn(10) == 1 { + value := distro.Uint64() + urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) + date = generateDate() + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to time.csv") + } + } + + } + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close time.csv") + } + return nil + +} + +func GenerateOptInField(requestedRecords int) error { + csvFile, err0 := os.Create("optin.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create optin.csv") + } + + writer := bufio.NewWriter(csvFile) + + var optString string + + for i := 0; i < requestedRecords; i++ { + + if rand.Intn(20) == 1 { + optString = "1" + } else { + optString = "0" + } + + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to optin.csv") + } + + if rand.Intn(10) == 1 { + if optString == "0" { + optString = "1" + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to optin.csv") + } + } else { + optString = "0" + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to optin.csv") + } + } + } + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close optin.csv") + } + return nil + +} + +func GenerateCountryField(requestedRecords int) error { + csvFile, err0 := os.Create("country.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create country.csv") + } + + writer := bufio.NewWriter(csvFile) + + // populate countries + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 245 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < requestedRecords; i++ { + if rand.Intn(2) == 1 { + value := distro.Uint64() + country := countryList[value] + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to country.csv") + } + + if rand.Intn(10) == 1 { + value = distro.Uint64() + country = countryList[value] + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to country.csv") + } + } + } + + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close country.csv") + } + return nil + +} + +func generateDate() string { + // Nov 22, 2021 = 1637616960 + // twenty years = 631138520 + // date min = 1006478440 + date := rand.Int63n(631138520) + 1006478440 + + t := time.Unix(date, 0) + YMDstring := t.Format("2006-01-02T15:04") + + return YMDstring +} diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go new file mode 100644 index 000000000..38462f5e5 --- /dev/null +++ b/qa/simulacraData/simulacra_data_test.go @@ -0,0 +1,69 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "testing" +) + +const testRecords int = 1000 + +func TestAge(t *testing.T) { + err := GenerateAgeField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestIP(t *testing.T) { + err := GenerateIPField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestIndentifer(t *testing.T) { + err := GenerateArbIdField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestOptIn(t *testing.T) { + err := GenerateOptInField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestCountry(t *testing.T) { + err := GenerateCountryField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestTime(t *testing.T) { + err := GenerateTimeField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestGenerateDate(t *testing.T) { + testDate := generateDate() + if testDate == "" { + t.Fatalf("error generating date") + } +} diff --git a/rbf/README.md b/rbf/README.md index 4f6efa072..cbc5b74a7 100644 --- a/rbf/README.md +++ b/rbf/README.md @@ -93,12 +93,12 @@ The leaf page contains a series of cells with the header of: [8] highbits [4] flag [4] child count - [*] array or RLE data + [*] array or RLE data or Handle (a pageno) to Bitmap Data -### Bitmap page +### Bitmap Data page -The data for the bitmap page takes up the entire 8KB. +The data for the bitmap data page takes up the entire 8KB. ## Proof of Concept Notes diff --git a/rbf/tx.go b/rbf/tx.go index efc664864..301987320 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -56,6 +56,27 @@ type Tx struct { // behavior where an existing container has all its bits cleared // but still sticks around in the database. DeleteEmptyContainer bool + + // It is possible for a modification of the free list to cause a page to + // be allocated or deallocated, which would modify the free list. + // + // For the case where pages need to be allocated during free list + // changes, we can trivially just allocate new pages and not use the + // free list. That's the simple case... + modifyingFreelist bool + // But removals can't be deferred/not-done like that. If a free list + // change causes us to deallocate a page (such as if we're *allocating* + // a page, which causes it to be *removed* from the free list), we really + // do need to record that, but if we try to do it during the update + // process, things could go horribly wrong. So, we have a transient list + // of page numbers which have been deallocated, but it happened *during* + // the modification of the free list. The top-level modification then + // processes them on its way out, using a defer. During the processing + // of this list, we *still* have the flag set, and we make a new list + // while processing the list, so if somehow a pending add to the list + // manages to trigger a *deallocation* (which I don't think should be + // happening), we'll process that one after the current list is processed. + pendingFreelistAdds []uint32 } func (tx *Tx) DBPath() string { @@ -312,7 +333,6 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error { if !strings.HasPrefix(name.(string), prefix) { continue } - // Deallocate all pages in the tree. if err := tx.deallocateTree(pgno.(uint32)); err != nil { return err @@ -903,16 +923,67 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er } } +// freelistCleanup handles things which we need to add to the free list, +// because they became free during the process of modifying the free list. +// It also marks us as done modifying the free list. Expected usage is that +// you set modifyingFreelist to true, then defer this. +// +// if you are modifying the free list, we can't further change the free +// list during that modification. for page allocations, we can just skip +// the free list check. for deallocations, though, we do need to mark them +// as freed at some point. so, if we're modifying the free list when +// a new freePgno happens, we stash the new pages in here, then apply +// them afterwards. so far as i know, this can actually only happen +// during an allocate, when we're removing entries from the free list, and +// the add path doesn't ever trigger it. so, when we remove entries from +// the free list, it's possible that doing so frees up pages that were +// part of the free list, and we then add them. but we don't have to worry +// about that removing things from the free list, because the add logic +// already just uses new pages rather than trying to use the free list +// when it knows the free list is involved. +func (tx *Tx) freelistCleanup(outErr *error) { + defer func() { + // no matter what, we're done with this after this, but we still + // want it set *while* we do this so nothing we do will have side + // effects that collide with what we're doing. + tx.modifyingFreelist = false + }() + if len(tx.pendingFreelistAdds) == 0 { + return + } + c := Cursor{tx: tx} + c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} + for len(tx.pendingFreelistAdds) > 0 { + var pass []uint32 + pass, tx.pendingFreelistAdds = tx.pendingFreelistAdds, nil + for _, pgno := range pass { + if changed, err := c.Add(uint64(pgno)); err != nil { + if outErr != nil && *outErr == nil { + *outErr = err + } + return + } else if !changed { + vprint.PanicOn(fmt.Sprintf("rbf.Tx.freePgno(): double free: %d", tx.pendingFreelistAdds)) + } + } + } +} + // allocatePgno returns a page number for a new available page. This page may be // pulled from the free list or, if no free pages are available, it will be // created by extending the file size. -func (tx *Tx) allocatePgno() (uint32, error) { +func (tx *Tx) allocatePgno() (_ uint32, outErr error) { + if tx.modifyingFreelist { + return tx.allocateNewPgno(), nil + } // Attempt to find page in freelist. pgno, err := tx.nextFreelistPageNo() if err != nil { return 0, err } else if pgno != 0 { + tx.modifyingFreelist = true + defer tx.freelistCleanup(&outErr) c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} if changed, err := c.Remove(uint64(pgno)); err != nil { @@ -922,11 +993,16 @@ func (tx *Tx) allocatePgno() (uint32, error) { } return pgno, nil } + // no freelist pages, fall back + return tx.allocateNewPgno(), nil +} +// allocateNewPgno requests a new page unconditionally, ignoring the free list. +func (tx *Tx) allocateNewPgno() uint32 { // Increment the total page count by one and return the last page. - pgno = readMetaPageN(tx.meta[:]) + pgno := readMetaPageN(tx.meta[:]) writeMetaPageN(tx.meta[:], pgno+1) - return pgno, nil + return pgno } func (tx *Tx) nextFreelistPageNo() (uint32, error) { @@ -952,10 +1028,16 @@ func (tx *Tx) nextFreelistPageNo() (uint32, error) { } // deallocate releases a page number to the freelist. -func (tx *Tx) freePgno(pgno uint32) error { +func (tx *Tx) freePgno(pgno uint32) (outErr error) { + if tx.modifyingFreelist { + tx.pendingFreelistAdds = append(tx.pendingFreelistAdds, pgno) + return nil + } c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} + tx.modifyingFreelist = true + defer tx.freelistCleanup(&outErr) if changed, err := c.Add(uint64(pgno)); err != nil { return err } else if !changed { @@ -979,9 +1061,16 @@ func (tx *Tx) deallocateTree(pgno uint32) error { return err } } - return nil + return tx.freePgno(pgno) case PageTypeLeaf: + for i, n := 0, readCellN(page); i < n; i++ { + if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr { + if err := tx.freePgno(toPgno(cell.Data)); err != nil { + return err + } + } + } return tx.freePgno(pgno) default: return fmt.Errorf("rbf.Tx.deallocateTree(): invalid page type: pgno=%d type=%d", pgno, typ) diff --git a/rbf/tx_test.go b/rbf/tx_test.go index f7cd78d6a..8a354a916 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/molecula/featurebase/v2/rbf" + "github.com/molecula/featurebase/v2/roaring" ) func TestTx_CommitRollback(t *testing.T) { @@ -219,6 +220,95 @@ func TestTx_DeleteBitmap(t *testing.T) { } } +// deallocateTree had a bug which caused a page to be marked neither free +// nor in-use. +func TestTx_DeallocateTree(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + var err error + + // Create bitmap & add value. + if err = tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + const N = 315 + slots := make([]uint64, N) + for i := range slots { + slots[i] = uint64(i) << 20 + } + if _, err = tx.Add("x", slots...); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatalf("check: %v", err) + } + if err = tx.DeleteBitmap("x"); err != nil { + t.Fatal(err) + } + if ok, err := tx.Contains("x", 0); err != nil { + t.Fatal(err) + } else if ok { + t.Fatal("expected no value in recreated bitmap") + } + if err = tx.Check(); err != nil { + t.Fatalf("check: %v", err) + } +} + +func TestTx_RecreateBitmap(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + + // Create bitmap & add value. + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + const N = 825000 + slots := make([]uint64, N) + for i := range slots { + slots[i] = uint64(i) << 20 + } + if _, err := tx.Add("x", slots...); err != nil { + t.Fatal(err) + } + err := tx.Commit() + if err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + // Delete bitmap, verifying that it's gone. + if err := tx.DeleteBitmap("x"); err != nil { + t.Fatal(err) + } else { + if ok, err := tx.Contains("x", 0); err != nil { + t.Fatal(err) + } else if ok { + t.Fatal("expected no value in recreated bitmap") + } + } + err = tx.Commit() + if err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + if _, err := tx.Add("x", slots...); err != nil { + t.Fatal(err) + } + err = tx.Commit() + if err != nil { + t.Fatal(err) + } +} + func TestTx_RenameBitmap(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) @@ -279,6 +369,83 @@ func TestTx_Add_Quick(t *testing.T) { }) } +func TestTx_DeallocateToFreeList(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + var err error + + // Create bitmap & add value. + if err = tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + if err = tx.CreateBitmap("y"); err != nil { + t.Fatal(err) + } + const N = 12274831 + slots := make([]uint64, N) + for i := range slots { + slots[i] = uint64(i) << 10 + } + bm := roaring.NewBitmap(slots...) + if _, err = tx.AddRoaring("x", bm); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + for i := 0; i < 500; i++ { + if _, err := tx.Add("y", uint64(i)<<16); err != nil { + t.Fatal(err) + } + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err := tx.DeleteBitmap("y"); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + // Delete bitmap, verifying that it's gone. + if err := tx.DeleteBitmap("x"); err != nil { + t.Fatal(err) + } else { + if ok, err := tx.Contains("x", 0); err != nil { + t.Fatal(err) + } else if ok { + t.Fatal("expected no value in recreated bitmap") + } + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + if _, err := tx.AddRoaring("x", bm); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } +} + func TestTx_AddRemove_Quick(t *testing.T) { if testing.Short() { t.Skip("-short enabled, skipping") @@ -517,3 +684,102 @@ func TestTx_CreateBitmap(t *testing.T) { } }) } + +func TestTx_DeleteBitmapsWithPrefix(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + prefix := "abc" + bitmapSize := 10000 + // var err error + // create a interleaved set up array and bitmap containers + + bits := make([]uint64, bitmapSize) + x := uint64(1) + for i := 0; i < len(bits); i++ { + bits[i] = x + x = x + 2 + } + ifError := func(err error) { + if err != nil { + t.Fatal(err) + } + } + checkInfos := func() { + tx := MustBegin(t, db, false) + defer tx.Rollback() + infos, err := tx.PageInfos() + ifError(err) + for pgno, info := range infos { + switch info := info.(type) { + case *rbf.MetaPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "meta") + fmt.Printf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo) + + case *rbf.RootRecordPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "rootrec") + fmt.Printf("next=%d\n", info.Next) + + case *rbf.LeafPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "leaf") + fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + + case *rbf.BranchPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "branch") + fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + + case *rbf.BitmapPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "bitmap") + fmt.Printf("-\n") + + case *rbf.FreePageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "free") + fmt.Printf("-\n") + + default: + t.Fatal(fmt.Sprintf("unexpected page info type %T", info)) + } + } + + } + populate := func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + for i := uint64(0); i < 16; i++ { + bm := roaring.NewBitmap() + if i%3 == 0 { + bm.Put(i, roaring.NewContainerBitmap(6144, bits)) + if _, err := tx.AddRoaring(prefix, bm); err != nil { + panic(err) + } + } else { + bm.Put(i, roaring.NewContainerArray([]uint16{2, 4, 5, 7})) + if _, err := tx.AddRoaring(prefix, bm); err != nil { + panic(err) + } + + } + } + ifError(tx.Commit()) + } + + checkInfos() + populate() + checkInfos() + ifError(db.Check()) + + tx := MustBegin(t, db, true) + tx.DeleteBitmapsWithPrefix(prefix) + ifError(tx.Commit()) + ifError(db.Check()) + checkInfos() + populate() + ifError(db.Check()) + checkInfos() + +} diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 7a952a937..ee51aeb0e 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -1,6 +1,6 @@ // Copyright 2014 The b Authors. All rights reserved. // Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// license that can be found in the NOTICE file. package roaring diff --git a/server.go b/server.go index 8058c25f4..bcb86f119 100644 --- a/server.go +++ b/server.go @@ -538,25 +538,17 @@ func (s *Server) SetAPI(api *API) { // UpAndDown brings the server up minimally and shuts it down // again; basically, it exists for testing holder open and close. func (s *Server) UpAndDown() error { - s.logger.Infof("open server. PID %v", os.Getpid()) - // Log startup err := s.holder.logStartup() if err != nil { log.Println(errors.Wrap(err, "logging startup")) } - - // Open holder. - if err := s.holder.Open(); err != nil { - return errors.Wrap(err, "opening Holder") + s.logger.Infof("open server. PID %v", os.Getpid()) + if err = s.Open(); err != nil { + return errors.Wrap(err, "starting server") } - - errh := s.holder.Close() - if errh != nil { - return errors.Wrap(errh, "closing holder") - } - - return nil + err = s.Close() + return errors.Wrap(err, "shutting down server") } // Open opens and initializes the server. diff --git a/server/pg.go b/server/pg.go index c2522f286..169f2e747 100644 --- a/server/pg.go +++ b/server/pg.go @@ -155,6 +155,24 @@ type PilosaQueryHandler struct { sqlVersion SqlVersion } +func pgWriteDistinctTimestamp(w pg.QueryResultWriter, val pilosa.DistinctTimestamp) error { + err := w.WriteHeader(pg.ColumnInfo{ + Name: val.Name, + Type: pg.TypeCharoid, + }) + if err != nil { + return errors.Wrap(err, "writing result header") + } + + for _, k := range val.Values { + err = w.WriteRowText(k) + if err != nil { + return errors.Wrap(err, "writing key") + } + } + return nil +} + func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error { err := w.WriteHeader(pg.ColumnInfo{ Name: "_id", @@ -313,7 +331,11 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error var v string switch { case g.Value != nil: - v = strconv.FormatInt(*g.Value, 10) + if g.FieldOptions.Type == pilosa.FieldTypeTimestamp { + v = pilosa.FormatTimestampNano(int64(*g.Value), g.FieldOptions.Base, g.FieldOptions.TimeUnit) + } else { + v = strconv.FormatInt(*g.Value, 10) + } case g.RowKey != "": v = g.RowKey default: @@ -551,7 +573,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { } return nil - + case pilosa.DistinctTimestamp: + return pgWriteDistinctTimestamp(w, result) case nil: return nil diff --git a/server/pg_internal_test.go b/server/pg_internal_test.go new file mode 100644 index 000000000..75024d509 --- /dev/null +++ b/server/pg_internal_test.go @@ -0,0 +1,67 @@ +// Copyright 2021 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "testing" + + pilosa "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/pg" +) + +// pg_internal_test.go tests unexported methods from server/pg.go + +// TestQueryResultWriter implements the QueryResultWriter interface for testing +type TestQueryResultWriter struct { + Header []pg.ColumnInfo + RowText []string + TagTag string +} + +func (t *TestQueryResultWriter) WriteHeader(headers ...pg.ColumnInfo) error { + for _, header := range headers { + t.Header = append(t.Header, header) + } + return nil +} + +func (t *TestQueryResultWriter) WriteRowText(rowTexts ...string) error { + for _, rowText := range rowTexts { + t.RowText = append(t.RowText, rowText) + } + return nil +} + +func (t *TestQueryResultWriter) Tag(tag string) { + t.TagTag = tag +} + +func TestPgWriteDistinctTimestamp(t *testing.T) { + w := TestQueryResultWriter{} + expected := pilosa.DistinctTimestamp{Name: "test", Values: []string{"date1", "date2", "date3"}} + pgWriteDistinctTimestamp(&w, expected) + if w.Header[0].Name != expected.Name { + t.Fatalf("Header Name is wrong. got %v, want %v", w.Header[0], expected.Name) + } + if w.Header[0].Type != pg.TypeCharoid { + t.Fatalf("Header Type is wrong. got %v, want %v", w.Header[0].Type, pg.TypeCharoid) + } + for i, value := range w.RowText { + if value != expected.Values[i] { + t.Fatalf("Value not written properly. got %v, want %v", value, expected.Values[i]) + } + } + +} diff --git a/server/server.go b/server/server.go index 3feaf4582..29eb20f1d 100644 --- a/server/server.go +++ b/server/server.go @@ -293,6 +293,7 @@ func (m *Command) UpAndDown() (err error) { if err != nil { return errors.Wrap(err, "setting up server") } + m.logger.Infof("bringing server up and shutting it down immediately") go func() { err := m.Handler.Serve() @@ -306,7 +307,7 @@ func (m *Command) UpAndDown() (err error) { return errors.Wrap(err, "bringing server up and down") } - m.logger.Errorf("brought up and shut down again") + m.logger.Infof("teardown complete") return nil } diff --git a/server/server_test.go b/server/server_test.go index 5e2a47dfa..a449d5838 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -531,6 +531,14 @@ func TestClusteringNodesReplica1(t *testing.T) { } } +func TestUpAndDown(t *testing.T) { + c := test.NewCommandNode(t) + err := c.UpAndDown() + if err != nil { + t.Fatalf("server up-and-down: %v", err) + } +} + func TestClusteringNodesReplica2(t *testing.T) { // Because this test shuts down 2 nodes, it needs to start as a 5-node // cluster in order to retain enough available nodes for raft leader diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..48659587f 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -60,8 +60,6 @@ func newCommand(tb testing.TB, opts ...server.CommandOption) *Command { // a problem with PDK tests which used pilosa/client as well. We put it at the // beginning of the option slice so that it can be overridden by user-passed // options. - // Also set TranslateFile MapSize to a smaller number so memory allocation - // does not fail on 32-bit systems. opts = append([]server.CommandOption{ server.OptCommandCloseTimeout(time.Millisecond * 2), }, opts...) @@ -315,7 +313,27 @@ func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { + // have to check each field Row individually because FieldOptions is getting set + for j := range expected[i].Group { + // Field:"ppa", RowID:0x3, RowKey:"", Value:(*int64)(nil), FieldOptions: + if !reflect.DeepEqual(expected[i].Group[j].Field, result.Group[j].Field) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].RowKey, result.Group[j].RowKey) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].Value, result.Group[j].Value) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } + + if !reflect.DeepEqual(expected[i].Count, result.Count) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Agg, result.Agg) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].DecimalAgg, result.DecimalAgg) { t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } diff --git a/util.go b/util.go index 6d4b8e09e..1db0c74fc 100644 --- a/util.go +++ b/util.go @@ -115,8 +115,9 @@ func roaringFragmentHasData(path string, index, field, view string, shard uint64 return } -// GetLoopProgress returns the estimated remaining time to iterate through some items -// as well as the loop completion percentage with the following parameters: +// GetLoopProgress returns the estimated remaining time to iterate through some +// items as well as the loop completion percentage with the following +// parameters: // the start time, the current time, the iteration, and the number of items func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) (remaining time.Duration, pctDone float64) { itemsLeft := total - (iteration + 1) @@ -124,3 +125,9 @@ func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) pctDone = (float64(iteration+1) / float64(total)) * 100 return time.Duration(avgItemTime * float64(itemsLeft)), pctDone } + +// FormatTimestampNano returns the string representation of a timestamp given: +// an epoch value, base, and time unit +func FormatTimestampNano(value, base int64, timeUnit string) string { + return time.Unix(0, (value+base)*TimeUnitNanos(timeUnit)).UTC().Format(time.RFC3339Nano) +} diff --git a/util_test.go b/util_test.go index c659053c5..14ed78c36 100644 --- a/util_test.go +++ b/util_test.go @@ -85,3 +85,21 @@ func TestGetLoopProgress(t *testing.T) { }) } } + +func TestFormatTimestampNano(t *testing.T) { + if FormatTimestampNano(0, 69, "s") != "1970-01-01T00:01:09Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(0, 420, "ms") != "1970-01-01T00:00:00.42Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(420, 0, "μs") != "1970-01-01T00:00:00.00000042Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(420, 69, "us") != "1970-01-01T00:00:00.000489Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(69, 420, "ns") != "1970-01-01T00:00:00.000000489Z" { + t.Fatal("Timestamp not formatted properly") + } +}