scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-09 22:51:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into pql-variables

Browse source
This commit is contained in:
Samir Patel 2022-02-09 14:40:18 -05:00 • committed by GitHub
commit 96de9834bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
111 changed files with 1787 additions and 5971 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
.gitlab/.gitlab-ci.yml
View file

@ -12,6 +12,17 @@ stages:
- build
- integration
- gauntlet
- post build
smoke build:
image: golang:$GOVERSION
stage: lint
allow_failure: false
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
- go build ./...
golangci-lint:
image: golangci/golangci-lint:v1.39.0
@ -21,7 +32,7 @@ golangci-lint:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- echo "Checking for issues in new code"
- golangci-lint run -v
- golangci-lint run
build lattice:
stage: test
@ -205,7 +216,7 @@ package for linux amd64:
GOARCH: "amd64"
script:
- echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
- apt update && apt install nfpm
- apt update && apt install nfpm=2.11.3
- make package
artifacts:
paths:
@ -222,7 +233,7 @@ package for linux arm64:
GOARCH: "arm64"
script:
- echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
- apt update && apt install nfpm
- apt update && apt install nfpm=2.11.3
- make package
artifacts:
paths:
@ -277,9 +288,18 @@ clustertests:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
allow_failure: true
script:
- make clustertests
authclustertests:
variables:
PROJECT: authclustertests_${CI_CONCURRENT_ID}
stage: integration
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- make authclustertests
external lookup tests:
@ -407,3 +427,40 @@ gauntlet:
after_script:
- ./qa/scripts/teardownSamsungGauntlet.sh
s3 dump:
stage: post build
variables:
PROFILE: "service-fb-ci"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
- aws configure set region "us-east-2"
- aws configure set aws_profile $PROFILE
- aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
- aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
- aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64
- aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_amd64
- aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
- aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
- aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64
- aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_arm64
- aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
- aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
- aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64
- aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_amd64
- aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
- aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
- aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64
- aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64
needs:
- job: build for darwin amd64
- job: build for darwin arm64
- job: build for linux amd64
- job: build for linux arm64

35
Dockerfile-clustertests-client Normal file
View file

@ -0,0 +1,35 @@
# This Dockerfile is used for cluster testing - it produces a much larger image
# and includes all of Go as well as some utilities.
FROM golang:1.16
LABEL maintainer "dev@pilosa.com"
COPY . /go/src/github.com/molecula/featurebase/
RUN cd /go/src/github.com/molecula/featurebase \
&& make install FLAGS="-a -mod=vendor"
# download pumba for fault injection
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
RUN chmod +x /pumba
# add docker client to pause/unpause nodes
RUN apt update
RUN apt install -y docker.io
# add docker-compose so tests can use it for stuff
ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
RUN chmod +x /usr/local/bin/docker-compose
RUN cp /go/bin/featurebase /featurebase
COPY NOTICE /NOTICE
COPY ./internal/clustertests /go/src/github.com/molecula/featurebase/internal/clustertests
EXPOSE 10101
VOLUME /data
ENTRYPOINT ["bash", "-c"]
CMD ["/featurebase", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

25
Makefile
View file

@ -79,9 +79,6 @@ testvsub-race:
cd ..; \
done
tour:
./tournament.sh
bench:
$(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
@ -159,13 +156,14 @@ clustertests: vendor
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
# Run the cluster tests with authentication enabled
DOCKER_COMPOSE_AUTH = docker-compose -p authclustertests
AUTH_ARGS="-c /go/src/github.com/molecula/featurebase/internal/clustertests/testdata/featurebase.conf"
authclustertests: vendor
$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down
$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml build
$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml run client1
$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down
$(eval PROJECT=authclustertests)
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
# Install Pilosa
install:
@ -349,14 +347,5 @@ install-gometalinter:
GO111MODULE=off gometalinter --install
GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
test-txstore-rbf:
PILOSA_STORAGE_BACKEND=rbf $(MAKE) testv-race
# WARNING: This feature is no longer being tested regularly in CI. The test is
# very slow and very expensive, and we're not sure it actually provides useful
# information now.
test-txstore-rbf_bolt:
PILOSA_STORAGE_BACKEND=rbf_bolt $(MAKE) testv-race
test-external-lookup:
$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)

4
api.go
View file

@ -2307,10 +2307,6 @@ func (api *API) Info() serverInfo {
}
}
func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) {
return api.holder.Inspect(ctx, req)
}
// GetTranslateEntryReader provides an entry reader for key translation logs starting at offset.
func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader")

89
api_test.go
View file

@ -5,11 +5,14 @@ import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"math/rand"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
@ -22,7 +25,6 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/test"
@ -36,21 +38,21 @@ func TestAPI_Import(t *testing.T) {
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -222,19 +224,19 @@ func TestAPI_ImportValue(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -529,7 +531,7 @@ func TestAPI_Ingest(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -648,7 +650,7 @@ func BenchmarkIngest(b *testing.B) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -709,7 +711,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -1363,7 +1365,9 @@ func TestVariousApiTranslateCalls(t *testing.T) {
if err != nil {
t.Fatalf("%v: could not create test index", err)
}
_, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false})
if _, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false}); err != nil {
t.Fatalf("creating field: %v", err)
}
t.Run("translateIndexDbOnNilIndex",
func(t *testing.T) {
err := api.TranslateIndexDB(context.Background(), "nonExistentIndex", 0, r)
@ -1430,7 +1434,7 @@ func TestAPI_RBFDebugInfo(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -1448,11 +1452,6 @@ func TestAPI_RBFDebugInfo(t *testing.T) {
func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.UserInfo {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
groupString, err := authn.ToGob64(groups)
if err != nil {
t.Fatalf("gobbing groups %v", err)
}
claims["molecula-idp-groups"] = groupString
claims["oid"] = "42"
claims["name"] = name
secretKey, _ := hex.DecodeString(secret)
@ -1473,7 +1472,6 @@ func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.Us
}
func TestAuth_MultiNode(t *testing.T) {
// create permissions file
permissions := `
"user-groups":
@ -1482,6 +1480,43 @@ func TestAuth_MultiNode(t *testing.T) {
"dca35310-ecda-4f23-86cd-876aee55906f":
"test": "write"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
adminUser := makeUser(t, []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "adminGroup"}}, "admin", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
adminCtx := context.WithValue(
context.Background(),
"userinfo",
adminUser,
)
readUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "readGroup"}}, "reader", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
readCtx := context.WithValue(
context.Background(),
"userinfo",
readUser,
)
writeUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906f", GroupName: "writeGroup"}}, "writer", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEED")
writeCtx := context.WithValue(
context.Background(),
"userinfo",
writeUser,
)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
token, ok := r.Header["Authorization"]
if !ok || len(token) == 0 {
http.Error(w, "BAD REQUEST", http.StatusBadRequest)
return
}
g := []authn.Group{}
switch token[0] {
case adminUser.Token:
g = adminUser.Groups
case readUser.Token:
g = readUser.Groups
case writeUser.Token:
g = writeUser.Groups
}
if err := json.NewEncoder(w).Encode(authn.Groups{Groups: g}); err != nil {
t.Fatalf("unexpected error marshalling groups response: %v", err)
}
}))
// authentication on
auth := server.Auth{
@ -1490,7 +1525,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
ClientSecret: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
AuthorizeURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
TokenURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
GroupEndpointURL: "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
GroupEndpointURL: srv.URL,
RedirectBaseURL: "https://localhost:10101",
LogoutURL: "https://login.microsoftonline.com/common/oauth2/v2.0/logout",
Scopes: []string{"https://graph.microsoft.com/.default", "offline_access"},
@ -1562,22 +1597,6 @@ f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
)
defer c.Close()
adminCtx := context.WithValue(
context.Background(),
"userinfo",
makeUser(t, []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "adminGroup"}}, "admin", config.Auth.SecretKey),
)
readCtx := context.WithValue(
context.Background(),
"userinfo",
makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "readGroup"}}, "reader", config.Auth.SecretKey),
)
writeCtx := context.WithValue(
context.Background(),
"userinfo",
makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906f", GroupName: "writeGroup"}}, "writer", config.Auth.SecretKey),
)
primaryAPI := c.GetPrimary().API
// needs internal/cluster/message

307
authn/authenticate.go
View file

@ -4,24 +4,55 @@
package authn
import (
"bytes"
"encoding/base64"
"encoding/gob"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/golang-jwt/jwt"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
"golang.org/x/oauth2"
)
func init() {
gob.Register([]Group{})
// cachedGroups is used to hold groups and when they were last cached
type cachedGroups struct {
cacheTime time.Time
groups []Group
}
// cacheToken is used to hold tokens and when they were added to the cache
type cachedToken struct {
cacheTime time.Time
token *oauth2.Token
}
// UserInfo holds the information about the user from the token
type UserInfo struct {
UserID string `json:"userid"`
UserName string `json:"username"`
Groups []Group `json:"groups"`
Expiry time.Time `json:"expiry"`
Token string `json:"token"`
}
// Group holds group information for an authenticated user
type Group struct {
GroupID string `json:"id"`
GroupName string `json:"displayName"`
}
// Groups holds a slice of Group for marshalling from JSON
type Groups struct {
Groups []Group `json:"value"`
}
// Auth holds state, configuration, and utilities needed for authentication.
@ -31,8 +62,13 @@ type Auth struct {
secretKey []byte
groupEndpoint string
logoutEndpoint string
fbURL string // fbURL is the domain FB is hosted on, used for post logout redirection
fbURL string // fbURL is the domain featurebase is hosted on, used for post logout redirection
oAuthConfig *oauth2.Config
cacheTTL time.Duration // cacheTTL is used to determine if a cached item should be refreshed or not
tokenTTR time.Duration // tokenTTR (time to refresh) is used to determine if a token should be refreshed or not
tokenCache map[string]cachedToken // tokenCache is a map of accessToken -> *oauth2.Token which we can use to refresh the tokens
groupsCache map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
lastCacheClean time.Time // last cache clean is the time that the cache was last cleaned
}
// NewAuth instantiates and returns a new Auth struct
@ -53,105 +89,124 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenUR
TokenURL: tokenURL,
},
},
tokenCache: map[string]cachedToken{},
groupsCache: map[string]cachedGroups{},
cacheTTL: 10 * time.Minute,
tokenTTR: 7 * time.Minute,
lastCacheClean: time.Now(),
}
if auth.secretKey, err = decodeHex(secretKey); err != nil {
return nil, errors.Wrap(err, "decoding secret key")
}
return auth, nil
}
// SecretKey is a convenient function to get the SecretKey from an Auth struct
func (a Auth) SecretKey() []byte {
return a.secretKey
}
// UserInfo holds the information about the user from the token
type UserInfo struct {
UserID string `json:"userid"`
UserName string `json:"username"`
Groups []Group `json:"groups"`
Expiry time.Time `json:"expiry"`
Token string `json:"token"`
}
// Group holds group information for an authenticated user
type Group struct {
GroupID string `json:"id"`
GroupName string `json:"displayName"`
}
// ToGob64 encodes a []Group to a string, returning string and nil on success (todd's idea)
// it has to be a string bc we're using it in a jwt.MapClaims which needs string-y things
func ToGob64(m []Group) (string, error) {
var b bytes.Buffer
if err := gob.NewEncoder(&b).Encode(m); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(b.Bytes()), nil
}
// FromGob64 converts a previously encoded []Group from a string to a []Group
// it has to be a string bc we're using it in a jwt.MapClaims which needs string-y things
func FromGob64(gobbed string) ([]Group, error) {
m := []Group{}
by, err := base64.StdEncoding.DecodeString(gobbed)
if err != nil {
return nil, err
}
b := bytes.Buffer{}
b.Write(by)
d := gob.NewDecoder(&b)
err = d.Decode(&m)
if err != nil {
return nil, err
}
return m, nil
}
// Groups holds a slice of Group for marshalling from Json
type Groups struct {
Groups []Group `json:"value"`
}
// Authenticate takes in a bearer token `bearer` and returns UserInfo from that token
func (a *Auth) Authenticate(bearer string) (*UserInfo, error) {
// parse the bearer token into a jwt.Token
// this also validates the token, and checks that it's not expired
token, err := jwt.Parse(bearer, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
// it is caller's responsibility to inform the user that the access token has been refreshed
func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, error) {
// clean up the cache every 30 minutes or so
if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute {
a.cleanCache()
}
if tkn, ok := a.tokenCache[bearer]; ok && (tkn.token.Expiry.Sub(time.Now()) <= a.tokenTTR || !tkn.token.Valid()) {
// refresh the token
resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
url.Values{
"grant_type": {"refresh_token"},
"refresh_token": {tkn.token.RefreshToken},
"client_id": {a.oAuthConfig.ClientID},
"client_secret": {a.oAuthConfig.ClientSecret},
},
)
if err != nil {
return nil, errors.Wrap(err, "refreshing token")
}
return a.secretKey, nil
})
if token == nil || token.Claims == nil || err != nil || !token.Valid {
defer resp.Body.Close()
var t oauth2.Token
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
return nil, errors.Wrap(err, "decoding refreshed token")
}
// update the cache
delete(a.tokenCache, bearer)
delete(a.groupsCache, bearer)
bearer = t.AccessToken
a.tokenCache[bearer] = cachedToken{time.Now(), &t}
}
// NOTE: we are using ParseUnverified here because the IDP validates the
// token's signature when we get the user's groups, we just need to make
// sure it's not expired and is well-formed
token, _, err := new(jwt.Parser).ParseUnverified(bearer, &jwt.MapClaims{})
// well-formed-ness check
if token == nil || token.Claims == nil || err != nil {
return nil, fmt.Errorf("parsing bearer token: %v", err)
}
userInfo := UserInfo{}
claims := token.Claims.(jwt.MapClaims)
userInfo.UserID = claims["oid"].(string)
userInfo.UserName = claims["name"].(string)
userInfo.Token = bearer
claims := *token.Claims.(*jwt.MapClaims)
g := claims["molecula-idp-groups"].(string)
groups, err := FromGob64(g)
if err != nil {
return nil, errors.Wrap(err, "decoding groups")
// expiry check
if exp, ok := claims["exp"].(string); ok {
if expiry, err := strconv.ParseInt(exp, 10, 64); err != nil || expiry < time.Now().UTC().Unix() {
return nil, fmt.Errorf("token is expired")
}
}
userInfo := UserInfo{
UserID: claims["oid"].(string),
UserName: claims["name"].(string),
Token: bearer,
Groups: []Group{},
}
if userInfo.Groups, err = a.getGroups(bearer); err != nil {
return nil, errors.Wrap(err, "getting groups")
}
userInfo.Groups = groups
return &userInfo, nil
}
// cleanCache removes old items from our cache
func (a *Auth) cleanCache() {
for bearer, tkn := range a.tokenCache {
// if it's been more than 24 hours since the token was cached
if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
// remove it from our cache
delete(a.tokenCache, bearer)
}
}
for bearer, tkn := range a.groupsCache {
// if it's been more than 24 hours since the groups were cached
if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
// remove it from our cache
delete(a.groupsCache, bearer)
}
}
a.lastCacheClean = time.Now()
}
// Login redirects a user to login to their configured oAuth authorize endpoint
func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
}
// Logout clears out user cookie and redirects user to IdP's logout endpoint
// Logout clears out the user's cookie, removes the token from our cache, and
// redirects user to IdP's logout endpoint
func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
// remove the bearer token from a.tokenCache and a.groupsCache
if bearer, err := r.Cookie(a.cookieName); err == nil {
delete(a.tokenCache, bearer.Value)
delete(a.groupsCache, bearer.Value)
}
// clear cookie
http.SetCookie(w, &http.Cookie{
Name: a.cookieName,
Value: "",
@ -161,84 +216,35 @@ func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(0, 0),
})
redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL)
http.Redirect(w, r, redirect, http.StatusTemporaryRedirect)
http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect)
}
// Redirect handles the oAuth /redirect endpoint. It gets user information from
// the identity provider and sets a secure cookie holding the user information
// signed by featurebase.
// Redirect handles the oAuth /redirect endpoint. It gets an access token and
// returns it to the user in the form of a cookie
func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
code := r.FormValue("code")
token, err := a.getToken(r, code)
token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline)
if err != nil {
a.logger.Warnf("getting token from IdP: %+v", err)
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
// enrich token with groups!
g, err := a.getGroups(token.AccessToken)
if err != nil {
a.logger.Warnf("getting groups from IdP: %+v", err)
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
a.tokenCache[token.AccessToken] = cachedToken{time.Now(), token}
// with vitamin G! (for groups)
enrichedTkn, err := a.addGroupMembership(token.AccessToken, g)
if err != nil {
a.logger.Warnf("enriching token with group membership: %+v", err)
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
a.setCookie(w, enrichedTkn, token.Expiry)
a.SetCookie(w, token.AccessToken, token.Expiry)
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}
// getToken exhanges authorization code for an oAuth2 token
func (a *Auth) getToken(r *http.Request, code string) (*oauth2.Token, error) {
token, err := a.oAuthConfig.Exchange(r.Context(), code)
if err != nil {
return nil, errors.Wrap(err, "exchanging auth code for token")
}
return token, nil
}
// addGroupMembership is only called in `a.Redirect`. It adds groups to a jwt's
// claims, and signs it using `a.secretKey`.
func (a *Auth) addGroupMembership(token string, g []Group) (string, error) {
// parse token into jwt
unenriched, _, err := new(jwt.Parser).ParseUnverified(token, jwt.MapClaims{})
if unenriched == nil || unenriched.Claims == nil || err != nil {
return "", fmt.Errorf("parsing bearer token: %v", err)
}
enriched := jwt.New(jwt.SigningMethodHS256)
enriched.Claims = unenriched.Claims
// parse groups into string format
claims := enriched.Claims.(jwt.MapClaims)
groupString, err := ToGob64(g)
if err != nil {
return "", errors.Wrap(err, "failed to serialize groups")
}
// stick it into jwt claims
claims["molecula-idp-groups"] = groupString
// get stringified and signed jwt
tokenStr, err := enriched.SignedString(a.secretKey)
if err != nil {
return "", errors.Wrap(err, "signing jwt")
}
return tokenStr, nil
}
// getGroups gets the group membership for a given token from configured IdP
func (a *Auth) getGroups(token string) ([]Group, error) {
var groups Groups
g, ok := a.groupsCache[token]
if ok && (time.Now().Sub(g.cacheTime) < a.cacheTTL) {
return g.groups, nil
}
req, err := http.NewRequest("GET", a.groupEndpoint, nil)
if err != nil {
return groups.Groups, errors.Wrap(err, "creating new request to group endpoint")
@ -251,19 +257,18 @@ func (a *Auth) getGroups(token string) ([]Group, error) {
}
defer response.Body.Close()
rawGroups, err := io.ReadAll(response.Body)
if err != nil {
return groups.Groups, errors.Wrap(err, "failed reading group membership response")
}
if err = json.Unmarshal(rawGroups, &groups); err != nil {
if err = json.NewDecoder(response.Body).Decode(&groups); err != nil {
return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
}
a.groupsCache[token] = cachedGroups{
cacheTime: time.Now(),
groups: groups.Groups,
}
return groups.Groups, nil
}
func (a *Auth) setCookie(w http.ResponseWriter, token string, expiry time.Time) error {
func (a *Auth) SetCookie(w http.ResponseWriter, token string, expiry time.Time) error {
http.SetCookie(w, &http.Cookie{
Name: a.cookieName,
Value: token,
@ -276,6 +281,20 @@ func (a *Auth) setCookie(w http.ResponseWriter, token string, expiry time.Time)
return nil
}
func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, token string) error {
cookies := []string{}
if c, ok := md["cookie"]; ok {
for _, cookie := range c {
if strings.HasPrefix(cookie, a.cookieName) {
cookie = a.cookieName + "=" + token
}
cookies = append(cookies, cookie)
}
}
md["cookie"] = cookies
return grpc.SetHeader(ctx, md)
}
func decodeHex(hexstr string) ([]byte, error) {
data, err := hex.DecodeString(hexstr)
if err != nil {

486
authn/authenticate_internal_test.go
View file

@ -2,19 +2,24 @@ package authn
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strconv"
"strings"
"testing"
"time"
"github.com/golang-jwt/jwt"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
"golang.org/x/oauth2"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
func NewTestAuth(t *testing.T) *Auth {
@ -47,12 +52,11 @@ func NewTestAuth(t *testing.T) *Auth {
}
return a
}
func TestAuth(t *testing.T) {
a := NewTestAuth(t)
t.Run("SetCookie", func(t *testing.T) {
w := httptest.NewRecorder()
err := a.setCookie(w, "a cookie value", time.Now().Add(time.Hour))
err := a.SetCookie(w, "a cookie value", time.Now().Add(time.Hour))
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
@ -65,6 +69,42 @@ func TestAuth(t *testing.T) {
t.Fatalf("path=%s, want %s", got, want)
}
})
t.Run("SetGRPCMetadata", func(t *testing.T) {
md := metadata.MD{
"cookie": []string{a.cookieName + "=something"},
}
ctx := grpc.NewContextWithServerTransportStream(
metadata.NewIncomingContext(context.TODO(),
md,
),
NewServerTransportStream(),
)
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
err := a.SetGRPCMetadata(ctx, md, "this is a token!")
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
md, ok = metadata.FromIncomingContext(ctx)
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
c, ok := md["cookie"]
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
var cookie string
for _, cookie = range c {
if strings.HasPrefix(cookie, a.cookieName) {
break
}
}
if exp, got := a.cookieName+"=this is a token!", cookie; got != exp {
t.Fatalf("expected '%v', got '%v'", exp, got)
}
})
t.Run("KeyLength", func(t *testing.T) {
_, err := NewAuth(
logger.NewStandardLogger(os.Stdout),
@ -88,13 +128,19 @@ func TestAuth(t *testing.T) {
t.Fatalf("expected %v, got %v", got, want)
}
})
}
func TestAuthenticate(t *testing.T) {
cases := []struct {
name string
uid string
uname string
exp interface{}
groups []Group
err error
name string
uid string
uname string
exp int64
refresh bool
errOnRefresh bool
malformed bool
groups []Group
err error
}{
{
name: "GoodToken",
@ -108,7 +154,13 @@ func TestAuth(t *testing.T) {
},
},
{
name: "ExpiredToken",
name: "Malformed",
malformed: true,
err: fmt.Errorf("parsing bearer token: token contains an invalid number of segments"),
},
{
name: "ExpiredTokenNoRefresh",
uid: "42",
uname: "A. Token",
groups: []Group{
@ -117,30 +169,99 @@ func TestAuth(t *testing.T) {
GroupName: "adminGroup",
},
},
exp: "-17764800",
err: errors.Wrap(fmt.Errorf("Token is expired"), "parsing bearer token"),
exp: -17764800,
err: fmt.Errorf("token is expired"),
},
{
name: "ExpiredTokenYesRefresh",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
refresh: true,
exp: -17764800,
},
{
name: "ExpiredTokenYesRefreshButError",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
refresh: true,
errOnRefresh: true,
exp: -17764800,
err: fmt.Errorf("decoding refreshed token: invalid character 'b' looking for beginning of value"),
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
groupString, err := ToGob64(test.groups)
if err != nil {
t.Fatalf("unexpected error when gobbing groups %v", err)
// setup the test
a := NewTestAuth(t)
token := ""
var err error
if !test.malformed {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = test.uid
claims["name"] = test.uname
if test.exp != 0 {
claims["exp"] = strconv.Itoa(int(test.exp))
}
token, err = tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
} else {
token = "asdfasdfasdfasdF"
}
claims["molecula-idp-groups"] = groupString
claims["oid"] = test.uid
claims["name"] = test.uname
if test.exp != nil {
claims["exp"] = test.exp
if len(test.groups) > 0 {
a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
}
token, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
if test.refresh {
var srv *httptest.Server
if !test.errOnRefresh {
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = test.uid
claims["name"] = test.uname
expiry := strconv.Itoa(int(time.Now().Add(2 * time.Hour).Unix()))
claims["exp"] = expiry
fresh, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
fmt.Fprintf(w, `{"access_token": "`+fresh+`", "refresh_token": "blah", "token_type": "bearer", "expires": `+expiry+` }`)
}))
} else {
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "bad", http.StatusInternalServerError)
}))
}
defer srv.Close()
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.tokenCache[token] = cachedToken{
time.Now(),
&oauth2.Token{
AccessToken: token,
RefreshToken: "blah",
Expiry: time.Unix(test.exp, 0),
},
}
}
uinfo, err := a.Authenticate(token)
// do the actual testing
uinfo, err := a.Authenticate(context.TODO(), token)
// okay this part kind of sucks bc we need to check errors and i
// dont want to write a whole new test for things that should have
// errors just to avoid this mess. errors.Is doesn't work either
@ -167,34 +288,124 @@ func TestAuth(t *testing.T) {
}
}
func TestGobs(t *testing.T) {
t.Run("goodGob!", func(t *testing.T) {
g := []Group{
{
GroupID: "groupA",
GroupName: "groupA-Name",
},
{
GroupID: "groupB",
GroupName: "groupB-Name",
},
{
GroupID: "groupC",
GroupName: "groupC-Name",
},
func TestAuthenticate_CleanCache(t *testing.T) {
// this deserves its own test bc it has gross setup required
t.Run("should clean", func(t *testing.T) {
a := NewTestAuth(t)
now := time.Now()
a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
a.lastCacheClean = now.Add(-45 * time.Minute)
_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
if a.lastCacheClean.Sub(now) <= time.Nanosecond {
t.Fatalf("cache should have been cleaned")
}
gobbed, err := ToGob64(g)
if err != nil {
t.Fatalf("could not gob %+v", g)
if _, ok := a.groupsCache["oldy"]; ok {
t.Errorf("oldy should have been deleted")
}
ungobbed, err := FromGob64(gobbed)
if err != nil {
t.Fatalf("could not ungob %+v", gobbed)
if _, ok := a.groupsCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
if !reflect.DeepEqual(ungobbed, g) {
t.Fatalf("expected %v, got %v", g, ungobbed)
if _, ok := a.tokenCache["oldy"]; ok {
t.Errorf("oldy should have been deleted")
}
if _, ok := a.tokenCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
})
t.Run("shouldn't clean", func(t *testing.T) {
a := NewTestAuth(t)
now := time.Now()
a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
a.lastCacheClean = now
_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
if a.lastCacheClean.Sub(now) >= time.Nanosecond {
t.Fatalf("cache should not have been cleaned")
}
if _, ok := a.groupsCache["oldy"]; !ok {
t.Errorf("oldy should not have been deleted")
}
if _, ok := a.groupsCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
if _, ok := a.tokenCache["oldy"]; !ok {
t.Errorf("oldy should not have been deleted")
}
if _, ok := a.tokenCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
})
}
func TestGetGroups(t *testing.T) {
a := NewTestAuth(t)
a.groupsCache = map[string]cachedGroups{
"the world is changed": {
cacheTime: time.Now(),
groups: []Group{
{
GroupID: "i feel it in the water",
GroupName: "i feel it in the earth",
},
},
},
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := json.Marshal(
Groups{
Groups: []Group{
{
GroupID: "much that once was is lost",
GroupName: "for none now live who remember it",
},
},
},
)
if err != nil {
t.Fatalf("unexpected error marshalling groups response: %v", err)
}
fmt.Fprintf(w, "%s", body)
}))
a.groupEndpoint = srv.URL
for name, test := range map[string]struct {
token string
groups []Group
}{
"InCache": {
token: "the world is changed",
groups: []Group{
{
GroupID: "i feel it in the water",
GroupName: "i feel it in the earth",
},
},
},
"NotInCache": {
token: "i smell it in the air",
groups: []Group{
{
GroupID: "much that once was is lost",
GroupName: "for none now live who remember it",
},
},
},
} {
t.Run(name, func(t *testing.T) {
if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) {
t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err)
}
})
}
}
func TestDecodeHex(t *testing.T) {
@ -224,68 +435,6 @@ func TestDecodeHex(t *testing.T) {
})
}
func TestAddGroupMembership(t *testing.T) {
cases := []struct {
name string
groups []Group
err error
}{
{
name: "emptyGroups",
groups: []Group{},
err: nil,
},
{
name: "happyPath",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
err: nil,
},
}
a := NewTestAuth(t)
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
tkn := jwt.New(jwt.SigningMethodHS256)
token, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
tokenWithGroups, err := a.addGroupMembership(token, test.groups)
// okay this part kind of sucks bc we need to check errors and i
// dont want to write a whole new test for things that should have
// errors just to avoid this mess. errors.Is doesn't work either
if (test.err == nil && err != nil) || (test.err != nil && err == nil) {
t.Fatalf("expected %v but got %v", test.err, err)
} else if test.err != nil && err != nil {
if test.err.Error() != err.Error() {
t.Fatalf("expected %v, but got %v", test.err, err)
} else {
return
}
}
parsed, _, err := new(jwt.Parser).ParseUnverified(tokenWithGroups, jwt.MapClaims{})
if err != nil {
t.Fatalf("unexpected error parsing token %v", err)
}
claims := parsed.Claims.(jwt.MapClaims)
groups, err := FromGob64(claims["molecula-idp-groups"].(string))
if err != nil {
t.Fatalf("unexpected error parsing groupString %v", err)
}
if !reflect.DeepEqual(groups, test.groups) {
t.Fatalf("expected %v, got %v", test.groups, groups)
}
})
}
}
func TestHandlers(t *testing.T) {
a := NewTestAuth(t)
t.Run("login", func(t *testing.T) {
@ -304,6 +453,19 @@ func TestHandlers(t *testing.T) {
t.Run("logout", func(t *testing.T) {
req := httptest.NewRequest("GET", "/logout", nil)
w := httptest.NewRecorder()
req.AddCookie(
&http.Cookie{
Name: a.cookieName,
Value: "test",
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(3000000, 0),
},
)
a.groupsCache["test"] = cachedGroups{}
a.tokenCache["test"] = cachedToken{time.Now(), &oauth2.Token{}}
a.Logout(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
@ -314,7 +476,7 @@ func TestHandlers(t *testing.T) {
t.Fatalf("expected %v, got %v", redirect, got.Path)
}
for _, c := range resp.Cookies() {
if c.Name == "molecula-chip" {
if c.Name == a.cookieName {
if c.Value != "" {
t.Fatalf("cookie not set to empty value!")
}
@ -326,5 +488,105 @@ func TestHandlers(t *testing.T) {
break
}
}
if _, ok := a.groupsCache["test"]; ok {
t.Fatalf("groups not deleted!")
}
if _, ok := a.tokenCache["test"]; ok {
t.Fatalf("token not deleted!")
}
})
t.Run("redirectGood", func(t *testing.T) {
req := httptest.NewRequest("GET", "/redirect", nil)
w := httptest.NewRecorder()
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = "user id"
claims["name"] = "user name"
expiresIn := 2 * time.Hour
exp := time.Now().Add(expiresIn)
expiry := strconv.Itoa(int(exp.Unix()))
claims["exp"] = expiry
fresh, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
freshToken := oauth2.Token{
AccessToken: fresh,
RefreshToken: "blah",
Expiry: exp,
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write([]byte(body))
}))
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.Redirect(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
t.Fatalf("expected redirect, got %v", resp.StatusCode)
}
if got, err := resp.Location(); err != nil || got.String() != "/" {
t.Fatalf("expected %v, got %v", "/", got.Path)
}
cachedToken := a.tokenCache[fresh].token
if cachedToken.AccessToken != freshToken.AccessToken {
t.Fatalf("expected %v, got %v", freshToken.AccessToken, cachedToken.AccessToken)
}
if cachedToken.RefreshToken != freshToken.RefreshToken {
t.Fatalf("expected %v, got %v", freshToken.RefreshToken, cachedToken.RefreshToken)
}
if cachedToken.Expiry.Sub(freshToken.Expiry) > time.Second {
t.Fatalf("expected %v, got %v", freshToken.Expiry, cachedToken.Expiry)
}
})
t.Run("redirectBad", func(t *testing.T) {
req := httptest.NewRequest("GET", "/redirect", nil)
w := httptest.NewRecorder()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Server Error", http.StatusInternalServerError)
}))
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.Redirect(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
}
})
}
// This type is used for mocking ServerTransportStreams in tests
type ServerTransportStream struct {
md metadata.MD
method string
}
func NewServerTransportStream() *ServerTransportStream {
return &ServerTransportStream{
md: metadata.MD{},
method: "test",
}
}
func (s *ServerTransportStream) Method() string {
return s.method
}
func (s *ServerTransportStream) SetHeader(md metadata.MD) error {
s.md = md
return nil
}
func (s *ServerTransportStream) SendHeader(md metadata.MD) error {
_ = md
return nil
}
func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
_ = md
return nil
}

300
client.go
View file

@ -1,300 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"io"
"time"
"github.com/molecula/featurebase/v3/ingest"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/topology"
)
// Bit represents the intersection of a row and a column. It can be specified by
// integer ids or string keys.
type Bit struct {
RowID uint64
ColumnID uint64
RowKey string
ColumnKey string
Timestamp int64
}
// FieldValue represents the value for a column within a
// range-encoded field.
type FieldValue struct {
ColumnID uint64
ColumnKey string
Value int64
}
// InternalClient should be implemented by any struct that enables any transport between nodes
// TODO: Refactor
// Note from Travis: Typically an interface containing more than two or three methods is an indication that
// something hasn't been architected correctly.
// While I understand that putting the entire Client behind an interface might require this many methods,
// I don't want to let it go unquestioned.
// Another note from Travis: I think we eventually want to unify `InternalClient` with
// the `github.com/molecula/featurebase/v3/client` client.
// Doing that may obviate the need to refactor this.
type InternalClient interface {
InternalQueryClient
AvailableShards(ctx context.Context, indexName string) ([]uint64, error)
MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error)
PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error)
Nodes(ctx context.Context) ([]*topology.Node, error)
Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureField(ctx context.Context, indexName string, fieldName string) error
EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error
ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string) error
CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error)
RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error)
MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error)
IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error
IDAllocDataReader(ctx context.Context) (io.ReadCloser, error)
IDAllocDataWriter(ctx context.Context, f io.Reader, primary *topology.Node) error
IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error)
FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error)
StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
FinishTransaction(ctx context.Context, id string) (*Transaction, error)
Transactions(ctx context.Context) (map[string]*Transaction, error)
GetTransaction(ctx context.Context, id string) (*Transaction, error)
GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error)
GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error)
// ImportFieldKeys and ImportIndexKeys are mainly used when
// restoring a backup. They take a readerFunc which returns a
// reader rather than taking an io.Reader directly to allow for
// efficient retries (rather than reading the entire request body
// into a buffer and reusing it). Reader returned from the func
// must be properly closed by the implementation.
ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error
ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error
// SetInternalAPI tells the client the API it should use for internal/loopback ops
// where applicable.
SetInternalAPI(api *API)
}
//===============
// InternalQueryClient is the internal interface for querying a node.
type InternalQueryClient interface {
SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error)
QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
// Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist.
TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error)
TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, id []uint64) ([]string, error)
FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, like string) ([]uint64, error)
}
type nopInternalQueryClient struct{}
func (nopInternalQueryClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) {
return nil, nil
}
func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) {
return nil, nil
}
func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, like string) ([]uint64, error) {
return nil, nil
}
func newNopInternalQueryClient() nopInternalQueryClient {
return nopInternalQueryClient{}
}
var _ InternalQueryClient = newNopInternalQueryClient()
//===============
type nopInternalClient struct{ nopInternalQueryClient }
func newNopInternalClient() nopInternalClient {
return nopInternalClient{}
}
var _ InternalClient = newNopInternalClient()
func (n nopInternalClient) AvailableShards(ctx context.Context, indexName string) ([]uint64, error) {
return nil, nil
}
func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error {
return nil
}
func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
return nil
}
func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n nopInternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error {
return nil
}
func (n nopInternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error {
return nil
}
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
return nil
}
func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
return nil, nil
}
func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error {
return nil
}
func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) IDAllocDataWriter(cctx context.Context, f io.Reader, primary *topology.Node) error {
return nil
}
func (n nopInternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
return nil
}
func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
return nil
}
func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
return nil
}
func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil }
func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
return nil, nil
}
func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
return nil, nil, nil
}
func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
return nil
}
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) {
return nil, nil
}
func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
return nil, nil
}
func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error {
return nil
}
func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error {
return nil
}
func (c nopInternalClient) SetInternalAPI(api *API) {
}

22
client/batch_test.go
View file

@ -359,7 +359,7 @@ func testTrimNull(t *testing.T, c *test.Cluster, client *Client) {
t.Fatalf("querying: %v", err)
}
for i, result := range resp.Results() {
if 1 == i {
if i == 1 {
if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) {
t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns)
}
@ -1187,26 +1187,6 @@ outer:
return nil
}
func isPermutationOfInt(one, two []uint64) error {
if len(one) != len(two) {
return errors.Errorf("different lengths %d and %d", len(one), len(two))
}
outer:
for _, vOne := range one {
for j, vTwo := range two {
if vOne == vTwo {
two = append(two[:j], two[j+1:]...)
continue outer
}
}
return errors.Errorf("%d in one but not two", vOne)
}
if len(two) != 0 {
return errors.Errorf("vals in two but not one: %v", two)
}
return nil
}
func TestQuantizedTime(t *testing.T) {
cases := []struct {
name string

4
cluster.go
View file

@ -102,7 +102,7 @@ type cluster struct { // nolint: maligned
logger logger.Logger
InternalClient InternalClient
InternalClient *InternalClient
confirmDownRetries int
confirmDownSleep time.Duration
@ -120,7 +120,7 @@ func newCluster() *cluster {
translationSyncer: NopTranslationSyncer,
InternalClient: newNopInternalClient(),
InternalClient: &InternalClient{}, // TODO might have to fill this out a bit
logger: logger.NopLogger,

6
cmd/badloader/badloader.go
View file

@ -13,7 +13,7 @@ import (
gohttp "net/http"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/vprint"
@ -22,7 +22,7 @@ import (
"strings"
)
func UploadTar(srcFile string, client *http.InternalClient) error {
func UploadTar(srcFile string, client *pilosa.InternalClient) error {
t0 := time.Now()
f, err := os.Open(srcFile)
if err != nil {
@ -114,7 +114,7 @@ func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := http.NewInternalClient(host, h)
c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{}))
vprint.PanicOn(err)
tarSrcPath := "q2.tar.gz"

33
cmd/check.go
View file

@ -1,33 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (
"context"
"fmt"
"io"
"github.com/spf13/cobra"
"github.com/molecula/featurebase/v3/ctl"
)
var checker *ctl.CheckCommand
func newCheckCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
checker = ctl.NewCheckCommand(stdin, stdout, stderr)
checkCmd := &cobra.Command{
Use: "check <path> [path2]...",
Short: "Do a consistency check on a FeatureBase data file.",
Long: `
Performs a consistency check on data files.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("path required")
}
checker.Paths = args
return checker.Run(context.Background())
},
}
return checkCmd
}

23
cmd/check_test.go
View file

@ -1,23 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (
"strings"
"testing"
)
func TestCheckHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "check", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "featurebase check") || err != nil {
t.Fatalf("Command 'check --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestCheckNoPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "check")
if !strings.Contains(err.Error(), "path required") {
t.Fatalf("Command 'check' without args should error but: err: '%v', output: '%v'", err, output)
}
}

43
cmd/convert.go
View file

@ -1,43 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (
"context"
"fmt"
"io"
"github.com/spf13/cobra"
"github.com/molecula/featurebase/v3/ctl"
)
var inspector *ctl.InspectCommand
func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
inspector = ctl.NewInspectCommand(stdin, stdout, stderr)
inspectCmd := &cobra.Command{
Use: "inspect",
Short: "Get stats on a FeatureBase data file.",
Long: `
Inspects a data file and provides stats.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("path required")
} else if len(args) > 1 {
return fmt.Errorf("only one path allowed")
}
inspector.Path = args[0]
return inspector.Run(context.Background())
},
}
flags := inspectCmd.Flags()
flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
return inspectCmd
}

29
cmd/inspect_test.go
View file

@ -1,29 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (
"strings"
"testing"
)
func TestInspectHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "featurebase inspect") || err != nil {
t.Fatalf("Command 'inspect --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestInspectNoPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect")
if !strings.Contains(err.Error(), "path required") {
t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
}
}
func TestInspectMultiPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect", "one", "two")
if !strings.Contains(err.Error(), "only one path") {
t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
}
}

10
cmd/pilosa-bench/main.go
View file

@ -16,8 +16,8 @@ import (
"strings"
"time"
"github.com/molecula/featurebase/v3"
phttp "github.com/molecula/featurebase/v3/http"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/encoding/proto"
"golang.org/x/sync/errgroup"
)
@ -78,7 +78,7 @@ func run(ctx context.Context, args []string) (err error) {
rand.Seed(0)
// Setup connection to pilosa.
client, err := phttp.NewInternalClient(*hostport, http.DefaultClient)
client, err := pilosa.NewInternalClient(*hostport, http.DefaultClient, pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
return err
}
@ -270,7 +270,7 @@ func generateTopKQuery(index, field string, from, to time.Time) string {
}
// loadFields returns a mapping of index/field names to field info & identifiers.
func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) {
func loadFields(ctx context.Context, client *pilosa.InternalClient) (map[fieldKey]*fieldInfo, error) {
indexes, err := client.Schema(ctx)
if err != nil {
return nil, err
@ -299,7 +299,7 @@ func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey
}
// fetchFieldIDs returns a list of field IDs or keys.
func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
func fetchFieldIDs(ctx context.Context, client *pilosa.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
if err != nil {
return nil, err

5
cmd/random-query/main.go
View file

@ -16,9 +16,10 @@ import (
"time"
"github.com/gogo/protobuf/proto"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/client"
"github.com/molecula/featurebase/v3/http"
fb_proto "github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/vprint"
@ -162,7 +163,7 @@ func main() {
func (cfg *RandomQueryConfig) Run() (err error) {
remoteClient := nethttp.DefaultClient
cli, err := http.NewInternalClient(cfg.HostPort, remoteClient)
cli, err := pilosa.NewInternalClient(cfg.HostPort, remoteClient, pilosa.WithSerializer(fb_proto.Serializer{}))
if err != nil {
return err
}

3
cmd/random-query/main_test.go
View file

@ -9,7 +9,6 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
@ -35,7 +34,7 @@ func Test_RandomQuery(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID(nodeid[0]),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(nReplicas),
)},
)

2
cmd/root.go
View file

@ -53,12 +53,10 @@ at https://docs.molecula.cloud/.
rc.AddCommand(newChkSumCommand(stdin, stdout, stderr))
rc.AddCommand(newBackupCommand(stdin, stdout, stderr))
rc.AddCommand(newRestoreCommand(stdin, stdout, stderr))
rc.AddCommand(newCheckCommand(stdin, stdout, stderr))
rc.AddCommand(newConfigCommand(stdin, stdout, stderr))
rc.AddCommand(newExportCommand(stdin, stdout, stderr))
rc.AddCommand(newGenerateConfigCommand(stdin, stdout, stderr))
rc.AddCommand(newImportCommand(stdin, stdout, stderr))
rc.AddCommand(newInspectCommand(stdin, stdout, stderr))
rc.AddCommand(newRBFCommand(stdin, stdout, stderr))
rc.AddCommand(newServeCmd(stdin, stdout, stderr))
rc.AddCommand(newHolderCmd(stdin, stdout, stderr))

23
cmd/server_test.go
View file

@ -3,10 +3,12 @@ package cmd_test
import (
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/felixge/fgprof"
"github.com/molecula/featurebase/v3/cmd"
_ "github.com/molecula/featurebase/v3/test"
"github.com/molecula/featurebase/v3/testhook"
@ -23,12 +25,11 @@ func TestServerHelp(t *testing.T) {
}
// I have no idea why the linter in ci is complaining about this being unused.
func nextPort() string { //nolint:unused
func nextPort() string {
return fmt.Sprintf(`"localhost:%d"`, 0)
}
func TestServerConfig(t *testing.T) {
t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
actualDataDir, err := testhook.TempDir(t, "")
failErr(t, err, "making data dir")
logFile, err := testhook.TempFile(t, "")
@ -106,7 +107,7 @@ func TestServerConfig(t *testing.T) {
},
// TEST 2
{
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true", "--translation.map-size", "100000"},
args: []string{"server", "--log-path", logFile.Name(), "--translation.map-size", "100000"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:19444"
@ -175,7 +176,9 @@ func TestServerConfig(t *testing.T) {
}
}
func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
// if you don't pass an empty dir as data-dir it will use the
// default... which might be full of data and cause the test to
// run super slow.
actualDataDir, err := testhook.TempDir(t, "")
failErr(t, err, "making data dir")
@ -203,6 +206,7 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
cfgFileContent: `
bind = ` + nextPort() + `
bind-grpc = ` + nextPort() + `
data-dir = "` + actualDataDir + `"
`,
validation: func() error {
v := validator{}
@ -218,6 +222,7 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
cfgFileContent: `
bind = ` + nextPort() + `
bind-grpc = ` + nextPort() + `
data-dir = "` + actualDataDir + `"
`,
validation: func() error {
v := validator{}
@ -228,7 +233,11 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
},
},
}
out, err := os.Create("myprof.prof")
if err != nil {
t.Fatalf("creating prof file: %v", err)
}
stop := fgprof.Start(out, fgprof.FormatPprof)
// run server tests
for i, test := range tests {
t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
@ -257,4 +266,8 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
test.reset()
})
}
err = stop()
if err != nil {
t.Fatalf("stopping profile: %v", err)
}
}

8
cmd/slurp/slurp.go
View file

@ -18,7 +18,7 @@ import (
"time"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/vprint"
)
@ -32,7 +32,7 @@ type stateMachine struct {
lastField string
lastShard uint64
state string
client *http.InternalClient
client *pilosa.InternalClient
start time.Time
profile string
@ -136,7 +136,7 @@ func (r *stateMachine) Upload() error {
return nil
}
func UploadTar(srcFile string, client *http.InternalClient, profile, host string) error {
func UploadTar(srcFile string, client *pilosa.InternalClient, profile, host string) error {
f, err := os.Open(srcFile)
if err != nil {
@ -192,7 +192,7 @@ func main() {
if profile != "" {
startProfile(host)
}
c, err := http.NewInternalClient(host, h)
c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{}))
vprint.PanicOn(err)
t0 := time.Now()

23
ctl/backup.go
View file

@ -13,7 +13,7 @@ import (
"time"
pilosa "github.com/molecula/featurebase/v3"
fb_http "github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/topology"
"github.com/pkg/errors"
@ -42,11 +42,14 @@ type BackupCommand struct { // nolint: maligned
// Amount of time after first failed request to continue retrying.
RetryPeriod time.Duration `json:"retry-period"`
// Response Header Timeout for HTTP Requests
HeaderTimeout time.Duration `json:"header-timeout"`
// Host:port on which to listen for pprof.
Pprof string `json:"pprof"`
// Reusable client.
client pilosa.InternalClient
client *pilosa.InternalClient
// Standard input/output
*pilosa.CmdIO
@ -59,10 +62,11 @@ type BackupCommand struct { // nolint: maligned
// NewBackupCommand returns a new instance of BackupCommand.
func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand {
return &BackupCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
Concurrency: 1,
RetryPeriod: time.Minute,
Pprof: "localhost:0",
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
Concurrency: 1,
RetryPeriod: time.Minute,
HeaderTimeout: time.Second * 3,
Pprof: "localhost:0",
}
}
@ -89,7 +93,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
}
// Create a client to the server.
client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod))
client, err := commandClient(cmd, pilosa.WithClientRetryPeriod(cmd.RetryPeriod), pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout))
if err != nil {
return fmt.Errorf("creating client: %w", err)
}
@ -285,7 +289,10 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string,
logger := cmd.Logger()
logger.Printf("backing up shard: index=%q id=%d", indexName, shard)
client := fb_http.NewInternalClientFromURI(&node.URI, fb_http.GetHTTPClient(cmd.tlsConfig), fb_http.WithClientRetryPeriod(cmd.RetryPeriod))
client := pilosa.NewInternalClientFromURI(&node.URI,
pilosa.GetHTTPClient(cmd.tlsConfig, pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)),
pilosa.WithClientRetryPeriod(cmd.RetryPeriod),
pilosa.WithSerializer(proto.Serializer{}))
rc, err := client.ShardReader(ctx, indexName, shard)
if err != nil {
return fmt.Errorf("fetching shard reader: %w", err)

122
ctl/check.go
View file

@ -1,122 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"syscall"
"github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
)
// CheckCommand represents a command for performing consistency checks on data files.
type CheckCommand struct {
// Data file paths.
Paths []string
// Standard input/output
*pilosa.CmdIO
}
// NewCheckCommand returns a new instance of CheckCommand.
func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand {
return &CheckCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run executes the check command.
func (cmd *CheckCommand) Run(_ context.Context) error {
for _, path := range cmd.Paths {
switch filepath.Ext(path) {
case "":
if err := cmd.checkBitmapFile(path); err != nil {
return errors.Wrap(err, "checking bitmap")
}
case ".cache":
if err := cmd.checkCacheFile(path); err != nil {
return errors.Wrap(err, "checking cache")
}
case ".snapshotting":
if err := cmd.checkSnapshotFile(path); err != nil {
return errors.Wrap(err, "checking snapshot")
}
}
}
return nil
}
// checkBitmapFile performs a consistency check on path for a roaring bitmap file.
func (cmd *CheckCommand) checkBitmapFile(path string) (err error) {
// Open file handle.
f, err := os.Open(path)
if err != nil {
return errors.Wrap(err, "opening file")
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return errors.Wrap(err, "statting file")
}
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
return errors.Wrap(err, "mmapping")
}
defer func() {
e := syscall.Munmap(data)
if e != nil {
fmt.Fprintf(cmd.Stderr, "WARNING: munmap failed: %v", e)
}
// don't overwrite another error with this, but also indicate
// this error.
if err == nil {
err = e
}
}()
// Attach the mmap file to the bitmap.
bm := roaring.NewBitmap()
if err := bm.UnmarshalBinary(data); err != nil {
return errors.Wrap(err, "unmarshalling")
}
// Perform consistency check.
if err := bm.Check(); err != nil {
// Print returned errors.
switch err := err.(type) {
case roaring.ErrorList:
for i := range err {
fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err[i].Error())
}
default:
fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err.Error())
}
}
// Print success message if no errors were found.
fmt.Fprintf(cmd.Stdout, "%s: ok\n", path)
return nil
}
// checkCacheFile performs a consistency check on path for a cache file.
func (cmd *CheckCommand) checkCacheFile(path string) error {
fmt.Fprintf(cmd.Stderr, "%s: ignoring cache file\n", path)
return nil
}
// checkSnapshotFile performs a consistency check on path for a snapshot file.
func (cmd *CheckCommand) checkSnapshotFile(path string) error {
fmt.Fprintf(cmd.Stderr, "%s: ignoring snapshot file\n", path)
return nil
}

95
ctl/check_test.go
View file

@ -1,95 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"bytes"
"io"
"os"
"strings"
"testing"
"context"
"github.com/molecula/featurebase/v3/testhook"
)
func TestCheckCommand_RunCacheFile(t *testing.T) {
fi, err := testhook.TempFile(t, "test*.cache")
if err != nil {
t.Fatalf("creating test file: %v", err)
}
cacheFile := fi.Name()
rder := []byte{}
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewCheckCommand(stdin, w, w)
cm.Paths = []string{cacheFile}
err = cm.Run(context.Background())
w.Close()
var buf bytes.Buffer
if _, err := io.Copy(&buf, r); err != nil {
t.Fatalf("copy: %v", err)
}
if !strings.Contains(buf.String(), "ignoring cache file") {
t.Fatalf("expect: ignoring cache file, actual: '%s'", err)
}
}
func TestCheckCommand_RunSnapshot(t *testing.T) {
fi, err := testhook.TempFile(t, "test*.snapshotting")
if err != nil {
t.Fatalf("creating test file: %v", err)
}
snapshotFile := fi.Name()
rder := []byte{}
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewCheckCommand(stdin, w, w)
cm.Paths = []string{snapshotFile}
err = cm.Run(context.Background())
w.Close()
var buf bytes.Buffer
if _, err := io.Copy(&buf, r); err != nil {
t.Fatalf("copy: %v", err)
}
if !strings.Contains(buf.String(), "ignoring snapshot file") {
t.Fatalf("expect: ignoring snapshot file, actual: '%s'", err)
}
}
func TestCheckCommand_Run(t *testing.T) {
file, err := testhook.TempFile(t, "run-command")
if err != nil {
t.Fatal(err)
}
fname := file.Name()
if _, err := file.Write([]byte("1234,1223")); err != nil {
t.Fatalf("writing to temp file: %v", err)
}
file.Close()
rder := []byte{}
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewCheckCommand(stdin, w, w)
cm.Paths = []string{fname}
err = cm.Run(context.Background())
w.Close()
var buf bytes.Buffer
if _, err := io.Copy(&buf, r); err != nil {
t.Fatalf("copy: %v", err)
}
expectedPrefix := "checking bitmap: unmarshalling: "
if !strings.HasPrefix(err.Error(), expectedPrefix) {
t.Fatalf("expect error: '%s...', actual: '%s'", expectedPrefix, err)
}
// Todo: need correct roaring file for happy path
}

2
ctl/chksum.go
View file

@ -20,7 +20,7 @@ type ChkSumCommand struct { // nolint: maligned
Host string `json:"host"`
// Reusable client.
client pilosa.InternalClient
client *pilosa.InternalClient
// Standard input/output
*pilosa.CmdIO

50
ctl/common.go
View file

@ -2,12 +2,10 @@
package ctl
import (
"net"
"time"
gohttp "net/http"
"github.com/molecula/featurebase/v3/http"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/server"
"github.com/pkg/errors"
@ -30,24 +28,50 @@ func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string,
flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
}
// default dial timeout is 30s for some reason which makes testing
// failures/retries really awkward. I don't think we need it that
// high, so I set it to 1s here... let's see what happens.
func clientOptions(client *gohttp.Client, dialer *net.Dialer) *gohttp.Client {
dialer.Timeout = time.Second * 1
return client
}
// AnyClientOption can be either pilosa.InternalClientOption or
// pilosa.ClientOption. The internal options are specific to the
// featurebase client, whereas the client options are applied to the
// Go HTTP client that gets used under the hood.
type AnyClientOption interface{}
// commandClient returns a pilosa.InternalHTTPClient for the command
func commandClient(cmd CommandWithTLSSupport, opts ...http.InternalClientOption) (*http.InternalClient, error) {
func commandClient(cmd CommandWithTLSSupport, opts ...AnyClientOption) (*pilosa.InternalClient, error) {
internalopts, clientopts, err := separateOptions(opts...)
if err != nil {
return nil, errors.Wrap(err, "separating client options")
}
// we default dial timeout to 3s in commandClient, but prepend it
// to the option list so other options can override it.
clientopts = append([]pilosa.ClientOption{pilosa.ClientDialTimeoutOption(time.Second * 3)}, clientopts...)
internalopts = append([]pilosa.InternalClientOption{pilosa.WithSerializer(proto.Serializer{})}, internalopts...)
tls := cmd.TLSConfiguration()
tlsConfig, err := server.GetTLSConfig(&tls, cmd.Logger())
if err != nil {
return nil, errors.Wrap(err, "getting tls config")
}
client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(tlsConfig, clientOptions), opts...)
client, err := pilosa.NewInternalClient(cmd.TLSHost(), pilosa.GetHTTPClient(tlsConfig, clientopts...), internalopts...)
if err != nil {
return nil, errors.Wrap(err, "getting internal client")
}
return client, err
}
// separateOptions splits the list of AnyClientOption into the two
// possible types.
func separateOptions(opts ...AnyClientOption) ([]pilosa.InternalClientOption, []pilosa.ClientOption, error) {
internalopts := []pilosa.InternalClientOption{}
clientopts := []pilosa.ClientOption{}
for _, opt := range opts {
if iopt, ok := opt.(pilosa.InternalClientOption); ok {
internalopts = append(internalopts, iopt)
continue
}
if copt, ok := opt.(pilosa.ClientOption); ok {
clientopts = append(clientopts, copt)
continue
}
return nil, nil, errors.Errorf("opt: %+v of type %[1]T must be an InternalClientOption or a ClientOption", opt)
}
return internalopts, clientopts, nil
}

4
ctl/import.go
View file

@ -11,7 +11,7 @@ import (
"strconv"
"time"
"github.com/molecula/featurebase/v3"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/server"
"github.com/pkg/errors"
@ -48,7 +48,7 @@ type ImportCommand struct { // nolint: maligned
Sort bool `json:"sort"`
// Reusable client.
client pilosa.InternalClient
client *pilosa.InternalClient
// Standard input/output
*pilosa.CmdIO

25
ctl/import_test.go
View file

@ -5,10 +5,12 @@ import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strings"
@ -582,6 +584,22 @@ func TestImport_AuthOn(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create query log file: %s", err)
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := json.Marshal(
authn.Groups{
Groups: []authn.Group{
{
GroupID: "group-id-test",
GroupName: "group-id-test",
},
},
},
)
if err != nil {
t.Fatalf("unexpected error marshalling groups response: %v", err)
}
fmt.Fprintf(w, "%s", body)
}))
auth := server.Auth{
Enable: true,
@ -589,7 +607,7 @@ func TestImport_AuthOn(t *testing.T) {
ClientSecret: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
AuthorizeURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
TokenURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
GroupEndpointURL: "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
GroupEndpointURL: srv.URL,
LogoutURL: "https://login.microsoftonline.com/common/oauth2/v2.0/logout",
Scopes: []string{"https://graph.microsoft.com/.default", "offline_access"},
SecretKey: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
@ -612,14 +630,13 @@ func TestImport_AuthOn(t *testing.T) {
conf.TLS.SkipVerify = true
commandOpts[i] = append(commandOpts[i], server.OptCommandConfig(conf))
}
a, err := authn.NewAuth(
logger.NewStandardLogger(os.Stdout),
"http://localhost:0/",
auth.Scopes,
auth.AuthorizeURL,
auth.TokenURL,
auth.GroupEndpointURL,
srv.URL,
auth.LogoutURL,
auth.ClientId,
auth.ClientSecret,
@ -632,8 +649,6 @@ func TestImport_AuthOn(t *testing.T) {
// make a valid token
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}})
claims["molecula-idp-groups"] = groupString
claims["oid"] = "42"
claims["name"] = "valid"
token, err := tkn.SignedString([]byte(a.SecretKey()))

394
ctl/inspect.go
View file

@ -1,394 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"context"
"encoding/binary"
"fmt"
"hash/fnv"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"text/tabwriter"
"time"
"unsafe"
"github.com/gogo/protobuf/proto"
"github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
)
// InspectCommand represents a command for inspecting fragment data files.
type InspectCommand struct {
// Path to data file
Path string
// don't list details of objects
Quiet bool
// list only this many objects
Max int
// Filters:
InspectOpts pilosa.InspectRequest
// Standard input/output
*pilosa.CmdIO
}
// NewInspectCommand returns a new instance of InspectCommand.
func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectCommand {
return &InspectCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
type pointerContext struct {
from, to uintptr
}
func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
var pointer string
if c.Mapped {
if c.Pointer >= p.from && c.Pointer < p.to {
pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
} else {
pointer = fmt.Sprintf("!0x%x!", c.Pointer)
}
} else {
pointer = fmt.Sprintf("0x%x", c.Pointer)
}
return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
}
func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) {
fmt.Fprintln(cmd.Stdout, " Ops:")
tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
printed := 0
for _, op := range info.OpDetails {
fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
printed++
if cmd.Max != 0 && printed >= cmd.Max {
break
}
}
tw.Flush()
}
func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) {
fmt.Fprintln(cmd.Stdout, " Containers:")
tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
c1s := info.Containers
c2s := info.OpContainers
l1 := len(c1s)
l2 := len(c2s)
i1 := 0
i2 := 0
var c1, c2 roaring.ContainerInfo
c1.Key = ^uint64(0)
c2.Key = ^uint64(0)
c1e := false
c2e := false
if i1 < l1 {
c1 = c1s[i1]
i1++
c1e = true
}
if i2 < l2 {
c2 = c2s[i2]
i2++
c2e = true
}
printed := 0
for c1e || c2e {
c1used := false
c2used := false
var key uint64
c1fmt := "-\t\t\t"
c2fmt := "-\t\t\t"
// If c2 exists, we'll always prefer its flags,
// if it doesn't, this gets overwritten.
flags := c2.Flags
if !c2e || (c1e && c1.Key < c2.Key) {
c1fmt = pC.pretty(c1)
key = c1.Key
c1used = true
flags = c1.Flags
} else if !c1e || (c2e && c2.Key < c1.Key) {
c2fmt = pC.pretty(c2)
key = c2.Key
c2used = true
} else {
// c1e and c2e both set, and neither key is < the other.
c1fmt = pC.pretty(c1)
c2fmt = pC.pretty(c2)
key = c1.Key
c1used = true
c2used = true
}
if c1used {
if i1 < l1 {
c1 = c1s[i1]
i1++
} else {
c1e = false
}
}
if c2used {
if i2 < l2 {
c2 = c2s[i2]
i2++
} else {
c2e = false
}
}
fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
printed++
if cmd.Max > 0 && printed >= cmd.Max {
break
}
}
tw.Flush()
}
// Run executes the inspect command.
func (cmd *InspectCommand) Run(ctx context.Context) error {
// Open file handle.
f, err := os.Open(cmd.Path)
if err != nil {
return errors.Wrap(err, "opening file")
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return errors.Wrap(err, "statting file")
}
if fi.IsDir() {
total := 0
infos, err := f.Readdir(0)
if err != nil {
return err
}
if len(infos) == 0 {
return errors.New("directory contains no files")
}
names := make([]string, len(infos))
nameToInfo := make(map[string]os.FileInfo, len(infos))
// find numeric-only names; we'll operate on
// either those, or the whole holder if we find
// a .topology file.
n := 0
for _, fi := range infos {
name := fi.Name()
if name == ".topology" {
return cmd.InspectHolder(ctx, cmd.Path)
}
if _, err := strconv.Atoi(name); err == nil {
names[n] = name
nameToInfo[name] = fi
n++
}
}
if n == 0 {
return fmt.Errorf("directory contains no fragments (looking for numeric names)")
}
names = names[:n]
fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n)
for _, name := range names {
f2, err := os.Open(filepath.Join(cmd.Path, name))
if err != nil {
return fmt.Errorf("opening %q: %v", name, err)
}
fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name)
err = cmd.InspectFile(f2, nameToInfo[name])
total++
f2.Close()
if err != nil {
return fmt.Errorf("inspecting %q: %v", name, err)
}
}
return nil
}
return cmd.InspectFile(f, fi)
}
// loadTopology is copied almost exactly from pilosa/cluster.go.
func loadTopology(path string) (topology pb.Topology, myID string, err error) {
buf, err := ioutil.ReadFile(filepath.Join(path, ".topology"))
if os.IsNotExist(err) {
return topology, myID, err
} else if err != nil {
return topology, myID, errors.Wrap(err, "reading file")
}
if err := proto.Unmarshal(buf, &topology); err != nil {
return topology, myID, errors.Wrap(err, "unmarshalling")
}
sort.Slice(topology.NodeIDs,
func(i, j int) bool {
return topology.NodeIDs[i] < topology.NodeIDs[j]
})
buf, err = ioutil.ReadFile(filepath.Join(path, ".id"))
if os.IsNotExist(err) {
return topology, myID, err
} else if err != nil {
return topology, myID, nil
}
myID = strings.TrimSpace(string(buf))
return topology, myID, nil
}
var partitions = make(map[string]map[uint64]int)
func findPartition(index string, shard uint64, partitionN int) (partition int) {
var shardMap map[uint64]int
var ok bool
if shardMap, ok = partitions[index]; !ok {
shardMap = make(map[uint64]int)
partitions[index] = shardMap
}
if partition, ok = shardMap[shard]; !ok {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], shard)
// Hash the bytes and mod by partition count.
h := fnv.New64a()
_, _ = h.Write([]byte(index))
_, _ = h.Write(buf[:])
partition = int(h.Sum64() % uint64(partitionN))
shardMap[shard] = partition
}
return partition
}
func findPartitionPath(path string, partitionN int) (int, error) {
parts := strings.Split(path, "/")
shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64)
if err != nil {
return 0, err
}
return findPartition(parts[0], shard, partitionN), nil
}
func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error {
holder := pilosa.NewHolder(path, nil)
holder.Opts.Inspect = true
holder.Opts.ReadOnly = true
err := holder.Open()
if err != nil {
return fmt.Errorf("%s: holder open: %v", path, err)
}
holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts)
if err != nil {
return fmt.Errorf("%s: inspect: %v", path, err)
}
myPartition := 0
topology, myID, err := loadTopology(path)
if err == nil {
fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID)
if len(topology.NodeIDs) > 1 {
fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID)
} else {
fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID)
}
found := false
for i := range topology.NodeIDs {
if topology.NodeIDs[i] == myID {
found = true
myPartition = i
break
}
}
if !found {
fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs)
}
} else {
fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err)
}
for _, name := range holderInfo.FragmentNames {
partition, err := findPartitionPath(name, len(topology.NodeIDs))
if err != nil {
fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err)
} else {
if partition == myPartition {
fmt.Fprintf(cmd.Stdout, "%s:\n", name)
} else {
fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition])
}
}
details := holderInfo.FragmentInfo[name]
cmd.DisplayInfo(details.BitmapInfo)
if details.BlockChecksums != nil {
fmt.Fprintf(cmd.Stdout, " Checksums [%d total]:\n", len(details.BlockChecksums))
for _, block := range details.BlockChecksums {
fmt.Fprintf(cmd.Stdout, " %8d: %x\n", block.ID, block.Checksum)
}
}
}
return nil
}
func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error {
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
return errors.Wrap(err, "mmapping")
}
defer func() {
err := syscall.Munmap(data)
if err != nil {
fmt.Fprintf(cmd.Stderr, "inspect command: munmap failed: %v", err)
}
}()
mappedFrom := uintptr(unsafe.Pointer(&data[0]))
mappedTo := mappedFrom + uintptr(len(data))
// Attach the mmap file to the bitmap.
t := time.Now()
fmt.Fprintf(cmd.Stderr, "inspecting bitmap...")
var info roaring.BitmapInfo
bitmap, _, err := roaring.InspectBinary(data, true, &info)
fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))
cmd.DisplayInfo(info)
if err != nil {
return errors.Wrap(err, "inspecting")
}
mappedIn, mappedOut, unmappedIn, errs, err := bitmap.SanityCheckMapping(mappedFrom, mappedTo)
if err != nil {
fmt.Fprintf(cmd.Stderr, "sanity check: %d mapped in, %d mapped out, %d unmapped in, %d errors\n",
mappedIn, mappedOut, unmappedIn, errs)
fmt.Fprintf(cmd.Stderr, "last error: %v\n", err)
}
return nil
}
func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) {
pC := pointerContext{
from: info.From,
to: info.To,
}
// Print top-level info.
fmt.Fprintf(cmd.Stdout, " Bitmap Info:\n")
fmt.Fprintf(cmd.Stdout, " Bits: %d\n", info.BitCount)
fmt.Fprintf(cmd.Stdout, " Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers))
fmt.Fprintf(cmd.Stdout, " Operations: %d (%d bits)\n", info.Ops, info.OpN)
fmt.Fprintln(cmd.Stdout, "")
// Print info for each container.
if !cmd.Quiet {
if info.ContainerCount > 0 {
cmd.PrintContainers(info, pC)
}
if info.Ops > 0 {
cmd.PrintOps(info)
}
}
}

48
ctl/inspect_test.go
View file

@ -1,48 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"bytes"
"context"
"io"
"os"
"strings"
"testing"
"github.com/molecula/featurebase/v3/testhook"
)
func TestInspectCommand_Run(t *testing.T) {
rder := []byte{}
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewInspectCommand(stdin, w, w)
file, err := testhook.TempFile(t, "inspectTest")
if err != nil {
t.Fatalf("Error creating tempfile: %s", err)
}
_, err = file.Write([]byte("12358267538963"))
if err != nil {
t.Fatalf("writing to tempfile: %v", err)
}
file.Close()
cm.Path = file.Name()
err = cm.Run(context.Background())
expectedError := "inspecting: "
if !strings.Contains(err.Error(), expectedError) {
t.Fatalf("expected error '%s', got '%v'", expectedError, err)
}
w.Close()
var buf bytes.Buffer
_, err = io.Copy(&buf, r)
if err != nil {
t.Fatalf("copying data: %v", err)
}
if !strings.Contains(buf.String(), "inspecting bitmap...") {
t.Fatalf("Inspect doesn't work: %s", err)
}
// Todo: need correct roaring file for happy path
}

5
ctl/restore.go
View file

@ -18,7 +18,6 @@ import (
"github.com/hashicorp/go-retryablehttp"
pilosa "github.com/molecula/featurebase/v3"
fb_http "github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/topology"
@ -44,7 +43,7 @@ type RestoreCommand struct {
Pprof string `json:"pprof"`
// Reusable client.
client pilosa.InternalClient
client *pilosa.InternalClient
// Standard input/output
*pilosa.CmdIO
@ -86,7 +85,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
return fmt.Errorf("parsing tls config: %w", err)
}
// Create a client to the server.
client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod))
client, err := commandClient(cmd, pilosa.WithClientRetryPeriod(cmd.RetryPeriod))
if err != nil {
return fmt.Errorf("creating client: %w", err)
}

9
ctl/server.go
View file

@ -2,7 +2,6 @@
package ctl
import (
"fmt"
"time"
"github.com/molecula/featurebase/v3/server"
@ -75,13 +74,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
// Storage
// Note: the default for --storage.backend must be kept "" empty string.
// Otherwise we cannot detect and honor the PILOSA_STORAGE_BACKEND env var
// over-ride.
// TODO: the comment above was carried over from the PILOSA_TXSRC flag, but
// we should confirm that this still applies.
flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring or rbf. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend))
flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, "Storage backend to use: 'rbf' is only supported value.")
flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk")
// RBF specific flags. See pilosa/rbf/cfg/cfg.go for definitions.

74
dbshard.go
View file

@ -67,9 +67,8 @@ type DBShard struct {
Shard uint64
Open bool
typ txtype
styp string
hasRoaring bool // if either of the types is roaringTxn
typ txtype
styp string
W DBWrapper
ParentDBIndex *DBIndex
@ -131,8 +130,7 @@ type DBPerShard struct {
// Easily see how many we have.
Flatmap map[flatkey]*DBShard
typ txtype
hasRoaring bool
typ txtype
txf *TxFactory
holder *Holder
@ -238,12 +236,6 @@ func newShardSet() *shardSet {
shardsMap: make(map[uint64]bool),
}
}
func newShardSetFromMap(m map[uint64]bool) *shardSet {
return &shardSet{
shardsMap: m,
shardsVer: 1,
}
}
func (per *DBPerShard) LoadExistingDBs() (err error) {
idxs := per.holder.Indexes()
@ -269,11 +261,6 @@ func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder
vprint.PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here")
}
hasRoaring := false
if typ == roaringTxn {
hasRoaring = true
}
d = &DBPerShard{
typ: typ,
HolderDir: holderDir,
@ -281,7 +268,6 @@ func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder
dbh: NewDBHolder(),
Flatmap: make(map[flatkey]*DBShard),
txf: txf,
hasRoaring: hasRoaring,
index2shards: newIndex2Shards(),
StorageConfig: holder.cfg.StorageConfig,
RBFConfig: holder.cfg.RBFConfig,
@ -407,10 +393,7 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In
}
dbs, ok = dbi.Shard[shard]
if dbs != nil && dbs.closed {
// roaring txn are nil/fake anyway. Don't freak out.
if per.typ != roaringTxn {
vprint.PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ))
}
vprint.PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ))
}
if !ok {
dbs = &DBShard{
@ -421,7 +404,6 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In
HolderPath: per.HolderDir,
idx: idx,
per: per,
hasRoaring: per.hasRoaring,
}
dbs.styp = per.typ.String()
dbi.Shard[shard] = dbs
@ -430,8 +412,6 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In
if !dbs.Open {
var registry DBRegistry
switch dbs.typ {
case roaringTxn:
registry = globalRoaringReg
case rbfTxn:
registry = globalRbfDBReg
registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig)
@ -470,8 +450,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir
return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData)
}
// if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover
// all the view paths under idx for type ty.
// requireData means open the database file and verify that at least one key is set.
// The returned sliceOfShards should not be modified. We will cache it for subsequent
// queries.
@ -485,14 +463,6 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
per.Mu.Lock()
defer per.Mu.Unlock()
if ty == roaringTxn && roaringViewPath != "" {
shardMap, err := roaringMapOfShards(roaringViewPath)
if err != nil {
return nil, err
}
return shardMap, nil
}
i2ss := per.index2shards
ss, ok := i2ss[idx.name]
@ -507,27 +477,6 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
// Upon return, cache the setOfShards value and reuse it next time
if ty == roaringTxn {
// INVAR: roaringViewPath == "", because the other case is
// handled above.
fields := idx.Fields()
for _, field := range fields {
for _, view := range field.views() {
shardMap, err := roaringMapOfShards(view.path)
if err != nil {
return nil,
errors.Wrap(err, fmt.Sprintf(
"TypedDBPerShardGetLocalShardsForIndex roaringTxn view.path='%v'", view.path))
}
for shard := range shardMap {
setOfShards.add(shard)
}
}
}
return setOfShards.CloneMaybe(), nil
}
// INVAR: not-roaring.
path := per.prefixForType(idx, ty)
ignoreEmpty := false
@ -627,19 +576,6 @@ func (vs *FieldView2Shards) getViewsForField(field string) map[string]*shardSet
return vs.m[field]
}
func (vs *FieldView2Shards) has(field, view string, shard uint64) bool {
vw, ok := vs.m[field]
if !ok {
return false
}
ss, ok := vw[view]
if !ok {
return false
}
shardMap := ss.CloneMaybe()
return shardMap[shard]
}
func (vs *FieldView2Shards) addViewShardSet(fv txkey.FieldView, ss *shardSet) {
f, ok := vs.m[fv.Field]
@ -730,8 +666,6 @@ func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView
ty := per.typ
switch ty {
case roaringTxn:
return roaringGetFieldView2Shards(idx)
default:
vs = NewFieldView2Shards()

71
dbshard_internal_test.go
View file

@ -71,7 +71,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
}
for _, src := range []string{"roaring", "rbf"} {
for _, src := range []string{"rbf"} {
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = src
holder := NewHolder(tmpdir, cfg)
@ -82,7 +82,6 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index)
PanicOn(err)
}
estd := "rick/fields/_exists/views/standard"
std := "rick/fields/f/views/standard"
shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
@ -93,65 +92,23 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
}
}
if src == "roaring" {
// check estd too
shards, err = holder.txf.GetShardsForIndex(idx, tmpdir+sep+estd, false)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
fvs, err := tx.GetSortedFieldViewList(idx, shard)
PanicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
if !shards[shard] {
panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
}
// expect these same two field/views for all 6 shards
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
if len(fvs) != 2 {
panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
}
// check GetSortedFieldViewList() and roaringGetFieldView2Shards()
vs, err := roaringGetFieldView2Shards(idx)
PanicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
fvs, err := tx.GetSortedFieldViewList(idx, shard)
PanicOn(err)
// expect these same two field/views for all 6 shards
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
if len(fvs) != 2 {
panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
}
if fvs[0] != expect0 {
panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
}
if fvs[1] != expect1 {
panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
}
for _, fv := range fvs {
if !vs.has(fv.Field, fv.View, shard) {
panic(fmt.Sprintf("vs did not contain fv='%#v' for shard %v", fv, shard))
}
}
tx.Rollback()
if fvs[0] != expect0 {
panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
}
} else {
// non-roaring: rbf
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
fvs, err := tx.GetSortedFieldViewList(idx, shard)
PanicOn(err)
// expect these same two field/views for all 6 shards
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
if len(fvs) != 2 {
panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
}
if fvs[0] != expect0 {
panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
}
if fvs[1] != expect1 {
panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
}
tx.Rollback()
if fvs[1] != expect1 {
panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1]))
}
tx.Rollback()
}
holder.Close()
}

5
dbshard_test.go
View file

@ -7,9 +7,8 @@ import (
"reflect"
"testing"
"github.com/molecula/featurebase/v3"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
@ -23,7 +22,7 @@ func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()

19
executor.go
View file

@ -55,7 +55,7 @@ type executor struct {
workCounter uint64
// Client used for remote requests.
client InternalQueryClient
client *InternalClient
// Maximum number of Set() or Clear() commands per request.
MaxWritesPerRequest int
@ -74,7 +74,7 @@ type executor struct {
// executorOption is a functional option type for pilosa.Executor
type executorOption func(e *executor) error
func optExecutorInternalQueryClient(c InternalQueryClient) executorOption {
func optExecutorInternalQueryClient(c *InternalClient) executorOption {
return func(e *executor) error {
e.client = c
return nil
@ -116,7 +116,6 @@ func emptyResult(c *pql.Call) interface{} {
// newExecutor returns a new instance of Executor.
func newExecutor(opts ...executorOption) *executor {
e := &executor{
client: newNopInternalQueryClient(),
workerPoolSize: 2,
}
for _, opt := range opts {
@ -1651,6 +1650,9 @@ func (d *DistinctTimestamp) Union(other DistinctTimestamp) DistinctTimestamp {
return DistinctTimestamp{Name: d.Name, Values: vals}
}
const ViewNotFound = Error("view not found")
const FragmentNotFound = Error("fragment not found")
func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) {
index := idx.Name()
tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard})
@ -3655,9 +3657,20 @@ func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index stri
}
// Apply bases.
//
// SUP-139: The group value is shared across multiple groups so we can't
// add the base to each one. Instead, we need to track which ones have been
// seen already and avoid adding to those again in the future.
for i, base := range bases {
m := make(map[*int64]struct{})
for _, r := range results {
if _, ok := m[r.Group[i].Value]; ok {
continue
}
*r.Group[i].Value += base
m[r.Group[i].Value] = struct{}{}
}
}

43
executor_test.go
View file

@ -29,11 +29,9 @@ import (
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/ctl"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/proto"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/storage"
"github.com/molecula/featurebase/v3/test"
"github.com/molecula/featurebase/v3/testhook"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
@ -1277,15 +1275,6 @@ func TestExecutor_Execute_Count(t *testing.T) {
}
func roaringOnlyTest(t *testing.T) {
src := pilosa.CurrentBackend()
if src == pilosa.RoaringTxn || (storage.DefaultBackend == pilosa.RoaringTxn && src == "") {
// okay to run, we are under roaring only
} else {
t.Skip("skip for everything but roaring")
}
}
// Ensure a set query can be executed.
func TestExecutor_Execute_Set(t *testing.T) {
t.Run("RowIDColumnID", func(t *testing.T) {
@ -3836,7 +3825,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
c := test.MustRunCluster(t, 1, []server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
),
})
defer c.Close()
@ -4227,7 +4216,7 @@ func TestExecutor_Execute_All(t *testing.T) {
c := test.MustRunCluster(t, 1, []server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
),
})
defer c.Close()
@ -6161,6 +6150,34 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
test.CheckGroupByOnKey(t, expected, results)
})
// SUP-139: GroupBy returns incorrect results when two or more Integer Range Fields are used to define the grouping
t.Run("CountByIntegersWithMinMax", func(t *testing.T) {
c.CreateField(t, "cbimm", pilosa.IndexOptions{}, "year", pilosa.OptFieldTypeInt(2019, 2020))
c.CreateField(t, "cbimm", pilosa.IndexOptions{}, "quarter", pilosa.OptFieldTypeInt(1, 4))
c.ImportIntID(t, "cbimm", "year", []test.IntID{{ID: 1, Val: 2019}, {ID: 2, Val: 2019}, {ID: 3, Val: 2019}, {ID: 4, Val: 2019}})
c.ImportIntID(t, "cbimm", "quarter", []test.IntID{{ID: 1, Val: 1}, {ID: 2, Val: 1}, {ID: 3, Val: 1}, {ID: 4, Val: 2}})
year2019 := int64(2019)
quarter1, quarter2 := int64(1), int64(2)
results := c.Query(t, "cbimm", `GroupBy(Rows(year), Rows(quarter))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t,
[]pilosa.GroupCount{
{Group: []pilosa.FieldRow{
{Field: "year", RowID: 0, Value: &year2019},
{Field: "quarter", RowID: 0, Value: &quarter1},
}, Count: 3},
{Group: []pilosa.FieldRow{
{Field: "year", RowID: 0, Value: &year2019},
{Field: "quarter", RowID: 0, Value: &quarter2},
}, Count: 1},
},
results,
)
})
}
for _, size := range []int{1, 3} {
t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) {

1
field_internal_test.go
View file

@ -247,7 +247,6 @@ func NewTestField(t testing.TB, opts FieldOption) *TestField {
}
cfg := DefaultHolderConfig()
cfg.StorageConfig.Backend = CurrentBackendOrDefault()
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
h := NewHolder(path, cfg)

347
fragment.go
View file

@ -3,7 +3,6 @@ package pilosa
import (
"archive/tar"
"bufio"
"bytes"
"container/heap"
"context"
@ -16,14 +15,11 @@ import (
"math/bits"
"os"
"path/filepath"
"runtime/debug"
"sort"
"strconv"
"strings"
"sync"
"syscall"
"time"
"unsafe"
"github.com/cespare/xxhash"
"github.com/gogo/protobuf/proto"
@ -59,18 +55,12 @@ const (
// width of roaring containers is 2^16
containerWidth = 1 << 16
// snapshotExt is the file extension used for an in-process snapshot.
snapshotExt = ".snapshotting"
// cacheExt is the file extension for persisted cache ids.
cacheExt = ".cache"
// HashBlockSize is the number of rows in a merkle hash block.
HashBlockSize = 100
// defaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
defaultFragmentMaxOpN = 10000
// Row ids used for boolean fields.
falseRowID = uint64(0)
trueRowID = uint64(1)
@ -132,23 +122,9 @@ type fragment struct {
// idx cached to avoid repeatedly looking it up everywhere.
idx *Index
// parent holder, used to find snapshot queue, etc.
// parent holder
holder *Holder
// debugging tool: addresses of current and previous maps
prevdata, currdata struct{ from, to uintptr }
// File-backed storage
flags byte // user-defined flags passed to roaring
storage *roaring.Bitmap
opN int // number of ops since snapshot (may be approximate for imports)
ops int // number of higher-level operations, as opposed to bit changes
snapshotPending bool // set to true when requesting a snapshot, set to false after snapshot completes
snapshotCond sync.Cond
snapshotErr error // error yielded by the last snapshot operation
snapshotStamp time.Time // timestamp of last snapshot
open bool // is this fragment actually open?
// Cache for row counts.
CacheType string // passed in by field
@ -164,11 +140,6 @@ type fragment struct {
// Cached checksums for each block.
checksums map[int][]byte
// Number of operations performed before performing a snapshot.
// This limits the size of fragments on the heap and flushes them to disk
// so that they can be mmapped and heap utilization can be kept low.
MaxOpN int
// Logger used for out-of-band log entries.
Logger logger.Logger
@ -177,8 +148,6 @@ type fragment struct {
mutexVector vector
stats stats.StatsClient
bitmapInfo *roaring.BitmapInfo
}
// newFragment returns a new instance of fragment.
@ -194,18 +163,15 @@ func newFragment(holder *Holder, spec fragSpec, shard uint64, flags byte) *fragm
fieldstr: spec.fieldstr,
fld: spec.field,
shard: shard,
flags: flags,
idx: idx,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
holder: holder,
MaxOpN: defaultFragmentMaxOpN,
stats: stats.NopStatsClient,
}
f.snapshotCond = sync.Cond{L: &f.mu}
return f
}
@ -241,30 +207,12 @@ func (f *fragment) Index() *Index {
return f.holder.Index(f.index())
}
func (f *fragment) inspect(params InspectRequestParams) (fi FragmentInfo) {
if f.bitmapInfo == nil {
fi.BitmapInfo = f.storage.Info(params.Containers)
} else {
fi.BitmapInfo = *f.bitmapInfo
}
if params.Checksum {
fi.BlockChecksums, _ = f.Blocks()
}
return fi
}
// Open opens the underlying storage.
func (f *fragment) Open() error {
f.mu.Lock()
defer f.mu.Unlock()
if err := func() error {
// Initialize storage in a function so we can close if anything goes wrong.
f.holder.Logger.Debugf("open storage for index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard)
if err := f.openStorage(true); err != nil {
return errors.Wrap(err, "opening storage")
}
// Fill cache with rows persisted to disk.
f.holder.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard)
if err := f.openCache(); err != nil {
@ -278,57 +226,12 @@ func (f *fragment) Open() error {
f.close()
return err
}
f.open = true
_ = testhook.Opened(f.holder.Auditor, f, nil)
f.holder.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard)
return nil
}
// emptyStorage is the common case for importStorage/applyStorage where they
// get no data. It tries to write the current storage to the provided file,
// which is assumed to be the file they didn't get any data from.
func (f *fragment) emptyStorage(file *os.File) (bool, error) {
if f.holder.Opts.ReadOnly {
return false, errors.New("can't flush/create storage for read-only holder")
}
// No data. We'll mark this for no mapping, clear any existing
// mapped containers, and set the Source to nil. We also have no
// ops.
f.opN = 0
f.ops = 0
f.storage.SetOps(0, 0)
f.storage.PreferMapping(false)
_, err := f.storage.RemapRoaringStorage(nil)
f.storage.SetSource(nil)
if err != nil {
return false, fmt.Errorf("applying/importing storage: no data, and clearing old mapping also failed: %v", err)
}
// Write the existing storage out to the file so it's
// a valid Roaring file thereafter. nothing to unmarshal.
// In the unlikely event that this happened even though we
// had significant data, we're not mapping it, but that's
// harmless even if it's not maximally efficient.
bi := bufio.NewWriter(file)
if _, err = f.storage.WriteTo(bi); err != nil {
return false, fmt.Errorf("init storage file: %s", err)
}
bi.Flush()
return false, nil
}
// openStorage opens the storage bitmap. Does nothing in RBF-world and will be removed soon.
func (f *fragment) openStorage(unmarshalData bool) error {
if !f.idx.NeedsSnapshot() {
f.currdata = struct{ from, to uintptr }{}
f.prevdata = f.currdata
return nil // openStorage becomes a noop under RBF, Badger, etc.
}
return nil
}
// openCache initializes the cache from row ids persisted to disk.
func (f *fragment) openCache() error {
// Determine cache type from field name.
@ -384,12 +287,6 @@ func (f *fragment) Close() error {
defer func() {
_ = testhook.Closed(f.holder.Auditor, f, nil)
}()
for f.snapshotPending {
f.snapshotCond.Wait()
}
// Note: snapshots won't progress on a closed fragment, so we
// wait until after a possible pending snapshot to close.
f.open = false
return f.close()
}
@ -400,28 +297,12 @@ func (f *fragment) close() error {
return errors.Wrap(err, "flushing cache")
}
// Close underlying storage.
if err := f.closeStorage(); err != nil {
f.holder.Logger.Errorf("fragment: error closing storage: err=%s, path=%s", err, f.path())
return errors.Wrap(err, "closing storage")
}
// Remove checksums.
f.checksums = nil
return nil
}
// closeStorage is essentially a no-op and will go away soon.
func (f *fragment) closeStorage() error {
// opN is determined by how many bit set/clear operations are in the storage
// write log, so once the storage is closed it should be 0. Opening new
// storage will set opN appropriately.
f.opN = 0
return nil
}
// mutexCheck checks for any entries in fragment which violate the mutex
// property of having only one value set for a given column ID.
func (f *fragment) mutexCheck(tx Tx, details bool, limit int) (map[uint64][]uint64, error) {
@ -544,9 +425,6 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
f.incrementOpN(1)
// If we're using a cache, update it. Otherwise skip the
// possibly-expensive count operation.
if f.CacheType != CacheTypeNone {
@ -596,9 +474,6 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
f.incrementOpN(1)
// If we're using a cache, update it. Otherwise skip the
// possibly-expensive count operation.
if f.CacheType != CacheTypeNone {
@ -665,8 +540,6 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
}
}
// Snapshot storage.
f.holder.SnapshotQueue.Enqueue(f)
f.stats.Count("setRow", 1, 1.0)
return changed, nil
@ -705,9 +578,6 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e
// Clear the row in cache.
f.cache.Add(rowID, 0)
// Snapshot storage.
f.holder.SnapshotQueue.Enqueue(f)
return changed, nil
}
@ -1962,7 +1832,7 @@ func (f *fragment) mergeBlock(tx Tx, id int, data []pairSet) (sets, clears []pai
return sets[1:], clears[1:], err
}
// bulkImport bulk imports a set of bits and then snapshots the storage.
// bulkImport bulk imports a set of bits.
// The cache is updated to reflect the new data.
func (f *fragment) bulkImport(tx Tx, rowIDs, columnIDs []uint64, options *ImportOptions) error {
// Verify that there are an equal number of row ids and column ids.
@ -2175,68 +2045,48 @@ func (p parallelSlices) Swap(i, j int) {
// snapshot of the fragment or just do in-memory updates while appending
// operations to the op log.
func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error {
//tx.AddN()
doFunc := func() error {
if len(set) > 0 {
f.stats.Count(MetricImportingN, int64(len(set)), 1)
if len(set) > 0 {
f.stats.Count(MetricImportingN, int64(len(set)), 1)
// TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...)
if err != nil {
return errors.Wrap(err, "adding positions")
}
f.stats.Count(MetricImportedN, int64(changedN), 1)
f.incrementOpN(changedN)
// TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...)
if err != nil {
return errors.Wrap(err, "adding positions")
}
f.stats.Count(MetricImportedN, int64(changedN), 1)
}
if len(clear) > 0 {
f.stats.Count(MetricClearingN, int64(len(clear)), 1)
changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...)
if err != nil {
return errors.Wrap(err, "clearing positions")
}
f.stats.Count(MetricClearedN, int64(changedN), 1)
f.incrementOpN(changedN)
if len(clear) > 0 {
f.stats.Count(MetricClearingN, int64(len(clear)), 1)
changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...)
if err != nil {
return errors.Wrap(err, "clearing positions")
}
f.stats.Count(MetricClearedN, int64(changedN), 1)
}
// Update cache counts for all affected rows.
for rowID := range rowSet {
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
if f.CacheType != CacheTypeNone {
start := rowID * ShardWidth
end := (rowID + 1) * ShardWidth
n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, start, end)
if err != nil {
return errors.Wrap(err, "CountRange")
}
f.cache.BulkAdd(rowID, n)
}
}
// Update cache counts for all affected rows.
for rowID := range rowSet {
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
if f.CacheType != CacheTypeNone {
f.cache.Invalidate()
}
return nil
}
err := doFunc()
if err != nil && f.storage != nil {
// we got an error. it's possible that the error indicates that something went wrong.
mappedIn, mappedOut, unmappedIn, errs, e2 := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to)
if errs != 0 {
f.holder.Logger.Errorf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v",
f.path(), mappedIn, mappedOut, unmappedIn, errs, e2)
if f.prevdata.from != f.currdata.from {
mappedIn, mappedOut, unmappedIn, errs, e2 = f.storage.SanityCheckMapping(f.prevdata.from, f.prevdata.to)
f.holder.Logger.Errorf("with previous map, storage would have %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v",
mappedIn, mappedOut, unmappedIn, errs, e2)
start := rowID * ShardWidth
end := (rowID + 1) * ShardWidth
n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, start, end)
if err != nil {
return errors.Wrap(err, "CountRange")
}
f.cache.BulkAdd(rowID, n)
}
}
return err
if f.CacheType != CacheTypeNone {
f.cache.Invalidate()
}
return nil
}
// sliceDifference removes everything from original that's found in remove,
@ -2474,7 +2324,7 @@ func (f *fragment) doImportRoaring(ctx context.Context, tx Tx, data []byte, clea
f.mu.RLock()
defer f.mu.RUnlock()
rowSize := uint64(1 << shardVsContainerExponent)
span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
span, _ := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
defer span.Finish()
var rowSet map[uint64]int
@ -2536,125 +2386,6 @@ func (f *fragment) importRoaringOverwrite(ctx context.Context, tx Tx, data []byt
return f.importRoaring(ctx, tx, data, false)
}
// incrementOpN increase the operation count by one.
// If the count exceeds the maximum allowed then a snapshot is performed.
func (f *fragment) incrementOpN(changed int) {
if changed <= 0 {
return
}
// don't count opN or ops if our index doesn't want snapshots
if !f.idx.NeedsSnapshot() {
return
}
f.opN += changed
f.ops++
if f.opN > f.MaxOpN {
f.holder.SnapshotQueue.Enqueue(f)
}
}
// Snapshot writes the storage bitmap to disk and reopens it. This may
// coexist with existing background-queue snapshotting; it does not remove
// things from the queue. You probably don't want to do this; use
// the snapshotQueue's Enqueue/Await.
func (f *fragment) Snapshot() error {
f.mu.Lock()
defer f.mu.Unlock()
return f.snapshot()
}
func track(start time.Time, message string, stats stats.StatsClient, logger logger.Logger) {
elapsed := time.Since(start)
logger.Debugf("%s took %s", message, elapsed)
stats.Timing(MetricSnapshotDurationSeconds, elapsed, 1.0)
}
// snapshot does the actual snapshot operation. it does not check or care
// about f.snapshotPending.
func (f *fragment) snapshot() (err error) {
if !f.idx.NeedsSnapshot() {
return nil
}
if !f.open {
return errors.New("snapshot request on closed fragment")
}
wouldPanic := debug.SetPanicOnFault(true)
defer func() {
debug.SetPanicOnFault(wouldPanic)
if r := recover(); r != nil {
if e2, ok := r.(error); ok {
err = e2
// special case: if we caught a page fault, we diagnose that directly. sadly,
// we can't see the actual values that were used to generate this, probably.
if e2.Error() == "runtime error: invalid memory address or nil pointer dereference" {
mappedIn, mappedOut, unmappedIn, errs, _ := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to)
f.holder.Logger.Errorf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total",
f.path(), mappedIn, mappedOut, unmappedIn, errs)
}
} else {
err = fmt.Errorf("non-error PanicOn: %v", r)
}
}
}()
_, err = unprotectedWriteToFragment(f, f.storage)
if err == nil {
f.snapshotStamp = time.Now()
}
return err
}
// unprotectedWriteToFragment writes the fragment f with bm as the data. It is unprotected, and
// f.mu must be locked when calling it.
func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err error) { // nolint: interfacer
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard)
start := time.Now()
defer track(start, completeMessage, f.stats, f.holder.Logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path() + snapshotExt
file, err := os.Create(snapshotPath)
if err != nil {
return n, fmt.Errorf("create snapshot file: %s", err)
}
// No deferred close, because we want to close it sooner than the
// end of this function.
// Write storage to snapshot.
bw := bufio.NewWriter(file)
if n, err = bm.WriteTo(bw); err != nil {
file.Close()
return n, fmt.Errorf("snapshot write to: %s", err)
}
if err := bw.Flush(); err != nil {
file.Close()
return n, fmt.Errorf("flush: %s", err)
}
// we close the file here so we don't still have it open when trying
// to open it in a moment.
file.Close()
// Move snapshot to data file location.
if err := os.Rename(snapshotPath, f.path()); err != nil {
return n, fmt.Errorf("rename snapshot: %s", err)
}
// if we reloaded from the file, we'd end up with this bitmap
// as our storage. so... let's use this bitmap. as our storage.
f.storage = bm
// Reopen storage.
if err := f.openStorage(false); err != nil {
return n, fmt.Errorf("open storage: %s", err)
}
// Reset operation count.
f.opN = 0
return n, nil
}
// RecalculateCache rebuilds the cache regardless of invalidate time delay.
func (f *fragment) RecalculateCache() {
f.mu.Lock()
@ -3570,14 +3301,6 @@ func bitsToRoaringData(ps pairSet) ([]byte, error) {
return buf.Bytes(), nil
}
func madvise(b []byte, advice int) error { // nolint: unparam
_, _, err := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice))
if err != 0 {
return err
}
return nil
}
// pairSet is a list of equal length row and column id lists.
type pairSet struct {
rowIDs []uint64

1007
fragment_internal_test.go
View file

File diff suppressed because it is too large Load diff

1
go.mod
View file

@ -52,6 +52,7 @@ require (
github.com/zeebo/blake3 v0.1.1
go.etcd.io/bbolt v1.3.5
go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 // indirect
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
golang.org/x/mod v0.4.2
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect

5
go.sum
View file

@ -413,8 +413,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 h1:3wPMTskHO3+O6jqTEXyFcsnuxMQOqYSaHsDxcbUXpqA=
golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -497,6 +498,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -507,6 +509,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

34
handler.go
View file

@ -76,9 +76,9 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
})
}
// Handler is the interface for the data handler, a wrapper around
// HandlerI is the interface for the data handler, a wrapper around
// Pilosa's data store.
type Handler interface {
type HandlerI interface {
Serve() error
Close() error
}
@ -94,7 +94,7 @@ func (n nopHandler) Close() error {
}
// NopHandler is a no-op implementation of the Handler interface.
var NopHandler Handler = nopHandler{}
var NopHandler HandlerI = nopHandler{}
// ImportValueRequest describes the import request structure
// for a value (BSI) import.
@ -410,31 +410,3 @@ type TranslateIDsRequest struct {
type TranslateIDsResponse struct {
Keys []string
}
// InspectRequestParams represents the parts of an InspectRequest that
// aren't generic holder filtering attributes.
type InspectRequestParams struct {
Containers bool // include container details
Checksum bool // perform checksums
}
// InspectRequest represents a request for a possibly-partial
// holder inspection, using a provided holder filter and inspect-specific
// parameters.
type InspectRequest struct {
HolderFilterParams
InspectRequestParams
}
// InspectResponse contains the structured results for an InspectRequest.
// It may some day be expanded to include metadata about views or indexes.
type InspectResponse struct {
Fragments []struct {
Index string
Field string
View string
Shard int64
Path string
Info *FragmentInfo
}
}

465
holder.go
View file

@ -7,10 +7,8 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
@ -85,8 +83,7 @@ type Holder struct {
// The interval at which the cached row ids are persisted to disk.
cacheFlushInterval time.Duration
Logger logger.Logger
SnapshotQueue SnapshotQueue
Logger logger.Logger
// Instantiates new translation stores
OpenTranslateStore OpenTranslateStoreFunc
@ -138,14 +135,6 @@ type Holder struct {
// HolderOpts holds information about the holder which other things might want
// to look up later while using the holder.
type HolderOpts struct {
// ReadOnly indicates that this holder's contents should not produce
// disk writes under any circumstances. It must be set before Open
// is called, and changing it is not supported.
ReadOnly bool
// If Inspect is set, we'll try to obtain additional information
// about fragments when opening them.
Inspect bool
// StorageBackend controls the tx/storage engine we instatiate. Set by
// server.go OptServerStorageConfig
StorageBackend string
@ -271,8 +260,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
Logger: cfg.Logger,
Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend},
SnapshotQueue: defaultSnapshotQueue,
Auditor: NewAuditor(),
path: path,
@ -298,280 +285,6 @@ func (h *Holder) IndexesPath() string {
return filepath.Join(h.path, IndexesDir)
}
type HolderInfo struct {
FragmentInfo map[string]FragmentInfo
FragmentNames []string
}
type regexpList []*regexp.Regexp
func newRegexpList(regexes string) (results regexpList, err error) {
if regexes == "" {
return nil, nil
}
for _, sub := range strings.Split(regexes, ",") {
re, err := regexp.Compile(sub)
if err != nil {
return nil, err
}
results = append(results, re)
}
return results, nil
}
func (rl regexpList) Match(haystack string) bool {
if rl == nil {
return true
}
for _, re := range rl {
if re.MatchString(haystack) {
return true
}
}
return false
}
// shardRange represents a series of shards
type shardRange struct {
min, max uint64
}
type shardRangeList []shardRange
func newShardRangeList(shards string) (results shardRangeList, err error) {
if shards == "" {
return nil, nil
}
for _, sub := range strings.Split(shards, ",") {
var sr shardRange
minMax := strings.Split(sub, "-")
if len(minMax) > 2 {
return nil, fmt.Errorf("invalid range %q", sub)
}
sr.min, err = strconv.ParseUint(minMax[0], 10, 64)
if err != nil {
return nil, err
}
sr.max = sr.min
if len(minMax) == 2 {
sr.max, err = strconv.ParseUint(minMax[0], 10, 64)
if err != nil {
return nil, err
}
}
if sr.max < sr.min {
return nil, fmt.Errorf("invalid range %q: max < min", sub)
}
results = append(results, sr)
}
return results, nil
}
func (sl shardRangeList) Match(shard uint64) bool {
if sl == nil {
return true
}
for _, sr := range sl {
if shard >= sr.min && shard <= sr.max {
return true
}
}
return false
}
// HolderFilter represents something that potentially filters out
// parts of a holder, indicating whether or not to process them,
// or recurse into them. It is permissible to recurse a thing
// without processing it, or process it without recursing it.
// For instance, something looking to accumulate statistics
// about views might return (true, false) from CheckView,
// while a fragment scanning operation would return (false, true)
// from everything above CheckFrag.
type HolderFilter interface {
CheckIndex(iname string) (process bool, recurse bool)
CheckField(iname, fname string) (process bool, recurse bool)
CheckView(iname, fname, vname string) (process bool, recurse bool)
CheckFragment(iname, fname, vname string, shard uint64) (process bool)
}
// HolderFilterAll is a placeholder type which always returns true for the
// check functions. You can embed it to make a HolderOperator which processes
// everything.
type HolderFilterAll struct{}
func (HolderFilterAll) CheckIndex(string) (bool, bool) {
return true, true
}
func (HolderFilterAll) CheckField(string, string) (bool, bool) {
return true, true
}
func (HolderFilterAll) CheckView(string, string, string) (bool, bool) {
return true, true
}
func (HolderFilterAll) CheckFragment(string, string, string, uint64) bool {
return true
}
// HolderProcessNone is a placeholder type which does nothing for the
// process functions. You can embed it to make a HolderOperator which
// does nothing, or embed it and provide your own ProcessFragment to
// do just that.
type HolderProcessNone struct{}
func (HolderProcessNone) ProcessIndex(*Index) error {
return nil
}
func (HolderProcessNone) ProcessField(*Field) error {
return nil
}
func (HolderProcessNone) ProcessView(*view) error {
return nil
}
func (HolderProcessNone) ProcessFragment(*fragment) error {
return nil
}
// HolderProcess represents something that has operations which can be
// performed on indexes, fields, views, and/or fragments.
type HolderProcess interface {
ProcessIndex(*Index) error
ProcessField(*Field) error
ProcessView(*view) error
ProcessFragment(*fragment) error
}
// HolderOperator is both a filter and a process. This is the general
// form of "I want to do something to some part of a holder."
type HolderOperator interface {
HolderFilter
HolderProcess
}
var _ HolderOperator = (*holderInspector)(nil)
type HolderFilterParams struct {
Indexes string
Fields string
Views string
Shards string
}
type holderFilterFull struct {
HolderFilterParams
indexRegexps regexpList
fieldRegexps regexpList
viewRegexps regexpList
shardRanges shardRangeList
}
type inspectRequestFull struct {
HolderFilter
params InspectRequestParams
}
func (i *holderFilterFull) CheckIndex(iname string) (process, recurse bool) {
return true, i.indexRegexps.Match(iname)
}
func (i *holderFilterFull) CheckField(iname, fname string) (process, recurse bool) {
return true, i.fieldRegexps.Match(fname)
}
func (i *holderFilterFull) CheckView(iname, fname, vname string) (process, recurse bool) {
return true, i.viewRegexps.Match(vname)
}
func (i *holderFilterFull) CheckFragment(iname, fname, vname string, shard uint64) (process bool) {
return i.shardRanges.Match(shard)
}
func NewHolderFilter(params HolderFilterParams) (result HolderFilter, err error) {
filter := &holderFilterFull{
HolderFilterParams: params,
}
filter.indexRegexps, err = newRegexpList(params.Indexes)
if err != nil {
return nil, err
}
filter.fieldRegexps, err = newRegexpList(params.Fields)
if err != nil {
return nil, err
}
filter.viewRegexps, err = newRegexpList(params.Views)
if err != nil {
return nil, err
}
filter.shardRanges, err = newShardRangeList(params.Shards)
if err != nil {
return nil, err
}
return filter, nil
}
func expandInspectRequest(req *InspectRequest) (*inspectRequestFull, error) {
filter, err := NewHolderFilter(req.HolderFilterParams)
if err != nil {
return nil, err
}
irf := &inspectRequestFull{
HolderFilter: filter,
params: req.InspectRequestParams,
}
return irf, nil
}
type holderInspector struct {
*inspectRequestFull
pathParts [3]string
path string
hi *HolderInfo
}
func (h *holderInspector) ProcessIndex(i *Index) error {
h.pathParts[0] = i.name
return nil
}
func (h *holderInspector) ProcessField(f *Field) error {
h.pathParts[1] = f.name
return nil
}
func (h *holderInspector) ProcessView(v *view) error {
h.pathParts[2] = v.name
h.path = strings.Join(h.pathParts[:], "/")
return nil
}
func (h *holderInspector) ProcessFragment(f *fragment) error {
path := h.path + "/" + strconv.FormatUint(f.shard, 10)
h.hi.FragmentInfo[path] = f.inspect(h.inspectRequestFull.params)
h.hi.FragmentNames = append(h.hi.FragmentNames, path)
return nil
}
func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) {
fullReq, err := expandInspectRequest(req)
if err != nil {
return nil, err
}
inspector := &holderInspector{
inspectRequestFull: fullReq,
hi: &HolderInfo{
FragmentInfo: make(map[string]FragmentInfo),
},
}
err = h.Process(ctx, inspector)
sort.Strings(inspector.hi.FragmentNames)
return inspector.hi, err
}
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.opening = true
@ -734,16 +447,15 @@ func (h *Holder) maybeSpool(msg Message) bool {
return true
}
// Activate runs the background tasks relevant to keeping a holder in a stable
// state, such as scanning it for needed snapshots, or flushing caches. This
// is separate from opening because, while a server would nearly always want
// to do this, other use cases (like consistency checks of a data directory)
// Activate runs the background tasks relevant to keeping a holder in
// a stable state, such as flushing caches. This is separate from
// opening because, while a server would nearly always want to do
// this, other use cases (like consistency checks of a data directory)
// need to avoid it even getting started.
func (h *Holder) Activate() {
// Periodically flush cache.
h.wg.Add(2)
h.wg.Add(1)
go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
go func() { defer h.wg.Done(); h.SnapshotQueue.ScanHolder(h, h.closing) }()
}
// checkForeignIndex is a check before applying a foreign
@ -777,7 +489,6 @@ func (h *Holder) processForeignIndexFields() error {
// Close closes all open fragments.
func (h *Holder) Close() error {
if h == nil {
return nil
}
@ -791,7 +502,6 @@ func (h *Holder) Close() error {
// Notify goroutines of closing and wait for completion.
close(h.closing)
h.wg.Wait()
for _, index := range h.Indexes() {
if err := index.Close(); err != nil {
return errors.Wrap(err, "closing index")
@ -809,10 +519,6 @@ func (h *Holder) Close() error {
h.opened.mu.Lock()
h.opened.ch = make(chan struct{})
h.opened.mu.Unlock()
if h.SnapshotQueue != nil {
h.SnapshotQueue.Stop()
h.SnapshotQueue = nil
}
if h.lookupDB != nil {
err := h.lookupDB.Close()
@ -827,13 +533,6 @@ func (h *Holder) Close() error {
return nil
}
func (h *Holder) NeedsSnapshot() bool {
h.mu.RLock()
defer h.mu.RUnlock()
return h.txf.NeedsSnapshot()
}
// HasData returns true if Holder contains at least one index.
// This is used to determine if the rebalancing of data is necessary
// when a node joins the cluster.
@ -1967,130 +1666,6 @@ func uint64InSlice(i uint64, s []uint64) bool {
return false
}
// Process loops through a holder based on the Check functions in op, calling
// the Process functions in op when indicated.
func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
var fieldNames, viewNames []string
var fragNums []uint64
indexes := h.Indexes()
for _, idx := range indexes {
if err = ctx.Err(); err != nil {
return err
}
if idx == nil {
continue
}
indexName := idx.name
process, recurse := op.CheckIndex(indexName)
if !process && !recurse {
continue
}
if err = ctx.Err(); err != nil {
return err
}
if process {
err = op.ProcessIndex(idx)
if err != nil {
return err
}
}
if !recurse {
continue
}
fieldNames = fieldNames[:0]
idx.mu.Lock()
for fieldName := range idx.fields {
fieldNames = append(fieldNames, fieldName)
}
idx.mu.Unlock()
for _, fieldName := range fieldNames {
if err = ctx.Err(); err != nil {
return err
}
process, recurse := op.CheckField(idx.name, fieldName)
if !process && !recurse {
continue
}
idx.mu.Lock()
field := idx.fields[fieldName]
idx.mu.Unlock()
if field == nil {
continue
}
if err = ctx.Err(); err != nil {
return err
}
if process {
err = op.ProcessField(field)
if err != nil {
return err
}
}
if !recurse {
continue
}
viewNames = viewNames[:0]
field.mu.Lock()
for viewName := range field.viewMap {
viewNames = append(viewNames, viewName)
}
field.mu.Unlock()
for _, viewName := range viewNames {
if err = ctx.Err(); err != nil {
return err
}
process, recurse := op.CheckView(indexName, fieldName, viewName)
if !process && !recurse {
continue
}
field.mu.Lock()
view := field.viewMap[viewName]
field.mu.Unlock()
if view == nil {
continue
}
if err = ctx.Err(); err != nil {
return err
}
if process {
err = op.ProcessView(view)
if err != nil {
return err
}
}
if !recurse {
continue
}
fragNums = fragNums[:0]
view.mu.Lock()
for fragNum := range view.fragments {
fragNums = append(fragNums, fragNum)
}
view.mu.Unlock()
for _, fragNum := range fragNums {
if err = ctx.Err(); err != nil {
return err
}
process := op.CheckFragment(indexName, fieldName, viewName, fragNum)
if !process {
continue
}
view.mu.Lock()
frag := view.fragments[fragNum]
view.mu.Unlock()
err = op.ProcessFragment(frag)
if err != nil {
return err
}
}
}
}
}
return nil
}
// used by Index.openFields(), enabling Tx / Txf by telling
// the holder about its own indexes.
func (h *Holder) addIndex(idx *Index) {
@ -2111,34 +1686,6 @@ func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) {
return h.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil
}
func (h *Holder) HasRoaringData() (has bool, err error) {
idxs := h.Indexes()
for _, idx := range idxs {
paths, err := listFilesUnderDir(idx.path, false, "", true)
if err != nil {
return false, errors.Wrap(err, "HasRoaringData listFilesUnderDir")
}
index := idx.name
for _, relpath := range paths {
field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
if err != nil {
continue // ignore .meta paths
}
abspath := idx.path + sep + relpath
hasData, err := roaringFragmentHasData(abspath, index, field, view, shard)
if err != nil {
return false, errors.Wrap(err, "HasRoaringData roaringFragmentHasData")
}
if hasData {
return true, nil
}
}
}
return
}
func decodeCreateIndexMessage(ser Serializer, b []byte) (*CreateIndexMessage, error) {
var cim CreateIndexMessage
if err := ser.Unmarshal(b, &cim); err != nil {

179
holder_internal_test.go
View file

@ -2,189 +2,12 @@
package pilosa
import (
"context"
"fmt"
"os"
"testing"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/testhook"
)
var _ = fmt.Printf
type testHolderOperator struct {
indexSeen, indexProcessed int
fieldSeen, fieldProcessed int
viewSeen, viewProcessed int
fragmentSeen, fragmentProcessed int
waitHere chan struct{}
}
func (t *testHolderOperator) CheckIndex(string) (bool, bool) {
t.indexSeen++
return true, true
}
func (t *testHolderOperator) CheckField(string, string) (bool, bool) {
t.fieldSeen++
return true, true
}
func (t *testHolderOperator) CheckView(string, string, string) (bool, bool) {
t.viewSeen++
return true, true
}
func (t *testHolderOperator) CheckFragment(string, string, string, uint64) bool {
t.fragmentSeen++
return true
}
func (t *testHolderOperator) ProcessIndex(*Index) error {
t.indexProcessed++
return nil
}
func (t *testHolderOperator) ProcessField(*Field) error {
t.fieldProcessed++
return nil
}
func (t *testHolderOperator) ProcessView(*view) error {
t.viewProcessed++
return nil
}
func (t *testHolderOperator) ProcessFragment(*fragment) error {
if t.waitHere != nil {
<-t.waitHere
}
t.fragmentProcessed++
return nil
}
func makeHolder(tb testing.TB, backend string) (*Holder, string, error) {
path, err := testhook.TempDir(tb, "pilosa-")
if err != nil {
return nil, "", err
}
cfg := mustHolderConfig()
if backend != "" {
cfg.StorageConfig.Backend = backend
cfg.StorageConfig.FsyncEnabled = false
}
h := NewHolder(path, cfg)
return h, path, h.Open()
}
func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault())
if err != nil {
t.Fatalf("setting bit: %v", err)
}
_, err = f.SetBit(nil, rowID, columnID, nil)
if err != nil {
t.Fatalf("setting bit: %v", err)
}
}
func TestHolderOperatorProcess(t *testing.T) {
h, path, err := makeHolder(t, "")
if err != nil {
t.Fatalf("creating holder: %v", err)
}
defer os.RemoveAll(path)
defer h.Close()
// Write bits to separate indexes.
testSetBit(t, h, "i0", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 12345678)
testOp := testHolderOperator{}
ctx := context.Background()
err = h.Process(ctx, &testOp)
if err != nil {
t.Fatalf("processing holder: %v", err)
}
expected := testHolderOperator{
indexSeen: 2, indexProcessed: 2,
fieldSeen: 2, fieldProcessed: 2,
viewSeen: 2, viewProcessed: 2,
fragmentSeen: 3, fragmentProcessed: 3,
}
if testOp != expected {
t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
}
}
func TestHolderOperatorCancel(t *testing.T) {
h, path, err := makeHolder(t, "")
if err != nil {
t.Fatalf("creating holder: %v", err)
}
defer os.RemoveAll(path)
defer h.Close()
// Write bits to separate indexes.
testSetBit(t, h, "i0", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 12345678)
// Here, we want to ensure that the operation gets cancelled
// successfully. In practice we expect it to process one fragment, then
// end up blocked on the waitHere, then get cancelled... But the
// waitHere blockage isn't really something holder.Process can do
// anything about, so we close the channel, so two fragments are
// processed. But in theory you could end up with only one fragment
// processed if this goroutine managed to cancel before the processor
// gets to the next fragment. Point is, it shouldn't hit all three,
// because the checks against the cancellation should fire before it
// gets there.
testOp := testHolderOperator{waitHere: make(chan struct{})}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
err = h.Process(ctx, &testOp)
close(done)
}()
testOp.waitHere <- struct{}{}
cancel()
close(testOp.waitHere)
<-done
if err != context.Canceled {
t.Fatalf("processing holder: expected context.Canceled, got %v", err)
}
testOp.waitHere = nil
expected := testHolderOperator{
indexSeen: 2, indexProcessed: 2,
fieldSeen: 2, fieldProcessed: 2,
viewSeen: 2, viewProcessed: 2,
fragmentSeen: 3, fragmentProcessed: 3,
}
if testOp == expected {
t.Fatalf("holder processor did not cancel. expected something other than %#v", expected)
}
}
// mustHolderConfig is meant to help minimize the number of places in the code
// where we're reading the PILOSA_STORAGE_BACKEND environment variable for
// testing purposes. Ideally we would handle this differently, but this is a
// first attempt at improving things. Note: the actual os.Getenv() call was
// moved to the CurrentBackend() function.
// mustHolderConfig sets up a default holder config for tests.
func mustHolderConfig() *HolderConfig {
cfg := DefaultHolderConfig()
if backend := CurrentBackend(); backend != "" {
_ = MustBackendToTxtype(backend)
cfg.StorageConfig.Backend = backend
}
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
cfg.Schemator = disco.InMemSchemator

111
holder_test.go
View file

@ -5,13 +5,12 @@ import (
"context"
"math"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"github.com/molecula/featurebase/v3"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/test"
@ -21,10 +20,7 @@ import (
// mustHolderConfig provides a default test-friendly holder config.
func mustHolderConfig() *pilosa.HolderConfig {
cfg := pilosa.DefaultHolderConfig()
if backend := pilosa.CurrentBackend(); backend != "" {
_ = pilosa.MustBackendToTxtype(backend)
cfg.StorageConfig.Backend = backend
}
cfg.StorageConfig.Backend = "rbf"
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
cfg.Schemator = disco.InMemSchemator
@ -55,109 +51,6 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
roaringOnlyTest(t)
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
h := test.MustOpenHolder(t)
defer h.Close()
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
var shard uint64
tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := tx.Commit(); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil {
t.Fatal(err)
}
defer func() {
_ = os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0644)
}()
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
roaringOnlyTest(t)
h := test.MustOpenHolder(t)
defer h.Close()
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
var shard uint64
tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := tx.Commit(); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: shard=0, err=opening storage: unmarshal storage") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
roaringOnlyTest(t)
h := test.MustOpenHolder(t)
defer h.Close()
idx, err := h.CreateIndex("foo", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
var shard uint64
tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard})
defer tx.Rollback()
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := tx.Commit(); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.IndexesPath(), "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err != nil {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ForeignIndex", func(t *testing.T) {
t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
h := test.MustOpenHolder(t)

13
http/error.go
View file

@ -1,13 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package http
// Error defines a standard application error.
type Error struct {
// Human-readable message.
Message string `json:"message"`
}
// Error returns the string representation of the error message.
func (e *Error) Error() string {
return e.Message
}

468
http/handler.go → http_handler.go
View file

File diff suppressed because it is too large Load diff

74
http/handler_internal_test.go → http_handler_internal_test.go
View file

@ -1,5 +1,5 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package http
package pilosa
import (
"bytes"
@ -7,7 +7,6 @@ import (
"encoding/json"
"io/ioutil"
"net/http"
gohttp "net/http"
"net/http/httptest"
"net/url"
"os"
@ -17,7 +16,6 @@ import (
"time"
"github.com/golang-jwt/jwt"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"golang.org/x/oauth2"
@ -33,9 +31,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
expected postIndexRequest
err string
}{
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}},
{json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}},
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}},
{json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{TrackExistence: true}}},
{json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: IndexOptions{TrackExistence: false}}},
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: IndexOptions{Keys: true, TrackExistence: true}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "unknown key: badKey:test"},
@ -107,8 +105,8 @@ func decimalPtr(d pql.Decimal) *pql.Decimal {
// Test fieldOption validation.
func TestFieldOptionValidation(t *testing.T) {
timeQuantum := pilosa.TimeQuantum("YMD")
defaultCacheSize := uint32(pilosa.DefaultCacheSize)
timeQuantum := TimeQuantum("YMD")
defaultCacheSize := uint32(DefaultCacheSize)
tests := []struct {
json string
expected postFieldRequest
@ -116,17 +114,17 @@ func TestFieldOptionValidation(t *testing.T) {
}{
// FieldType: Set
{json: `{"options": {}}`, expected: postFieldRequest{Options: fieldOptions{
Type: pilosa.FieldTypeSet,
CacheType: stringPtr(pilosa.DefaultCacheType),
Type: FieldTypeSet,
CacheType: stringPtr(DefaultCacheType),
CacheSize: &defaultCacheSize,
}}},
{json: `{"options": {"type": "set"}}`, expected: postFieldRequest{Options: fieldOptions{
Type: pilosa.FieldTypeSet,
CacheType: stringPtr(pilosa.DefaultCacheType),
Type: FieldTypeSet,
CacheType: stringPtr(DefaultCacheType),
CacheSize: &defaultCacheSize,
}}},
{json: `{"options": {"type": "set", "cacheType": "lru"}}`, expected: postFieldRequest{Options: fieldOptions{
Type: pilosa.FieldTypeSet,
Type: FieldTypeSet,
CacheType: stringPtr("lru"),
CacheSize: &defaultCacheSize,
}}},
@ -138,7 +136,7 @@ func TestFieldOptionValidation(t *testing.T) {
{json: `{"options": {"type": "int"}}`, err: "min is required for field type int"},
{json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"},
{json: `{"options": {"type": "int", "min": 0, "max": 1001}}`, expected: postFieldRequest{Options: fieldOptions{
Type: pilosa.FieldTypeInt,
Type: FieldTypeInt,
Min: decimalPtr(pql.NewDecimal(0, 0)),
Max: decimalPtr(pql.NewDecimal(1001, 0)),
}}},
@ -149,7 +147,7 @@ func TestFieldOptionValidation(t *testing.T) {
// FieldType: Time
{json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"},
{json: `{"options": {"type": "time", "timeQuantum": "YMD"}}`, expected: postFieldRequest{Options: fieldOptions{
Type: pilosa.FieldTypeTime,
Type: FieldTypeTime,
TimeQuantum: &timeQuantum,
}}},
{json: `{"options": {"type": "time", "timeQuantum": "YMD", "min": 0}}`, err: "min does not apply to field type time"},
@ -189,7 +187,7 @@ func readResponse(w *httptest.ResponseRecorder) ([]byte, error) {
func TestAuthentication(t *testing.T) {
type evaluate func(w *httptest.ResponseRecorder, data []byte)
type endpoint func(w gohttp.ResponseWriter, r *gohttp.Request)
type endpoint func(w http.ResponseWriter, r *http.Request)
var (
ClientId = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
@ -230,8 +228,6 @@ func TestAuthentication(t *testing.T) {
// make a valid token
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "thing", GroupName: "whatever"}})
claims["molecula-idp-groups"] = groupString
claims["oid"] = "42"
claims["name"] = "todd"
validToken, err := tkn.SignedString([]byte(secretKey))
@ -255,7 +251,7 @@ func TestAuthentication(t *testing.T) {
}
expiredToken = "Bearer " + expiredToken
validCookie := &gohttp.Cookie{
validCookie := &http.Cookie{
Name: "molecula-chip",
Value: token.AccessToken,
Path: "/",
@ -276,7 +272,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
method string
yamlData string
token string
cookie *gohttp.Cookie
cookie *http.Cookie
handler endpoint
fn evaluate
}{
@ -337,7 +333,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
handler: h.handleCheckAuthentication,
fn: func(w *httptest.ResponseRecorder, data []byte) {
// no valid token in header == Unauthorized
if w.Result().StatusCode != gohttp.StatusUnauthorized {
if w.Result().StatusCode != http.StatusUnauthorized {
t.Errorf("expected http code 401, got: %+v", w.Result().StatusCode)
}
},
@ -379,7 +375,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
token: "",
handler: h.handleUserInfo,
fn: func(w *httptest.ResponseRecorder, data []byte) {
if got := w.Result().StatusCode; got != gohttp.StatusForbidden {
if got := w.Result().StatusCode; got != http.StatusForbidden {
t.Errorf("expected 403, got %v", got)
}
},
@ -487,7 +483,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
path: "/index/{index}/query",
kind: "middleware",
cookie: validCookie,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
handler: func(w http.ResponseWriter, r *http.Request) {
f := hOff.chkAuthZ(hOff.handlePostQuery, authz.Admin)
f(w, r)
},
@ -501,9 +497,9 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
name: "MW-CreateIndexInsufficientPerms",
path: "/index/abcd",
kind: "bearer",
method: gohttp.MethodPost,
method: http.MethodPost,
token: validToken,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
handler: func(w http.ResponseWriter, r *http.Request) {
h := h
var p authz.GroupPermissions
if err := p.ReadPermissionsFile(strings.NewReader(permissions1)); err != nil {
@ -515,7 +511,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
f(w, r)
},
fn: func(w *httptest.ResponseRecorder, data []byte) {
if got, want := w.Result().StatusCode, gohttp.StatusForbidden; got != want {
if got, want := w.Result().StatusCode, http.StatusForbidden; got != want {
t.Errorf("expected %v, got %v", want, got)
}
},
@ -527,13 +523,13 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
path: "/index/{index}/query",
kind: "bearer",
token: validToken,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
handler: func(w http.ResponseWriter, r *http.Request) {
h := h
f := h.chkAuthZ(h.handlePostQuery, authz.Write)
f(w, r)
},
fn: func(w *httptest.ResponseRecorder, data []byte) {
if got, want := w.Result().StatusCode, gohttp.StatusInternalServerError; got != want {
if got, want := w.Result().StatusCode, http.StatusInternalServerError; got != want {
t.Errorf("expected %v, got %v", want, got)
}
},
@ -543,7 +539,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
path: "/index/{index}/query",
kind: "bearer",
token: validToken,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
handler: func(w http.ResponseWriter, r *http.Request) {
h := h
var p authz.GroupPermissions
if err := p.ReadPermissionsFile(strings.NewReader(permissions1)); err != nil {
@ -554,7 +550,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
f(w, r)
},
fn: func(w *httptest.ResponseRecorder, data []byte) {
if got, want := w.Result().StatusCode, gohttp.StatusBadRequest; got != want {
if got, want := w.Result().StatusCode, http.StatusBadRequest; got != want {
t.Errorf("expected %v, got: %+v", want, got)
}
},
@ -565,7 +561,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
switch test.kind {
case "type1", "middleware":
t.Run(test.name, func(t *testing.T) {
r := httptest.NewRequest(gohttp.MethodGet, test.path, nil)
r := httptest.NewRequest(http.MethodGet, test.path, nil)
w := httptest.NewRecorder()
if test.cookie != nil {
r.AddCookie(test.cookie)
@ -579,7 +575,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
})
case "type2":
t.Run(test.name, func(t *testing.T) {
r := httptest.NewRequest(gohttp.MethodGet, test.path, nil)
r := httptest.NewRequest(http.MethodGet, test.path, nil)
w := httptest.NewRecorder()
r.Form = url.Values{}
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
@ -597,7 +593,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
case "bearer":
t.Run(test.name, func(t *testing.T) {
if test.method == "" {
test.method = gohttp.MethodGet
test.method = http.MethodGet
}
r := httptest.NewRequest(test.method, test.path, nil)
w := httptest.NewRecorder()
@ -628,8 +624,6 @@ func TestChkAuthN(t *testing.T) {
// make a valid token
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "thing", GroupName: "whatever"}})
claims["molecula-idp-groups"] = groupString
claims["oid"] = "42"
claims["name"] = "A. Token"
validToken, err := tkn.SignedString(a.SecretKey())
@ -639,15 +633,7 @@ func TestChkAuthN(t *testing.T) {
validToken = "Bearer " + validToken
// make an invalid token
invalidKey, err := hex.DecodeString("DEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEED")
if err != nil {
t.Fatal(err)
}
invalidToken, err := tkn.SignedString(invalidKey)
if err != nil {
t.Fatal(err)
}
invalidToken = "Bearer " + invalidToken
invalidToken := "Bearer " + "thisis.a.bad.token"
// make an expired token
claims["exp"] = "1"

24
http/handler_test.go → http_handler_test.go
View file

@ -1,5 +1,5 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package http_test
package pilosa_test
import (
"encoding/json"
@ -10,17 +10,17 @@ import (
"testing"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
)
func TestHandlerOptions(t *testing.T) {
_, err := http.NewHandler()
_, err := pilosa.NewHandler()
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
_, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{}))
_, err = pilosa.NewHandler(pilosa.OptHandlerAPI(&pilosa.API{}))
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
@ -30,24 +30,30 @@ func TestHandlerOptions(t *testing.T) {
t.Fatalf("creating listener: %v", err)
}
_, err = http.NewHandler(http.OptHandlerListener(ln, ln.Addr().String()))
_, err = pilosa.NewHandler(pilosa.OptHandlerListener(ln, ln.Addr().String()))
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
_, err = pilosa.NewHandler(pilosa.OptHandlerListener(ln, ln.Addr().String()), pilosa.OptHandlerSerializer(proto.Serializer{}), pilosa.OptHandlerSerializer(proto.RoaringSerializer))
if err == nil {
t.Fatalf("expected error making handler without enough options, got nil")
}
}
func TestMarshalUnmarshalTransactionResponse(t *testing.T) {
tests := []struct {
name string
tr *http.TransactionResponse
tr *pilosa.TransactionResponse
}{
{
name: "nil transaction",
tr: &http.TransactionResponse{},
tr: &pilosa.TransactionResponse{},
},
{
name: "empty transaction",
tr: &http.TransactionResponse{Transaction: &pilosa.Transaction{}},
tr: &pilosa.TransactionResponse{Transaction: &pilosa.Transaction{}},
},
}
@ -58,7 +64,7 @@ func TestMarshalUnmarshalTransactionResponse(t *testing.T) {
t.Fatalf("marshaling: %v", err)
}
mytr := &http.TransactionResponse{}
mytr := &pilosa.TransactionResponse{}
err = json.Unmarshal(data, mytr)
if err != nil {
t.Fatalf("unmarshalling: %v", err)

29
http/translator.go → http_translator.go
View file

@ -1,5 +1,5 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package http
package pilosa
import (
"bytes"
@ -12,25 +12,24 @@ import (
"reflect"
"sync"
"github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/logger"
)
func GetOpenTranslateReaderFunc(client *http.Client) pilosa.OpenTranslateReaderFunc {
func GetOpenTranslateReaderFunc(client *http.Client) OpenTranslateReaderFunc {
return GetOpenTranslateReaderWithLockerFunc(client, nopLocker{})
}
func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) pilosa.OpenTranslateReaderFunc {
func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) OpenTranslateReaderFunc {
lockType := reflect.TypeOf(locker)
if lockType.Kind() == reflect.Ptr {
lockType = lockType.Elem()
}
return func(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) {
return func(ctx context.Context, nodeURL string, offsets TranslateOffsetMap) (TranslateEntryReader, error) {
return openTranslateReader(ctx, nodeURL, offsets, client, reflect.New(lockType).Interface().(sync.Locker))
}
}
func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap, client *http.Client, locker sync.Locker) (pilosa.TranslateEntryReader, error) {
func openTranslateReader(ctx context.Context, nodeURL string, offsets TranslateOffsetMap, client *http.Client, locker sync.Locker) (TranslateEntryReader, error) {
r := NewTranslateEntryReader(ctx, client)
r.locker = locker
@ -47,9 +46,9 @@ type nopLocker struct{}
func (nopLocker) Lock() {}
func (nopLocker) Unlock() {}
// TranslateEntryReader represents an implementation of pilosa.TranslateEntryReader.
// TranslateEntryReader represents an implementation of TranslateEntryReader.
// It consolidates all index & field translate entries into a single reader.
type TranslateEntryReader struct {
type HTTPTranslateEntryReader struct {
locker sync.Locker
ctx context.Context
@ -60,7 +59,7 @@ type TranslateEntryReader struct {
// Lookup of offsets for each index & field.
// Must be set before calling Open().
Offsets pilosa.TranslateOffsetMap
Offsets TranslateOffsetMap
// URL to stream entries from.
// Must be set before calling Open().
@ -72,17 +71,17 @@ type TranslateEntryReader struct {
}
// NewTranslateEntryReader returns a new instance of TranslateEntryReader.
func NewTranslateEntryReader(ctx context.Context, client *http.Client) *TranslateEntryReader {
func NewTranslateEntryReader(ctx context.Context, client *http.Client) *HTTPTranslateEntryReader {
if client == nil {
client = http.DefaultClient
}
r := &TranslateEntryReader{locker: nopLocker{}, HTTPClient: client, Logger: logger.NopLogger}
r := &HTTPTranslateEntryReader{locker: nopLocker{}, HTTPClient: client, Logger: logger.NopLogger}
r.ctx, r.cancel = context.WithCancel(ctx)
return r
}
// Open initiates the reader.
func (r *TranslateEntryReader) Open() error {
func (r *HTTPTranslateEntryReader) Open() error {
// Serialize map of offsets to request body.
requestBody, err := json.Marshal(r.Offsets)
if err != nil {
@ -107,7 +106,7 @@ func (r *TranslateEntryReader) Open() error {
// Handle error codes.
if resp.StatusCode == http.StatusNotImplemented {
r.body.Close()
return pilosa.ErrNotImplemented
return ErrNotImplemented
} else if resp.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(resp.Body)
r.body.Close()
@ -117,7 +116,7 @@ func (r *TranslateEntryReader) Open() error {
}
// Close stops the reader.
func (r *TranslateEntryReader) Close() error {
func (r *HTTPTranslateEntryReader) Close() error {
if r.cancel != nil {
r.cancel()
}
@ -132,7 +131,7 @@ func (r *TranslateEntryReader) Close() error {
// ReadEntry reads the next entry from the stream into entry.
// Returns io.EOF at the end of the stream.
func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
func (r *HTTPTranslateEntryReader) ReadEntry(entry *TranslateEntry) error {
r.locker.Lock()
defer r.locker.Unlock()

11
http/translator_test.go → http_translator_test.go
View file

@ -1,5 +1,5 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package http_test
package pilosa_test
import (
"context"
@ -8,8 +8,7 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/http"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/test"
)
@ -41,7 +40,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
}
// Connect to server and stream all available data.
r := http.NewTranslateEntryReader(context.Background(), nil)
r := pilosa.NewTranslateEntryReader(context.Background(), nil)
r.URL = primary.URL()
// Wait to ensure writes make it to translate store
@ -123,7 +122,7 @@ func BenchmarkReadEntryNoMutex(b *testing.B) {
defer teardown()
for n := 0; n < b.N; n++ {
r, err := http.GetOpenTranslateReaderFunc(nil)(ctx, url, offset)
r, err := pilosa.GetOpenTranslateReaderFunc(nil)(ctx, url, offset)
if err != nil {
b.Fatalf("opening translate reader: %+v", err)
}
@ -138,7 +137,7 @@ func BenchmarkReadEntryWithMutex(b *testing.B) {
defer teardown()
for n := 0; n < b.N; n++ {
r, err := http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})(ctx, url, offset)
r, err := pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})(ctx, url, offset)
if err != nil {
b.Fatalf("opening translate reader: %+v", err)
}

4
index.go
View file

@ -93,10 +93,6 @@ func (i *Index) NewTx(txo Txo) Tx {
return i.holder.txf.NewTx(txo)
}
func (i *Index) NeedsSnapshot() bool {
return i.holder.txf.NeedsSnapshot()
}
// CreatedAt is an timestamp for a specific version of an index.
func (i *Index) CreatedAt() int64 {
i.mu.RLock()

11
index_internal_test.go
View file

@ -28,14 +28,3 @@ func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index {
return index
}
// reopen closes the index and reopens it.
func (i *Index) reopen() error {
if err := i.Close(); err != nil {
return err
}
if err := i.Open(); err != nil {
return err
}
return nil
}

77
internal/authclustertests/docker-compose.yml
View file

@ -1,77 +0,0 @@
version: '2'
services:
pilosa1:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
environment:
- PILOSA_NAME=pilosa1
- PILOSA_ETCD_DIR=/root/.etcd
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
- PILOSA_CLUSTER_REPLICAS=3
networks:
- pilosanet
command:
- "/featurebase server --bind pilosa1:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf"
pilosa2:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
environment:
- PILOSA_NAME=pilosa2
- PILOSA_ETCD_DIR=/root/.etcd
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
- PILOSA_CLUSTER_REPLICAS=3
networks:
- pilosanet
command:
- "/featurebase server --bind pilosa2:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf"
pilosa3:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
environment:
- PILOSA_NAME=pilosa3
- PILOSA_ETCD_DIR=/root/.etcd
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
- PILOSA_CLUSTER_REPLICAS=3
networks:
- pilosanet
command:
- "/featurebase server --bind pilosa3:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf"
client1:
build:
context: .
dockerfile: ../clustertests/Dockerfile
depends_on:
- "pilosa1"
- "pilosa2"
- "pilosa3"
environment:
- ENABLE_PILOSA_CLUSTER_TESTS=1
- GO111MODULE=on
- PROJECT=authclustertests
- ENABLE_AUTH=1
networks:
- pilosanet
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command:
- "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests"
networks:
pilosanet:

3
internal/clustertests/Dockerfile
View file

@ -1,3 +0,0 @@
FROM ptest
COPY . /go/src/github.com/molecula/featurebase/internal/clustertests

7
internal/clustertests/Dockerfile-fakeIDP Normal file
View file

@ -0,0 +1,7 @@
FROM golang:latest
WORKDIR /
COPY fakeidp ./
RUN go build .
ENTRYPOINT ["/fakeidp"]

120
internal/clustertests/cluster_test.go
View file

@ -2,12 +2,14 @@
package clustertest
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"testing"
"time"
@ -15,37 +17,36 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/disco"
picli "github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
)
// container turns a docker-compose service name into a container name
// assuming the project name is set in the enviroment as PROJECT. This
// refers to the "-p" argument to docker-compose. NOTE: this assumes
// docker-compose joins the project name with a separating
// underscore... this may not always be true as I've seen a dash used
// as well, but I think it is true in recent versions.
func container(svc string) string {
// container turns a docker-compose service name into a container ID
// by calling "docker-compose ps"
func container(t *testing.T, svc string) string {
project := "clustertests"
if os.Getenv("ENABLE_AUTH") == "1" {
project = "authclustertests"
}
if p := os.Getenv("PROJECT"); p != "" {
project = p
}
return project + "_" + svc + "_1"
stdout, stderr, err := runCmd("docker-compose", "-p", project, "ps", "-q", svc)
if err != nil {
t.Fatalf("couldn't construct container name, err: %v, stderr:\n%s\nstdout:\n%s", err, stderr, stdout)
}
name := strings.Trim(stdout, "\n")
return name
}
func GetAuthToken(t *testing.T) string {
t.Helper()
var (
ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
LogoutURL = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
AuthorizeURL = "fakeidp:10101/authorize"
TokenURL = "fakeidp:10101/token"
GroupEndpointURL = "fakeidp:10101/groups"
LogoutURL = "fakeidp:10101/logout"
Scopes = []string{"https://graph.microsoft.com/.default", "offline_access"}
Key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
)
@ -62,12 +63,13 @@ func GetAuthToken(t *testing.T) string {
ClientSecret,
Key,
)
if err != nil {
t.Fatalf("NewAuth: %v", err)
}
// make a valid token
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}})
claims["molecula-idp-groups"] = groupString
claims["oid"] = "42"
claims["name"] = "valid"
token, err := tkn.SignedString([]byte(a.SecretKey()))
@ -87,15 +89,15 @@ func TestClusterStuff(t *testing.T) {
auth = true
}
cli1, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil))
cli1, err := pilosa.NewInternalClient("pilosa1:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
t.Fatalf("getting client: %v", err)
}
cli2, err := picli.NewInternalClient("pilosa2:10101", picli.GetHTTPClient(nil))
cli2, err := pilosa.NewInternalClient("pilosa2:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
t.Fatalf("getting client: %v", err)
}
cli3, err := picli.NewInternalClient("pilosa3:10101", picli.GetHTTPClient(nil))
cli3, err := pilosa.NewInternalClient("pilosa3:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
t.Fatalf("getting client: %v", err)
}
@ -134,7 +136,7 @@ func TestClusterStuff(t *testing.T) {
}
// Check query results from each node.
for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
for i, cli := range []*pilosa.InternalClient{cli1, cli2, cli3} {
r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
if err != nil {
t.Fatalf("count querying pilosa%d: %v", i, err)
@ -144,24 +146,20 @@ func TestClusterStuff(t *testing.T) {
}
}
t.Run("long pause", func(t *testing.T) {
pcmd := exec.Command("/pumba", "pause", container("pilosa3"), "--duration", "10s")
pcmd.Stdout = os.Stdout
pcmd.Stderr = os.Stderr
if err := sendCmd("docker", "pause", container(t, "pilosa3")); err != nil {
t.Fatalf("sending pause: %v", err)
}
t.Log("pausing pilosa3 for 10s")
if err := pcmd.Start(); err != nil {
t.Fatalf("starting pumba command: %v", err)
time.Sleep(time.Second * 10)
if err := sendCmd("docker", "unpause", container(t, "pilosa3")); err != nil {
t.Fatalf("sending unpause: %v", err)
}
if err := pcmd.Wait(); err != nil {
t.Fatalf("waiting on pumba pause cmd: %v", err)
}
t.Log("done with pause, waiting for stability")
waitForStatus(t, cli1.Status, string(disco.ClusterStateNormal), 30, time.Second, ctx)
t.Log("done waiting for stability")
// Check query results from each node.
for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
for i, cli := range []*pilosa.InternalClient{cli1, cli2, cli3} {
r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
if err != nil {
t.Fatalf("count querying pilosa%d: %v", i, err)
@ -174,7 +172,7 @@ func TestClusterStuff(t *testing.T) {
t.Run("backup", func(t *testing.T) {
// do backup with node 1 down, but restart it after a few seconds
if err := sendCmd("docker", "stop", container("pilosa1")); err != nil {
if err := sendCmd("docker", "stop", container(t, "pilosa1")); err != nil {
t.Fatalf("sending stop command: %v", err)
}
var backupCmd *exec.Cmd
@ -192,7 +190,7 @@ func TestClusterStuff(t *testing.T) {
}
}
time.Sleep(time.Second * 5)
if err = sendCmd("docker", "start", container("pilosa1")); err != nil {
if err = sendCmd("docker", "start", container(t, "pilosa1")); err != nil {
t.Fatalf("sending start command: %v", err)
}
@ -228,18 +226,27 @@ func TestClusterStuff(t *testing.T) {
}
}
time.Sleep(time.Millisecond * 50)
if err = sendCmd("docker", "stop", container("pilosa2")); err != nil {
if err = sendCmd("docker", "stop", container(t, "pilosa2")); err != nil {
t.Fatalf("sending stop command: %v", err)
}
time.Sleep(time.Second * 10)
if err = sendCmd("docker", "start", container("pilosa2")); err != nil {
if err = sendCmd("docker", "start", container(t, "pilosa2")); err != nil {
t.Fatalf("sending stop command: %v", err)
}
if err := restoreCmd.Wait(); err != nil {
t.Fatalf("restore failed: %v", err)
}
if err = sendCmd("docker", "pause", container(t, "pilosa1")); err != nil {
t.Fatalf("sending pause command: %v", err)
}
if err = sendCmd("docker", "pause", container(t, "pilosa2")); err != nil {
t.Fatalf("sending pause command: %v", err)
}
if err = sendCmd("docker", "pause", container(t, "pilosa3")); err != nil {
t.Fatalf("sending pause command: %v", err)
}
// now do backup with all nodes down and too short a timeout
// so it fails. Has be to be all 3 because the cluster has
// replicas=3 and the backup command will retry on replicas.
@ -254,27 +261,19 @@ func TestClusterStuff(t *testing.T) {
t.Fatalf("sending second backup command: %v", err)
}
}
time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail
if err = sendCmd("docker", "stop", container("pilosa1")); err != nil {
t.Fatalf("sending stop command: %v", err)
}
if err = sendCmd("docker", "stop", container("pilosa2")); err != nil {
t.Fatalf("sending stop command: %v", err)
}
if err = sendCmd("docker", "stop", container("pilosa3")); err != nil {
t.Fatalf("sending stop command: %v", err)
}
time.Sleep(time.Second * 5)
t.Logf("sleeping 8s")
time.Sleep(time.Second * 8)
t.Logf("restarting FB nodes")
if err = sendCmd("docker", "start", container("pilosa1")); err != nil {
t.Fatalf("sending start command: %v", err)
if err = sendCmd("docker", "unpause", container(t, "pilosa1")); err != nil {
t.Fatalf("sending unpause command: %v", err)
}
if err = sendCmd("docker", "start", container("pilosa2")); err != nil {
t.Fatalf("sending start command: %v", err)
if err = sendCmd("docker", "unpause", container(t, "pilosa2")); err != nil {
t.Fatalf("sending unpause command: %v", err)
}
if err = sendCmd("docker", "start", container("pilosa3")); err != nil {
t.Fatalf("sending start command: %v", err)
if err = sendCmd("docker", "unpause", container(t, "pilosa3")); err != nil {
t.Fatalf("sending unpause command: %v", err)
}
if err = backupCmd.Wait(); err == nil {
t.Fatal("backup command should have errored but didn't")
@ -308,3 +307,14 @@ func waitForStatus(t *testing.T, stator func(context.Context) (string, error), s
t.Fatalf("waited %s for status: %s, got: %s", waited.String(), status, s)
}
}
// runCmd is a helper which uses os.Exec to run a command and returns
// stdout and stderr as separate strings, and any error returned from
// Command.Run
func runCmd(name string, args ...string) (sout, serr string, err error) {
cmd := exec.Command(name, args...)
stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
cmd.Stdout, cmd.Stderr = stdout, stderr
err = cmd.Run()
return stdout.String(), stderr.String(), errors.Wrap(err, "running command")
}

22
internal/clustertests/docker-compose.yml
View file

@ -4,7 +4,6 @@ services:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
environment:
- PILOSA_NAME=pilosa1
- PILOSA_ETCD_DIR=/root/.etcd
@ -17,12 +16,11 @@ services:
networks:
- pilosanet
command:
- "/featurebase server --bind pilosa1:10101"
- "/featurebase server --bind pilosa1:10101 ${CLUSTERTESTS_FB_ARGS}"
pilosa2:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
environment:
- PILOSA_NAME=pilosa2
- PILOSA_ETCD_DIR=/root/.etcd
@ -35,12 +33,11 @@ services:
networks:
- pilosanet
command:
- "/featurebase server --bind pilosa2:10101"
- "/featurebase server --bind pilosa2:10101 ${CLUSTERTESTS_FB_ARGS}"
pilosa3:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
environment:
- PILOSA_NAME=pilosa3
- PILOSA_ETCD_DIR=/root/.etcd
@ -53,25 +50,32 @@ services:
networks:
- pilosanet
command:
- "/featurebase server --bind pilosa3:10101"
- "/featurebase server --bind pilosa3:10101 ${CLUSTERTESTS_FB_ARGS}"
client1:
build:
context: .
context: ../..
dockerfile: Dockerfile-clustertests-client
depends_on:
- "pilosa1"
- "pilosa2"
- "pilosa3"
- "fakeidp"
environment:
- ENABLE_PILOSA_CLUSTER_TESTS=1
- GO111MODULE=on
- PROJECT=${PROJECT}
- ENABLE_AUTH=0
- ENABLE_AUTH=${ENABLE_AUTH}
networks:
- pilosanet
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command:
- "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests"
fakeidp:
build:
context: .
dockerfile: Dockerfile-fakeIDP
networks:
- pilosanet
networks:
pilosanet:

5
internal/clustertests/fakeidp/go.mod Normal file
View file

@ -0,0 +1,5 @@
module fakeidp
go 1.17
require github.com/golang-jwt/jwt v3.2.2+incompatible

2
internal/clustertests/fakeidp/go.sum Normal file
View file

@ -0,0 +1,2 @@
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=

44
internal/clustertests/fakeidp/server.go Normal file
View file

@ -0,0 +1,44 @@
package main
import (
"encoding/hex"
"log"
"net/http"
"strconv"
"time"
"github.com/golang-jwt/jwt"
)
func groups(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"value":[{"id":"group-id-test","displayName":"group-id-test"}]}`))
}
func token(w http.ResponseWriter, req *http.Request) {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = "42"
claims["name"] = "valid"
expiresIn := 2 * time.Hour
claims["exp"] = strconv.Itoa(int(time.Now().Add(expiresIn).Unix()))
k, err := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
if err != nil {
log.Fatalf("i am not equipped to handle this!!! %v", err)
}
fresh, err := tkn.SignedString(k)
if err != nil {
log.Fatalf("i am not equipped to handle this!!! %v", err)
}
body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write([]byte(body))
}
func main() {
http.HandleFunc("/groups", groups)
http.HandleFunc("/token", token)
log.Println("FAKEIDP SERVER UP AND RUNNING")
log.Fatal(http.ListenAndServe(":10101", nil))
}

26
internal/clustertests/pause_node_test.go
View file

@ -17,7 +17,7 @@ import (
pilosa "github.com/molecula/featurebase/v3"
boltdb "github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/topology"
"github.com/pkg/errors"
@ -43,18 +43,18 @@ func sendCmd(cmd string, args ...string) error {
return nil
}
func unpauseNode(node string) error {
unpauseArgs := []string{"container", "unpause", container(node)}
func unpauseNode(t *testing.T, node string) error {
unpauseArgs := []string{"container", "unpause", container(t, node)}
return sendCmd("docker", unpauseArgs...)
}
func pauseNode(node string) error {
pauseArgs := []string{"container", "pause", container(node)}
func pauseNode(t *testing.T, node string) error {
pauseArgs := []string{"container", "pause", container(t, node)}
return sendCmd("docker", pauseArgs...)
}
type keyInserter struct {
client *http.InternalClient
client *pilosa.InternalClient
uri *net.URI
index string
keys []string
@ -69,10 +69,10 @@ func getAddress(node string) string {
return node + ":10101"
}
func getClients(addrs []string) ([]*http.InternalClient, error) {
clients := make([]*http.InternalClient, 0, len(addrs))
func getClients(addrs []string) ([]*pilosa.InternalClient, error) {
clients := make([]*pilosa.InternalClient, 0, len(addrs))
for _, addr := range addrs {
c, err := http.NewInternalClient(addr, http.GetHTTPClient(nil))
c, err := pilosa.NewInternalClient(addr, pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
return nil, err
}
@ -93,7 +93,7 @@ func getURIsFromAddresses(addrs []string) ([]*net.URI, error) {
return uris, nil
}
func readIndexTranslateData(ctx context.Context, client *http.InternalClient, dirPath, index string, partition int) error {
func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient, dirPath, index string, partition int) error {
// read translateStore contents from endpoint
r, err := client.IndexTranslateDataReader(ctx, index, partition)
if err != nil {
@ -177,7 +177,7 @@ var errOpRetriable = errors.New("If operation failed on this error, it can be re
func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, keys []string) error {
// get client that's connected to node
address := getAddress(node)
client, err := http.NewInternalClient(address, http.GetHTTPClient(nil))
client, err := pilosa.NewInternalClient(address, pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
return err
}
@ -346,7 +346,7 @@ func TestPauseReplica(t *testing.T) {
// pause node
t.Logf("pause %s", nodeToPause)
err = pauseNode(nodeToPause)
err = pauseNode(t, nodeToPause)
if err != nil {
t.Fatalf("error on pause node %s: %v", nodeToPause, err)
}
@ -369,7 +369,7 @@ func TestPauseReplica(t *testing.T) {
// wait for cluster status to get back to normal
t.Logf("unpause %s", nodeToPause)
err = unpauseNode(nodeToPause)
err = unpauseNode(t, nodeToPause)
if err != nil {
t.Fatalf("error on unpause node %s: %v", nodeToPause, err)
}

0
internal/authclustertests/testdata/certs/README.md → internal/clustertests/testdata/certs/README.md vendored
View file

0
internal/authclustertests/testdata/certs/localhost.crt → internal/clustertests/testdata/certs/localhost.crt vendored
View file

0
internal/authclustertests/testdata/certs/localhost.csr → internal/clustertests/testdata/certs/localhost.csr vendored
View file

0
internal/authclustertests/testdata/certs/localhost.key → internal/clustertests/testdata/certs/localhost.key vendored
View file

0
internal/authclustertests/testdata/certs/pilosa-ca.crl → internal/clustertests/testdata/certs/pilosa-ca.crl vendored
View file

0
internal/authclustertests/testdata/certs/pilosa-ca.crt → internal/clustertests/testdata/certs/pilosa-ca.crt vendored
View file

0
internal/authclustertests/testdata/certs/pilosa-ca.key → internal/clustertests/testdata/certs/pilosa-ca.key vendored
View file

10
internal/authclustertests/testdata/featurebase.conf → internal/clustertests/testdata/featurebase.conf vendored
View file

@ -298,8 +298,8 @@
# Suffix should contain .crt or .pem
[tls]
certificate = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.crt"
key = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.key"
certificate = "/go/src/github.com/molecula/featurebase/internal/clustertests/testdata/certs/localhost.crt"
key = "/go/src/github.com/molecula/featurebase/internal/clustertests/testdata/certs/localhost.key"
# ==============================================================================
# Tracing Section
@ -373,11 +373,11 @@
client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
client-secret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
authorize-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/authorize"
token-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/token"
group-endpoint-url = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
token-url="http://fakeidp:10101/token"
group-endpoint-url = "http://fakeidp:10101/groups"
logout-url = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
scopes = ["https://graph.microsoft.com/.default", "offline_access"]
secret-key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
permissions = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/permissions.yaml"
permissions = "/go/src/github.com/molecula/featurebase/internal/clustertests/testdata/permissions.yaml"
query-log-path = "query-log-test.log"
redirect-base-url = "https://localhost:10101"

0
internal/authclustertests/testdata/permissions.yaml → internal/clustertests/testdata/permissions.yaml vendored
View file

299
http/client.go → internal_client.go
View file

File diff suppressed because it is too large Load diff

48
http/client_test.go → internal_client_test.go
View file

@ -1,5 +1,5 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package http_test
package pilosa_test
import (
"bufio"
@ -15,7 +15,7 @@ import (
"github.com/davecgh/go-spew/spew"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
@ -122,9 +122,9 @@ func TestClient_MultiNode(t *testing.T) {
// Connect to each node to compare results.
client := make([]*Client, 3)
client[0] = MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil))
client[1] = MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil))
client[2] = MustNewClient(c.GetNode(2).URL(), http.GetHTTPClient(nil))
client[0] = MustNewClient(c.GetNode(0).URL(), pilosa.GetHTTPClient(nil))
client[1] = MustNewClient(c.GetNode(1).URL(), pilosa.GetHTTPClient(nil))
client[2] = MustNewClient(c.GetNode(2).URL(), pilosa.GetHTTPClient(nil))
topN := 4
queryRequest := &pilosa.QueryRequest{
@ -188,7 +188,7 @@ func TestClient_Export(t *testing.T) {
cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
cmd.MustCreateField(t, "unkeyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000))
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
data := []pilosa.Bit{
{RowID: 1, ColumnID: 100, RowKey: "row1", ColumnKey: "col100"},
{RowID: 1, ColumnID: 101, RowKey: "row1", ColumnKey: "col101"},
@ -376,7 +376,7 @@ func TestClient_Import(t *testing.T) {
recIDs := []uint64{0, 3, 7}
valueIDs := []uint64{0, 3, 7}
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
// set API to point at the local node
c.SetInternalAPI(cmd.API)
@ -532,7 +532,7 @@ func TestClient_ImportRoaring(t *testing.T) {
// Send import request.
host := cluster.GetNode(0).URL()
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")
if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
@ -656,7 +656,7 @@ func TestClient_ImportRoaring_MultiView(t *testing.T) {
// Send import request.
host := cluster.GetNode(0).URL()
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportRoaringRequest{Views: map[string][]byte{}}
req.Views["a"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100")
req.Views["b"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100")
@ -681,7 +681,7 @@ func TestClient_ImportKeys(t *testing.T) {
cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
// Send import request.
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
baseReq := &pilosa.ImportRequest{
Index: "keyed",
Field: "keyedf",
@ -774,8 +774,8 @@ func TestClient_ImportKeys(t *testing.T) {
cmd0.MustCreateField(t, "keyed", "keyedf1", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys())
// Send import request.
c0 := MustNewClient(host0, http.GetHTTPClient(nil))
c1 := MustNewClient(host1, http.GetHTTPClient(nil))
c0 := MustNewClient(host0, pilosa.GetHTTPClient(nil))
c1 := MustNewClient(host1, pilosa.GetHTTPClient(nil))
// Import to node0.
t.Run("Import node0", func(t *testing.T) {
@ -852,7 +852,7 @@ func TestClient_ImportKeys(t *testing.T) {
}
// Send import request.
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportValueRequest{
Index: "i",
Field: "f",
@ -931,7 +931,7 @@ func TestClient_ImportIDs(t *testing.T) {
}
// Send import request.
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportValueRequest{
Index: idxName,
Field: fldName,
@ -999,7 +999,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Send import request.
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportValueRequest{
Index: "i",
Field: "f",
@ -1078,7 +1078,7 @@ func TestClient_ImportExistence(t *testing.T) {
}
// Send import request.
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportRequest{
Index: "iset",
Field: "fset",
@ -1114,7 +1114,7 @@ func TestClient_ImportExistence(t *testing.T) {
}
// Send import request.
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
req := &pilosa.ImportValueRequest{
Index: "iint",
Field: "fint",
@ -1155,7 +1155,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
// Set a bit on a different shard.
hldr.SetBit("i", "f", 0, 1)
c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil))
blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", "standard", 0)
if err != nil {
t.Fatal(err)
@ -1180,7 +1180,7 @@ func TestClient_CreateDecimalField(t *testing.T) {
defer cluster.Close()
cmd := cluster.GetNode(0)
c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil))
index := "cdf"
err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{})
@ -1290,8 +1290,8 @@ func TestClientTransactions(t *testing.T) {
coord := c.GetPrimary()
other := c.GetNonPrimary()
client0 := MustNewClient(coord.URL(), http.GetHTTPClient(nil))
client1 := MustNewClient(other.URL(), http.GetHTTPClient(nil))
client0 := MustNewClient(coord.URL(), pilosa.GetHTTPClient(nil))
client1 := MustNewClient(other.URL(), pilosa.GetHTTPClient(nil))
// can create, list, get, and finish a transaction
var expDeadline time.Time
@ -1444,12 +1444,12 @@ func TestClientTransactions(t *testing.T) {
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*http.InternalClient
*pilosa.InternalClient
}
// MustNewClient returns a new instance of Client. Panic on error.
func MustNewClient(host string, h *gohttp.Client) *Client {
c, err := http.NewInternalClient(host, h)
c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
panic(err)
}
@ -1497,7 +1497,7 @@ func TestClient_ImportRoaringExists(t *testing.T) {
}
// Send import request.
host := node.URL()
c := MustNewClient(host, http.GetHTTPClient(nil))
c := MustNewClient(host, pilosa.GetHTTPClient(nil))
// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")

68
mmap_test.go
View file

@ -1,68 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"math/rand"
"runtime"
"testing"
"github.com/molecula/featurebase/v3/logger"
)
type cv struct {
cols []uint64
vals []int64
}
func forceSnapshotsCheckMapping(t *testing.T) {
depth := uint64(6)
f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewStandard, 0)
tx.Rollback()
f.Logger = logger.NewLogfLogger(t)
defer f.Clean(t)
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
for i := 0; i < f.MaxOpN; i++ {
_, _ = f.setBit(tx, 0, uint64(32*i))
}
// force snapshot so we get a mmapped row...
err := f.Snapshot()
if err != nil {
t.Fatalf("initial snapshot error: %v", err)
}
values := make([]cv, 1024)
for i := range values {
cols := make([]uint64, 128)
vals := make([]int64, 128)
for j := range cols {
// pick values in the first 16 cols of each of the 16
// shards in a default shardwidth, so each set will
// probably change some values from the previous one.
cols[j] = uint64(((rand.Int63n(16) & int64(i>>2)) << 16) + rand.Int63n(16))
vals[j] = int64(rand.Int63n(1 << depth))
}
values[i] = cv{cols, vals}
}
// modify the original bitmap, until it causes a snapshot, which
// then invalidates the other map...
for i := 0; i < 32; i++ {
cv := values[i%len(values)]
// periodically force gc, so if we have a small pool of maps
// we'll go in and out of mapping mode
if i%5 == 0 {
runtime.GC()
}
err := f.importValue(tx, cv.cols, cv.vals, depth, (i%3 == 1))
if err != nil {
t.Fatalf("importValue[%d]: %v", i, err)
}
err = f.Snapshot()
if err != nil {
t.Fatalf("snapshot[%d]: %v", i, err)
}
}
}

19
pilosa.go
View file

@ -2,13 +2,11 @@
package pilosa
import (
"os"
"regexp"
"time"
"github.com/molecula/featurebase/v3/disco"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/storage"
"github.com/pkg/errors"
)
@ -157,20 +155,3 @@ func AddressWithDefaults(addr string) (*pnet.URI, error) {
}
return pnet.NewURIFromAddress(addr)
}
// CurrentBackend is one step in an attempt to centralize (and either minimize
// or completely remove), the calls to environment variables throughout the
// tests. Ideally we could get rid of this and rely completely on the
// configuration parameters.
func CurrentBackend() string {
return os.Getenv("PILOSA_STORAGE_BACKEND")
}
// CurrentBackendOrDefault tries the environment variable first, but falls back
// to the default backend if the environment variable is empty.
func CurrentBackendOrDefault() string {
if backend := os.Getenv("PILOSA_STORAGE_BACKEND"); backend != "" {
return backend
}
return storage.DefaultBackend
}

10
pprof.go
View file

@ -19,10 +19,7 @@ import (
// commented out—in holder.go.
func CPUProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
backend := CurrentBackend()
if backend == "" {
backend = storage.DefaultBackend
}
backend := storage.DefaultBackend
path := outpath + "." + backend
f, err := os.Create(path)
vprint.PanicOn(err)
@ -45,10 +42,7 @@ func CPUProfileForDur(dur time.Duration, outpath string) {
// commented out—in holder.go.
func MemProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
backend := CurrentBackend()
if backend == "" {
backend = storage.DefaultBackend
}
backend := storage.DefaultBackend
path := outpath + "." + backend
f, err := os.Create(path)
vprint.PanicOn(err)

3
qa/scripts/runSamsungGauntlet.sh
View file

@ -2,9 +2,6 @@
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
# requires TF_VAR_gitlab_token env var to be set
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
# requires TF_VAR_branch env var to be set
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi

2
qa/scripts/runSmokeTest.sh
View file

@ -2,8 +2,6 @@
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
# requires TF_VAR_gitlab_token env var to be set
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
# requires TF_VAR_branch env var to be set
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi

3
qa/scripts/setupSamsungGauntlet.sh
View file

@ -3,9 +3,6 @@
# To run script: ./setupSamsungGauntlet.sh
export TF_IN_AUTOMATION=1
# requires TF_VAR_gitlab_token env var to be set
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
# requires TF_VAR_branch env var to be set
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi

3
qa/scripts/setupSmokeTest.sh
View file

@ -3,9 +3,6 @@
# To run script: ./setupSmokeTest.sh
export TF_IN_AUTOMATION=1
# requires TF_VAR_gitlab_token env var to be set
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
# requires TF_VAR_branch env var to be set
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi

1
qa/scripts/teardownSamsungGauntlet.sh
View file

@ -1,7 +1,6 @@
#!/bin/bash
# To run script: ./teardownSamsungGauntlet.sh
# requires TF_VAR_gitlab_token env var to be set
cd qa/tf/gauntlet/samsung
export TF_IN_AUTOMATION=1

2
qa/scripts/teardownSmokeTest.sh
View file

@ -1,8 +1,6 @@
#!/bin/bash
# To run script: ./teardownSmokeTest.sh
# requires TF_VAR_gitlab_token env var to be set
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
# requires TF_VAR_branch env var to be set
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi

3
qa/scripts/testSmokeTest.sh
View file

@ -1,8 +1,5 @@
#!/bin/bash
# requires TF_VAR_gitlab_token env var to be set
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
# requires TF_VAR_branch env var to be set
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi

7
qa/scripts/utilCluster.sh
View file

@ -125,15 +125,14 @@ executeGeneralNodeConfigCommands() {
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data/featurebase"
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /data/featurebase"
# TODO handle different archs
echo "Getting featurebase binary (https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64)..."
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "curl --fail --header 'PRIVATE-TOKEN: ${TF_VAR_gitlab_token}' -o /home/ec2-user/featurebase_linux_arm64 https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64"
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase_linux_arm64 ec2-user@${NODEIP}:
if (( $? != 0 ))
then
echo "Unable to get featurebase binary"
echo "featurebase binary copy failed"
exit 1
fi
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64"
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64"
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase"

5
qa/tf/ci/smoketest/variables.tf
View file

@ -8,11 +8,6 @@ variable "profile" {
type = string
}
variable "gitlab_token" {
description = "The API token for taking to Gitlab API - expected to come from an env variable."
type = string
}
variable "cluster_prefix" {
type = string
description = "This is a identifier that will be prefixed to created resources"

5
qa/tf/gauntlet/samsung/variables.tf
View file

@ -8,11 +8,6 @@ variable "profile" {
type = string
}
variable "gitlab_token" {
description = "The API token for taking to Gitlab API - expected to come from an env variable."
type = string
}
variable "cluster_prefix" {
type = string
description = "This is a identifier that will be prefixed to created resources"

1
rbf/db.go
View file

@ -704,7 +704,6 @@ func (db *DB) afterCurrentTx(callback func()) {
defer db.mu.Unlock()
txw.callback()
}()
return
}
// removeTx removes an active transaction from the database. it obtains

2
roaring/filter.go
View file

@ -579,7 +579,6 @@ func (b *BitmapRowFilterMultiFilter) ConsiderData(key FilterKey, data *Container
// offsets the input bitmap's containers have, it matches them against
// corresponding keys.
type BitmapBitmapFilter struct {
filter *Bitmap // We don't use this while iterating, but in ludicrous edge cases it might be holding a generation we need. TODO @seebs I don't understand why this mentions generations
containers []*Container
nextOffsets []uint64
callback func(uint64) error
@ -629,7 +628,6 @@ func (b *BitmapBitmapFilter) ConsiderData(key FilterKey, data *Container) Filter
// because offset-within-row is what we care about.
func NewBitmapBitmapFilter(filter *Bitmap, callback func(uint64) error) *BitmapBitmapFilter {
b := &BitmapBitmapFilter{
filter: filter,
callback: callback,
containers: make([]*Container, rowWidth),
nextOffsets: make([]uint64, rowWidth),

53
roaring/roaring.go
View file

@ -167,7 +167,6 @@ type ContainerIterator interface {
// Bitmap represents a roaring bitmap.
type Bitmap struct {
Containers Containers
Source Source
// User-defined flags.
Flags byte
@ -248,7 +247,6 @@ func (b *Bitmap) Freeze() *Bitmap {
// Create a copy of the bitmap structure.
other := &Bitmap{
Containers: b.Containers.Freeze(),
Source: b.Source,
}
return other
@ -609,20 +607,13 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap {
hi0, hi1 := highbits(start), highbits(end)
citer, _ := b.Containers.Iterator(hi0)
other := NewSliceBitmap()
mappedAny := false
for citer.Next() {
k, c := citer.Value()
if k >= hi1 {
break
}
if c.Mapped() {
mappedAny = true
}
other.Containers.Put(off+(k-hi0), c.Freeze())
}
if b.Source != nil && mappedAny {
other.Source = b.Source
}
return other
}
@ -661,7 +652,6 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
// Intersect returns the intersection of b and other.
func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
output := NewBitmap()
usedB, usedOther := false, false
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)
i, j := iiter.Next(), jiter.Next()
@ -676,26 +666,12 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
kj, cj = jiter.Value()
} else { // ki == kj
newC := intersect(ci, cj)
if newC == ci {
usedB = true
}
if newC == cj {
usedOther = true
}
output.Containers.Put(ki, newC)
i, j = iiter.Next(), jiter.Next()
ki, ci = iiter.Value()
kj, cj = jiter.Value()
}
}
switch {
case usedB && usedOther:
output.Source = MergeSources(b.Source, other.Source)
case usedB:
output.Source = b.Source
case usedOther:
output.Source = other.Source
}
return output
}
@ -1192,43 +1168,26 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) {
func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) {
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)
usedB, usedOther := false, false
i, j := iiter.Next(), jiter.Next()
ki, ci := iiter.Value()
kj, cj := jiter.Value()
for i || j {
if i && (!j || ki < kj) {
target.Containers.Put(ki, ci.Freeze())
usedB = true
i = iiter.Next()
ki, ci = iiter.Value()
} else if j && (!i || ki > kj) {
target.Containers.Put(kj, cj.Freeze())
usedOther = true
j = jiter.Next()
kj, cj = jiter.Value()
} else { // ki == kj
newC := union(ci, cj)
target.Containers.Put(ki, newC)
if newC == ci {
usedB = true
}
if newC == cj {
usedOther = true
}
i, j = iiter.Next(), jiter.Next()
ki, ci = iiter.Value()
kj, cj = jiter.Value()
}
}
switch {
case usedB && usedOther:
target.Source = MergeSources(b.Source, other.Source)
case usedB:
target.Source = b.Source
case usedOther:
target.Source = other.Source
}
}
// unionInPlace stores the union of b and others into b. The others will
@ -1324,14 +1283,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
bitmapIters = make(handledIters, 0, requiredSliceSize)
}
var sources []Source
if b.Source != nil {
sources = append(sources, b.Source)
}
for _, other := range others {
if other.Source != nil {
sources = append(sources, other.Source)
}
otherIter, _ := other.Containers.Iterator(0)
if otherIter.Next() {
bitmapIters = append(bitmapIters, handledIter{
@ -1341,8 +1293,6 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
})
}
}
// new bitmap might have containers from any of those bitmaps in it
b.Source = MergeSources(sources...)
// Loop until we've exhausted every iter.
hasNext := true
@ -1505,9 +1455,6 @@ func (b *Bitmap) singleDifference(other *Bitmap) *Bitmap {
// Xor returns the bitwise exclusive or of b and other.
func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
output := NewBitmap()
// Xor can end up with containers from either parent if the other
// had no container or an empty container.
output.Source = MergeSources(b.Source, other.Source)
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)

85
roaring/source.go
View file

@ -1,85 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package roaring
import (
"strings"
)
// A Source represents the source a given bitmap gets its data from,
// such as a memory-mapped file. When combining bitmaps, we might
// track them together in a single combined-source of some sort.
type Source interface {
ID() string
Dead() bool
}
// MergeSources combines sources. If you have two bitmaps, and you're
// combining them, then the combination's source is a combination of
// those two sources.
func MergeSources(sources ...Source) Source {
sourceCount := 0
totalCount := 0
var lastSource Source
for _, s := range sources {
if s == nil {
continue
}
lastSource = s
if s, ok := s.(combinedSource); ok {
sourceCount++
totalCount += len(s)
} else {
sourceCount++
totalCount++
}
}
// if there's no sources (this includes all sources being
// empty combinedSources), we don't have a source.
if totalCount == 0 {
return nil
}
// if there's exactly one source, combined or otherwise, that's
// fine, we'll just return it.
if sourceCount == 1 {
return lastSource
}
// make a new combinedSource, flattening any combinedSources
// already present.
newSources := make([]Source, 0, totalCount)
for _, s := range sources {
if s == nil {
continue
}
if s, ok := s.(combinedSource); ok {
newSources = append(newSources, s...)
} else {
newSources = append(newSources, s)
}
}
return combinedSource(newSources)
}
// SetSource tells the bitmap what source to associate with new things it
// creates. This is possibly logically incorrect.
func (b *Bitmap) SetSource(s Source) {
b.Source = s
}
type combinedSource []Source
func (c combinedSource) ID() string {
ids := make([]string, len(c))
for i := range c {
ids[i] = c[i].ID()
}
return strings.Join(ids, ",")
}
func (c combinedSource) Dead() bool {
for i := range c {
if c[i].Dead() {
return true
}
}
return false
}

662
rrtx.go
View file

@ -1,662 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/storage"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
)
// RoaringTx represents a fake transaction object for Roaring storage.
type RoaringTx struct {
write bool
Index *Index
Field *Field
fragment *fragment
o Txo
sn int64 // serial number
done bool
mu sync.Mutex // protect done as it changes state
w *RoaringWrapper
}
func (tx *RoaringTx) Type() string {
return RoaringTxn
}
// based on view.openFragments()
func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err error) {
shardMap = make(map[uint64]bool)
path := filepath.Join(optionalViewPath, "fragments")
file, err := os.Open(path)
if os.IsNotExist(err) {
return
} else if err != nil {
return nil, errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
//vv("rrtx next fi = '%v'", fi.Name())
if fi.IsDir() {
continue
}
name := fi.Name()
if strings.HasSuffix(name, ".cache") {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(name), 10, 64)
if err != nil {
//vv("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
//panic(fmt.Sprintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name()))
//tx.Index.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
continue
}
shardMap[shard] = true
}
return
}
// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE
// the transaction Commits or Rollsback.
func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.bitmap(index, field, view, shard)
vprint.PanicOn(err)
return b.Iterator()
}
// ImportRoaringBits return values changed and rowSet will be inaccurate if
// the data []byte is supplied. This mimics the traditional roaring-per-file
// and should be faster.
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
f, err := tx.getFragment(index, field, view, shard)
if err != nil {
return 0, nil, err
}
changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize)
return
}
func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
}
// Rollback
func (tx *RoaringTx) Rollback() {
tx.w.CleanupTx(tx)
}
// Commit
func (tx *RoaringTx) Commit() error {
tx.w.CleanupTx(tx)
return nil
}
func (tx *RoaringTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.bitmap(index, field, view, shard)
}
func (tx *RoaringTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
return b.Containers.Get(key), nil
}
func (tx *RoaringTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.Containers.Put(key, c)
return nil
}
func (tx *RoaringTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.Containers.Remove(key)
return nil
}
func (tx *RoaringTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
//vv("RoaringTx.Add(index='%v', shard='%v') stack=\n%v", index, shard, stack())
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
// Note: do not replace b.AddN() with b.DirectAddN().
// DirectAddN() does not do op-log operations inside roaring, so the
// on-disk representation no longer matches the in-memory operations.
count, err := b.AddN(a...)
return count, err
}
func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.RemoveN(a...)
}
func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return false, err
}
return b.Contains(v), nil
}
func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, false, errors.Wrap(err, "getting bitmap")
}
//vv("b bitmap back from bitmap(index='%v', field='%v', view='%v', shard='%v')='%#v'", index, field, view, shard, b.Slice())
citer, found = b.Containers.Iterator(key)
return citer, found, nil
}
func (tx *RoaringTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
return b.ForEach(fn)
}
func (tx *RoaringTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
return b.ForEachRange(start, end, fn)
}
func (tx *RoaringTx) Count(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.Count(), nil
}
func (tx *RoaringTx) Max(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.Max(), nil
}
func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, false, err
}
v, ok := b.Min()
return v, ok, nil
}
func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.CountRange(start, end), nil
}
func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
return b.OffsetRange(offset, start, end), nil
}
// getFragment is used by IncrementOpN() and by bitmap()
func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) {
// If a fragment is attached, always use it. Since it was set at Tx creation,
// it is highly likely to be correct.
if tx.fragment != nil {
// but still a basic sanity check.
if tx.fragment.index() != index ||
tx.fragment.field() != field ||
tx.fragment.view() != view ||
tx.fragment.shard != shard {
// still insist that index and shard match, since that is the current scope of all Tx.
if tx.fragment.index() != index ||
tx.fragment.shard != shard {
panic(fmt.Sprintf("different fragment cached vs requested. index='%v', field='%v'; view='%v'; shard='%v'; tx.fragment='%#v'", index, field, view, shard, tx.fragment))
}
// cannot use this fragment.
tx.fragment = nil
} else {
return tx.fragment, nil
}
}
// If a field is attached, start from there.
// Otherwise look up the field from the index.
f := tx.Field
if f == nil {
// we cannot assume that the tx.Index that we "started" on is the same
// as the index we are being queried; it might be foreign: TestExecutor_ForeignIndex
// So go through the holder
idx := tx.Index.holder.Index(index)
if idx == nil {
// only thing we can try is the cached index, and hope we aren't being asked for a foreign index.
f = tx.Index.Field(field)
if f == nil {
return nil, newNotFoundError(ErrFieldNotFound, field)
}
} else {
if f = idx.Field(field); f == nil {
return nil, newNotFoundError(ErrFieldNotFound, field)
}
}
}
// INVAR: f is not nil.
v := f.view(view)
if v == nil {
return nil, errors.Wrapf(ViewNotFound, "getting %s", view)
}
frag := v.Fragment(shard)
if frag == nil {
return nil, errors.Wrapf(FragmentNotFound, "field:%q, view:%q, shard:%d", field, view, shard)
}
// Note: we cannot cache frag into tx.fragment.
// Empirically, it breaks 245 top-level pilosa tests.
// tx.fragment = frag // breaks the world.
return frag, nil
}
const ViewNotFound = Error("view not found")
const FragmentNotFound = Error("fragment not found")
func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
frag, err := tx.getFragment(index, field, view, shard)
if err != nil {
return nil, errors.Wrap(err, "getFragment")
}
return frag.storage, nil
}
func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) {
vs = NewFieldView2Shards()
// A) open the index directory
f, err := os.Open(idx.FieldsPath())
if err != nil {
return nil, errors.Wrap(err, "opening directory")
}
defer f.Close()
fieldFIs, err := f.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading directory")
}
//vv("roaringGetFieldView2Shards A) opened index path '%v'", idx.path)
// B) read the name of each field under the index
for _, loopFieldFi := range fieldFIs {
fieldFI := loopFieldFi
if !fieldFI.IsDir() {
continue
}
field := fieldFI.Name()
//vv("roaringGetFieldView2Shards B) on field '%v'", field)
fieldPath := filepath.Join(idx.FieldsPath(), field)
viewsDir := filepath.Join(fieldPath, "views")
file, err := os.Open(viewsDir)
if os.IsNotExist(err) {
//return nil
continue
} else if err != nil {
return nil, errors.Wrapf(err, "opening view directory '%v'", viewsDir)
}
defer file.Close()
// C) read the name of each view under the field
viewFIs, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrapf(err, "reading views directory '%v'", viewsDir)
}
for _, viewFI := range viewFIs {
if !viewFI.IsDir() {
continue
}
view := viewFI.Name()
roaringViewPath := filepath.Join(viewsDir, view)
shardMap, err := roaringMapOfShards(roaringViewPath)
if err != nil {
return nil, errors.Wrapf(err, "reading view path directory '%v'", roaringViewPath)
}
if len(shardMap) == 0 {
//vv("roaringGetFieldView2Shards C) SAVED SPACE! field '%v' view '%v' had no shards", field, view)
continue
}
ss := newShardSetFromMap(shardMap)
fv := txkey.FieldView{Field: field, View: view}
vs.addViewShardSet(fv, ss)
//vv("roaringGetFieldView2Shards C) added field '%v' view '%v' with shards '%#v'", field, view, ss.shards)
}
}
return
}
// inefficient for roaring. Instead use the roaringGetFieldView2Shards() above.
func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
// A) open the index directory
f, err := os.Open(idx.FieldsPath())
if err != nil {
return nil, errors.Wrap(err, "opening directory")
}
defer f.Close()
fieldFIs, err := f.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading directory")
}
//vv("A) shard %v, opened index path '%v'", shard, idx.path)
// B) read the name of each field under the index
for _, loopFieldFi := range fieldFIs {
fieldFI := loopFieldFi
if !fieldFI.IsDir() {
continue
}
field := fieldFI.Name()
//vv("B) on field '%v'", field)
fieldPath := filepath.Join(idx.FieldsPath(), field)
viewsDir := filepath.Join(fieldPath, "views")
file, err := os.Open(viewsDir)
if os.IsNotExist(err) {
//return nil
continue
} else if err != nil {
return nil, errors.Wrapf(err, "opening view directory '%v'", viewsDir)
}
defer file.Close()
// C) read the name of each view under the field
viewFIs, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrapf(err, "reading views directory '%v'", viewsDir)
}
for _, viewFI := range viewFIs {
if !viewFI.IsDir() {
continue
}
view := viewFI.Name()
roaringViewPath := filepath.Join(viewsDir, view)
shardMap, err := roaringMapOfShards(roaringViewPath)
if err != nil {
return nil, errors.Wrapf(err, "reading view path directory '%v'", roaringViewPath)
}
if len(shardMap) == 0 {
continue
}
// once we know we have data for this shard!
if shardMap[shard] {
fv := txkey.FieldView{Field: field, View: view}
//vv("C) adding fv '%#v'", fv)
fvs = append(fvs, fv)
}
}
}
// directory stuff isn't returned in sorted order, we must sort.
sort.Slice(fvs, func(i, j int) bool {
if fvs[i].Field < fvs[j].Field {
return true
}
if fvs[i].Field > fvs[j].Field {
return false
}
return fvs[i].View < fvs[j].View
})
return
}
func (tx *RoaringTx) GetFieldSizeBytes(index, field string) (uint64, error) {
return 0, nil
}
//////// registrar and wrapper machinery
// roaringRegistrar mirrors the machinery expected
// for all backends for the roaring files approach.
//
type roaringRegistrar struct {
mu sync.Mutex
mp map[*RoaringWrapper]bool
path2db map[string]*RoaringWrapper
}
func (r *roaringRegistrar) Size() int {
r.mu.Lock()
defer r.mu.Unlock()
nmp := len(r.mp)
npa := len(r.path2db)
if nmp != npa {
panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa))
}
return nmp
}
var globalRoaringReg *roaringRegistrar = newRoaringRegistrar()
func newRoaringRegistrar() *roaringRegistrar {
return &roaringRegistrar{
mp: make(map[*RoaringWrapper]bool),
path2db: make(map[string]*RoaringWrapper),
}
}
func (r *roaringRegistrar) unprotectedRegister(w *RoaringWrapper) {
r.mp[w] = true
r.path2db[w.path] = w
}
// unregister removes w from r
func (r *roaringRegistrar) unregister(w *RoaringWrapper) {
r.mu.Lock()
delete(r.mp, w)
delete(r.path2db, w.path)
r.mu.Unlock()
}
// openRoaringDB will check the registry and make a new instance only
// if one does not exist for its path0. Otherwise it returns
// the existing instance.
func (r *roaringRegistrar) OpenDBWrapper(path string, doAllocZero bool, _ *storage.Config) (DBWrapper, error) {
r.mu.Lock()
defer r.mu.Unlock()
w, ok := r.path2db[path]
if ok {
return w, nil
}
// otherwise, make a new roaring and store it in globalRoaringReg
w = &RoaringWrapper{
reg: r,
path: path,
}
r.unprotectedRegister(w)
return w, nil
}
func (w *RoaringWrapper) SetHolder(h *Holder) {
w.h = h
}
func (w *RoaringWrapper) Path() string {
return w.path
}
func (w *RoaringWrapper) HasData() (has bool, err error) {
return w.h.HasRoaringData()
}
func (w *RoaringWrapper) CleanupTx(tx Tx) {
r := tx.(*RoaringTx)
r.mu.Lock()
defer r.mu.Unlock()
if r.done {
return
}
r.done = true
}
func (w *RoaringWrapper) OpenListString() (r string) {
return "RoaringWrapper.OpenListString() not yet implemented"
}
func (w *RoaringWrapper) CloseDB() error {
return errors.New("CloseDB not supported in roaring")
}
func (w *RoaringWrapper) OpenDB() error {
return errors.New("OpenDB not supported in roaring")
}
// statically confirm that RoaringTx satisfies the Tx interface.
var _ Tx = (*RoaringTx)(nil)
// RoaringWrapper provides the NewTx() method.
type RoaringWrapper struct {
muDb sync.Mutex
path string
h *Holder
reg *roaringRegistrar
// make RoaringWrapper.Close() idempotent, avoiding panic on double Close()
closed bool
}
var globalNextTxSnRoaring int64
func (w *RoaringWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
sn := atomic.AddInt64(&globalNextTxSnRoaring, 1)
return &RoaringTx{
write: o.Write,
Field: o.Field,
Index: o.Index,
fragment: o.Fragment,
o: o,
sn: sn,
w: w,
}, nil
}
// Close shuts down the Roaring database.
func (w *RoaringWrapper) Close() (err error) {
w.muDb.Lock()
defer w.muDb.Unlock()
if !w.closed {
w.reg.unregister(w)
w.closed = true
}
return nil
}
func (w *RoaringWrapper) IsClosed() (closed bool) {
w.muDb.Lock()
closed = w.closed
w.muDb.Unlock()
return
}
func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error {
//vv("RoaringWrapper.DeleteField(index = '%v', field = '%v', fieldPath = '%v'", index, field, fieldPath)
// match txn sn count vs lmdb/etc.
atomic.AddInt64(&globalNextTxSnRoaring, 1)
err := os.RemoveAll(fieldPath)
if err != nil {
return errors.Wrap(err, "removing directory")
}
return nil
}
func (w *RoaringWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
// match txn sn count vs lmdb/etc.
atomic.AddInt64(&globalNextTxSnRoaring, 1)
fragment, ok := frag.(*fragment)
if !ok {
return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
}
// Delete fragment file.
if err := os.Remove(fragment.path()); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
}
return nil
}

30
server.go
View file

@ -64,11 +64,10 @@ type Server struct { // nolint: maligned
schemator disco.Schemator
// External
systemInfo SystemInfo
gcNotifier GCNotifier
logger logger.Logger
queryLogger logger.Logger
snapshotQueue SnapshotQueue
systemInfo SystemInfo
gcNotifier GCNotifier
logger logger.Logger
queryLogger logger.Logger
nodeID string
uri pnet.URI
@ -87,7 +86,7 @@ type Server struct { // nolint: maligned
// HolderConfig stashes server options that are really Holder options.
holderConfig *HolderConfig
defaultClient InternalClient
defaultClient *InternalClient
dataDir string
// Threshold for logging long-running queries
@ -194,7 +193,7 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption {
// OptServerInternalClient is a functional option on Server
// used to set the implementation of InternalClient.
func OptServerInternalClient(c InternalClient) ServerOption {
func OptServerInternalClient(c *InternalClient) ServerOption {
return func(s *Server) error {
s.defaultClient = c
s.cluster.InternalClient = c
@ -406,7 +405,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
cluster: cluster,
diagnostics: newDiagnosticsCollector(defaultDiagnosticServer),
systemInfo: newNopSystemInfo(),
defaultClient: nopInternalClient{},
defaultClient: &InternalClient{}, // TODO may need to make this a valid thing
gcNotifier: NopGCNotifier,
@ -512,7 +511,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
return s, nil
}
func (s *Server) InternalClient() InternalClient {
func (s *Server) InternalClient() *InternalClient {
return s.defaultClient
}
@ -544,13 +543,6 @@ func (s *Server) UpAndDown() error {
func (s *Server) Open() error {
s.logger.Infof("open server. PID %v", os.Getpid())
if s.holder.NeedsSnapshot() {
// Start background monitoring.
s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
} else {
s.snapshotQueue = defaultSnapshotQueue //TODO (twg) rethink this
}
// Log startup
err := s.holder.logStartup()
if err != nil {
@ -612,7 +604,6 @@ func (s *Server) Open() error {
return errors.Wrap(err, "opening Holder")
}
// bring up the background tasks for the holder.
s.holder.SnapshotQueue = s.snapshotQueue
s.holder.Activate()
// if we joined existing cluster then broadcast "resize on add" message
if initState == disco.InitialClusterStateExisting {
@ -743,11 +734,6 @@ func (s *Server) Close() error {
if s.holder != nil {
errh = s.holder.Close()
}
if s.snapshotQueue != nil {
s.holder.SnapshotQueue = nil
s.snapshotQueue.Stop()
s.snapshotQueue = nil
}
// prefer to return holder error over cluster
// error. This order is somewhat arbitrary. It would be better if we had

15
server/grpc.go
View file

@ -1598,6 +1598,12 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
return nil, err
}
LogQuery(ctx, info.FullMethod, req, server.logger)
// reset the molecula-chip cookie just in case the token was refreshed
md, ok := metadata.FromIncomingContext(ctx)
if uinfo, yeah := ctx.Value("userinfo").(*authn.UserInfo); ok && yeah {
server.auth.SetGRPCMetadata(ctx, md, uinfo.Token)
}
return handler(ctx, req)
},
))
@ -1607,6 +1613,11 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
if err != nil {
return err
}
// reset the molecula-chip cookie just in case the token was refreshed
md, ok := metadata.FromIncomingContext(ctx)
if uinfo, yeah := ctx.Value("userinfo").(*authn.UserInfo); ok && yeah {
server.auth.SetGRPCMetadata(ctx, md, uinfo.Token)
}
return handler(srv, &wrappedStream{ss, ctx})
},
))
@ -1645,7 +1656,7 @@ func LogQuery(ctx context.Context, method string, req interface{}, logger logger
}
switch r := req.(type) {
case *pb.QueryPQLRequest:
logger.Infof("GRPC: %v, %v, %v, %v, %v, %s", ip, ua, method, uinfo.UserID, uinfo.UserName, r.Pql)
logger.Infof("GRPC: %v, %v, %v, %v, %v, [%s]%s", ip, ua, method, uinfo.UserID, uinfo.UserName, r.Index, r.Pql)
case *pb.QuerySQLRequest:
logger.Infof("GRPC: %v, %v, %v, %v, %v, %s", ip, ua, method, uinfo.UserID, uinfo.UserName, r.Sql)
default:
@ -1697,7 +1708,7 @@ func Valid(ctx context.Context, auth *authn.Auth) (context.Context, error) {
}
token := strings.TrimPrefix(authorization[0], "Bearer ")
uinfo, err := auth.Authenticate(token)
uinfo, err := auth.Authenticate(ctx, token)
if err != nil {
return ctx, status.Errorf(codes.Unauthenticated, err.Error())
}

8
server/grpc_test.go
View file

@ -1055,15 +1055,13 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
// make a valid token
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
groupString, _ := authn.ToGob64(groups)
claims["molecula-idp-groups"] = groupString
claims["oid"] = "42"
claims["name"] = name
secretKey, _ := hex.DecodeString(auth.SecretKey)
validToken, err := tkn.SignedString(secretKey)
if err != nil {
panic(err)
t.Fatalf("unexpected error creating token %v", err)
}
validToken = "Bearer " + validToken
@ -1453,8 +1451,8 @@ func TestLogQuery(t *testing.T) {
},
{
name: "QueryPQLReq",
req: &pb.QueryPQLRequest{Pql: "Count(All())"},
expected: fmt.Sprintf("GRPC: %v, %v, %v, %v, %v, %v\n", "", []string{}, "test!", uinfo.UserID, uinfo.UserName, "Count(All())"),
req: &pb.QueryPQLRequest{Index: "index", Pql: "Count(All())"},
expected: fmt.Sprintf("GRPC: %v, %v, %v, %v, %v, %v\n", "", []string{}, "test!", uinfo.UserID, uinfo.UserName, "[index]Count(All())"),
},
}
for _, test := range cases {

17
server/handler_test.go
View file

@ -21,7 +21,6 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
@ -31,7 +30,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) {
cluster := test.MustRunCluster(t, 3)
defer cluster.Close()
cmd := cluster.GetNode(0)
h := cmd.Handler.(*http.Handler).Handler
h := cmd.Handler.(*pilosa.Handler).Handler
t.Run("PostSchema", func(t *testing.T) {
w := httptest.NewRecorder()
@ -70,7 +69,7 @@ func TestHandler_Endpoints(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
h := cmd.Handler.(*http.Handler).Handler
h := cmd.Handler.(*pilosa.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
@ -1120,7 +1119,7 @@ func TestHandler_Endpoints(t *testing.T) {
clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
defer clus.Close()
w = httptest.NewRecorder()
h1 := clus.GetNode(0).Handler.(*http.Handler).Handler
h1 := clus.GetNode(0).Handler.(*pilosa.Handler).Handler
h1.ServeHTTP(w, req)
result = w.Result()
@ -1383,7 +1382,7 @@ func TestHandler_Endpoints(t *testing.T) {
clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
defer clus.Close()
w = httptest.NewRecorder()
h := clus.GetNode(0).Handler.(*http.Handler).Handler
h := clus.GetNode(0).Handler.(*pilosa.Handler).Handler
h.ServeHTTP(w, req)
result = w.Result()
@ -1402,11 +1401,11 @@ func TestCluster_TranslateStore(t *testing.T) {
cluster.Nodes[0] = test.NewCommandNode(t,
server.OptCommandServerOptions(
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
),
)
if err := cluster.GetIdleNode(0).Start(); err != nil {
if err := cluster.Start(); err != nil {
t.Fatalf("starting node 0: %v", err)
}
defer cluster.GetIdleNode(0).Close() // nolint: errcheck
@ -1423,7 +1422,7 @@ func TestClusterTranslator(t *testing.T) {
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
@ -1487,7 +1486,7 @@ func TestClusterTranslator(t *testing.T) {
// defer cluster.Close()
// cmd := cluster.GetNode(0)
// h := cmd.Handler.(*http.Handler).Handler
// h := cmd.Handler.(*pilosa.Handler).Handler
// w := httptest.NewRecorder()

40
server/server.go
View file

@ -36,7 +36,6 @@ import (
petcd "github.com/molecula/featurebase/v3/etcd"
"github.com/molecula/featurebase/v3/gcnotify"
"github.com/molecula/featurebase/v3/gopsutil"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/logger"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/prometheus"
@ -74,7 +73,7 @@ type Command struct {
logger loggerLogger
queryLogger loggerLogger
Handler pilosa.Handler
Handler pilosa.HandlerI
grpcServer *grpcServer
grpcLn net.Listener
API *pilosa.API
@ -405,7 +404,7 @@ func (m *Command) SetupServer() error {
// Save listenURI for later reference.
m.listenURI = uri
c := http.GetHTTPClient(m.tlsConfig)
c := pilosa.GetHTTPClient(m.tlsConfig)
// Get advertise address as uri.
advertiseURI, err := pilosa.AddressWithDefaults(m.Config.Advertise)
@ -473,7 +472,7 @@ func (m *Command) SetupServer() error {
pilosa.OptServerDiagnosticsInterval(diagnosticsInterval),
pilosa.OptServerExecutorPoolSize(m.Config.WorkerPoolSize),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})),
pilosa.OptServerOpenIDAllocator(pilosa.OpenIDAllocator),
pilosa.OptServerLogger(m.logger),
pilosa.OptServerQueryLogger(m.queryLogger),
@ -498,9 +497,9 @@ func (m *Command) SetupServer() error {
serverOptions = append(serverOptions, m.serverOptions...)
if m.Config.Auth.Enable {
serverOptions = append(serverOptions, pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c, http.WithSecretKey(m.Config.Auth.SecretKey))))
serverOptions = append(serverOptions, pilosa.OptServerInternalClient(pilosa.NewInternalClientFromURI(uri, c, pilosa.WithSecretKey(m.Config.Auth.SecretKey), pilosa.WithSerializer(proto.Serializer{}))))
} else {
serverOptions = append(serverOptions, pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)))
serverOptions = append(serverOptions, pilosa.OptServerInternalClient(pilosa.NewInternalClientFromURI(uri, c, pilosa.WithSerializer(proto.Serializer{}))))
}
m.Server, err = pilosa.NewServer(serverOptions...)
@ -548,7 +547,7 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "setting up queryLogger")
}
m.queryLogger.Infof("Featurebase Server Started")
m.queryLogger.Infof("Starting Featurebase...")
m.queryLogger.Infof("Group with admin level access: %v", p.Admin)
m.queryLogger.Infof("Permissions: %+v", p.Permissions)
@ -572,18 +571,23 @@ func (m *Command) SetupServer() error {
OptGRPCServerPerm(&p),
OptGRPCServerQueryLogger(m.queryLogger),
)
if err != nil {
return errors.Wrap(err, "getting grpcServer")
}
m.Handler, err = http.NewHandler(
http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
http.OptHandlerAPI(m.API),
http.OptHandlerLogger(m.logger),
http.OptHandlerQueryLogger(m.queryLogger),
http.OptHandlerFileSystem(&statik.FileSystem{}),
http.OptHandlerListener(m.ln, m.Config.Advertise),
http.OptHandlerCloseTimeout(m.closeTimeout),
http.OptHandlerMiddleware(m.grpcServer.middleware(m.Config.Handler.AllowedOrigins)),
http.OptHandlerAuthN(m.auth),
http.OptHandlerAuthZ(&p),
m.Handler, err = pilosa.NewHandler(
pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
pilosa.OptHandlerAPI(m.API),
pilosa.OptHandlerLogger(m.logger),
pilosa.OptHandlerQueryLogger(m.queryLogger),
pilosa.OptHandlerFileSystem(&statik.FileSystem{}),
pilosa.OptHandlerListener(m.ln, m.Config.Advertise),
pilosa.OptHandlerCloseTimeout(m.closeTimeout),
pilosa.OptHandlerMiddleware(m.grpcServer.middleware(m.Config.Handler.AllowedOrigins)),
pilosa.OptHandlerAuthN(m.auth),
pilosa.OptHandlerAuthZ(&p),
pilosa.OptHandlerSerializer(proto.Serializer{}),
pilosa.OptHandlerRoaringSerializer(proto.RoaringSerializer),
)
return errors.Wrap(err, "new handler")
}

6
server/server_test.go
View file

@ -19,7 +19,7 @@ import (
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/http"
"github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/server"
@ -54,7 +54,7 @@ func TestMain_Set_Quick(t *testing.T) {
defer m.Close()
// Create client.
client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
client, err := pilosa.NewInternalClient(m.API.Node().URI.HostPort(), pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{}))
client.SetInternalAPI(m.API)
if err != nil {
t.Fatal(err)
@ -904,7 +904,7 @@ func TestQueryingWithQuotesAndStuff(t *testing.T) {
m := test.RunCommand(t)
defer m.Close()
client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
client, err := pilosa.NewInternalClient(m.API.Node().URI.HostPort(), pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{}))
client.SetInternalAPI(m.API)
if err != nil {
t.Fatal(err)

18
server_internal_test.go
View file

@ -2,7 +2,6 @@
package pilosa
import (
"runtime"
"testing"
"time"
@ -10,23 +9,6 @@ import (
"github.com/molecula/featurebase/v3/testhook"
)
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
roaringOnlyTest(t)
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported countOpenFiles test on Windows.")
}
count, err := countOpenFiles()
if err != nil {
t.Errorf("countOpenFiles failed: %s", err)
}
if count == 0 {
t.Error("countOpenFiles returned invalid value 0.")
}
}
func TestMonitorAntiEntropyZero(t *testing.T) {
td, err := testhook.TempDirInDir(t, *TempDir, "")

495
snapshotqueue.go
View file

@ -1,495 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"fmt"
"io"
"math/bits"
"os"
"sync"
"sync/atomic"
"time"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/testhook"
"github.com/pkg/errors"
)
// snapshotQueue is a thing which can handle enqueuing snapshots. A snapshot
// queue distinguishes between high-priority requests, which get satisfied
// by the next available worker, and regular requests, which get enqueued
// if there's space in the queue, and otherwise dropped. There's also a
// separate background task to scan a holder for fragments which may need
// snapshots, but which is processed only when the queue is empty, and only
// slowly. "Await" awaits an existing snapshot if one is already enqueued.
// "Immediate" tries to do one right away. (If one's already enqueued, this
// can leave it in the queue, which will ignore anything that shows up with
// the request flag cleared.)
//
// Await, Enqueue, and Immediate should be called only with the fragment lock
// held.
//
// If you create a queue, it should get stopped at some point. The
// atomicSnapshotQueue implementation used as defaultSnapshotQueue has
// a Start function which will tell you whether it actually started a
// queue. This logic exists because in a normal server case, you probably
// want the queue to be shut down as part of server shutdown, but if you're
// running cluster tests, you probably want to start and shop the queue as
// part of the test, not stop it when any server terminates.
//
// It's less likely to be desireable to start/stop individual queues,
// because fragments use the defaultSnapshotQueue anyway. This design
// needs revisiting.
type SnapshotQueue interface {
Immediate(*fragment) error
Enqueue(*fragment)
Await(*fragment) error
ScanHolder(*Holder, chan struct{})
Stop()
}
// queuelessSnapshotQueue isn't a snapshot queue, but it satisfies the
// interface.
type queuelessSnapshotQueue struct{}
func (q *queuelessSnapshotQueue) Enqueue(f *fragment) {
// We don't actually try to enqueue the snapshot; it breaks things
// if a snapshot gets caused during a transaction.
}
func (q *queuelessSnapshotQueue) Await(f *fragment) error {
return nil
}
func (q *queuelessSnapshotQueue) Immediate(f *fragment) error {
return f.snapshot()
}
func (q *queuelessSnapshotQueue) ScanHolder(h *Holder, done chan struct{}) {
}
func (q *queuelessSnapshotQueue) Stop() {
}
var defaultSnapshotQueue = &queuelessSnapshotQueue{}
// newSnapshotQueue makes a new snapshot queue, of depth N, with
// w worker threads.
func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue {
ctx, cancel := context.WithCancel(context.Background())
sq := &prioritySnapshotQueue{
normal: make(chan snapshotRequest, n),
urgent: make(chan snapshotRequest),
background: make(chan snapshotRequest),
ctx: ctx,
cancel: cancel,
maxOpN: 10000,
logger: l,
}
if sq.logger == nil {
sq.logger = logger.NewStandardLogger(os.Stderr)
}
_ = testhook.Opened(NewAuditor(), sq, nil)
sq.spawnWorkers(w)
return sq
}
type snapshotRequest struct {
frag *fragment
when time.Time
}
// prioritySnapshotQueue gives preference to "immediate" requests, and
// dispreference to "background" requests from ScanHolder. It timestamps
// requests, so it can discard a request if the most recent snapshot is
// newer than the request. The snapshotPending flag in the fragment is
// used to track that a given fragment thinks it has been successfully
// enqueued. Background requests are not considered enqueued, since
// they'll never get processed if there's anything else. In normal workloads,
// immediate/urgent snapshots should be rare, but we'll happily drop
// most requests on the floor; the scanner should pick them up once things
// are quiet.
type prioritySnapshotQueue struct {
logger logger.Logger
urgent chan snapshotRequest
normal chan snapshotRequest
background chan snapshotRequest
ctx context.Context
cancel context.CancelFunc
mu sync.RWMutex
scanWG, workerWG sync.WaitGroup
maxOpN int
observedOpN [16]uint32
stats struct {
enqueued uint32
skipped uint32
}
stopped bool
}
func (sq *prioritySnapshotQueue) spawnWorkers(w int) {
sq.mu.Lock()
defer sq.mu.Unlock()
if sq.ctx.Err() != nil {
sq.logger.Infof("prioritySnapshotQueue worker: already done")
return
}
sq.workerWG.Add(w)
for i := 0; i < w; i++ {
go sq.worker(sq.ctx, sq.urgent, sq.normal, sq.background)
}
}
func (sq *prioritySnapshotQueue) worker(ctx context.Context, urgent, normal, background chan snapshotRequest) {
defer sq.workerWG.Done()
done := ctx.Done()
ok := true
var req snapshotRequest
for ok {
req.frag = nil
select {
case _, ok = <-done:
case req, ok = <-urgent:
default:
select {
case _, ok = <-done:
case req, ok = <-urgent:
case req, ok = <-normal:
default:
select {
case _, ok = <-done:
case req, ok = <-urgent:
case req, ok = <-normal:
case req, ok = <-background:
}
}
}
if req.frag != nil {
sq.process(req)
}
}
}
// process actually runs a fragment. it will do this if either the fragment
// has a pending snapshot, or the force flag is set.
func (sq *prioritySnapshotQueue) process(req snapshotRequest) {
f := req.frag
f.mu.Lock()
defer f.mu.Unlock()
if f.snapshotStamp.Before(req.when) {
f.snapshotErr = f.snapshot()
if f.snapshotErr != nil {
fmt.Printf("ERROR: snapshot error: %v\n", f.snapshotErr)
sq.logger.Errorf("snapshot error: %v", f.snapshotErr)
}
f.snapshotPending = false
f.snapshotCond.Broadcast()
}
}
// Stop shuts down the snapshot queue. It first marks it as done, causing
// the background scanner(s), if any, to shut down, then waits for them, then
// closes and nils the queues. The background scanner has to get stopped
// because otherwise it might try to write to those closed queues.
func (sq *prioritySnapshotQueue) Stop() {
sq.mu.Lock()
defer sq.mu.Unlock()
if sq.stopped {
return
}
sq.stopped = true
sq.cancel()
// scanners need to be done before we close the other channels.
sq.scanWG.Wait()
close(sq.normal)
sq.normal = nil
close(sq.urgent)
sq.urgent = nil
close(sq.background)
sq.background = nil
_ = testhook.Closed(NewAuditor(), sq, nil)
enqueued := atomic.LoadUint32(&sq.stats.enqueued)
skipped := atomic.LoadUint32(&sq.stats.skipped)
if skipped > 0 || enqueued > 1 {
sq.logger.Infof("snapshot queue: enqueued %d, skipped %d\n", sq.stats.enqueued, sq.stats.skipped)
}
}
// Enqueue tries to add a fragment to the queue, if the fragment is not already
// enqueued. You should hold a lock on the fragment when calling this.
func (sq *prioritySnapshotQueue) Enqueue(f *fragment) {
if f.snapshotPending {
return
}
sq.observeOpN(uint32(f.opN))
sq.mu.RLock()
defer sq.mu.RUnlock()
if sq.normal == nil {
sq.logger.Infof("requested snapshot after snapshot queue was closed")
return
}
// we have to set this before enqueing, because it's
// otherwise possible that we're at the head of the queue,
// and the recipient gets the fragment before we execute the
// line after the send.
f.snapshotPending = true
// try to enqueue snapshot
select {
case sq.normal <- snapshotRequest{frag: f, when: time.Now()}:
atomic.AddUint32(&sq.stats.enqueued, 1)
return
default:
atomic.AddUint32(&sq.stats.skipped, 1)
f.snapshotPending = false
return
}
}
// Await returns when f is not pending a snapshot. Call with the fragment lock
// held. Await waits on a condition variable inside f, associated with the
// fragment's lock, so this does not conflict with the lock being used for
// snapshots.
//
// Note that workers don't stop just because the queue's been stopped; only
// the background scanner is stopped. So an Await shouldn't block forever
// even if the queue gets shut down. If you're reading this, possibly that
// analysis is incorrect.
func (sq *prioritySnapshotQueue) Await(f *fragment) (err error) {
for f.snapshotPending {
f.snapshotCond.Wait()
}
err, f.snapshotErr = f.snapshotErr, nil
return err
}
// Immediate forces an immediate snapshot of the given fragment. Call with
// the fragment locked. If the queue is already closing, the fragment does
// not get snapshotted.
func (sq *prioritySnapshotQueue) Immediate(f *fragment) error {
sq.mu.RLock()
// no deferred unlock, because we want to unlock this before calling Await.
// Not because that needs this lock, but because once we're that far, we
// *don't* need this lock anymore so someone else should have it.
if sq.urgent == nil {
sq.mu.RUnlock()
sq.logger.Errorf("requested immediate snapshot after snapshot queue was closed")
return errors.New("requested immediate snapshot after snapshot queue was closed")
}
f.snapshotPending = true
sq.observeOpN(uint32(f.opN))
req := snapshotRequest{frag: f, when: time.Now()}
// if the fragment was already in the work queue, it's *possible*
// that the only available worker just picked it off the queue, and
// is now waiting on getting the fragment's lock, so it can run
// a snapshot. So we let go of the lock on the fragment, send the
// request, then request the fragment lock again, because Await will
// be sleeping on the condition variable associated with the lock,
// which means it needs to hold the lock so it can let it go during
// the wait... No, really, this made sense.
f.mu.Unlock()
sq.urgent <- req
sq.mu.RUnlock()
f.mu.Lock()
return sq.Await(f)
}
// ScanHolder spawns a goroutine which iterates through the holder's
// indexes/fields/views/fragments, looking for fragments which have OpN
// high enough to justify a snapshot but don't seem to have one pending.
// It then dumps these in the low priority background queue.
func (sq *prioritySnapshotQueue) ScanHolder(h *Holder, done chan struct{}) {
sq.mu.Lock()
sq.scanWG.Add(1)
go sq.scanHolderWorker(h, sq.background, done)
sq.mu.Unlock()
}
// observeOpN reports that a given value of opN was "observed", meaning,
// we encountered a fragment which had that value. This happens for every
// enqueue/immediate, including enqueue attempts which fail to actually
// enter the queue, and it also happens for fragments noticed by the background
// scan but which don't have high enough opN to trigger a snapshot.
func (sq *prioritySnapshotQueue) observeOpN(n uint32) {
// aka "log2(n) + 1", or 0 for n==0
pow2 := 32 - bits.LeadingZeros32(n)
// 15 == 16384. Our usual fragment maxOpN is 10k, so most fragments
// should end up in the 8k-16k bucket, rather than the 16k+ bucket,
// unless we've got a lot of ingests with large batches going on,
// in which case the 16k bucket will win.
if pow2 > 15 {
pow2 = 15
}
// store in inverse order so the lowest slot in the array is the
// highest cardinality
atomic.AddUint32(&sq.observedOpN[15-pow2], 1)
}
// computeMaxOpN tries to pick a reasonable new maxOpN for the background
// scan to use. On a quiet system, we want to gradually lower opN, picking
// the fragments with the highest opN values first, because those offer the
// largest benefit. So, whenever we check a fragment in the background, if we
// *don't* snapshot it, we'll "observe" its OpN value, and then we pick a
// value which picks up at least 1/4 of them.
//
// If there's ingest activity, the Immediate and Enqueue operations will
// "observe" the OpN of fragments submitted to them. This can drive OpN back
// up, if those fragments frequently have very high opN values, which reflects
// the fact that we have enough of that activity that we don't need the
// background scanner adding more.
//
// If we have enough ingest activity that the background scanner never actually
// gets to submit work, we'll rarely get here, because the background scanner
// will block until there's no snapshots pending for the normal workload.
// When we do, we'll probably pick a MaxOpN which is dominated by the ingest
// workload's opN values. So for instance, if everything coming in from the
// ingest workload has 10k or more items, because that's the default fragment
// maxOpN, that will probably set the background snapshot queue value to 8k.
func (sq *prioritySnapshotQueue) computeMaxOpN() {
sq.logger.Debugf("observedOpN by power of 2: %d\n", sq.observedOpN[:])
total := uint32(0)
for i := range sq.observedOpN {
total += atomic.LoadUint32(&sq.observedOpN[i])
}
target := (total / 4) + 1
subTotal := uint32(0)
for i := range sq.observedOpN {
v := atomic.LoadUint32(&sq.observedOpN[i])
subTotal += v
if subTotal >= target {
prevMaxOpN := sq.maxOpN
sq.maxOpN = (1 << (15 - uint(i))) / 2
if sq.maxOpN > 0 {
sq.maxOpN--
}
if prevMaxOpN != sq.maxOpN {
sq.logger.Infof("background scan: %d/%d fragments considered have opN %d or higher\n",
subTotal, total, sq.maxOpN)
}
break
}
}
// It's conceptually possible that we'll miss a couple of observations
// here but that's not really important. This is all pretty approximate.
for i := range sq.observedOpN {
atomic.StoreUint32(&sq.observedOpN[i], 0)
}
}
// prioritySnapshotQueueScanner is the data type that implements HolderOperator
// and represents a single scan of a holder, with a given maxOpN.
type prioritySnapshotQueueScanner struct {
HolderFilterAll
HolderProcessNone
sq *prioritySnapshotQueue
holder *Holder
queue chan snapshotRequest
ctx context.Context
maxOpN int
seen, hits, counter int
}
func (s *prioritySnapshotQueueScanner) ProcessFragment(f *fragment) error {
if f == nil {
return nil
}
s.seen++
// we can't defer this reasonably, because otherwise we'll keep
// the fragment locked forever if we end up trying to send it
// to the queue, but the workers are busy on other fragments.
f.mu.Lock()
open := f.open
snapshotPending, opN := f.snapshotPending, f.opN
f.mu.Unlock()
// a pending snapshot is one that is either in the normal or
// immediate queue, or is trying to get into the normal queue
// and about to fail, but either way, it already got observed
// there, so we don't need to observe it here. A closed fragment
// doesn't matter to us -- it should be a transient state that
// happens during a shutdown, or shouldn't happen, but we don't
// care about it.
if snapshotPending || !open {
return nil
}
if opN <= s.maxOpN {
// observe the value but don't do a snapshot
s.sq.observeOpN(uint32(opN))
s.counter++
if s.counter == 1000 {
select {
case <-time.After(1 * time.Second):
case <-s.ctx.Done():
return io.EOF
}
s.counter = 0
}
return nil
}
// we don't observe values when we decide to trigger a snapshot,
// because those values will be changing anyway. we could also
// observe them as zero, but that's also sort of wrong.
s.hits++
select {
case s.queue <- snapshotRequest{frag: f, when: time.Now()}:
s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path())
case <-s.ctx.Done():
return io.EOF
}
return nil
}
func contextMergedWithStructChan(ctx context.Context, ch chan struct{}) (context.Context, context.CancelFunc) {
canCancel, cancel := context.WithCancel(ctx)
go func() {
select {
case <-ctx.Done():
cancel()
case <-ch:
cancel()
case <-canCancel.Done():
// don't need to cancel, but do need to exit this
// function
}
}()
return canCancel, cancel
}
// scanHolderWorker is a background task that scans a holder looking for
// fragments which need snapshots taken. It's the cleanup task for snapshots
// that would have been requested by Enqueue, but the queue was full.
func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) {
defer sq.scanWG.Done()
ctx, cancel := contextMergedWithStructChan(sq.ctx, done)
defer cancel()
scanner := &prioritySnapshotQueueScanner{
sq: sq,
holder: h,
queue: background,
ctx: sq.ctx,
maxOpN: sq.maxOpN,
}
for {
err := h.Process(ctx, scanner)
if err != nil {
return
}
if scanner.hits > 0 {
sq.logger.Infof("background scan: %d/%d fragments needed snapshots\n", scanner.hits, scanner.seen)
scanner.hits = 0
} else {
sq.logger.Debugf("background scan: no fragments needed snapshots, waiting\n")
// No reason to be active if we're not finding anything.
select {
case <-time.After(60 * time.Second):
case <-ctx.Done():
return
}
}
scanner.seen = 0
sq.computeMaxOpN()
scanner.maxOpN = sq.maxOpN
}
}

Some files were not shown because too many files have changed in this diff Show more