diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index da44fa181..4c0cb0c3b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,6 +12,17 @@ stages: - build - integration - gauntlet + - post build + +smoke build: + image: golang:$GOVERSION + stage: lint + allow_failure: false + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)" + - go build ./... golangci-lint: image: golangci/golangci-lint:v1.39.0 @@ -21,7 +32,7 @@ golangci-lint: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Checking for issues in new code" - - golangci-lint run -v + - golangci-lint run build lattice: stage: test @@ -205,7 +216,7 @@ package for linux amd64: GOARCH: "amd64" script: - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list - - apt update && apt install nfpm + - apt update && apt install nfpm=2.11.3 - make package artifacts: paths: @@ -222,7 +233,7 @@ package for linux arm64: GOARCH: "arm64" script: - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list - - apt update && apt install nfpm + - apt update && apt install nfpm=2.11.3 - make package artifacts: paths: @@ -277,9 +288,18 @@ clustertests: - shell rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - allow_failure: true script: - make clustertests + +authclustertests: + variables: + PROJECT: authclustertests_${CI_CONCURRENT_ID} + stage: integration + tags: + - shell + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: - make authclustertests external lookup tests: @@ -407,3 +427,40 @@ gauntlet: after_script: - ./qa/scripts/teardownSamsungGauntlet.sh +s3 dump: + stage: post build + variables: + PROFILE: "service-fb-ci" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + tags: + - shell + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64 + - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64 + - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64 + - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_amd64 + - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64 + - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 + - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64 + - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_arm64 + - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64 + - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64 + - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64 + - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_amd64 + - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64 + - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64 + - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64 + - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64 + needs: + - job: build for darwin amd64 + - job: build for darwin arm64 + - job: build for linux amd64 + - job: build for linux arm64 diff --git a/Dockerfile-clustertests-client b/Dockerfile-clustertests-client new file mode 100644 index 000000000..553bffe03 --- /dev/null +++ b/Dockerfile-clustertests-client @@ -0,0 +1,35 @@ +# This Dockerfile is used for cluster testing - it produces a much larger image +# and includes all of Go as well as some utilities. + +FROM golang:1.16 + +LABEL maintainer "dev@pilosa.com" + +COPY . /go/src/github.com/molecula/featurebase/ + +RUN cd /go/src/github.com/molecula/featurebase \ + && make install FLAGS="-a -mod=vendor" + +# download pumba for fault injection +ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba +RUN chmod +x /pumba + +# add docker client to pause/unpause nodes +RUN apt update +RUN apt install -y docker.io + +# add docker-compose so tests can use it for stuff +ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose +RUN chmod +x /usr/local/bin/docker-compose + +RUN cp /go/bin/featurebase /featurebase + +COPY NOTICE /NOTICE + +COPY ./internal/clustertests /go/src/github.com/molecula/featurebase/internal/clustertests + +EXPOSE 10101 +VOLUME /data + +ENTRYPOINT ["bash", "-c"] +CMD ["/featurebase", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] diff --git a/Makefile b/Makefile index 5ee669d17..a35dc42cd 100644 --- a/Makefile +++ b/Makefile @@ -79,9 +79,6 @@ testvsub-race: cd ..; \ done -tour: - ./tournament.sh - bench: $(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) @@ -159,13 +156,14 @@ clustertests: vendor $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down # Run the cluster tests with authentication enabled -DOCKER_COMPOSE_AUTH = docker-compose -p authclustertests +AUTH_ARGS="-c /go/src/github.com/molecula/featurebase/internal/clustertests/testdata/featurebase.conf" authclustertests: vendor - $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down - $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml build - $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3 - $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml run client1 - $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down + $(eval PROJECT=authclustertests) + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3 + PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1 + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down # Install Pilosa install: @@ -349,14 +347,5 @@ install-gometalinter: GO111MODULE=off gometalinter --install GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode -test-txstore-rbf: - PILOSA_STORAGE_BACKEND=rbf $(MAKE) testv-race - -# WARNING: This feature is no longer being tested regularly in CI. The test is -# very slow and very expensive, and we're not sure it actually provides useful -# information now. -test-txstore-rbf_bolt: - PILOSA_STORAGE_BACKEND=rbf_bolt $(MAKE) testv-race - test-external-lookup: $(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN) diff --git a/api.go b/api.go index 5766701a9..45892dce1 100644 --- a/api.go +++ b/api.go @@ -2307,10 +2307,6 @@ func (api *API) Info() serverInfo { } } -func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) { - return api.holder.Inspect(ctx, req) -} - // GetTranslateEntryReader provides an entry reader for key translation logs starting at offset. func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader") diff --git a/api_test.go b/api_test.go index 0c7c63287..cd2595064 100644 --- a/api_test.go +++ b/api_test.go @@ -5,11 +5,14 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "errors" "fmt" "io" "math" "math/rand" + "net/http" + "net/http/httptest" "os" "path/filepath" "reflect" @@ -22,7 +25,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/shardwidth" "github.com/molecula/featurebase/v3/test" @@ -36,21 +38,21 @@ func TestAPI_Import(t *testing.T) { pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -222,19 +224,19 @@ func TestAPI_ImportValue(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -529,7 +531,7 @@ func TestAPI_Ingest(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -648,7 +650,7 @@ func BenchmarkIngest(b *testing.B) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -709,7 +711,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -1363,7 +1365,9 @@ func TestVariousApiTranslateCalls(t *testing.T) { if err != nil { t.Fatalf("%v: could not create test index", err) } - _, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false}) + if _, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false}); err != nil { + t.Fatalf("creating field: %v", err) + } t.Run("translateIndexDbOnNilIndex", func(t *testing.T) { err := api.TranslateIndexDB(context.Background(), "nonExistentIndex", 0, r) @@ -1430,7 +1434,7 @@ func TestAPI_RBFDebugInfo(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -1448,11 +1452,6 @@ func TestAPI_RBFDebugInfo(t *testing.T) { func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.UserInfo { tkn := jwt.New(jwt.SigningMethodHS256) claims := tkn.Claims.(jwt.MapClaims) - groupString, err := authn.ToGob64(groups) - if err != nil { - t.Fatalf("gobbing groups %v", err) - } - claims["molecula-idp-groups"] = groupString claims["oid"] = "42" claims["name"] = name secretKey, _ := hex.DecodeString(secret) @@ -1473,7 +1472,6 @@ func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.Us } func TestAuth_MultiNode(t *testing.T) { - // create permissions file permissions := ` "user-groups": @@ -1482,6 +1480,43 @@ func TestAuth_MultiNode(t *testing.T) { "dca35310-ecda-4f23-86cd-876aee55906f": "test": "write" admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` + adminUser := makeUser(t, []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "adminGroup"}}, "admin", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + adminCtx := context.WithValue( + context.Background(), + "userinfo", + adminUser, + ) + readUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "readGroup"}}, "reader", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + readCtx := context.WithValue( + context.Background(), + "userinfo", + readUser, + ) + writeUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906f", GroupName: "writeGroup"}}, "writer", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEED") + writeCtx := context.WithValue( + context.Background(), + "userinfo", + writeUser, + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token, ok := r.Header["Authorization"] + if !ok || len(token) == 0 { + http.Error(w, "BAD REQUEST", http.StatusBadRequest) + return + } + g := []authn.Group{} + switch token[0] { + case adminUser.Token: + g = adminUser.Groups + case readUser.Token: + g = readUser.Groups + case writeUser.Token: + g = writeUser.Groups + } + if err := json.NewEncoder(w).Encode(authn.Groups{Groups: g}); err != nil { + t.Fatalf("unexpected error marshalling groups response: %v", err) + } + })) // authentication on auth := server.Auth{ @@ -1490,7 +1525,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` ClientSecret: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", AuthorizeURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize", TokenURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token", - GroupEndpointURL: "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true", + GroupEndpointURL: srv.URL, RedirectBaseURL: "https://localhost:10101", LogoutURL: "https://login.microsoftonline.com/common/oauth2/v2.0/logout", Scopes: []string{"https://graph.microsoft.com/.default", "offline_access"}, @@ -1562,22 +1597,6 @@ f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== ) defer c.Close() - adminCtx := context.WithValue( - context.Background(), - "userinfo", - makeUser(t, []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "adminGroup"}}, "admin", config.Auth.SecretKey), - ) - readCtx := context.WithValue( - context.Background(), - "userinfo", - makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "readGroup"}}, "reader", config.Auth.SecretKey), - ) - writeCtx := context.WithValue( - context.Background(), - "userinfo", - makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906f", GroupName: "writeGroup"}}, "writer", config.Auth.SecretKey), - ) - primaryAPI := c.GetPrimary().API // needs internal/cluster/message diff --git a/authn/authenticate.go b/authn/authenticate.go index 202369a90..50373199c 100644 --- a/authn/authenticate.go +++ b/authn/authenticate.go @@ -4,24 +4,55 @@ package authn import ( - "bytes" - "encoding/base64" - "encoding/gob" + "context" "encoding/hex" "encoding/json" "fmt" - "io" "net/http" + "net/url" + "strconv" + "strings" "time" "github.com/golang-jwt/jwt" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + "github.com/molecula/featurebase/v3/logger" "github.com/pkg/errors" "golang.org/x/oauth2" ) -func init() { - gob.Register([]Group{}) +// cachedGroups is used to hold groups and when they were last cached +type cachedGroups struct { + cacheTime time.Time + groups []Group +} + +// cacheToken is used to hold tokens and when they were added to the cache +type cachedToken struct { + cacheTime time.Time + token *oauth2.Token +} + +// UserInfo holds the information about the user from the token +type UserInfo struct { + UserID string `json:"userid"` + UserName string `json:"username"` + Groups []Group `json:"groups"` + Expiry time.Time `json:"expiry"` + Token string `json:"token"` +} + +// Group holds group information for an authenticated user +type Group struct { + GroupID string `json:"id"` + GroupName string `json:"displayName"` +} + +// Groups holds a slice of Group for marshalling from JSON +type Groups struct { + Groups []Group `json:"value"` } // Auth holds state, configuration, and utilities needed for authentication. @@ -31,8 +62,13 @@ type Auth struct { secretKey []byte groupEndpoint string logoutEndpoint string - fbURL string // fbURL is the domain FB is hosted on, used for post logout redirection + fbURL string // fbURL is the domain featurebase is hosted on, used for post logout redirection oAuthConfig *oauth2.Config + cacheTTL time.Duration // cacheTTL is used to determine if a cached item should be refreshed or not + tokenTTR time.Duration // tokenTTR (time to refresh) is used to determine if a token should be refreshed or not + tokenCache map[string]cachedToken // tokenCache is a map of accessToken -> *oauth2.Token which we can use to refresh the tokens + groupsCache map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships + lastCacheClean time.Time // last cache clean is the time that the cache was last cleaned } // NewAuth instantiates and returns a new Auth struct @@ -53,105 +89,124 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenUR TokenURL: tokenURL, }, }, + tokenCache: map[string]cachedToken{}, + groupsCache: map[string]cachedGroups{}, + cacheTTL: 10 * time.Minute, + tokenTTR: 7 * time.Minute, + lastCacheClean: time.Now(), } if auth.secretKey, err = decodeHex(secretKey); err != nil { return nil, errors.Wrap(err, "decoding secret key") } - return auth, nil } +// SecretKey is a convenient function to get the SecretKey from an Auth struct func (a Auth) SecretKey() []byte { return a.secretKey } -// UserInfo holds the information about the user from the token -type UserInfo struct { - UserID string `json:"userid"` - UserName string `json:"username"` - Groups []Group `json:"groups"` - Expiry time.Time `json:"expiry"` - Token string `json:"token"` -} - -// Group holds group information for an authenticated user -type Group struct { - GroupID string `json:"id"` - GroupName string `json:"displayName"` -} - -// ToGob64 encodes a []Group to a string, returning string and nil on success (todd's idea) -// it has to be a string bc we're using it in a jwt.MapClaims which needs string-y things -func ToGob64(m []Group) (string, error) { - var b bytes.Buffer - if err := gob.NewEncoder(&b).Encode(m); err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(b.Bytes()), nil -} - -// FromGob64 converts a previously encoded []Group from a string to a []Group -// it has to be a string bc we're using it in a jwt.MapClaims which needs string-y things -func FromGob64(gobbed string) ([]Group, error) { - m := []Group{} - by, err := base64.StdEncoding.DecodeString(gobbed) - if err != nil { - return nil, err - } - b := bytes.Buffer{} - b.Write(by) - d := gob.NewDecoder(&b) - err = d.Decode(&m) - if err != nil { - return nil, err - } - return m, nil -} - -// Groups holds a slice of Group for marshalling from Json -type Groups struct { - Groups []Group `json:"value"` -} - // Authenticate takes in a bearer token `bearer` and returns UserInfo from that token -func (a *Auth) Authenticate(bearer string) (*UserInfo, error) { - // parse the bearer token into a jwt.Token - // this also validates the token, and checks that it's not expired - token, err := jwt.Parse(bearer, func(token *jwt.Token) (interface{}, error) { - if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) +// it is caller's responsibility to inform the user that the access token has been refreshed +func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, error) { + // clean up the cache every 30 minutes or so + if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute { + a.cleanCache() + } + + if tkn, ok := a.tokenCache[bearer]; ok && (tkn.token.Expiry.Sub(time.Now()) <= a.tokenTTR || !tkn.token.Valid()) { + // refresh the token + resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL, + url.Values{ + "grant_type": {"refresh_token"}, + "refresh_token": {tkn.token.RefreshToken}, + "client_id": {a.oAuthConfig.ClientID}, + "client_secret": {a.oAuthConfig.ClientSecret}, + }, + ) + if err != nil { + return nil, errors.Wrap(err, "refreshing token") } - return a.secretKey, nil - }) - if token == nil || token.Claims == nil || err != nil || !token.Valid { + defer resp.Body.Close() + var t oauth2.Token + if err := json.NewDecoder(resp.Body).Decode(&t); err != nil { + return nil, errors.Wrap(err, "decoding refreshed token") + } + + // update the cache + delete(a.tokenCache, bearer) + delete(a.groupsCache, bearer) + bearer = t.AccessToken + a.tokenCache[bearer] = cachedToken{time.Now(), &t} + } + + // NOTE: we are using ParseUnverified here because the IDP validates the + // token's signature when we get the user's groups, we just need to make + // sure it's not expired and is well-formed + token, _, err := new(jwt.Parser).ParseUnverified(bearer, &jwt.MapClaims{}) + // well-formed-ness check + if token == nil || token.Claims == nil || err != nil { return nil, fmt.Errorf("parsing bearer token: %v", err) } - userInfo := UserInfo{} - claims := token.Claims.(jwt.MapClaims) - userInfo.UserID = claims["oid"].(string) - userInfo.UserName = claims["name"].(string) - userInfo.Token = bearer + claims := *token.Claims.(*jwt.MapClaims) - g := claims["molecula-idp-groups"].(string) - groups, err := FromGob64(g) - if err != nil { - return nil, errors.Wrap(err, "decoding groups") + // expiry check + if exp, ok := claims["exp"].(string); ok { + if expiry, err := strconv.ParseInt(exp, 10, 64); err != nil || expiry < time.Now().UTC().Unix() { + return nil, fmt.Errorf("token is expired") + } + } + + userInfo := UserInfo{ + UserID: claims["oid"].(string), + UserName: claims["name"].(string), + Token: bearer, + Groups: []Group{}, + } + + if userInfo.Groups, err = a.getGroups(bearer); err != nil { + return nil, errors.Wrap(err, "getting groups") } - userInfo.Groups = groups return &userInfo, nil } +// cleanCache removes old items from our cache +func (a *Auth) cleanCache() { + for bearer, tkn := range a.tokenCache { + // if it's been more than 24 hours since the token was cached + if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour { + // remove it from our cache + delete(a.tokenCache, bearer) + } + } + for bearer, tkn := range a.groupsCache { + // if it's been more than 24 hours since the groups were cached + if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour { + // remove it from our cache + delete(a.groupsCache, bearer) + } + } + a.lastCacheClean = time.Now() +} + // Login redirects a user to login to their configured oAuth authorize endpoint func (a *Auth) Login(w http.ResponseWriter, r *http.Request) { authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL) http.Redirect(w, r, authURL, http.StatusTemporaryRedirect) } -// Logout clears out user cookie and redirects user to IdP's logout endpoint +// Logout clears out the user's cookie, removes the token from our cache, and +// redirects user to IdP's logout endpoint func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) { + // remove the bearer token from a.tokenCache and a.groupsCache + if bearer, err := r.Cookie(a.cookieName); err == nil { + delete(a.tokenCache, bearer.Value) + delete(a.groupsCache, bearer.Value) + } + // clear cookie http.SetCookie(w, &http.Cookie{ Name: a.cookieName, Value: "", @@ -161,84 +216,35 @@ func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) { SameSite: http.SameSiteStrictMode, Expires: time.Unix(0, 0), }) - redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL) - http.Redirect(w, r, redirect, http.StatusTemporaryRedirect) + + http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect) } -// Redirect handles the oAuth /redirect endpoint. It gets user information from -// the identity provider and sets a secure cookie holding the user information -// signed by featurebase. +// Redirect handles the oAuth /redirect endpoint. It gets an access token and +// returns it to the user in the form of a cookie func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) { - code := r.FormValue("code") - token, err := a.getToken(r, code) + token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline) if err != nil { a.logger.Warnf("getting token from IdP: %+v", err) http.Error(w, "Bad Request", http.StatusBadRequest) return } - // enrich token with groups! - g, err := a.getGroups(token.AccessToken) - if err != nil { - a.logger.Warnf("getting groups from IdP: %+v", err) - http.Error(w, "Bad Request", http.StatusBadRequest) - return - } + a.tokenCache[token.AccessToken] = cachedToken{time.Now(), token} - // with vitamin G! (for groups) - enrichedTkn, err := a.addGroupMembership(token.AccessToken, g) - if err != nil { - a.logger.Warnf("enriching token with group membership: %+v", err) - http.Error(w, "Bad Request", http.StatusBadRequest) - return - } - - a.setCookie(w, enrichedTkn, token.Expiry) + a.SetCookie(w, token.AccessToken, token.Expiry) http.Redirect(w, r, "/", http.StatusTemporaryRedirect) } -// getToken exhanges authorization code for an oAuth2 token -func (a *Auth) getToken(r *http.Request, code string) (*oauth2.Token, error) { - token, err := a.oAuthConfig.Exchange(r.Context(), code) - if err != nil { - return nil, errors.Wrap(err, "exchanging auth code for token") - } - return token, nil -} - -// addGroupMembership is only called in `a.Redirect`. It adds groups to a jwt's -// claims, and signs it using `a.secretKey`. -func (a *Auth) addGroupMembership(token string, g []Group) (string, error) { - // parse token into jwt - unenriched, _, err := new(jwt.Parser).ParseUnverified(token, jwt.MapClaims{}) - if unenriched == nil || unenriched.Claims == nil || err != nil { - return "", fmt.Errorf("parsing bearer token: %v", err) - } - - enriched := jwt.New(jwt.SigningMethodHS256) - enriched.Claims = unenriched.Claims - // parse groups into string format - claims := enriched.Claims.(jwt.MapClaims) - groupString, err := ToGob64(g) - if err != nil { - return "", errors.Wrap(err, "failed to serialize groups") - } - // stick it into jwt claims - claims["molecula-idp-groups"] = groupString - - // get stringified and signed jwt - tokenStr, err := enriched.SignedString(a.secretKey) - if err != nil { - return "", errors.Wrap(err, "signing jwt") - } - - return tokenStr, nil -} - // getGroups gets the group membership for a given token from configured IdP func (a *Auth) getGroups(token string) ([]Group, error) { var groups Groups + g, ok := a.groupsCache[token] + if ok && (time.Now().Sub(g.cacheTime) < a.cacheTTL) { + return g.groups, nil + } + req, err := http.NewRequest("GET", a.groupEndpoint, nil) if err != nil { return groups.Groups, errors.Wrap(err, "creating new request to group endpoint") @@ -251,19 +257,18 @@ func (a *Auth) getGroups(token string) ([]Group, error) { } defer response.Body.Close() - rawGroups, err := io.ReadAll(response.Body) - if err != nil { - return groups.Groups, errors.Wrap(err, "failed reading group membership response") - } - - if err = json.Unmarshal(rawGroups, &groups); err != nil { + if err = json.NewDecoder(response.Body).Decode(&groups); err != nil { return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response") } + a.groupsCache[token] = cachedGroups{ + cacheTime: time.Now(), + groups: groups.Groups, + } return groups.Groups, nil } -func (a *Auth) setCookie(w http.ResponseWriter, token string, expiry time.Time) error { +func (a *Auth) SetCookie(w http.ResponseWriter, token string, expiry time.Time) error { http.SetCookie(w, &http.Cookie{ Name: a.cookieName, Value: token, @@ -276,6 +281,20 @@ func (a *Auth) setCookie(w http.ResponseWriter, token string, expiry time.Time) return nil } +func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, token string) error { + cookies := []string{} + if c, ok := md["cookie"]; ok { + for _, cookie := range c { + if strings.HasPrefix(cookie, a.cookieName) { + cookie = a.cookieName + "=" + token + } + cookies = append(cookies, cookie) + } + } + md["cookie"] = cookies + return grpc.SetHeader(ctx, md) +} + func decodeHex(hexstr string) ([]byte, error) { data, err := hex.DecodeString(hexstr) if err != nil { diff --git a/authn/authenticate_internal_test.go b/authn/authenticate_internal_test.go index d4fd63c6c..44c192d28 100644 --- a/authn/authenticate_internal_test.go +++ b/authn/authenticate_internal_test.go @@ -2,19 +2,24 @@ package authn import ( "bytes" + "context" "encoding/hex" + "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "reflect" + "strconv" "strings" "testing" "time" "github.com/golang-jwt/jwt" "github.com/molecula/featurebase/v3/logger" - "github.com/pkg/errors" + "golang.org/x/oauth2" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) func NewTestAuth(t *testing.T) *Auth { @@ -47,12 +52,11 @@ func NewTestAuth(t *testing.T) *Auth { } return a } - func TestAuth(t *testing.T) { a := NewTestAuth(t) t.Run("SetCookie", func(t *testing.T) { w := httptest.NewRecorder() - err := a.setCookie(w, "a cookie value", time.Now().Add(time.Hour)) + err := a.SetCookie(w, "a cookie value", time.Now().Add(time.Hour)) if err != nil { t.Fatalf("expected no errors, got: %v", err) } @@ -65,6 +69,42 @@ func TestAuth(t *testing.T) { t.Fatalf("path=%s, want %s", got, want) } }) + t.Run("SetGRPCMetadata", func(t *testing.T) { + md := metadata.MD{ + "cookie": []string{a.cookieName + "=something"}, + } + ctx := grpc.NewContextWithServerTransportStream( + metadata.NewIncomingContext(context.TODO(), + md, + ), + NewServerTransportStream(), + ) + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + t.Fatalf("expected ok, got: %v", ok) + } + err := a.SetGRPCMetadata(ctx, md, "this is a token!") + if err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + md, ok = metadata.FromIncomingContext(ctx) + if !ok { + t.Fatalf("expected ok, got: %v", ok) + } + c, ok := md["cookie"] + if !ok { + t.Fatalf("expected ok, got: %v", ok) + } + var cookie string + for _, cookie = range c { + if strings.HasPrefix(cookie, a.cookieName) { + break + } + } + if exp, got := a.cookieName+"=this is a token!", cookie; got != exp { + t.Fatalf("expected '%v', got '%v'", exp, got) + } + }) t.Run("KeyLength", func(t *testing.T) { _, err := NewAuth( logger.NewStandardLogger(os.Stdout), @@ -88,13 +128,19 @@ func TestAuth(t *testing.T) { t.Fatalf("expected %v, got %v", got, want) } }) +} + +func TestAuthenticate(t *testing.T) { cases := []struct { - name string - uid string - uname string - exp interface{} - groups []Group - err error + name string + uid string + uname string + exp int64 + refresh bool + errOnRefresh bool + malformed bool + groups []Group + err error }{ { name: "GoodToken", @@ -108,7 +154,13 @@ func TestAuth(t *testing.T) { }, }, { - name: "ExpiredToken", + name: "Malformed", + malformed: true, + err: fmt.Errorf("parsing bearer token: token contains an invalid number of segments"), + }, + + { + name: "ExpiredTokenNoRefresh", uid: "42", uname: "A. Token", groups: []Group{ @@ -117,30 +169,99 @@ func TestAuth(t *testing.T) { GroupName: "adminGroup", }, }, - exp: "-17764800", - err: errors.Wrap(fmt.Errorf("Token is expired"), "parsing bearer token"), + exp: -17764800, + err: fmt.Errorf("token is expired"), + }, + { + name: "ExpiredTokenYesRefresh", + uid: "42", + uname: "A. Token", + groups: []Group{ + { + GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + GroupName: "adminGroup", + }, + }, + refresh: true, + exp: -17764800, + }, + { + name: "ExpiredTokenYesRefreshButError", + uid: "42", + uname: "A. Token", + groups: []Group{ + { + GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + GroupName: "adminGroup", + }, + }, + refresh: true, + errOnRefresh: true, + exp: -17764800, + err: fmt.Errorf("decoding refreshed token: invalid character 'b' looking for beginning of value"), }, } for _, test := range cases { t.Run(test.name, func(t *testing.T) { - tkn := jwt.New(jwt.SigningMethodHS256) - claims := tkn.Claims.(jwt.MapClaims) - groupString, err := ToGob64(test.groups) - if err != nil { - t.Fatalf("unexpected error when gobbing groups %v", err) + // setup the test + a := NewTestAuth(t) + token := "" + var err error + if !test.malformed { + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = test.uid + claims["name"] = test.uname + if test.exp != 0 { + claims["exp"] = strconv.Itoa(int(test.exp)) + } + token, err = tkn.SignedString(a.SecretKey()) + if err != nil { + t.Fatalf("unexpected error when signing token %v", err) + } + } else { + token = "asdfasdfasdfasdF" } - claims["molecula-idp-groups"] = groupString - claims["oid"] = test.uid - claims["name"] = test.uname - if test.exp != nil { - claims["exp"] = test.exp + if len(test.groups) > 0 { + a.groupsCache[token] = cachedGroups{time.Now(), test.groups} } - token, err := tkn.SignedString(a.SecretKey()) - if err != nil { - t.Fatalf("unexpected error when signing token %v", err) + if test.refresh { + var srv *httptest.Server + if !test.errOnRefresh { + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = test.uid + claims["name"] = test.uname + expiry := strconv.Itoa(int(time.Now().Add(2 * time.Hour).Unix())) + claims["exp"] = expiry + fresh, err := tkn.SignedString(a.SecretKey()) + if err != nil { + t.Fatalf("unexpected error when signing token %v", err) + } + + a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups} + fmt.Fprintf(w, `{"access_token": "`+fresh+`", "refresh_token": "blah", "token_type": "bearer", "expires": `+expiry+` }`) + })) + } else { + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "bad", http.StatusInternalServerError) + })) + } + defer srv.Close() + a.oAuthConfig.Endpoint.TokenURL = srv.URL + a.tokenCache[token] = cachedToken{ + time.Now(), + &oauth2.Token{ + AccessToken: token, + RefreshToken: "blah", + Expiry: time.Unix(test.exp, 0), + }, + } } - uinfo, err := a.Authenticate(token) + // do the actual testing + uinfo, err := a.Authenticate(context.TODO(), token) // okay this part kind of sucks bc we need to check errors and i // dont want to write a whole new test for things that should have // errors just to avoid this mess. errors.Is doesn't work either @@ -167,34 +288,124 @@ func TestAuth(t *testing.T) { } } -func TestGobs(t *testing.T) { - t.Run("goodGob!", func(t *testing.T) { - g := []Group{ - { - GroupID: "groupA", - GroupName: "groupA-Name", - }, - { - GroupID: "groupB", - GroupName: "groupB-Name", - }, - { - GroupID: "groupC", - GroupName: "groupC-Name", - }, +func TestAuthenticate_CleanCache(t *testing.T) { + // this deserves its own test bc it has gross setup required + t.Run("should clean", func(t *testing.T) { + a := NewTestAuth(t) + now := time.Now() + a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}} + a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}} + a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}} + a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}} + a.lastCacheClean = now.Add(-45 * time.Minute) + + _, _ = a.Authenticate(context.TODO(), "this doesn't matter") + if a.lastCacheClean.Sub(now) <= time.Nanosecond { + t.Fatalf("cache should have been cleaned") } - gobbed, err := ToGob64(g) - if err != nil { - t.Fatalf("could not gob %+v", g) + if _, ok := a.groupsCache["oldy"]; ok { + t.Errorf("oldy should have been deleted") } - ungobbed, err := FromGob64(gobbed) - if err != nil { - t.Fatalf("could not ungob %+v", gobbed) + if _, ok := a.groupsCache["goldy"]; !ok { + t.Errorf("goldy should not have been deleted") } - if !reflect.DeepEqual(ungobbed, g) { - t.Fatalf("expected %v, got %v", g, ungobbed) + if _, ok := a.tokenCache["oldy"]; ok { + t.Errorf("oldy should have been deleted") + } + if _, ok := a.tokenCache["goldy"]; !ok { + t.Errorf("goldy should not have been deleted") } }) + t.Run("shouldn't clean", func(t *testing.T) { + a := NewTestAuth(t) + now := time.Now() + a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}} + a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}} + a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}} + a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}} + a.lastCacheClean = now + + _, _ = a.Authenticate(context.TODO(), "this doesn't matter") + if a.lastCacheClean.Sub(now) >= time.Nanosecond { + t.Fatalf("cache should not have been cleaned") + } + if _, ok := a.groupsCache["oldy"]; !ok { + t.Errorf("oldy should not have been deleted") + } + if _, ok := a.groupsCache["goldy"]; !ok { + t.Errorf("goldy should not have been deleted") + } + if _, ok := a.tokenCache["oldy"]; !ok { + t.Errorf("oldy should not have been deleted") + } + if _, ok := a.tokenCache["goldy"]; !ok { + t.Errorf("goldy should not have been deleted") + } + }) + +} + +func TestGetGroups(t *testing.T) { + a := NewTestAuth(t) + a.groupsCache = map[string]cachedGroups{ + "the world is changed": { + cacheTime: time.Now(), + groups: []Group{ + { + GroupID: "i feel it in the water", + GroupName: "i feel it in the earth", + }, + }, + }, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := json.Marshal( + Groups{ + Groups: []Group{ + { + GroupID: "much that once was is lost", + GroupName: "for none now live who remember it", + }, + }, + }, + ) + if err != nil { + t.Fatalf("unexpected error marshalling groups response: %v", err) + } + fmt.Fprintf(w, "%s", body) + })) + a.groupEndpoint = srv.URL + + for name, test := range map[string]struct { + token string + groups []Group + }{ + "InCache": { + token: "the world is changed", + groups: []Group{ + { + GroupID: "i feel it in the water", + GroupName: "i feel it in the earth", + }, + }, + }, + "NotInCache": { + token: "i smell it in the air", + groups: []Group{ + { + GroupID: "much that once was is lost", + GroupName: "for none now live who remember it", + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) { + t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err) + } + }) + } + } func TestDecodeHex(t *testing.T) { @@ -224,68 +435,6 @@ func TestDecodeHex(t *testing.T) { }) } -func TestAddGroupMembership(t *testing.T) { - cases := []struct { - name string - groups []Group - err error - }{ - { - name: "emptyGroups", - groups: []Group{}, - err: nil, - }, - { - name: "happyPath", - groups: []Group{ - { - GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", - GroupName: "adminGroup", - }, - }, - err: nil, - }, - } - a := NewTestAuth(t) - for _, test := range cases { - t.Run(test.name, func(t *testing.T) { - tkn := jwt.New(jwt.SigningMethodHS256) - token, err := tkn.SignedString(a.SecretKey()) - if err != nil { - t.Fatalf("unexpected error when signing token %v", err) - } - - tokenWithGroups, err := a.addGroupMembership(token, test.groups) - // okay this part kind of sucks bc we need to check errors and i - // dont want to write a whole new test for things that should have - // errors just to avoid this mess. errors.Is doesn't work either - if (test.err == nil && err != nil) || (test.err != nil && err == nil) { - t.Fatalf("expected %v but got %v", test.err, err) - } else if test.err != nil && err != nil { - if test.err.Error() != err.Error() { - t.Fatalf("expected %v, but got %v", test.err, err) - } else { - return - } - } - parsed, _, err := new(jwt.Parser).ParseUnverified(tokenWithGroups, jwt.MapClaims{}) - if err != nil { - t.Fatalf("unexpected error parsing token %v", err) - } - - claims := parsed.Claims.(jwt.MapClaims) - groups, err := FromGob64(claims["molecula-idp-groups"].(string)) - if err != nil { - t.Fatalf("unexpected error parsing groupString %v", err) - } - - if !reflect.DeepEqual(groups, test.groups) { - t.Fatalf("expected %v, got %v", test.groups, groups) - } - }) - } -} - func TestHandlers(t *testing.T) { a := NewTestAuth(t) t.Run("login", func(t *testing.T) { @@ -304,6 +453,19 @@ func TestHandlers(t *testing.T) { t.Run("logout", func(t *testing.T) { req := httptest.NewRequest("GET", "/logout", nil) w := httptest.NewRecorder() + req.AddCookie( + &http.Cookie{ + Name: a.cookieName, + Value: "test", + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + Expires: time.Unix(3000000, 0), + }, + ) + a.groupsCache["test"] = cachedGroups{} + a.tokenCache["test"] = cachedToken{time.Now(), &oauth2.Token{}} a.Logout(w, req) resp := w.Result() if resp.StatusCode != http.StatusTemporaryRedirect { @@ -314,7 +476,7 @@ func TestHandlers(t *testing.T) { t.Fatalf("expected %v, got %v", redirect, got.Path) } for _, c := range resp.Cookies() { - if c.Name == "molecula-chip" { + if c.Name == a.cookieName { if c.Value != "" { t.Fatalf("cookie not set to empty value!") } @@ -326,5 +488,105 @@ func TestHandlers(t *testing.T) { break } } + if _, ok := a.groupsCache["test"]; ok { + t.Fatalf("groups not deleted!") + } + if _, ok := a.tokenCache["test"]; ok { + t.Fatalf("token not deleted!") + } }) + t.Run("redirectGood", func(t *testing.T) { + req := httptest.NewRequest("GET", "/redirect", nil) + w := httptest.NewRecorder() + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = "user id" + claims["name"] = "user name" + expiresIn := 2 * time.Hour + exp := time.Now().Add(expiresIn) + expiry := strconv.Itoa(int(exp.Unix())) + claims["exp"] = expiry + fresh, err := tkn.SignedString(a.SecretKey()) + if err != nil { + t.Fatalf("unexpected error when signing token %v", err) + } + freshToken := oauth2.Token{ + AccessToken: fresh, + RefreshToken: "blah", + Expiry: exp, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}` + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + w.Write([]byte(body)) + })) + a.oAuthConfig.Endpoint.TokenURL = srv.URL + a.Redirect(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusTemporaryRedirect { + t.Fatalf("expected redirect, got %v", resp.StatusCode) + } + if got, err := resp.Location(); err != nil || got.String() != "/" { + t.Fatalf("expected %v, got %v", "/", got.Path) + } + cachedToken := a.tokenCache[fresh].token + if cachedToken.AccessToken != freshToken.AccessToken { + t.Fatalf("expected %v, got %v", freshToken.AccessToken, cachedToken.AccessToken) + } + if cachedToken.RefreshToken != freshToken.RefreshToken { + t.Fatalf("expected %v, got %v", freshToken.RefreshToken, cachedToken.RefreshToken) + } + if cachedToken.Expiry.Sub(freshToken.Expiry) > time.Second { + t.Fatalf("expected %v, got %v", freshToken.Expiry, cachedToken.Expiry) + } + }) + + t.Run("redirectBad", func(t *testing.T) { + req := httptest.NewRequest("GET", "/redirect", nil) + w := httptest.NewRecorder() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Server Error", http.StatusInternalServerError) + })) + a.oAuthConfig.Endpoint.TokenURL = srv.URL + a.Redirect(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected BadRequest, got %v", resp.StatusCode) + } + }) + +} + +// This type is used for mocking ServerTransportStreams in tests +type ServerTransportStream struct { + md metadata.MD + method string +} + +func NewServerTransportStream() *ServerTransportStream { + return &ServerTransportStream{ + md: metadata.MD{}, + method: "test", + } +} + +func (s *ServerTransportStream) Method() string { + return s.method +} + +func (s *ServerTransportStream) SetHeader(md metadata.MD) error { + s.md = md + return nil +} + +func (s *ServerTransportStream) SendHeader(md metadata.MD) error { + _ = md + return nil +} + +func (s *ServerTransportStream) SetTrailer(md metadata.MD) error { + _ = md + return nil } diff --git a/client.go b/client.go deleted file mode 100644 index 4f0f40ecd..000000000 --- a/client.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "context" - "io" - "time" - - "github.com/molecula/featurebase/v3/ingest" - pnet "github.com/molecula/featurebase/v3/net" - "github.com/molecula/featurebase/v3/topology" -) - -// Bit represents the intersection of a row and a column. It can be specified by -// integer ids or string keys. -type Bit struct { - RowID uint64 - ColumnID uint64 - RowKey string - ColumnKey string - Timestamp int64 -} - -// FieldValue represents the value for a column within a -// range-encoded field. -type FieldValue struct { - ColumnID uint64 - ColumnKey string - Value int64 -} - -// InternalClient should be implemented by any struct that enables any transport between nodes -// TODO: Refactor -// Note from Travis: Typically an interface containing more than two or three methods is an indication that -// something hasn't been architected correctly. -// While I understand that putting the entire Client behind an interface might require this many methods, -// I don't want to let it go unquestioned. -// Another note from Travis: I think we eventually want to unify `InternalClient` with -// the `github.com/molecula/featurebase/v3/client` client. -// Doing that may obviate the need to refactor this. -type InternalClient interface { - InternalQueryClient - - AvailableShards(ctx context.Context, indexName string) ([]uint64, error) - MaxShardByIndex(ctx context.Context) (map[string]uint64, error) - Schema(ctx context.Context) ([]*IndexInfo, error) - PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error - CreateIndex(ctx context.Context, index string, opt IndexOptions) error - FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) - PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error) - Nodes(ctx context.Context) ([]*topology.Node, error) - Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) - Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error - EnsureIndex(ctx context.Context, name string, options IndexOptions) error - EnsureField(ctx context.Context, indexName string, fieldName string) error - EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error - ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error - ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error - CreateField(ctx context.Context, index, field string) error - CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error - FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) - SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error - RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) - RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) - ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error - ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) - MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) - IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error - - IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) - IDAllocDataWriter(ctx context.Context, f io.Reader, primary *topology.Node) error - IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) - FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) - - StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) - FinishTransaction(ctx context.Context, id string) (*Transaction, error) - Transactions(ctx context.Context) (map[string]*Transaction, error) - GetTransaction(ctx context.Context, id string) (*Transaction, error) - - GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) - GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) - - // ImportFieldKeys and ImportIndexKeys are mainly used when - // restoring a backup. They take a readerFunc which returns a - // reader rather than taking an io.Reader directly to allow for - // efficient retries (rather than reading the entire request body - // into a buffer and reusing it). Reader returned from the func - // must be properly closed by the implementation. - ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error - ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error - - // SetInternalAPI tells the client the API it should use for internal/loopback ops - // where applicable. - SetInternalAPI(api *API) -} - -//=============== - -// InternalQueryClient is the internal interface for querying a node. -type InternalQueryClient interface { - SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) - - QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) - - // Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist. - TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) - TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, id []uint64) ([]string, error) - - FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) - FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) - - CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) - CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) - - MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, like string) ([]uint64, error) -} - -type nopInternalQueryClient struct{} - -func (nopInternalQueryClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) { - return nil, nil -} - -func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { - return nil, nil -} - -func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) { - return nil, nil -} - -func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) { - return nil, nil -} - -func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) { - return nil, nil -} - -func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) { - return nil, nil -} - -func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) { - return nil, nil -} - -func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) { - return nil, nil -} - -func (n nopInternalQueryClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, like string) ([]uint64, error) { - return nil, nil -} - -func newNopInternalQueryClient() nopInternalQueryClient { - return nopInternalQueryClient{} -} - -var _ InternalQueryClient = newNopInternalQueryClient() - -//=============== - -type nopInternalClient struct{ nopInternalQueryClient } - -func newNopInternalClient() nopInternalClient { - return nopInternalClient{} -} - -var _ InternalClient = newNopInternalClient() - -func (n nopInternalClient) AvailableShards(ctx context.Context, indexName string) ([]uint64, error) { - return nil, nil -} - -func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) { - return nil, nil -} -func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } -func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error { - return nil -} - -func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { - return nil -} -func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) { - return nil, nil -} -func (n nopInternalClient) PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error) { - return nil, nil -} -func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { - return nil, nil -} -func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { - return nil, nil -} -func (n nopInternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error { - return nil -} -func (n nopInternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error { - return nil -} - -func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { - return nil -} - -func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { - return nil, nil -} - -func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { - return nil -} - -func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) IDAllocDataWriter(cctx context.Context, f io.Reader, primary *topology.Node) error { - return nil -} - -func (n nopInternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { - return nil -} -func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { - return nil -} -func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { - return nil -} -func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { - return nil -} -func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } -func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { - return nil -} -func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { - return nil, nil -} -func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { - return nil, nil, nil -} -func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error { - return nil -} -func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) { - return nil, nil -} -func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) { - return nil, nil -} -func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) { - return nil, nil -} -func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) { - return nil, nil -} -func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) { - return nil, nil -} - -func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) { - return nil, nil -} - -func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) { - return nil, nil -} -func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error { - return nil -} - -func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error { - return nil -} - -func (c nopInternalClient) SetInternalAPI(api *API) { -} diff --git a/client/batch_test.go b/client/batch_test.go index 8823a5099..3436ec1fc 100644 --- a/client/batch_test.go +++ b/client/batch_test.go @@ -359,7 +359,7 @@ func testTrimNull(t *testing.T, c *test.Cluster, client *Client) { t.Fatalf("querying: %v", err) } for i, result := range resp.Results() { - if 1 == i { + if i == 1 { if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) { t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns) } @@ -1187,26 +1187,6 @@ outer: return nil } -func isPermutationOfInt(one, two []uint64) error { - if len(one) != len(two) { - return errors.Errorf("different lengths %d and %d", len(one), len(two)) - } -outer: - for _, vOne := range one { - for j, vTwo := range two { - if vOne == vTwo { - two = append(two[:j], two[j+1:]...) - continue outer - } - } - return errors.Errorf("%d in one but not two", vOne) - } - if len(two) != 0 { - return errors.Errorf("vals in two but not one: %v", two) - } - return nil -} - func TestQuantizedTime(t *testing.T) { cases := []struct { name string diff --git a/cluster.go b/cluster.go index 1f45d2c15..687c6ca8e 100644 --- a/cluster.go +++ b/cluster.go @@ -102,7 +102,7 @@ type cluster struct { // nolint: maligned logger logger.Logger - InternalClient InternalClient + InternalClient *InternalClient confirmDownRetries int confirmDownSleep time.Duration @@ -120,7 +120,7 @@ func newCluster() *cluster { translationSyncer: NopTranslationSyncer, - InternalClient: newNopInternalClient(), + InternalClient: &InternalClient{}, // TODO might have to fill this out a bit logger: logger.NopLogger, diff --git a/cmd/badloader/badloader.go b/cmd/badloader/badloader.go index 642137575..035b70dea 100644 --- a/cmd/badloader/badloader.go +++ b/cmd/badloader/badloader.go @@ -13,7 +13,7 @@ import ( gohttp "net/http" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/vprint" @@ -22,7 +22,7 @@ import ( "strings" ) -func UploadTar(srcFile string, client *http.InternalClient) error { +func UploadTar(srcFile string, client *pilosa.InternalClient) error { t0 := time.Now() f, err := os.Open(srcFile) if err != nil { @@ -114,7 +114,7 @@ func main() { host := "127.0.0.1:10101" h := &gohttp.Client{} - c, err := http.NewInternalClient(host, h) + c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) vprint.PanicOn(err) tarSrcPath := "q2.tar.gz" diff --git a/cmd/check.go b/cmd/check.go deleted file mode 100644 index f22dcd3a0..000000000 --- a/cmd/check.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package cmd - -import ( - "context" - "fmt" - "io" - - "github.com/spf13/cobra" - - "github.com/molecula/featurebase/v3/ctl" -) - -var checker *ctl.CheckCommand - -func newCheckCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command { - checker = ctl.NewCheckCommand(stdin, stdout, stderr) - checkCmd := &cobra.Command{ - Use: "check [path2]...", - Short: "Do a consistency check on a FeatureBase data file.", - Long: ` -Performs a consistency check on data files. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - return fmt.Errorf("path required") - } - checker.Paths = args - return checker.Run(context.Background()) - }, - } - return checkCmd -} diff --git a/cmd/check_test.go b/cmd/check_test.go deleted file mode 100644 index a6abf0529..000000000 --- a/cmd/check_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package cmd_test - -import ( - "strings" - "testing" -) - -func TestCheckHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "check", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "featurebase check") || err != nil { - t.Fatalf("Command 'check --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestCheckNoPath(t *testing.T) { - output, err := ExecNewRootCommand(t, "check") - if !strings.Contains(err.Error(), "path required") { - t.Fatalf("Command 'check' without args should error but: err: '%v', output: '%v'", err, output) - } -} diff --git a/cmd/convert.go b/cmd/convert.go deleted file mode 100644 index 2a9f5711a..000000000 --- a/cmd/convert.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package cmd - -import ( - "context" - "fmt" - "io" - - "github.com/spf13/cobra" - - "github.com/molecula/featurebase/v3/ctl" -) - -var inspector *ctl.InspectCommand - -func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - inspector = ctl.NewInspectCommand(stdin, stdout, stderr) - - inspectCmd := &cobra.Command{ - Use: "inspect", - Short: "Get stats on a FeatureBase data file.", - Long: ` -Inspects a data file and provides stats. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - return fmt.Errorf("path required") - } else if len(args) > 1 { - return fmt.Errorf("only one path allowed") - } - inspector.Path = args[0] - return inspector.Run(context.Background()) - }, - } - flags := inspectCmd.Flags() - flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers") - flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)") - flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes") - flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views") - flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields") - flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards") - return inspectCmd -} diff --git a/cmd/inspect_test.go b/cmd/inspect_test.go deleted file mode 100644 index 33dd616d6..000000000 --- a/cmd/inspect_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package cmd_test - -import ( - "strings" - "testing" -) - -func TestInspectHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "inspect", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "featurebase inspect") || err != nil { - t.Fatalf("Command 'inspect --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestInspectNoPath(t *testing.T) { - output, err := ExecNewRootCommand(t, "inspect") - if !strings.Contains(err.Error(), "path required") { - t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output) - } -} - -func TestInspectMultiPath(t *testing.T) { - output, err := ExecNewRootCommand(t, "inspect", "one", "two") - if !strings.Contains(err.Error(), "only one path") { - t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output) - } -} diff --git a/cmd/pilosa-bench/main.go b/cmd/pilosa-bench/main.go index 15c7f4e6b..c68e97e14 100644 --- a/cmd/pilosa-bench/main.go +++ b/cmd/pilosa-bench/main.go @@ -16,8 +16,8 @@ import ( "strings" "time" - "github.com/molecula/featurebase/v3" - phttp "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/encoding/proto" "golang.org/x/sync/errgroup" ) @@ -78,7 +78,7 @@ func run(ctx context.Context, args []string) (err error) { rand.Seed(0) // Setup connection to pilosa. - client, err := phttp.NewInternalClient(*hostport, http.DefaultClient) + client, err := pilosa.NewInternalClient(*hostport, http.DefaultClient, pilosa.WithSerializer(proto.Serializer{})) if err != nil { return err } @@ -270,7 +270,7 @@ func generateTopKQuery(index, field string, from, to time.Time) string { } // loadFields returns a mapping of index/field names to field info & identifiers. -func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) { +func loadFields(ctx context.Context, client *pilosa.InternalClient) (map[fieldKey]*fieldInfo, error) { indexes, err := client.Schema(ctx) if err != nil { return nil, err @@ -299,7 +299,7 @@ func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey } // fetchFieldIDs returns a list of field IDs or keys. -func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) { +func fetchFieldIDs(ctx context.Context, client *pilosa.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) { resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`}) if err != nil { return nil, err diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 282e9034b..27529e782 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -16,9 +16,10 @@ import ( "time" "github.com/gogo/protobuf/proto" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/client" - "github.com/molecula/featurebase/v3/http" + fb_proto "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/pb" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/vprint" @@ -162,7 +163,7 @@ func main() { func (cfg *RandomQueryConfig) Run() (err error) { remoteClient := nethttp.DefaultClient - cli, err := http.NewInternalClient(cfg.HostPort, remoteClient) + cli, err := pilosa.NewInternalClient(cfg.HostPort, remoteClient, pilosa.WithSerializer(fb_proto.Serializer{})) if err != nil { return err } diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go index 2c847217f..b29582f1a 100644 --- a/cmd/random-query/main_test.go +++ b/cmd/random-query/main_test.go @@ -9,7 +9,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck @@ -35,7 +34,7 @@ func Test_RandomQuery(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID(nodeid[0]), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), pilosa.OptServerReplicaN(nReplicas), )}, ) diff --git a/cmd/root.go b/cmd/root.go index f32cc5875..5feee57d0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -53,12 +53,10 @@ at https://docs.molecula.cloud/. rc.AddCommand(newChkSumCommand(stdin, stdout, stderr)) rc.AddCommand(newBackupCommand(stdin, stdout, stderr)) rc.AddCommand(newRestoreCommand(stdin, stdout, stderr)) - rc.AddCommand(newCheckCommand(stdin, stdout, stderr)) rc.AddCommand(newConfigCommand(stdin, stdout, stderr)) rc.AddCommand(newExportCommand(stdin, stdout, stderr)) rc.AddCommand(newGenerateConfigCommand(stdin, stdout, stderr)) rc.AddCommand(newImportCommand(stdin, stdout, stderr)) - rc.AddCommand(newInspectCommand(stdin, stdout, stderr)) rc.AddCommand(newRBFCommand(stdin, stdout, stderr)) rc.AddCommand(newServeCmd(stdin, stdout, stderr)) rc.AddCommand(newHolderCmd(stdin, stdout, stderr)) diff --git a/cmd/server_test.go b/cmd/server_test.go index f2d3cf057..91263c0ed 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -3,10 +3,12 @@ package cmd_test import ( "fmt" + "os" "strings" "testing" "time" + "github.com/felixge/fgprof" "github.com/molecula/featurebase/v3/cmd" _ "github.com/molecula/featurebase/v3/test" "github.com/molecula/featurebase/v3/testhook" @@ -23,12 +25,11 @@ func TestServerHelp(t *testing.T) { } // I have no idea why the linter in ci is complaining about this being unused. -func nextPort() string { //nolint:unused +func nextPort() string { return fmt.Sprintf(`"localhost:%d"`, 0) } func TestServerConfig(t *testing.T) { - t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.") actualDataDir, err := testhook.TempDir(t, "") failErr(t, err, "making data dir") logFile, err := testhook.TempFile(t, "") @@ -106,7 +107,7 @@ func TestServerConfig(t *testing.T) { }, // TEST 2 { - args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true", "--translation.map-size", "100000"}, + args: []string{"server", "--log-path", logFile.Name(), "--translation.map-size", "100000"}, env: map[string]string{}, cfgFileContent: ` bind = "localhost:19444" @@ -175,7 +176,9 @@ func TestServerConfig(t *testing.T) { } } func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { - t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.") + // if you don't pass an empty dir as data-dir it will use the + // default... which might be full of data and cause the test to + // run super slow. actualDataDir, err := testhook.TempDir(t, "") failErr(t, err, "making data dir") @@ -203,6 +206,7 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { cfgFileContent: ` bind = ` + nextPort() + ` bind-grpc = ` + nextPort() + ` + data-dir = "` + actualDataDir + `" `, validation: func() error { v := validator{} @@ -218,6 +222,7 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { cfgFileContent: ` bind = ` + nextPort() + ` bind-grpc = ` + nextPort() + ` + data-dir = "` + actualDataDir + `" `, validation: func() error { v := validator{} @@ -228,7 +233,11 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { }, }, } - + out, err := os.Create("myprof.prof") + if err != nil { + t.Fatalf("creating prof file: %v", err) + } + stop := fgprof.Start(out, fgprof.FormatPprof) // run server tests for i, test := range tests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { @@ -257,4 +266,8 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { test.reset() }) } + err = stop() + if err != nil { + t.Fatalf("stopping profile: %v", err) + } } diff --git a/cmd/slurp/slurp.go b/cmd/slurp/slurp.go index 8b4eaef85..dfd01f063 100644 --- a/cmd/slurp/slurp.go +++ b/cmd/slurp/slurp.go @@ -18,7 +18,7 @@ import ( "time" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/vprint" ) @@ -32,7 +32,7 @@ type stateMachine struct { lastField string lastShard uint64 state string - client *http.InternalClient + client *pilosa.InternalClient start time.Time profile string @@ -136,7 +136,7 @@ func (r *stateMachine) Upload() error { return nil } -func UploadTar(srcFile string, client *http.InternalClient, profile, host string) error { +func UploadTar(srcFile string, client *pilosa.InternalClient, profile, host string) error { f, err := os.Open(srcFile) if err != nil { @@ -192,7 +192,7 @@ func main() { if profile != "" { startProfile(host) } - c, err := http.NewInternalClient(host, h) + c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) vprint.PanicOn(err) t0 := time.Now() diff --git a/ctl/backup.go b/ctl/backup.go index ff5a93307..aed077a37 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -13,7 +13,7 @@ import ( "time" pilosa "github.com/molecula/featurebase/v3" - fb_http "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/topology" "github.com/pkg/errors" @@ -42,11 +42,14 @@ type BackupCommand struct { // nolint: maligned // Amount of time after first failed request to continue retrying. RetryPeriod time.Duration `json:"retry-period"` + // Response Header Timeout for HTTP Requests + HeaderTimeout time.Duration `json:"header-timeout"` + // Host:port on which to listen for pprof. Pprof string `json:"pprof"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO @@ -59,10 +62,11 @@ type BackupCommand struct { // nolint: maligned // NewBackupCommand returns a new instance of BackupCommand. func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand { return &BackupCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - Concurrency: 1, - RetryPeriod: time.Minute, - Pprof: "localhost:0", + CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), + Concurrency: 1, + RetryPeriod: time.Minute, + HeaderTimeout: time.Second * 3, + Pprof: "localhost:0", } } @@ -89,7 +93,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) { } // Create a client to the server. - client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) + client, err := commandClient(cmd, pilosa.WithClientRetryPeriod(cmd.RetryPeriod), pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)) if err != nil { return fmt.Errorf("creating client: %w", err) } @@ -285,7 +289,10 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string, logger := cmd.Logger() logger.Printf("backing up shard: index=%q id=%d", indexName, shard) - client := fb_http.NewInternalClientFromURI(&node.URI, fb_http.GetHTTPClient(cmd.tlsConfig), fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) + client := pilosa.NewInternalClientFromURI(&node.URI, + pilosa.GetHTTPClient(cmd.tlsConfig, pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)), + pilosa.WithClientRetryPeriod(cmd.RetryPeriod), + pilosa.WithSerializer(proto.Serializer{})) rc, err := client.ShardReader(ctx, indexName, shard) if err != nil { return fmt.Errorf("fetching shard reader: %w", err) diff --git a/ctl/check.go b/ctl/check.go deleted file mode 100644 index 655394757..000000000 --- a/ctl/check.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ctl - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - "syscall" - - "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/roaring" - "github.com/pkg/errors" -) - -// CheckCommand represents a command for performing consistency checks on data files. -type CheckCommand struct { - // Data file paths. - Paths []string - - // Standard input/output - *pilosa.CmdIO -} - -// NewCheckCommand returns a new instance of CheckCommand. -func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *CheckCommand { - return &CheckCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the check command. -func (cmd *CheckCommand) Run(_ context.Context) error { - for _, path := range cmd.Paths { - switch filepath.Ext(path) { - case "": - if err := cmd.checkBitmapFile(path); err != nil { - return errors.Wrap(err, "checking bitmap") - } - - case ".cache": - if err := cmd.checkCacheFile(path); err != nil { - return errors.Wrap(err, "checking cache") - } - - case ".snapshotting": - if err := cmd.checkSnapshotFile(path); err != nil { - return errors.Wrap(err, "checking snapshot") - } - } - } - - return nil -} - -// checkBitmapFile performs a consistency check on path for a roaring bitmap file. -func (cmd *CheckCommand) checkBitmapFile(path string) (err error) { - // Open file handle. - f, err := os.Open(path) - if err != nil { - return errors.Wrap(err, "opening file") - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return errors.Wrap(err, "statting file") - } - - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - return errors.Wrap(err, "mmapping") - } - defer func() { - e := syscall.Munmap(data) - if e != nil { - fmt.Fprintf(cmd.Stderr, "WARNING: munmap failed: %v", e) - } - // don't overwrite another error with this, but also indicate - // this error. - if err == nil { - err = e - } - }() - // Attach the mmap file to the bitmap. - bm := roaring.NewBitmap() - if err := bm.UnmarshalBinary(data); err != nil { - return errors.Wrap(err, "unmarshalling") - } - - // Perform consistency check. - if err := bm.Check(); err != nil { - // Print returned errors. - switch err := err.(type) { - case roaring.ErrorList: - for i := range err { - fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err[i].Error()) - } - default: - fmt.Fprintf(cmd.Stdout, "%s: %s\n", path, err.Error()) - } - } - - // Print success message if no errors were found. - fmt.Fprintf(cmd.Stdout, "%s: ok\n", path) - - return nil -} - -// checkCacheFile performs a consistency check on path for a cache file. -func (cmd *CheckCommand) checkCacheFile(path string) error { - fmt.Fprintf(cmd.Stderr, "%s: ignoring cache file\n", path) - return nil -} - -// checkSnapshotFile performs a consistency check on path for a snapshot file. -func (cmd *CheckCommand) checkSnapshotFile(path string) error { - fmt.Fprintf(cmd.Stderr, "%s: ignoring snapshot file\n", path) - return nil -} diff --git a/ctl/check_test.go b/ctl/check_test.go deleted file mode 100644 index 229fcff44..000000000 --- a/ctl/check_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ctl - -import ( - "bytes" - "io" - "os" - "strings" - "testing" - - "context" - - "github.com/molecula/featurebase/v3/testhook" -) - -func TestCheckCommand_RunCacheFile(t *testing.T) { - fi, err := testhook.TempFile(t, "test*.cache") - if err != nil { - t.Fatalf("creating test file: %v", err) - } - cacheFile := fi.Name() - - rder := []byte{} - stdin := bytes.NewReader(rder) - r, w, _ := os.Pipe() - cm := NewCheckCommand(stdin, w, w) - cm.Paths = []string{cacheFile} - - err = cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { - t.Fatalf("copy: %v", err) - } - - if !strings.Contains(buf.String(), "ignoring cache file") { - t.Fatalf("expect: ignoring cache file, actual: '%s'", err) - } -} - -func TestCheckCommand_RunSnapshot(t *testing.T) { - fi, err := testhook.TempFile(t, "test*.snapshotting") - if err != nil { - t.Fatalf("creating test file: %v", err) - } - snapshotFile := fi.Name() - - rder := []byte{} - stdin := bytes.NewReader(rder) - r, w, _ := os.Pipe() - cm := NewCheckCommand(stdin, w, w) - cm.Paths = []string{snapshotFile} - - err = cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { - t.Fatalf("copy: %v", err) - } - - if !strings.Contains(buf.String(), "ignoring snapshot file") { - t.Fatalf("expect: ignoring snapshot file, actual: '%s'", err) - } -} - -func TestCheckCommand_Run(t *testing.T) { - file, err := testhook.TempFile(t, "run-command") - if err != nil { - t.Fatal(err) - } - fname := file.Name() - if _, err := file.Write([]byte("1234,1223")); err != nil { - t.Fatalf("writing to temp file: %v", err) - } - file.Close() - - rder := []byte{} - stdin := bytes.NewReader(rder) - r, w, _ := os.Pipe() - cm := NewCheckCommand(stdin, w, w) - cm.Paths = []string{fname} - - err = cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { - t.Fatalf("copy: %v", err) - } - - expectedPrefix := "checking bitmap: unmarshalling: " - if !strings.HasPrefix(err.Error(), expectedPrefix) { - t.Fatalf("expect error: '%s...', actual: '%s'", expectedPrefix, err) - } - // Todo: need correct roaring file for happy path -} diff --git a/ctl/chksum.go b/ctl/chksum.go index 5b10d56c9..037970644 100644 --- a/ctl/chksum.go +++ b/ctl/chksum.go @@ -20,7 +20,7 @@ type ChkSumCommand struct { // nolint: maligned Host string `json:"host"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO diff --git a/ctl/common.go b/ctl/common.go index 028a1250a..ae6551e53 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -2,12 +2,10 @@ package ctl import ( - "net" "time" - gohttp "net/http" - - "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/server" "github.com/pkg/errors" @@ -30,24 +28,50 @@ func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string, flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections") } -// default dial timeout is 30s for some reason which makes testing -// failures/retries really awkward. I don't think we need it that -// high, so I set it to 1s here... let's see what happens. -func clientOptions(client *gohttp.Client, dialer *net.Dialer) *gohttp.Client { - dialer.Timeout = time.Second * 1 - return client -} +// AnyClientOption can be either pilosa.InternalClientOption or +// pilosa.ClientOption. The internal options are specific to the +// featurebase client, whereas the client options are applied to the +// Go HTTP client that gets used under the hood. +type AnyClientOption interface{} // commandClient returns a pilosa.InternalHTTPClient for the command -func commandClient(cmd CommandWithTLSSupport, opts ...http.InternalClientOption) (*http.InternalClient, error) { +func commandClient(cmd CommandWithTLSSupport, opts ...AnyClientOption) (*pilosa.InternalClient, error) { + internalopts, clientopts, err := separateOptions(opts...) + if err != nil { + return nil, errors.Wrap(err, "separating client options") + } + + // we default dial timeout to 3s in commandClient, but prepend it + // to the option list so other options can override it. + clientopts = append([]pilosa.ClientOption{pilosa.ClientDialTimeoutOption(time.Second * 3)}, clientopts...) + internalopts = append([]pilosa.InternalClientOption{pilosa.WithSerializer(proto.Serializer{})}, internalopts...) tls := cmd.TLSConfiguration() tlsConfig, err := server.GetTLSConfig(&tls, cmd.Logger()) if err != nil { return nil, errors.Wrap(err, "getting tls config") } - client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(tlsConfig, clientOptions), opts...) + client, err := pilosa.NewInternalClient(cmd.TLSHost(), pilosa.GetHTTPClient(tlsConfig, clientopts...), internalopts...) if err != nil { return nil, errors.Wrap(err, "getting internal client") } return client, err } + +// separateOptions splits the list of AnyClientOption into the two +// possible types. +func separateOptions(opts ...AnyClientOption) ([]pilosa.InternalClientOption, []pilosa.ClientOption, error) { + internalopts := []pilosa.InternalClientOption{} + clientopts := []pilosa.ClientOption{} + for _, opt := range opts { + if iopt, ok := opt.(pilosa.InternalClientOption); ok { + internalopts = append(internalopts, iopt) + continue + } + if copt, ok := opt.(pilosa.ClientOption); ok { + clientopts = append(clientopts, copt) + continue + } + return nil, nil, errors.Errorf("opt: %+v of type %[1]T must be an InternalClientOption or a ClientOption", opt) + } + return internalopts, clientopts, nil +} diff --git a/ctl/import.go b/ctl/import.go index 3b18b8499..09bc4e0c4 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -11,7 +11,7 @@ import ( "strconv" "time" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/server" "github.com/pkg/errors" @@ -48,7 +48,7 @@ type ImportCommand struct { // nolint: maligned Sort bool `json:"sort"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO diff --git a/ctl/import_test.go b/ctl/import_test.go index 0511100db..28a91a683 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -5,10 +5,12 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" "io" "io/ioutil" "net/http" + "net/http/httptest" "os" "reflect" "strings" @@ -582,6 +584,22 @@ func TestImport_AuthOn(t *testing.T) { if err != nil { t.Fatalf("Failed to create query log file: %s", err) } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := json.Marshal( + authn.Groups{ + Groups: []authn.Group{ + { + GroupID: "group-id-test", + GroupName: "group-id-test", + }, + }, + }, + ) + if err != nil { + t.Fatalf("unexpected error marshalling groups response: %v", err) + } + fmt.Fprintf(w, "%s", body) + })) auth := server.Auth{ Enable: true, @@ -589,7 +607,7 @@ func TestImport_AuthOn(t *testing.T) { ClientSecret: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", AuthorizeURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize", TokenURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token", - GroupEndpointURL: "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true", + GroupEndpointURL: srv.URL, LogoutURL: "https://login.microsoftonline.com/common/oauth2/v2.0/logout", Scopes: []string{"https://graph.microsoft.com/.default", "offline_access"}, SecretKey: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", @@ -612,14 +630,13 @@ func TestImport_AuthOn(t *testing.T) { conf.TLS.SkipVerify = true commandOpts[i] = append(commandOpts[i], server.OptCommandConfig(conf)) } - a, err := authn.NewAuth( logger.NewStandardLogger(os.Stdout), "http://localhost:0/", auth.Scopes, auth.AuthorizeURL, auth.TokenURL, - auth.GroupEndpointURL, + srv.URL, auth.LogoutURL, auth.ClientId, auth.ClientSecret, @@ -632,8 +649,6 @@ func TestImport_AuthOn(t *testing.T) { // make a valid token tkn := jwt.New(jwt.SigningMethodHS256) claims := tkn.Claims.(jwt.MapClaims) - groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}}) - claims["molecula-idp-groups"] = groupString claims["oid"] = "42" claims["name"] = "valid" token, err := tkn.SignedString([]byte(a.SecretKey())) diff --git a/ctl/inspect.go b/ctl/inspect.go deleted file mode 100644 index 73d197501..000000000 --- a/ctl/inspect.go +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ctl - -import ( - "context" - "encoding/binary" - "fmt" - "hash/fnv" - "io" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "syscall" - "text/tabwriter" - "time" - "unsafe" - - "github.com/gogo/protobuf/proto" - "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/pb" - "github.com/molecula/featurebase/v3/roaring" - "github.com/pkg/errors" -) - -// InspectCommand represents a command for inspecting fragment data files. -type InspectCommand struct { - // Path to data file - Path string - // don't list details of objects - Quiet bool - // list only this many objects - Max int - // Filters: - InspectOpts pilosa.InspectRequest - - // Standard input/output - *pilosa.CmdIO -} - -// NewInspectCommand returns a new instance of InspectCommand. -func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *InspectCommand { - return &InspectCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -type pointerContext struct { - from, to uintptr -} - -func (p *pointerContext) pretty(c roaring.ContainerInfo) string { - var pointer string - if c.Mapped { - if c.Pointer >= p.from && c.Pointer < p.to { - pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from) - } else { - pointer = fmt.Sprintf("!0x%x!", c.Pointer) - } - } else { - pointer = fmt.Sprintf("0x%x", c.Pointer) - } - return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer) -} - -func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) { - fmt.Fprintln(cmd.Stdout, " Ops:") - tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE") - printed := 0 - for _, op := range info.OpDetails { - fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size) - printed++ - if cmd.Max != 0 && printed >= cmd.Max { - break - } - } - tw.Flush() -} - -func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) { - fmt.Fprintln(cmd.Stdout, " Containers:") - tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n") - fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS") - c1s := info.Containers - c2s := info.OpContainers - l1 := len(c1s) - l2 := len(c2s) - i1 := 0 - i2 := 0 - var c1, c2 roaring.ContainerInfo - c1.Key = ^uint64(0) - c2.Key = ^uint64(0) - c1e := false - c2e := false - if i1 < l1 { - c1 = c1s[i1] - i1++ - c1e = true - } - if i2 < l2 { - c2 = c2s[i2] - i2++ - c2e = true - } - printed := 0 - for c1e || c2e { - c1used := false - c2used := false - var key uint64 - c1fmt := "-\t\t\t" - c2fmt := "-\t\t\t" - // If c2 exists, we'll always prefer its flags, - // if it doesn't, this gets overwritten. - flags := c2.Flags - if !c2e || (c1e && c1.Key < c2.Key) { - c1fmt = pC.pretty(c1) - key = c1.Key - c1used = true - flags = c1.Flags - } else if !c1e || (c2e && c2.Key < c1.Key) { - c2fmt = pC.pretty(c2) - key = c2.Key - c2used = true - } else { - // c1e and c2e both set, and neither key is < the other. - c1fmt = pC.pretty(c1) - c2fmt = pC.pretty(c2) - key = c1.Key - c1used = true - c2used = true - } - if c1used { - if i1 < l1 { - c1 = c1s[i1] - i1++ - } else { - c1e = false - } - } - if c2used { - if i2 < l2 { - c2 = c2s[i2] - i2++ - } else { - c2e = false - } - } - fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags) - printed++ - if cmd.Max > 0 && printed >= cmd.Max { - break - } - } - tw.Flush() -} - -// Run executes the inspect command. -func (cmd *InspectCommand) Run(ctx context.Context) error { - // Open file handle. - f, err := os.Open(cmd.Path) - if err != nil { - return errors.Wrap(err, "opening file") - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return errors.Wrap(err, "statting file") - } - if fi.IsDir() { - total := 0 - infos, err := f.Readdir(0) - if err != nil { - return err - } - if len(infos) == 0 { - return errors.New("directory contains no files") - } - - names := make([]string, len(infos)) - nameToInfo := make(map[string]os.FileInfo, len(infos)) - // find numeric-only names; we'll operate on - // either those, or the whole holder if we find - // a .topology file. - n := 0 - for _, fi := range infos { - name := fi.Name() - if name == ".topology" { - return cmd.InspectHolder(ctx, cmd.Path) - } - if _, err := strconv.Atoi(name); err == nil { - names[n] = name - nameToInfo[name] = fi - n++ - } - } - if n == 0 { - return fmt.Errorf("directory contains no fragments (looking for numeric names)") - } - names = names[:n] - fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n) - for _, name := range names { - f2, err := os.Open(filepath.Join(cmd.Path, name)) - if err != nil { - return fmt.Errorf("opening %q: %v", name, err) - } - fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name) - err = cmd.InspectFile(f2, nameToInfo[name]) - total++ - f2.Close() - if err != nil { - return fmt.Errorf("inspecting %q: %v", name, err) - } - } - return nil - } - return cmd.InspectFile(f, fi) -} - -// loadTopology is copied almost exactly from pilosa/cluster.go. -func loadTopology(path string) (topology pb.Topology, myID string, err error) { - buf, err := ioutil.ReadFile(filepath.Join(path, ".topology")) - if os.IsNotExist(err) { - return topology, myID, err - } else if err != nil { - return topology, myID, errors.Wrap(err, "reading file") - } - if err := proto.Unmarshal(buf, &topology); err != nil { - return topology, myID, errors.Wrap(err, "unmarshalling") - } - sort.Slice(topology.NodeIDs, - func(i, j int) bool { - return topology.NodeIDs[i] < topology.NodeIDs[j] - }) - buf, err = ioutil.ReadFile(filepath.Join(path, ".id")) - if os.IsNotExist(err) { - return topology, myID, err - } else if err != nil { - return topology, myID, nil - } - myID = strings.TrimSpace(string(buf)) - return topology, myID, nil -} - -var partitions = make(map[string]map[uint64]int) - -func findPartition(index string, shard uint64, partitionN int) (partition int) { - var shardMap map[uint64]int - var ok bool - if shardMap, ok = partitions[index]; !ok { - shardMap = make(map[uint64]int) - partitions[index] = shardMap - } - if partition, ok = shardMap[shard]; !ok { - var buf [8]byte - binary.BigEndian.PutUint64(buf[:], shard) - - // Hash the bytes and mod by partition count. - h := fnv.New64a() - _, _ = h.Write([]byte(index)) - _, _ = h.Write(buf[:]) - partition = int(h.Sum64() % uint64(partitionN)) - shardMap[shard] = partition - } - return partition -} - -func findPartitionPath(path string, partitionN int) (int, error) { - parts := strings.Split(path, "/") - shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64) - if err != nil { - return 0, err - } - return findPartition(parts[0], shard, partitionN), nil -} - -func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error { - holder := pilosa.NewHolder(path, nil) - holder.Opts.Inspect = true - holder.Opts.ReadOnly = true - err := holder.Open() - if err != nil { - return fmt.Errorf("%s: holder open: %v", path, err) - } - holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts) - if err != nil { - return fmt.Errorf("%s: inspect: %v", path, err) - } - myPartition := 0 - topology, myID, err := loadTopology(path) - if err == nil { - fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID) - if len(topology.NodeIDs) > 1 { - fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID) - } else { - fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID) - } - found := false - for i := range topology.NodeIDs { - if topology.NodeIDs[i] == myID { - found = true - myPartition = i - break - } - } - if !found { - fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs) - } - } else { - fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err) - } - for _, name := range holderInfo.FragmentNames { - partition, err := findPartitionPath(name, len(topology.NodeIDs)) - if err != nil { - fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err) - } else { - if partition == myPartition { - fmt.Fprintf(cmd.Stdout, "%s:\n", name) - } else { - fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition]) - } - } - details := holderInfo.FragmentInfo[name] - cmd.DisplayInfo(details.BitmapInfo) - if details.BlockChecksums != nil { - fmt.Fprintf(cmd.Stdout, " Checksums [%d total]:\n", len(details.BlockChecksums)) - for _, block := range details.BlockChecksums { - fmt.Fprintf(cmd.Stdout, " %8d: %x\n", block.ID, block.Checksum) - } - } - } - return nil -} - -func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error { - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - return errors.Wrap(err, "mmapping") - } - defer func() { - err := syscall.Munmap(data) - if err != nil { - fmt.Fprintf(cmd.Stderr, "inspect command: munmap failed: %v", err) - } - }() - mappedFrom := uintptr(unsafe.Pointer(&data[0])) - mappedTo := mappedFrom + uintptr(len(data)) - // Attach the mmap file to the bitmap. - t := time.Now() - fmt.Fprintf(cmd.Stderr, "inspecting bitmap...") - var info roaring.BitmapInfo - bitmap, _, err := roaring.InspectBinary(data, true, &info) - fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) - cmd.DisplayInfo(info) - if err != nil { - return errors.Wrap(err, "inspecting") - } - mappedIn, mappedOut, unmappedIn, errs, err := bitmap.SanityCheckMapping(mappedFrom, mappedTo) - if err != nil { - fmt.Fprintf(cmd.Stderr, "sanity check: %d mapped in, %d mapped out, %d unmapped in, %d errors\n", - mappedIn, mappedOut, unmappedIn, errs) - fmt.Fprintf(cmd.Stderr, "last error: %v\n", err) - } - return nil -} - -func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) { - pC := pointerContext{ - from: info.From, - to: info.To, - } - - // Print top-level info. - fmt.Fprintf(cmd.Stdout, " Bitmap Info:\n") - fmt.Fprintf(cmd.Stdout, " Bits: %d\n", info.BitCount) - fmt.Fprintf(cmd.Stdout, " Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers)) - fmt.Fprintf(cmd.Stdout, " Operations: %d (%d bits)\n", info.Ops, info.OpN) - fmt.Fprintln(cmd.Stdout, "") - - // Print info for each container. - if !cmd.Quiet { - if info.ContainerCount > 0 { - cmd.PrintContainers(info, pC) - } - if info.Ops > 0 { - cmd.PrintOps(info) - } - } -} diff --git a/ctl/inspect_test.go b/ctl/inspect_test.go deleted file mode 100644 index 3528edbe7..000000000 --- a/ctl/inspect_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ctl - -import ( - "bytes" - "context" - "io" - "os" - "strings" - "testing" - - "github.com/molecula/featurebase/v3/testhook" -) - -func TestInspectCommand_Run(t *testing.T) { - rder := []byte{} - stdin := bytes.NewReader(rder) - r, w, _ := os.Pipe() - - cm := NewInspectCommand(stdin, w, w) - file, err := testhook.TempFile(t, "inspectTest") - if err != nil { - t.Fatalf("Error creating tempfile: %s", err) - } - _, err = file.Write([]byte("12358267538963")) - if err != nil { - t.Fatalf("writing to tempfile: %v", err) - } - file.Close() - cm.Path = file.Name() - err = cm.Run(context.Background()) - expectedError := "inspecting: " - if !strings.Contains(err.Error(), expectedError) { - t.Fatalf("expected error '%s', got '%v'", expectedError, err) - } - - w.Close() - var buf bytes.Buffer - _, err = io.Copy(&buf, r) - if err != nil { - t.Fatalf("copying data: %v", err) - } - if !strings.Contains(buf.String(), "inspecting bitmap...") { - t.Fatalf("Inspect doesn't work: %s", err) - } - - // Todo: need correct roaring file for happy path -} diff --git a/ctl/restore.go b/ctl/restore.go index 8f370ba0e..0c8cb51b0 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -18,7 +18,6 @@ import ( "github.com/hashicorp/go-retryablehttp" pilosa "github.com/molecula/featurebase/v3" - fb_http "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/topology" @@ -44,7 +43,7 @@ type RestoreCommand struct { Pprof string `json:"pprof"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO @@ -86,7 +85,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) { return fmt.Errorf("parsing tls config: %w", err) } // Create a client to the server. - client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) + client, err := commandClient(cmd, pilosa.WithClientRetryPeriod(cmd.RetryPeriod)) if err != nil { return fmt.Errorf("creating client: %w", err) } diff --git a/ctl/server.go b/ctl/server.go index 98391f761..2d8de2df2 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -2,7 +2,6 @@ package ctl import ( - "fmt" "time" "github.com/molecula/featurebase/v3/server" @@ -75,13 +74,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per ns.") flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/ of events.") - // Storage - // Note: the default for --storage.backend must be kept "" empty string. - // Otherwise we cannot detect and honor the PILOSA_STORAGE_BACKEND env var - // over-ride. - // TODO: the comment above was carried over from the PILOSA_TXSRC flag, but - // we should confirm that this still applies. - flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring or rbf. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend)) + flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, "Storage backend to use: 'rbf' is only supported value.") flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk") // RBF specific flags. See pilosa/rbf/cfg/cfg.go for definitions. diff --git a/dbshard.go b/dbshard.go index ee738458f..6c1945fdb 100644 --- a/dbshard.go +++ b/dbshard.go @@ -67,9 +67,8 @@ type DBShard struct { Shard uint64 Open bool - typ txtype - styp string - hasRoaring bool // if either of the types is roaringTxn + typ txtype + styp string W DBWrapper ParentDBIndex *DBIndex @@ -131,8 +130,7 @@ type DBPerShard struct { // Easily see how many we have. Flatmap map[flatkey]*DBShard - typ txtype - hasRoaring bool + typ txtype txf *TxFactory holder *Holder @@ -238,12 +236,6 @@ func newShardSet() *shardSet { shardsMap: make(map[uint64]bool), } } -func newShardSetFromMap(m map[uint64]bool) *shardSet { - return &shardSet{ - shardsMap: m, - shardsVer: 1, - } -} func (per *DBPerShard) LoadExistingDBs() (err error) { idxs := per.holder.Indexes() @@ -269,11 +261,6 @@ func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder vprint.PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here") } - hasRoaring := false - if typ == roaringTxn { - hasRoaring = true - } - d = &DBPerShard{ typ: typ, HolderDir: holderDir, @@ -281,7 +268,6 @@ func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder dbh: NewDBHolder(), Flatmap: make(map[flatkey]*DBShard), txf: txf, - hasRoaring: hasRoaring, index2shards: newIndex2Shards(), StorageConfig: holder.cfg.StorageConfig, RBFConfig: holder.cfg.RBFConfig, @@ -407,10 +393,7 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In } dbs, ok = dbi.Shard[shard] if dbs != nil && dbs.closed { - // roaring txn are nil/fake anyway. Don't freak out. - if per.typ != roaringTxn { - vprint.PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ)) - } + vprint.PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ)) } if !ok { dbs = &DBShard{ @@ -421,7 +404,6 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In HolderPath: per.HolderDir, idx: idx, per: per, - hasRoaring: per.hasRoaring, } dbs.styp = per.typ.String() dbi.Shard[shard] = dbs @@ -430,8 +412,6 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In if !dbs.Open { var registry DBRegistry switch dbs.typ { - case roaringTxn: - registry = globalRoaringReg case rbfTxn: registry = globalRbfDBReg registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) @@ -470,8 +450,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData) } -// if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover -// all the view paths under idx for type ty. // requireData means open the database file and verify that at least one key is set. // The returned sliceOfShards should not be modified. We will cache it for subsequent // queries. @@ -485,14 +463,6 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r per.Mu.Lock() defer per.Mu.Unlock() - if ty == roaringTxn && roaringViewPath != "" { - shardMap, err := roaringMapOfShards(roaringViewPath) - if err != nil { - return nil, err - } - return shardMap, nil - } - i2ss := per.index2shards ss, ok := i2ss[idx.name] @@ -507,27 +477,6 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r // Upon return, cache the setOfShards value and reuse it next time - if ty == roaringTxn { - // INVAR: roaringViewPath == "", because the other case is - // handled above. - fields := idx.Fields() - for _, field := range fields { - for _, view := range field.views() { - shardMap, err := roaringMapOfShards(view.path) - if err != nil { - return nil, - errors.Wrap(err, fmt.Sprintf( - "TypedDBPerShardGetLocalShardsForIndex roaringTxn view.path='%v'", view.path)) - } - for shard := range shardMap { - setOfShards.add(shard) - } - } - } - return setOfShards.CloneMaybe(), nil - } - // INVAR: not-roaring. - path := per.prefixForType(idx, ty) ignoreEmpty := false @@ -627,19 +576,6 @@ func (vs *FieldView2Shards) getViewsForField(field string) map[string]*shardSet return vs.m[field] } -func (vs *FieldView2Shards) has(field, view string, shard uint64) bool { - vw, ok := vs.m[field] - if !ok { - return false - } - ss, ok := vw[view] - if !ok { - return false - } - shardMap := ss.CloneMaybe() - return shardMap[shard] -} - func (vs *FieldView2Shards) addViewShardSet(fv txkey.FieldView, ss *shardSet) { f, ok := vs.m[fv.Field] @@ -730,8 +666,6 @@ func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView ty := per.typ switch ty { - case roaringTxn: - return roaringGetFieldView2Shards(idx) default: vs = NewFieldView2Shards() diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index ceb63ab89..38f752425 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -71,7 +71,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet) } - for _, src := range []string{"roaring", "rbf"} { + for _, src := range []string{"rbf"} { cfg := mustHolderConfig() cfg.StorageConfig.Backend = src holder := NewHolder(tmpdir, cfg) @@ -82,7 +82,6 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index) PanicOn(err) } - estd := "rick/fields/_exists/views/standard" std := "rick/fields/f/views/standard" shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false) @@ -93,65 +92,23 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards)) } } - if src == "roaring" { - // check estd too - shards, err = holder.txf.GetShardsForIndex(idx, tmpdir+sep+estd, false) + for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) + fvs, err := tx.GetSortedFieldViewList(idx, shard) PanicOn(err) - for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { - if !shards[shard] { - panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards)) - } + // expect these same two field/views for all 6 shards + expect0 := txkey.FieldView{Field: "_exists", View: "standard"} + expect1 := txkey.FieldView{Field: "f", View: "standard"} + if len(fvs) != 2 { + panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src)) } - - // check GetSortedFieldViewList() and roaringGetFieldView2Shards() - vs, err := roaringGetFieldView2Shards(idx) - PanicOn(err) - - for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) - fvs, err := tx.GetSortedFieldViewList(idx, shard) - PanicOn(err) - // expect these same two field/views for all 6 shards - expect0 := txkey.FieldView{Field: "_exists", View: "standard"} - expect1 := txkey.FieldView{Field: "f", View: "standard"} - if len(fvs) != 2 { - panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src)) - } - if fvs[0] != expect0 { - panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0])) - } - if fvs[1] != expect1 { - panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1])) - } - - for _, fv := range fvs { - if !vs.has(fv.Field, fv.View, shard) { - panic(fmt.Sprintf("vs did not contain fv='%#v' for shard %v", fv, shard)) - } - } - tx.Rollback() + if fvs[0] != expect0 { + panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0])) } - } else { - // non-roaring: rbf - - for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) - fvs, err := tx.GetSortedFieldViewList(idx, shard) - PanicOn(err) - // expect these same two field/views for all 6 shards - expect0 := txkey.FieldView{Field: "_exists", View: "standard"} - expect1 := txkey.FieldView{Field: "f", View: "standard"} - if len(fvs) != 2 { - panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src)) - } - if fvs[0] != expect0 { - panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0])) - } - if fvs[1] != expect1 { - panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1])) - } - tx.Rollback() + if fvs[1] != expect1 { + panic(fmt.Sprintf("expected fvs[1]='%#v', but got '%#v'", expect1, fvs[1])) } + tx.Rollback() } holder.Close() } diff --git a/dbshard_test.go b/dbshard_test.go index 6f7281301..95ff92eea 100644 --- a/dbshard_test.go +++ b/dbshard_test.go @@ -7,9 +7,8 @@ import ( "reflect" "testing" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck @@ -23,7 +22,7 @@ func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) { pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() diff --git a/executor.go b/executor.go index 4409637a0..1736ce94a 100644 --- a/executor.go +++ b/executor.go @@ -55,7 +55,7 @@ type executor struct { workCounter uint64 // Client used for remote requests. - client InternalQueryClient + client *InternalClient // Maximum number of Set() or Clear() commands per request. MaxWritesPerRequest int @@ -74,7 +74,7 @@ type executor struct { // executorOption is a functional option type for pilosa.Executor type executorOption func(e *executor) error -func optExecutorInternalQueryClient(c InternalQueryClient) executorOption { +func optExecutorInternalQueryClient(c *InternalClient) executorOption { return func(e *executor) error { e.client = c return nil @@ -116,7 +116,6 @@ func emptyResult(c *pql.Call) interface{} { // newExecutor returns a new instance of Executor. func newExecutor(opts ...executorOption) *executor { e := &executor{ - client: newNopInternalQueryClient(), workerPoolSize: 2, } for _, opt := range opts { @@ -1651,6 +1650,9 @@ func (d *DistinctTimestamp) Union(other DistinctTimestamp) DistinctTimestamp { return DistinctTimestamp{Name: d.Name, Values: vals} } +const ViewNotFound = Error("view not found") +const FragmentNotFound = Error("fragment not found") + func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { index := idx.Name() tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) @@ -3655,9 +3657,20 @@ func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index stri } // Apply bases. + // + // SUP-139: The group value is shared across multiple groups so we can't + // add the base to each one. Instead, we need to track which ones have been + // seen already and avoid adding to those again in the future. for i, base := range bases { + m := make(map[*int64]struct{}) + for _, r := range results { + if _, ok := m[r.Group[i].Value]; ok { + continue + } + *r.Group[i].Value += base + m[r.Group[i].Value] = struct{}{} } } diff --git a/executor_test.go b/executor_test.go index 1c534ac52..f1da0402f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -29,11 +29,9 @@ import ( "github.com/molecula/featurebase/v3/boltdb" "github.com/molecula/featurebase/v3/ctl" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/proto" "github.com/molecula/featurebase/v3/server" - "github.com/molecula/featurebase/v3/storage" "github.com/molecula/featurebase/v3/test" "github.com/molecula/featurebase/v3/testhook" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck @@ -1277,15 +1275,6 @@ func TestExecutor_Execute_Count(t *testing.T) { } -func roaringOnlyTest(t *testing.T) { - src := pilosa.CurrentBackend() - if src == pilosa.RoaringTxn || (storage.DefaultBackend == pilosa.RoaringTxn && src == "") { - // okay to run, we are under roaring only - } else { - t.Skip("skip for everything but roaring") - } -} - // Ensure a set query can be executed. func TestExecutor_Execute_Set(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { @@ -3836,7 +3825,7 @@ func TestExecutor_Execute_Existence(t *testing.T) { c := test.MustRunCluster(t, 1, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), ), }) defer c.Close() @@ -4227,7 +4216,7 @@ func TestExecutor_Execute_All(t *testing.T) { c := test.MustRunCluster(t, 1, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), ), }) defer c.Close() @@ -6161,6 +6150,34 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { test.CheckGroupByOnKey(t, expected, results) }) + // SUP-139: GroupBy returns incorrect results when two or more Integer Range Fields are used to define the grouping + t.Run("CountByIntegersWithMinMax", func(t *testing.T) { + c.CreateField(t, "cbimm", pilosa.IndexOptions{}, "year", pilosa.OptFieldTypeInt(2019, 2020)) + c.CreateField(t, "cbimm", pilosa.IndexOptions{}, "quarter", pilosa.OptFieldTypeInt(1, 4)) + + c.ImportIntID(t, "cbimm", "year", []test.IntID{{ID: 1, Val: 2019}, {ID: 2, Val: 2019}, {ID: 3, Val: 2019}, {ID: 4, Val: 2019}}) + c.ImportIntID(t, "cbimm", "quarter", []test.IntID{{ID: 1, Val: 1}, {ID: 2, Val: 1}, {ID: 3, Val: 1}, {ID: 4, Val: 2}}) + + year2019 := int64(2019) + quarter1, quarter2 := int64(1), int64(2) + + results := c.Query(t, "cbimm", `GroupBy(Rows(year), Rows(quarter))`).Results[0].(*pilosa.GroupCounts).Groups() + + test.CheckGroupBy(t, + []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{ + {Field: "year", RowID: 0, Value: &year2019}, + {Field: "quarter", RowID: 0, Value: &quarter1}, + }, Count: 3}, + {Group: []pilosa.FieldRow{ + {Field: "year", RowID: 0, Value: &year2019}, + {Field: "quarter", RowID: 0, Value: &quarter2}, + }, Count: 1}, + }, + results, + ) + + }) } for _, size := range []int{1, 3} { t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) { diff --git a/field_internal_test.go b/field_internal_test.go index da1ba40bd..98dea321b 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -247,7 +247,6 @@ func NewTestField(t testing.TB, opts FieldOption) *TestField { } cfg := DefaultHolderConfig() - cfg.StorageConfig.Backend = CurrentBackendOrDefault() cfg.StorageConfig.FsyncEnabled = false cfg.RBFConfig.FsyncEnabled = false h := NewHolder(path, cfg) diff --git a/fragment.go b/fragment.go index 41b1f6a79..54d68eed9 100644 --- a/fragment.go +++ b/fragment.go @@ -3,7 +3,6 @@ package pilosa import ( "archive/tar" - "bufio" "bytes" "container/heap" "context" @@ -16,14 +15,11 @@ import ( "math/bits" "os" "path/filepath" - "runtime/debug" "sort" "strconv" "strings" "sync" - "syscall" "time" - "unsafe" "github.com/cespare/xxhash" "github.com/gogo/protobuf/proto" @@ -59,18 +55,12 @@ const ( // width of roaring containers is 2^16 containerWidth = 1 << 16 - // snapshotExt is the file extension used for an in-process snapshot. - snapshotExt = ".snapshotting" - // cacheExt is the file extension for persisted cache ids. cacheExt = ".cache" // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 - // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. - defaultFragmentMaxOpN = 10000 - // Row ids used for boolean fields. falseRowID = uint64(0) trueRowID = uint64(1) @@ -132,23 +122,9 @@ type fragment struct { // idx cached to avoid repeatedly looking it up everywhere. idx *Index - // parent holder, used to find snapshot queue, etc. + // parent holder holder *Holder - // debugging tool: addresses of current and previous maps - prevdata, currdata struct{ from, to uintptr } - - // File-backed storage - flags byte // user-defined flags passed to roaring - storage *roaring.Bitmap - opN int // number of ops since snapshot (may be approximate for imports) - ops int // number of higher-level operations, as opposed to bit changes - snapshotPending bool // set to true when requesting a snapshot, set to false after snapshot completes - snapshotCond sync.Cond - snapshotErr error // error yielded by the last snapshot operation - snapshotStamp time.Time // timestamp of last snapshot - open bool // is this fragment actually open? - // Cache for row counts. CacheType string // passed in by field @@ -164,11 +140,6 @@ type fragment struct { // Cached checksums for each block. checksums map[int][]byte - // Number of operations performed before performing a snapshot. - // This limits the size of fragments on the heap and flushes them to disk - // so that they can be mmapped and heap utilization can be kept low. - MaxOpN int - // Logger used for out-of-band log entries. Logger logger.Logger @@ -177,8 +148,6 @@ type fragment struct { mutexVector vector stats stats.StatsClient - - bitmapInfo *roaring.BitmapInfo } // newFragment returns a new instance of fragment. @@ -194,18 +163,15 @@ func newFragment(holder *Holder, spec fragSpec, shard uint64, flags byte) *fragm fieldstr: spec.fieldstr, fld: spec.field, shard: shard, - flags: flags, idx: idx, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, holder: holder, - MaxOpN: defaultFragmentMaxOpN, stats: stats.NopStatsClient, } - f.snapshotCond = sync.Cond{L: &f.mu} return f } @@ -241,30 +207,12 @@ func (f *fragment) Index() *Index { return f.holder.Index(f.index()) } -func (f *fragment) inspect(params InspectRequestParams) (fi FragmentInfo) { - if f.bitmapInfo == nil { - fi.BitmapInfo = f.storage.Info(params.Containers) - } else { - fi.BitmapInfo = *f.bitmapInfo - } - if params.Checksum { - fi.BlockChecksums, _ = f.Blocks() - } - return fi -} - // Open opens the underlying storage. func (f *fragment) Open() error { f.mu.Lock() defer f.mu.Unlock() if err := func() error { - // Initialize storage in a function so we can close if anything goes wrong. - f.holder.Logger.Debugf("open storage for index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard) - if err := f.openStorage(true); err != nil { - return errors.Wrap(err, "opening storage") - } - // Fill cache with rows persisted to disk. f.holder.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard) if err := f.openCache(); err != nil { @@ -278,57 +226,12 @@ func (f *fragment) Open() error { f.close() return err } - f.open = true _ = testhook.Opened(f.holder.Auditor, f, nil) f.holder.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard) return nil } -// emptyStorage is the common case for importStorage/applyStorage where they -// get no data. It tries to write the current storage to the provided file, -// which is assumed to be the file they didn't get any data from. -func (f *fragment) emptyStorage(file *os.File) (bool, error) { - if f.holder.Opts.ReadOnly { - return false, errors.New("can't flush/create storage for read-only holder") - } - // No data. We'll mark this for no mapping, clear any existing - // mapped containers, and set the Source to nil. We also have no - // ops. - f.opN = 0 - f.ops = 0 - f.storage.SetOps(0, 0) - - f.storage.PreferMapping(false) - _, err := f.storage.RemapRoaringStorage(nil) - f.storage.SetSource(nil) - if err != nil { - return false, fmt.Errorf("applying/importing storage: no data, and clearing old mapping also failed: %v", err) - } - // Write the existing storage out to the file so it's - // a valid Roaring file thereafter. nothing to unmarshal. - // In the unlikely event that this happened even though we - // had significant data, we're not mapping it, but that's - // harmless even if it's not maximally efficient. - bi := bufio.NewWriter(file) - if _, err = f.storage.WriteTo(bi); err != nil { - return false, fmt.Errorf("init storage file: %s", err) - } - bi.Flush() - return false, nil -} - -// openStorage opens the storage bitmap. Does nothing in RBF-world and will be removed soon. -func (f *fragment) openStorage(unmarshalData bool) error { - if !f.idx.NeedsSnapshot() { - f.currdata = struct{ from, to uintptr }{} - f.prevdata = f.currdata - return nil // openStorage becomes a noop under RBF, Badger, etc. - } - - return nil -} - // openCache initializes the cache from row ids persisted to disk. func (f *fragment) openCache() error { // Determine cache type from field name. @@ -384,12 +287,6 @@ func (f *fragment) Close() error { defer func() { _ = testhook.Closed(f.holder.Auditor, f, nil) }() - for f.snapshotPending { - f.snapshotCond.Wait() - } - // Note: snapshots won't progress on a closed fragment, so we - // wait until after a possible pending snapshot to close. - f.open = false return f.close() } @@ -400,28 +297,12 @@ func (f *fragment) close() error { return errors.Wrap(err, "flushing cache") } - // Close underlying storage. - if err := f.closeStorage(); err != nil { - f.holder.Logger.Errorf("fragment: error closing storage: err=%s, path=%s", err, f.path()) - return errors.Wrap(err, "closing storage") - } - // Remove checksums. f.checksums = nil return nil } -// closeStorage is essentially a no-op and will go away soon. -func (f *fragment) closeStorage() error { - // opN is determined by how many bit set/clear operations are in the storage - // write log, so once the storage is closed it should be 0. Opening new - // storage will set opN appropriately. - f.opN = 0 - - return nil -} - // mutexCheck checks for any entries in fragment which violate the mutex // property of having only one value set for a given column ID. func (f *fragment) mutexCheck(tx Tx, details bool, limit int) (map[uint64][]uint64, error) { @@ -544,9 +425,6 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo // Invalidate block checksum. delete(f.checksums, int(rowID/HashBlockSize)) - // Increment number of operations until snapshot is required. - f.incrementOpN(1) - // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. if f.CacheType != CacheTypeNone { @@ -596,9 +474,6 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b // Invalidate block checksum. delete(f.checksums, int(rowID/HashBlockSize)) - // Increment number of operations until snapshot is required. - f.incrementOpN(1) - // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. if f.CacheType != CacheTypeNone { @@ -665,8 +540,6 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo } } - // Snapshot storage. - f.holder.SnapshotQueue.Enqueue(f) f.stats.Count("setRow", 1, 1.0) return changed, nil @@ -705,9 +578,6 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e // Clear the row in cache. f.cache.Add(rowID, 0) - // Snapshot storage. - f.holder.SnapshotQueue.Enqueue(f) - return changed, nil } @@ -1962,7 +1832,7 @@ func (f *fragment) mergeBlock(tx Tx, id int, data []pairSet) (sets, clears []pai return sets[1:], clears[1:], err } -// bulkImport bulk imports a set of bits and then snapshots the storage. +// bulkImport bulk imports a set of bits. // The cache is updated to reflect the new data. func (f *fragment) bulkImport(tx Tx, rowIDs, columnIDs []uint64, options *ImportOptions) error { // Verify that there are an equal number of row ids and column ids. @@ -2175,68 +2045,48 @@ func (p parallelSlices) Swap(i, j int) { // snapshot of the fragment or just do in-memory updates while appending // operations to the op log. func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error { - //tx.AddN() - doFunc := func() error { - if len(set) > 0 { - f.stats.Count(MetricImportingN, int64(len(set)), 1) + if len(set) > 0 { + f.stats.Count(MetricImportingN, int64(len(set)), 1) - // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions - changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...) - if err != nil { - return errors.Wrap(err, "adding positions") - } - f.stats.Count(MetricImportedN, int64(changedN), 1) - f.incrementOpN(changedN) + // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions + changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...) + if err != nil { + return errors.Wrap(err, "adding positions") } + f.stats.Count(MetricImportedN, int64(changedN), 1) + } - if len(clear) > 0 { - f.stats.Count(MetricClearingN, int64(len(clear)), 1) - changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...) - if err != nil { - return errors.Wrap(err, "clearing positions") - } - f.stats.Count(MetricClearedN, int64(changedN), 1) - f.incrementOpN(changedN) + if len(clear) > 0 { + f.stats.Count(MetricClearingN, int64(len(clear)), 1) + changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...) + if err != nil { + return errors.Wrap(err, "clearing positions") } + f.stats.Count(MetricClearedN, int64(changedN), 1) + } - // Update cache counts for all affected rows. - for rowID := range rowSet { - // Invalidate block checksum. - delete(f.checksums, int(rowID/HashBlockSize)) - - if f.CacheType != CacheTypeNone { - start := rowID * ShardWidth - end := (rowID + 1) * ShardWidth - - n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, start, end) - if err != nil { - return errors.Wrap(err, "CountRange") - } - - f.cache.BulkAdd(rowID, n) - } - } + // Update cache counts for all affected rows. + for rowID := range rowSet { + // Invalidate block checksum. + delete(f.checksums, int(rowID/HashBlockSize)) if f.CacheType != CacheTypeNone { - f.cache.Invalidate() - } - return nil - } - err := doFunc() - if err != nil && f.storage != nil { - // we got an error. it's possible that the error indicates that something went wrong. - mappedIn, mappedOut, unmappedIn, errs, e2 := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to) - if errs != 0 { - f.holder.Logger.Errorf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v", - f.path(), mappedIn, mappedOut, unmappedIn, errs, e2) - if f.prevdata.from != f.currdata.from { - mappedIn, mappedOut, unmappedIn, errs, e2 = f.storage.SanityCheckMapping(f.prevdata.from, f.prevdata.to) - f.holder.Logger.Errorf("with previous map, storage would have %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total, last %v", - mappedIn, mappedOut, unmappedIn, errs, e2) + start := rowID * ShardWidth + end := (rowID + 1) * ShardWidth + + n, err := tx.CountRange(f.index(), f.field(), f.view(), f.shard, start, end) + if err != nil { + return errors.Wrap(err, "CountRange") } + + f.cache.BulkAdd(rowID, n) } } - return err + + if f.CacheType != CacheTypeNone { + f.cache.Invalidate() + } + return nil } // sliceDifference removes everything from original that's found in remove, @@ -2474,7 +2324,7 @@ func (f *fragment) doImportRoaring(ctx context.Context, tx Tx, data []byte, clea f.mu.RLock() defer f.mu.RUnlock() rowSize := uint64(1 << shardVsContainerExponent) - span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits") + span, _ := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits") defer span.Finish() var rowSet map[uint64]int @@ -2536,125 +2386,6 @@ func (f *fragment) importRoaringOverwrite(ctx context.Context, tx Tx, data []byt return f.importRoaring(ctx, tx, data, false) } -// incrementOpN increase the operation count by one. -// If the count exceeds the maximum allowed then a snapshot is performed. -func (f *fragment) incrementOpN(changed int) { - if changed <= 0 { - return - } - // don't count opN or ops if our index doesn't want snapshots - if !f.idx.NeedsSnapshot() { - return - } - f.opN += changed - f.ops++ - if f.opN > f.MaxOpN { - f.holder.SnapshotQueue.Enqueue(f) - } -} - -// Snapshot writes the storage bitmap to disk and reopens it. This may -// coexist with existing background-queue snapshotting; it does not remove -// things from the queue. You probably don't want to do this; use -// the snapshotQueue's Enqueue/Await. -func (f *fragment) Snapshot() error { - f.mu.Lock() - defer f.mu.Unlock() - return f.snapshot() -} - -func track(start time.Time, message string, stats stats.StatsClient, logger logger.Logger) { - elapsed := time.Since(start) - logger.Debugf("%s took %s", message, elapsed) - stats.Timing(MetricSnapshotDurationSeconds, elapsed, 1.0) -} - -// snapshot does the actual snapshot operation. it does not check or care -// about f.snapshotPending. -func (f *fragment) snapshot() (err error) { - if !f.idx.NeedsSnapshot() { - return nil - } - if !f.open { - return errors.New("snapshot request on closed fragment") - } - wouldPanic := debug.SetPanicOnFault(true) - defer func() { - debug.SetPanicOnFault(wouldPanic) - if r := recover(); r != nil { - if e2, ok := r.(error); ok { - err = e2 - // special case: if we caught a page fault, we diagnose that directly. sadly, - // we can't see the actual values that were used to generate this, probably. - if e2.Error() == "runtime error: invalid memory address or nil pointer dereference" { - mappedIn, mappedOut, unmappedIn, errs, _ := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to) - f.holder.Logger.Errorf("transaction failed on %s. storage has %d mapped in range, %d mapped out of range, %d unmapped in range, %d errors total", - f.path(), mappedIn, mappedOut, unmappedIn, errs) - } - } else { - err = fmt.Errorf("non-error PanicOn: %v", r) - } - } - }() - _, err = unprotectedWriteToFragment(f, f.storage) - if err == nil { - f.snapshotStamp = time.Now() - } - return err -} - -// unprotectedWriteToFragment writes the fragment f with bm as the data. It is unprotected, and -// f.mu must be locked when calling it. -func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err error) { // nolint: interfacer - completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard) - start := time.Now() - defer track(start, completeMessage, f.stats, f.holder.Logger) - - // Create a temporary file to snapshot to. - snapshotPath := f.path() + snapshotExt - file, err := os.Create(snapshotPath) - if err != nil { - return n, fmt.Errorf("create snapshot file: %s", err) - } - // No deferred close, because we want to close it sooner than the - // end of this function. - - // Write storage to snapshot. - bw := bufio.NewWriter(file) - if n, err = bm.WriteTo(bw); err != nil { - file.Close() - return n, fmt.Errorf("snapshot write to: %s", err) - } - - if err := bw.Flush(); err != nil { - file.Close() - return n, fmt.Errorf("flush: %s", err) - } - - // we close the file here so we don't still have it open when trying - // to open it in a moment. - file.Close() - - // Move snapshot to data file location. - if err := os.Rename(snapshotPath, f.path()); err != nil { - return n, fmt.Errorf("rename snapshot: %s", err) - } - - // if we reloaded from the file, we'd end up with this bitmap - // as our storage. so... let's use this bitmap. as our storage. - f.storage = bm - - // Reopen storage. - if err := f.openStorage(false); err != nil { - return n, fmt.Errorf("open storage: %s", err) - } - - // Reset operation count. - f.opN = 0 - - return n, nil -} - // RecalculateCache rebuilds the cache regardless of invalidate time delay. func (f *fragment) RecalculateCache() { f.mu.Lock() @@ -3570,14 +3301,6 @@ func bitsToRoaringData(ps pairSet) ([]byte, error) { return buf.Bytes(), nil } -func madvise(b []byte, advice int) error { // nolint: unparam - _, _, err := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice)) - if err != 0 { - return err - } - return nil -} - // pairSet is a list of equal length row and column id lists. type pairSet struct { rowIDs []uint64 diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e079c05ef..6fd803954 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -8,13 +8,9 @@ import ( "fmt" "io" "io/ioutil" - "math" "math/rand" "os" - "path/filepath" "reflect" - "runtime" - "runtime/debug" "sort" "strconv" "strings" @@ -25,7 +21,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" - "github.com/molecula/featurebase/v3/storage" "github.com/molecula/featurebase/v3/testhook" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck "github.com/pkg/errors" @@ -1025,50 +1020,44 @@ func BenchmarkFragment_ImportValue(b *testing.B) { // // We test a variety of combinations of the number of separate updates(imports), // the number of bits in the import, the number of rows in the fragment (which -// is a pretty good proxy for fragment size on disk), and the MaxOpN on the -// fragment which controls how many set bits occur before a snapshot is done. If -// the number of bits in a given import is greater than MaxOpN, bulkImport will -// always go through the standard snapshotting import path. +// is a pretty good proxy for fragment size on disk). func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { for _, numUpdates := range []int{100} { for _, bitsPerUpdate := range []int{100, 1000} { for _, numRows := range []int{1000, 100000, 1000000} { - for _, opN := range []int{1, 5000, 50000} { - b.Run(fmt.Sprintf("Rows%dUpdates%dBits%dOpN%d", numRows, numUpdates, bitsPerUpdate, opN), func(b *testing.B) { - for a := 0; a < b.N; a++ { - b.StopTimer() - // build the update data set all at once - this will get applied - // to a fragment in numUpdates batches - updateRows := make([]uint64, numUpdates*bitsPerUpdate) - updateCols := make([]uint64, numUpdates*bitsPerUpdate) - for i := 0; i < numUpdates*bitsPerUpdate; i++ { - updateRows[i] = uint64(rand.Int63n(int64(numRows))) // row id - updateCols[i] = uint64(rand.Int63n(ShardWidth)) // column id - } - f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") - _ = idx - f.MaxOpN = opN - defer f.Clean(b) - - err := f.importRoaringT(tx, getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) - if err != nil { - b.Fatalf("importing base data for benchmark: %v", err) - } - b.StartTimer() - for i := 0; i < numUpdates; i++ { - err := f.bulkImportStandard(tx, - updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], - updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], - &ImportOptions{}, - ) - if err != nil { - b.Fatalf("doing small bulk import: %v", err) - } - } - tx.Rollback() // don't exhaust the Tx space under b.N iterations. + b.Run(fmt.Sprintf("Rows%dUpdates%dBits%d", numRows, numUpdates, bitsPerUpdate), func(b *testing.B) { + for a := 0; a < b.N; a++ { + b.StopTimer() + // build the update data set all at once - this will get applied + // to a fragment in numUpdates batches + updateRows := make([]uint64, numUpdates*bitsPerUpdate) + updateCols := make([]uint64, numUpdates*bitsPerUpdate) + for i := 0; i < numUpdates*bitsPerUpdate; i++ { + updateRows[i] = uint64(rand.Int63n(int64(numRows))) // row id + updateCols[i] = uint64(rand.Int63n(ShardWidth)) // column id } - }) - } + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") + _ = idx + defer f.Clean(b) + + err := f.importRoaringT(tx, getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) + if err != nil { + b.Fatalf("importing base data for benchmark: %v", err) + } + b.StartTimer() + for i := 0; i < numUpdates; i++ { + err := f.bulkImportStandard(tx, + updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], + updateRows[bitsPerUpdate*i:bitsPerUpdate*(i+1)], + &ImportOptions{}, + ) + if err != nil { + b.Fatalf("doing small bulk import: %v", err) + } + } + tx.Rollback() // don't exhaust the Tx space under b.N iterations. + } + }) } } } @@ -1078,33 +1067,30 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { for _, numUpdates := range []int{100} { for _, bitsPerUpdate := range []uint64{100, 1000} { for _, numRows := range []uint64{1000, 100000, 1000000} { - for _, opN := range []int{1, 5000, 50000} { - b.Run(fmt.Sprintf("Rows%dUpdates%dBits%dOpN%d", numRows, numUpdates, bitsPerUpdate, opN), func(b *testing.B) { - for a := 0; a < b.N; a++ { - b.StopTimer() - // build the update data set all at once - this will get applied - // to a fragment in numUpdates batches - f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") - _ = idx - f.MaxOpN = opN - defer f.Clean(b) + b.Run(fmt.Sprintf("Rows%dUpdates%dBits%d", numRows, numUpdates, bitsPerUpdate), func(b *testing.B) { + for a := 0; a < b.N; a++ { + b.StopTimer() + // build the update data set all at once - this will get applied + // to a fragment in numUpdates batches + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") + _ = idx + defer f.Clean(b) - err := f.importRoaringT(tx, getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) + err := f.importRoaringT(tx, getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) + if err != nil { + b.Fatalf("importing base data for benchmark: %v", err) + } + for i := 0; i < numUpdates; i++ { + data := getUpdataRoaring(numRows, bitsPerUpdate, int64(i)) + b.StartTimer() + err := f.importRoaringT(tx, data, false) + b.StopTimer() if err != nil { - b.Fatalf("importing base data for benchmark: %v", err) - } - for i := 0; i < numUpdates; i++ { - data := getUpdataRoaring(numRows, bitsPerUpdate, int64(i)) - b.StartTimer() - err := f.importRoaringT(tx, data, false) - b.StopTimer() - if err != nil { - b.Fatalf("doing small roaring import: %v", err) - } + b.Fatalf("doing small roaring import: %v", err) } } - }) - } + } + }) } } } @@ -1131,70 +1117,34 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { updateVals[i] = int64(rand.Int63n(1 << 21)) } - for _, opN := range []int{1, 5000, 50000} { - b.Run(fmt.Sprintf("Updates%dVals%dOpN%d", numUpdates, valsPerUpdate, opN), func(b *testing.B) { - for i := 0; i < b.N; i++ { - b.StopTimer() - f, _, tx := mustOpenBSIFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0) - f.MaxOpN = opN + b.Run(fmt.Sprintf("Updates%dVals%d", numUpdates, valsPerUpdate), func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + f, _, tx := mustOpenBSIFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0) - err := f.importValue(tx, initialCols, initialVals, 21, false) - if err != nil { - b.Fatalf("initial value import: %v", err) - } - b.StartTimer() - for j := 0; j < numUpdates; j++ { - err := f.importValue(tx, - updateCols[valsPerUpdate*j:valsPerUpdate*(j+1)], - updateVals[valsPerUpdate*j:valsPerUpdate*(j+1)], - 21, - false, - ) - if err != nil { - b.Fatalf("importing values: %v", err) - } - } - tx.Rollback() // don't exhaust the Tx over the b.N iterations. + err := f.importValue(tx, initialCols, initialVals, 21, false) + if err != nil { + b.Fatalf("initial value import: %v", err) } - }) - } - + b.StartTimer() + for j := 0; j < numUpdates; j++ { + err := f.importValue(tx, + updateCols[valsPerUpdate*j:valsPerUpdate*(j+1)], + updateVals[valsPerUpdate*j:valsPerUpdate*(j+1)], + 21, + false, + ) + if err != nil { + b.Fatalf("importing values: %v", err) + } + } + tx.Rollback() // don't exhaust the Tx over the b.N iterations. + } + }) } } } -// Ensure a fragment can snapshot correctly. -func TestFragment_Snapshot(t *testing.T) { - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") - defer f.Clean(t) - - // Set and then clear bits on the fragment. - if _, err := f.setBit(tx, 1000, 1); err != nil { - t.Fatal(err) - } else if _, err := f.setBit(tx, 1000, 2); err != nil { - t.Fatal(err) - } else if _, err := f.clearBit(tx, 1000, 1); err != nil { - t.Fatal(err) - } - PanicOn(tx.Commit()) - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - // Snapshot bitmap and verify data. - if err := f.Snapshot(); err != nil { - t.Fatal(err) - } else if n := f.mustRow(tx, 1000).Count(); n != 1 { - t.Fatalf("unexpected count: %d", n) - } - - // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { - t.Fatal(err) - } else if n := f.mustRow(tx, 1000).Count(); n != 1 { - t.Fatalf("unexpected count (reopen): %d", n) - } -} - // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") @@ -1577,91 +1527,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { } } -// Ensure a fragment's cache can be persisted between restarts. -func TestFragment_RankCache_Persistence(t *testing.T) { - roaringOnlyTest(t) - - index := mustOpenIndex(t, IndexOptions{}) - defer index.Close() - - // Create field. - field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) - if err != nil { - t.Fatal(err) - } - - // Create view. - view, err := field.createViewIfNotExists(viewStandard) - if err != nil { - t.Fatal(err) - } - - // Create fragment. - f, err := view.CreateFragmentIfNotExists(0) - if err != nil { - t.Fatal(err) - } - - // Obtain transaction. - tx := index.holder.txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - // Set bits on the fragment. - for i := uint64(0); i < 1000; i++ { - if _, err := f.setBit(tx, i, 0); err != nil { - t.Fatal(err) - } - } - - PanicOn(tx.Commit()) - tx = index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - // Verify correct cache type and size. - if cache, ok := f.cache.(*rankCache); !ok { - t.Fatalf("unexpected cache: %T", f.cache) - } else if cache.Len() != 1000 { - t.Fatalf("unexpected cache len: %d", cache.Len()) - } - - // Reopen the index. - if err := index.reopen(); err != nil { - t.Fatal(err) - } - - // Re-fetch fragment. - f = index.Field("f").view(viewStandard).Fragment(0) - - // Re-verify correct cache type and size. - if cache, ok := f.cache.(*rankCache); !ok { - t.Fatalf("unexpected cache: %T", f.cache) - } else if cache.Len() != 1000 { - t.Fatalf("unexpected cache len: %d", cache.Len()) - } -} - -func roaringOnlyTest(t *testing.T) { - src := CurrentBackend() - if src == RoaringTxn || (storage.DefaultBackend == RoaringTxn && src == "") { - // okay to run, we are under roaring only - } else { - t.Skip("skip for everything but roaring") - } -} - -func roaringOnlyBenchmark(b *testing.B) { - src := CurrentBackend() - if src == RoaringTxn || (storage.DefaultBackend == RoaringTxn && src == "") { - // okay to run, we are under roaring only - } else { - b.Skip("skip for everything but roaring") - } -} - // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - // roaringOnlyTest(t) - f0, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f0.Clean(t) @@ -1752,7 +1619,6 @@ func BenchmarkFragment_Blocks(b *testing.B) { func BenchmarkFragment_IntersectionCount(b *testing.B) { f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") defer f.Clean(b) - f.MaxOpN = math.MaxInt32 // Generate some intersecting data. for i := 0; i < 10000; i += 2 { @@ -1770,11 +1636,6 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() - // Snapshot to disk before benchmarking. - if err := f.Snapshot(); err != nil { - b.Fatal(err) - } - // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1834,35 +1695,6 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { } } -func TestFragment_Snapshot_Run(t *testing.T) { - roaringOnlyTest(t) - - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") - _ = idx - defer f.Clean(t) - - // Set bits on the fragment. - for i := uint64(1); i < 3; i++ { - if _, err := f.setBit(tx, 1000, i); err != nil { - t.Fatal(err) - } - } - - // Snapshot bitmap and verify data. - if err := f.Snapshot(); err != nil { - t.Fatal(err) - } else if n := f.mustRow(tx, 1000).Count(); n != 2 { - t.Fatalf("unexpected count: %d", n) - } - - // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { - t.Fatal(err) - } else if n := f.mustRow(tx, 1000).Count(); n != 2 { - t.Fatalf("unexpected count (reopen): %d", n) - } -} - // Ensure a fragment can set mutually exclusive values. func TestFragment_SetMutex(t *testing.T) { f, _, tx := mustOpenMutexFragment(t, "i", "f", viewStandard, 0, "") @@ -2684,77 +2516,6 @@ func makeTestFragSpec(path, index, field, view0 string) fragSpec { } } -func BenchmarkFragment_Snapshot(b *testing.B) { - if *FragmentPath == "" { - b.Skip("no fragment specified") - } - - b.ReportAllocs() - // Open the fragment specified by the path. - f := newFragment(newTestHolder(b), makeTestFragSpec(*FragmentPath, "i", "f", viewStandard), 0, 0) - if err := f.Open(); err != nil { - b.Fatal(err) - } - defer f.Clean(b) - b.ResetTimer() - - // Reset timer and execute benchmark. - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - err := f.Snapshot() - if err != nil { - b.Fatalf("unexpected count (reopen): %s", err) - } - } -} - -func BenchmarkFragment_FullSnapshot(b *testing.B) { - f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") - _ = idx - tx.Rollback() - defer f.Clean(b) - - // Generate some intersecting data. - maxX := ShardWidth / 2 - sz := maxX - rows := make([]uint64, sz) - cols := make([]uint64, sz) - - options := &ImportOptions{} - max := 0 - for row := 0; row < 100; row++ { - val := 1 - i := 0 - for col := 0; col < ShardWidth/2; col++ { - rows[i] = uint64(row) - cols[i] = uint64(val) - val += 2 - i++ - } - - tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - if err := f.bulkImport(tx, rows, cols, options); err != nil { - b.Fatalf("Error Building Sample: %s", err) - } - tx.Rollback() - if row > max { - max = row - } - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - if err := f.Snapshot(); err != nil { - b.Fatal(err) - } - } -} - func BenchmarkFragment_Import(b *testing.B) { b.StopTimer() maxX := ShardWidth * 5 * 2 @@ -2813,11 +2574,6 @@ func BenchmarkImportRoaring(b *testing.B) { err := f.importRoaringT(tx, data, false) if err != nil { - // we don't actually particularly - // care whether this succeeds, - // but if it's happening we want - // it to be done. - _ = f.holder.SnapshotQueue.Await(f) f.Clean(b) b.Fatalf("import error: %v", err) } @@ -2859,9 +2615,6 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { defer txs[j].Rollback() err := frags[j].importRoaringT(txs[j], data[j], false) - // error unimportant if it happened, but we want - // any snapshots to have finished. - _ = frags[j].holder.SnapshotQueue.Await(frags[j]) return err }) } @@ -2879,68 +2632,6 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { } } } -func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { - roaringOnlyBenchmark(b) - if testing.Short() { - b.SkipNow() - } - for _, numRows := range rowCases { - for _, numCols := range colCases { - data := getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth) - updata := getUpdataRoaring(numRows, numCols, 1) - for _, concurrency := range concurrencyCases { - for _, cacheType := range cacheCases { - b.Run(fmt.Sprintf("Rows%dCols%dConcurrency%dCache_%s", numRows, numCols, concurrency, cacheType), func(b *testing.B) { - b.StopTimer() - frags := make([]*fragment, concurrency) - txs := make([]Tx, concurrency) - for i := 0; i < b.N; i++ { - for j := 0; j < concurrency; j++ { - frags[j], _, txs[j] = mustOpenFragment(b, "i", "f", viewStandard, uint64(j), cacheType) - - // the cost of actually doing the op log for the large initial data set - // is excessive. force storage into snapshotted state, then use import - // to generate an op log and/or snapshot. - // note: skipped for rbf, bolt, lmdb, above. - _, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0) - if err != nil { - b.Fatalf("importing roaring: %v", err) - } - err = frags[j].holder.SnapshotQueue.Immediate(frags[j]) - if err != nil { - b.Fatalf("snapshot after import: %v", err) - } - } - eg := errgroup.Group{} - b.StartTimer() - for j := 0; j < concurrency; j++ { - j := j - eg.Go(func() error { - defer txs[j].Rollback() - - err := frags[j].importRoaringT(txs[j], updata, false) - err2 := frags[j].holder.SnapshotQueue.Await(frags[j]) - if err == nil { - err = err2 - } - return err - }) - } - err := eg.Wait() - if err != nil { - b.Errorf("importing fragment: %v", err) - } - b.StopTimer() - for j := 0; j < concurrency; j++ { - frags[j].Clean(b) - } - } - }) - } - } - } - } -} func BenchmarkImportStandard(b *testing.B) { for _, cacheType := range cacheCases { @@ -2984,29 +2675,18 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { f, idx, tx := mustOpenFragment(b, "i", fmt.Sprintf("r%dc%dcache_%s", numRows, numCols, cacheType), viewStandard, 0, cacheType) _ = idx - // the cost of actually doing the op log for the large initial data set - // is excessive. force storage into snapshotted state, then use import - // to generate an op log and/or snapshot. itr, err := roaring.NewRoaringIterator(data) PanicOn(err) _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0) if err != nil { b.Errorf("import error: %v", err) } - err = f.holder.SnapshotQueue.Immediate(f) - if err != nil { - b.Errorf("snapshot after import error: %v", err) - } b.StartTimer() err = f.importRoaringT(tx, updata, false) if err != nil { f.Clean(b) b.Errorf("import error: %v", err) } - err = f.holder.SnapshotQueue.Await(f) - if err != nil { - b.Errorf("snapshot after import error: %v", err) - } b.StopTimer() var stat os.FileInfo var statTarget io.Writer @@ -3160,9 +2840,6 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { //nf, idx, tx := mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) idx := fragTestMustOpenIndex("i", th, IndexOptions{}) - if th.NeedsSnapshot() { - th.SnapshotQueue = newSnapshotQueue(1, 1, nil) - } // XXX TODO: newFragment is using the wrong path here, we should fix that someday. f := newFragment(th, makeTestFragSpec(fi.Name(), "i", "f", viewStandard), 0, 0) defer f.Clean(b) @@ -3433,58 +3110,10 @@ func BenchmarkFileWrite(b *testing.B) { } -///////////////////////////////////////////////////////////////////// - -// not called under Tx stores b/c f.idx.NeedsSnapshot() in Clean() avoids it. -func (f *fragment) sanityCheck(t testing.TB) { - newBM := roaring.NewFileBitmap() - file, err := os.Open(f.path()) - if err != nil { - t.Fatalf("sanityCheck couldn't open file %s: %v", f.path(), err) - } - defer file.Close() - data, err := ioutil.ReadAll(file) - if err != nil { - t.Fatalf("sanityCheck couldn't read fragment %s: %v", f.path(), err) - } - err = newBM.UnmarshalBinary(data) - if err != nil { - t.Fatalf("sanityCheck couldn't unmarshal fragment %s: %v", f.path(), err) - } - // Refactor fragment.storage - // note: not called for rbf, see above. - if equal, reason := newBM.BitwiseEqual(f.storage); !equal { - t.Fatalf("fragment %s: unmarshalled bitmap different: %v", f.path(), reason) - } -} - // Clean used to delete fragments, but doesn't anymore -- deleting is // handled by the testhook.TempDir when appropriate. +// TODO(jaffee): this can likely go away entirely... it was doing snapshot/source/generation stuff that it no longer needs to. func (f *fragment) Clean(t testing.TB) { - f.mu.Lock() - // we need to ensure that we unlock the mutex before terminating - // the clean operation, but we need it held during the sanity - // check or else, in some cases, the background snapshot queue - // can decide to pick it up. - func() { - // should we skip snapshot queue stuff under bolt/rbf? - defer f.mu.Unlock() - - // rbf doesn't need snapshot, so this stuff is skipped. - // The snapshot queue stuff doesn't work under rbf. - if f.idx.NeedsSnapshot() { - err := f.holder.SnapshotQueue.Await(f) - if err != nil { - t.Fatalf("snapshot failed before sanity check: %v", err) - } - f.sanityCheck(t) - if f.storage != nil && f.storage.Source != nil { - if f.storage.Source.Dead() { - t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path(), f.storage.Source.ID()) - } - } - } - }() errc := f.Close() if errc != nil { t.Fatalf("error closing fragment: %v", errc) @@ -3513,7 +3142,7 @@ func newTestHolder(tb testing.TB) *Holder { testhook.Cleanup(tb, func() { h.Close() }) - //h.SnapshotQueue = newSnapshotQueue(1, 1, nil) + return h } @@ -3547,9 +3176,6 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6 th := newTestHolder(tb) idx := fragTestMustOpenIndex(index, th, IndexOptions{}) - if th.NeedsSnapshot() { - th.SnapshotQueue = newSnapshotQueue(1, 1, nil) - } fragDir := fmt.Sprintf("%v/%v/views/%v/fragments/", idx.path, field, view) PanicOn(os.MkdirAll(fragDir, 0777)) @@ -4253,83 +3879,6 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { }) } -func TestUnionInPlaceMapped(t *testing.T) { - roaringOnlyTest(t) - - f, _, _ := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) - // note: clean has to be deferred first, because it has to run with - // the lock *not* held, because it is sometimes so it has to grab the - // lock... - defer f.Clean(t) - - f.mu.Lock() - defer f.mu.Unlock() - r0 := rand.New(rand.NewSource(2)) - r1 := rand.New(rand.NewSource(1)) - data0 := randPositions(1000000, r0) - setBM0 := roaring.NewBitmap() - setBM0.OpWriter = nil - _, err := setBM0.Add(data0...) - if err != nil { - t.Fatalf("adding bits: %v", err) - } - count0 := setBM0.Count() - - data1 := randPositions(1000000, r1) - setBM1 := roaring.NewBitmap() - setBM1.OpWriter = nil - _, err = setBM1.Add(data1...) - if err != nil { - t.Fatalf("adding bits: %v", err) - } - count1 := setBM1.Count() - - // now we write setBM0 into f.storage. - _, err = unprotectedWriteToFragment(f, setBM0) - if err != nil { - t.Fatalf("trying to flush fragment to disk: %v", err) - } - countF := f.storage.Count() - - f.storage.UnionInPlace(setBM1) - countUnion := f.storage.Count() - - // UnionInPlace produces no ops log, we have to make it snapshot, to - // ensure that the on-disk representation is correct. Note, UIP is - // not used for things that are modifying real fragments, usually; - // it's used only in computation of things that usually don't go to - // disk, which is why we handle this specially in testing and not - // generically. - err = f.holder.SnapshotQueue.Immediate(f) - if err != nil { - t.Fatalf("snapshot after union-in-place: %v", err) - } - - if count0 != countF { - t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF) - } - min := count0 - if count1 > min { - min = count1 - } - max := count0 + count1 - // We don't know how many bits we should have, because of overlap, - // but it should be between the size of the largest bitmap and the - // sum of the bitmaps. - if countUnion < min || countUnion > max { - t.Fatalf("union of sets with cardinality %d and %d should be between %d and %d, got %d", - count0, count1, min, max, countUnion) - } -} - -func randPositions(n int, r *rand.Rand) []uint64 { - ret := make([]uint64, n) - for i := 0; i < n; i++ { - ret[i] = uint64(r.Int63n(ShardWidth)) - } - return ret -} - func TestFragmentPositionsForValue(t *testing.T) { f, _, _ := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) defer f.Clean(t) @@ -4950,218 +4499,12 @@ func TestFragmentBSISigned(t *testing.T) { }) } -func TestImportClearRestart(t *testing.T) { - roaringOnlyTest(t) - - tests := []struct { - rows []uint64 - cols []uint64 - }{ - { - rows: []uint64{1}, - cols: []uint64{1}, - }, - { - rows: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 1}, - cols: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 500000}, - }, - { - rows: []uint64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - cols: []uint64{0, 65535, 65536, 131071, 131072, 196607, 196608, 262143, 262144, 1000000}, - }, - { - rows: []uint64{1, 2, 20, 200, 2000, 200000}, - cols: []uint64{1, 1, 1, 1, 1, 1}, - }, - } - for i, test := range tests { - for _, maxOpN := range []int{0, 10000} { - t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) { - testrows, testcols := make([]uint64, len(test.rows)), make([]uint64, len(test.rows)) - copy(testrows, test.rows) - copy(testcols, test.cols) - exp := make(map[uint64]map[uint64]struct{}) // row num to cols - if len(testrows) != len(testcols) { - t.Fatalf("bad test spec-need same number of rows/cols, %d/%d", len(testrows), len(testcols)) - } - // set up expected data - expOpN := 0 - for i := range testrows { - row, col := testrows[i], testcols[i] - cols, ok := exp[row] - if !ok { - exp[row] = make(map[uint64]struct{}) - cols = exp[row] - } - if _, ok = cols[col]; !ok { - expOpN++ - cols[col] = struct{}{} - } - } - - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") - _ = idx - f.MaxOpN = maxOpN - - err := f.bulkImport(tx, testrows, testcols, &ImportOptions{}) - if err != nil { - t.Fatalf("initial small import: %v", err) - } - if idx.holder.txf.TxType() == RoaringTxn { - if expOpN <= maxOpN && f.opN != expOpN { - t.Errorf("unexpected opN - %d is not %d", f.opN, expOpN) - } - } - check(t, tx, f, exp) - - err = f.Close() - if err != nil { - t.Fatalf("closing fragment: %v", err) - } - PanicOn(tx.Commit()) - - err = f.Open() - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - if err != nil { - t.Fatalf("reopening fragment: %v", err) - } - - if idx.holder.txf.TxType() == RoaringTxn { - if expOpN <= maxOpN && f.opN != expOpN { - t.Errorf("unexpected opN after close/open %d is not %d", f.opN, expOpN) - } - } - - check(t, tx, f, exp) - - h := newTestHolder(t) - idx2, err := h.CreateIndex("i", IndexOptions{}) - _ = idx2 - PanicOn(err) - - // OVERWRITING the f.path with a new fragment - f2 := newFragment(h, makeTestFragSpec(f.path(), "i", "f", viewStandard), 0, 0) - f2.MaxOpN = maxOpN - f2.CacheType = f.CacheType - - PanicOn(tx.Commit()) // match the f.closeStorage which overlaps the f2 creation. - - tx2 := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2, Shard: f2.shard}) - defer tx2.Rollback() - - err = f.Close() - if err != nil { - t.Fatalf("closing storage: %v", err) - } - - err = f2.Open() - if err != nil { - t.Fatalf("opening new fragment: %v", err) - } - - if idx.holder.txf.TxType() == RoaringTxn { - if expOpN <= maxOpN && f2.opN != expOpN { - t.Errorf("unexpected opN after close/open %d is not %d", f2.opN, expOpN) - } - } - - check(t, tx2, f2, exp) - - copy(testrows, test.rows) - copy(testcols, test.cols) - err = f2.bulkImport(tx2, testrows, testcols, &ImportOptions{Clear: true}) - if err != nil { - t.Fatalf("clearing imported data: %v", err) - } - - // clear exp, but leave rows in so we re-query them in `check` - for row := range exp { - exp[row] = nil - } - - check(t, tx2, f2, exp) - - PanicOn(tx2.Commit()) - - h3 := NewHolder(filepath.Dir(f2.path()), mustHolderConfig()) - testhook.Cleanup(t, func() { - h3.Close() - }) - - idx3, err := h3.CreateIndex("i", IndexOptions{}) - _ = idx3 - PanicOn(err) - - f3 := newFragment(h3, makeTestFragSpec(f2.path(), "i", "f", viewStandard), 0, 0) - f3.MaxOpN = maxOpN - f3.CacheType = f.CacheType - - tx3 := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f3, Shard: f3.shard}) - defer tx3.Rollback() - - err = f2.Close() - if err != nil { - t.Fatalf("f2 closing storage: %v", err) - } - - err = f3.Open() - if err != nil { - t.Fatalf("opening f3: %v", err) - } - defer f3.Clean(t) - - check(t, tx3, f3, exp) - - }) - - } - } -} - -func check(t *testing.T, tx Tx, f *fragment, exp map[uint64]map[uint64]struct{}) { - - for rowID, colsExp := range exp { - colsAct := f.mustRow(tx, rowID).Columns() - if len(colsAct) != len(colsExp) { - t.Errorf("row %d len mismatch got: %d exp:%d", rowID, len(colsAct), len(colsExp)) - } - for _, colAct := range colsAct { - if _, ok := colsExp[colAct]; !ok { - t.Errorf("extra column: %d", colAct) - } - } - for colExp := range colsExp { - found := false - for _, colAct := range colsAct { - if colExp == colAct { - found = true - break - } - } - if !found { - t.Errorf("expected %d, but not found", colExp) - } - } - } - -} - func TestImportValueConcurrent(t *testing.T) { f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) defer f.Clean(t) // we will be making a new Tx each time, so we can rollback the default provided one. tx.Rollback() - ty := idx.holder.txf.TxTyp() - switch ty { - case roaringTxn: - t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + - "roaring because the lack of transactional consistency " + - "from Roaring-per-file will create false comparison " + - "failures.")) - } - eg := &errgroup.Group{} for i := 0; i < 4; i++ { i := i @@ -5201,38 +4544,29 @@ func TestImportMultipleValues(t *testing.T) { } for i, test := range tests { - for _, maxOpN := range []int{0, 10000} { // test small/large write - t.Run(fmt.Sprintf("%dLowOpN", i), func(t *testing.T) { - f, _, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) - f.MaxOpN = maxOpN - defer f.Clean(t) + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + f, _, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) + defer f.Clean(t) - err := f.importValue(tx, test.cols, test.vals, test.depth, false) + err := f.importValue(tx, test.cols, test.vals, test.depth, false) + if err != nil { + t.Fatalf("importing values: %v", err) + } + + for i := range test.checkCols { + cc, cv := test.checkCols[i], test.checkVals[i] + n, exists, err := f.value(tx, cc, test.depth) if err != nil { - t.Fatalf("importing values: %v", err) + t.Fatalf("getting value: %v", err) } - - // probably too slow, would hit disk alot: - //PanicOn(tx.Commit()) - //tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard:f.shard, ShardSet:true}) - //defer tx.Rollback() - - for i := range test.checkCols { - cc, cv := test.checkCols[i], test.checkVals[i] - n, exists, err := f.value(tx, cc, test.depth) - if err != nil { - t.Fatalf("getting value: %v", err) - } - if !exists { - t.Errorf("column %d should exist", cc) - } - if n != cv { - t.Errorf("wrong value: %d is not %d", n, cv) - } + if !exists { + t.Errorf("column %d should exist", cc) } - }) - - } + if n != cv { + t.Errorf("wrong value: %d is not %d", n, cv) + } + } + }) } } @@ -5264,35 +4598,32 @@ func TestImportValueRowCache(t *testing.T) { } for i, test := range tests { - for _, maxOpN := range []int{1, 10000} { - t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) { - f, _, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) - f.MaxOpN = maxOpN - defer f.Clean(t) + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + f, _, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) + defer f.Clean(t) - // First import (tc1) - if err := f.importValue(tx, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { - t.Fatalf("importing values: %v", err) - } + // First import (tc1) + if err := f.importValue(tx, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { + t.Fatalf("importing values: %v", err) + } - if r, err := f.rangeOp(tx, pql.GT, test.tc1.depth, 0); err != nil { - t.Error("getting range of values") - } else if !reflect.DeepEqual(r.Columns(), test.tc1.checkCols) { - t.Errorf("wrong column values. expected: %v, but got: %v", test.tc1.checkCols, r.Columns()) - } + if r, err := f.rangeOp(tx, pql.GT, test.tc1.depth, 0); err != nil { + t.Error("getting range of values") + } else if !reflect.DeepEqual(r.Columns(), test.tc1.checkCols) { + t.Errorf("wrong column values. expected: %v, but got: %v", test.tc1.checkCols, r.Columns()) + } - // Second import (tc2) - if err := f.importValue(tx, test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil { - t.Fatalf("importing values: %v", err) - } + // Second import (tc2) + if err := f.importValue(tx, test.tc2.cols, test.tc2.vals, test.tc2.depth, false); err != nil { + t.Fatalf("importing values: %v", err) + } - if r, err := f.rangeOp(tx, pql.GT, test.tc2.depth, 0); err != nil { - t.Error("getting range of values") - } else if !reflect.DeepEqual(r.Columns(), test.tc2.checkCols) { - t.Errorf("wrong column values. expected: %v, but got: %v", test.tc2.checkCols, r.Columns()) - } - }) - } + if r, err := f.rangeOp(tx, pql.GT, test.tc2.depth, 0); err != nil { + t.Error("getting range of values") + } else if !reflect.DeepEqual(r.Columns(), test.tc2.checkCols) { + t.Errorf("wrong column values. expected: %v, but got: %v", test.tc2.checkCols, r.Columns()) + } + }) } } @@ -5335,64 +4666,6 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { t.Logf("%d", acc) } -func TestRemapCache(t *testing.T) { - f, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") - defer f.Close() - index, field, view, shard := f.index(), f.field(), f.view(), f.shard - - // request a PanicOn that doesn't kill the program on fault - wouldFault := debug.SetPanicOnFault(true) - defer func() { - debug.SetPanicOnFault(wouldFault) - if r := recover(); r != nil { - if err, ok := r.(error); ok { - // special case: if we caught a page fault, we diagnose that directly. sadly, - // we can't see the actual values that were used to generate this, probably. - if err.Error() == "runtime error: invalid memory address or nil pointer dereference" { - t.Fatalf("segfault trapped during remap test (expected failure mode)") - } - } - t.Fatalf("unexpected PanicOn: %v", r) - } - }() - - // create a container - _, err := tx.Add(index, field, view, shard, 65537) - if err != nil { - t.Fatalf("storage add: %v", err) - } - // cause the container to be mapped - err = f.Snapshot() - if err != nil { - t.Fatalf("storage snapshot: %v", err) - } - // freeze the row - _ = f.mustRow(tx, 0) - // add a bit that isn't in that container, so that container doesn't - // change - _, err = tx.Add(index, field, view, shard, 2) - if err != nil { - t.Fatalf("storage add: %v", err) - } - // make the original container be the most recent, thus cached, container - _, err = f.bit(tx, 0, 65537) - if err != nil { - t.Fatalf("storage bit check: %v", err) - } - // force snapshot, remapping the containers - err = f.Snapshot() - if err != nil { - t.Fatalf("storage snapshot: %v", err) - } - // get rid of the old mapping - runtime.GC() - // try to read that container again - _, err = f.bit(tx, 0, 65537) - if err != nil { - t.Fatalf("storage bit check: %v", err) - } -} - func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx @@ -6010,47 +5283,3 @@ func TestSliceDifference(t *testing.T) { compareSlices(t, name, tc.expected, result) } } - -func TestBitmapGrowth(t *testing.T) { - roaringOnlyTest(t) - f, _, tx := mustOpenFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0, "") - path := f.path() - defer f.Clean(t) - const values = 500 - cols := make([]uint64, values) - vals := make([]int64, values) - for i := range cols { - cols[i] = uint64(rand.Int63n(65536)) - vals[i] = rand.Int63n(24) - } - err := f.importValue(tx, cols, vals, 7, false) - if err != nil { - t.Fatalf("importing values: %v", err) - } - info, err := os.Stat(path) - if err != nil { - t.Fatalf("statting %s: %v", path, err) - } - prevSize := info.Size() - prevOpN := f.opN - err = f.importValue(tx, cols, vals, 7, false) - if err != nil { - t.Fatalf("importing values: %v", err) - } - info, err = os.Stat(path) - if err != nil { - t.Fatalf("statting %s: %v", path, err) - } - deltaSize := info.Size() - prevSize - deltaOpN := f.opN - prevOpN - // This is somewhat arbitrary, but the issue tested for was that - // opN would grow by 0 or 1 with multiple KB of actual ops written. - // If deltaOpN is at least 20, we'll probably see snapshots happening - // at least occasionally, and if deltaSize is under 1024, the writes - // are probably going to be small enough that the regular backlog of - // snapshotting catches them anyway. - if deltaSize > 1024 && deltaOpN < 20 { - t.Fatalf("bitmap grew by %d bytes but OpN only grew by %d", - deltaSize, deltaOpN) - } -} diff --git a/go.mod b/go.mod index f1d6d285b..529f2a29d 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/zeebo/blake3 v0.1.1 go.etcd.io/bbolt v1.3.5 go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b + golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 // indirect golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 golang.org/x/mod v0.4.2 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect diff --git a/go.sum b/go.sum index 2778ff697..79fc9f1f6 100644 --- a/go.sum +++ b/go.sum @@ -413,8 +413,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 h1:3wPMTskHO3+O6jqTEXyFcsnuxMQOqYSaHsDxcbUXpqA= +golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -497,6 +498,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -507,6 +509,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/handler.go b/handler.go index 725225f5f..c39e03780 100644 --- a/handler.go +++ b/handler.go @@ -76,9 +76,9 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { }) } -// Handler is the interface for the data handler, a wrapper around +// HandlerI is the interface for the data handler, a wrapper around // Pilosa's data store. -type Handler interface { +type HandlerI interface { Serve() error Close() error } @@ -94,7 +94,7 @@ func (n nopHandler) Close() error { } // NopHandler is a no-op implementation of the Handler interface. -var NopHandler Handler = nopHandler{} +var NopHandler HandlerI = nopHandler{} // ImportValueRequest describes the import request structure // for a value (BSI) import. @@ -410,31 +410,3 @@ type TranslateIDsRequest struct { type TranslateIDsResponse struct { Keys []string } - -// InspectRequestParams represents the parts of an InspectRequest that -// aren't generic holder filtering attributes. -type InspectRequestParams struct { - Containers bool // include container details - Checksum bool // perform checksums -} - -// InspectRequest represents a request for a possibly-partial -// holder inspection, using a provided holder filter and inspect-specific -// parameters. -type InspectRequest struct { - HolderFilterParams - InspectRequestParams -} - -// InspectResponse contains the structured results for an InspectRequest. -// It may some day be expanded to include metadata about views or indexes. -type InspectResponse struct { - Fragments []struct { - Index string - Field string - View string - Shard int64 - Path string - Info *FragmentInfo - } -} diff --git a/holder.go b/holder.go index cfbd50801..18cd7154c 100644 --- a/holder.go +++ b/holder.go @@ -7,10 +7,8 @@ import ( "fmt" "os" "path/filepath" - "regexp" "runtime" "sort" - "strconv" "strings" "sync" "time" @@ -85,8 +83,7 @@ type Holder struct { // The interval at which the cached row ids are persisted to disk. cacheFlushInterval time.Duration - Logger logger.Logger - SnapshotQueue SnapshotQueue + Logger logger.Logger // Instantiates new translation stores OpenTranslateStore OpenTranslateStoreFunc @@ -138,14 +135,6 @@ type Holder struct { // HolderOpts holds information about the holder which other things might want // to look up later while using the holder. type HolderOpts struct { - // ReadOnly indicates that this holder's contents should not produce - // disk writes under any circumstances. It must be set before Open - // is called, and changing it is not supported. - ReadOnly bool - // If Inspect is set, we'll try to obtain additional information - // about fragments when opening them. - Inspect bool - // StorageBackend controls the tx/storage engine we instatiate. Set by // server.go OptServerStorageConfig StorageBackend string @@ -271,8 +260,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { Logger: cfg.Logger, Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend}, - SnapshotQueue: defaultSnapshotQueue, - Auditor: NewAuditor(), path: path, @@ -298,280 +285,6 @@ func (h *Holder) IndexesPath() string { return filepath.Join(h.path, IndexesDir) } -type HolderInfo struct { - FragmentInfo map[string]FragmentInfo - FragmentNames []string -} - -type regexpList []*regexp.Regexp - -func newRegexpList(regexes string) (results regexpList, err error) { - if regexes == "" { - return nil, nil - } - for _, sub := range strings.Split(regexes, ",") { - re, err := regexp.Compile(sub) - if err != nil { - return nil, err - } - results = append(results, re) - } - return results, nil -} - -func (rl regexpList) Match(haystack string) bool { - if rl == nil { - return true - } - for _, re := range rl { - if re.MatchString(haystack) { - return true - } - } - return false -} - -// shardRange represents a series of shards -type shardRange struct { - min, max uint64 -} - -type shardRangeList []shardRange - -func newShardRangeList(shards string) (results shardRangeList, err error) { - if shards == "" { - return nil, nil - } - for _, sub := range strings.Split(shards, ",") { - var sr shardRange - minMax := strings.Split(sub, "-") - if len(minMax) > 2 { - return nil, fmt.Errorf("invalid range %q", sub) - } - sr.min, err = strconv.ParseUint(minMax[0], 10, 64) - if err != nil { - return nil, err - } - sr.max = sr.min - if len(minMax) == 2 { - sr.max, err = strconv.ParseUint(minMax[0], 10, 64) - if err != nil { - return nil, err - } - } - if sr.max < sr.min { - return nil, fmt.Errorf("invalid range %q: max < min", sub) - } - results = append(results, sr) - } - return results, nil -} - -func (sl shardRangeList) Match(shard uint64) bool { - if sl == nil { - return true - } - for _, sr := range sl { - if shard >= sr.min && shard <= sr.max { - return true - } - } - return false -} - -// HolderFilter represents something that potentially filters out -// parts of a holder, indicating whether or not to process them, -// or recurse into them. It is permissible to recurse a thing -// without processing it, or process it without recursing it. -// For instance, something looking to accumulate statistics -// about views might return (true, false) from CheckView, -// while a fragment scanning operation would return (false, true) -// from everything above CheckFrag. -type HolderFilter interface { - CheckIndex(iname string) (process bool, recurse bool) - CheckField(iname, fname string) (process bool, recurse bool) - CheckView(iname, fname, vname string) (process bool, recurse bool) - CheckFragment(iname, fname, vname string, shard uint64) (process bool) -} - -// HolderFilterAll is a placeholder type which always returns true for the -// check functions. You can embed it to make a HolderOperator which processes -// everything. -type HolderFilterAll struct{} - -func (HolderFilterAll) CheckIndex(string) (bool, bool) { - return true, true -} - -func (HolderFilterAll) CheckField(string, string) (bool, bool) { - return true, true -} - -func (HolderFilterAll) CheckView(string, string, string) (bool, bool) { - return true, true -} - -func (HolderFilterAll) CheckFragment(string, string, string, uint64) bool { - return true -} - -// HolderProcessNone is a placeholder type which does nothing for the -// process functions. You can embed it to make a HolderOperator which -// does nothing, or embed it and provide your own ProcessFragment to -// do just that. -type HolderProcessNone struct{} - -func (HolderProcessNone) ProcessIndex(*Index) error { - return nil -} - -func (HolderProcessNone) ProcessField(*Field) error { - return nil -} - -func (HolderProcessNone) ProcessView(*view) error { - return nil -} - -func (HolderProcessNone) ProcessFragment(*fragment) error { - return nil -} - -// HolderProcess represents something that has operations which can be -// performed on indexes, fields, views, and/or fragments. -type HolderProcess interface { - ProcessIndex(*Index) error - ProcessField(*Field) error - ProcessView(*view) error - ProcessFragment(*fragment) error -} - -// HolderOperator is both a filter and a process. This is the general -// form of "I want to do something to some part of a holder." -type HolderOperator interface { - HolderFilter - HolderProcess -} - -var _ HolderOperator = (*holderInspector)(nil) - -type HolderFilterParams struct { - Indexes string - Fields string - Views string - Shards string -} - -type holderFilterFull struct { - HolderFilterParams - indexRegexps regexpList - fieldRegexps regexpList - viewRegexps regexpList - shardRanges shardRangeList -} - -type inspectRequestFull struct { - HolderFilter - params InspectRequestParams -} - -func (i *holderFilterFull) CheckIndex(iname string) (process, recurse bool) { - return true, i.indexRegexps.Match(iname) -} - -func (i *holderFilterFull) CheckField(iname, fname string) (process, recurse bool) { - return true, i.fieldRegexps.Match(fname) -} - -func (i *holderFilterFull) CheckView(iname, fname, vname string) (process, recurse bool) { - return true, i.viewRegexps.Match(vname) -} - -func (i *holderFilterFull) CheckFragment(iname, fname, vname string, shard uint64) (process bool) { - return i.shardRanges.Match(shard) -} - -func NewHolderFilter(params HolderFilterParams) (result HolderFilter, err error) { - filter := &holderFilterFull{ - HolderFilterParams: params, - } - filter.indexRegexps, err = newRegexpList(params.Indexes) - if err != nil { - return nil, err - } - filter.fieldRegexps, err = newRegexpList(params.Fields) - if err != nil { - return nil, err - } - filter.viewRegexps, err = newRegexpList(params.Views) - if err != nil { - return nil, err - } - filter.shardRanges, err = newShardRangeList(params.Shards) - if err != nil { - return nil, err - } - return filter, nil -} - -func expandInspectRequest(req *InspectRequest) (*inspectRequestFull, error) { - filter, err := NewHolderFilter(req.HolderFilterParams) - if err != nil { - return nil, err - } - irf := &inspectRequestFull{ - HolderFilter: filter, - params: req.InspectRequestParams, - } - return irf, nil -} - -type holderInspector struct { - *inspectRequestFull - pathParts [3]string - path string - hi *HolderInfo -} - -func (h *holderInspector) ProcessIndex(i *Index) error { - h.pathParts[0] = i.name - return nil -} - -func (h *holderInspector) ProcessField(f *Field) error { - h.pathParts[1] = f.name - return nil -} - -func (h *holderInspector) ProcessView(v *view) error { - h.pathParts[2] = v.name - h.path = strings.Join(h.pathParts[:], "/") - return nil -} - -func (h *holderInspector) ProcessFragment(f *fragment) error { - path := h.path + "/" + strconv.FormatUint(f.shard, 10) - h.hi.FragmentInfo[path] = f.inspect(h.inspectRequestFull.params) - h.hi.FragmentNames = append(h.hi.FragmentNames, path) - return nil -} - -func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) { - fullReq, err := expandInspectRequest(req) - if err != nil { - return nil, err - } - inspector := &holderInspector{ - inspectRequestFull: fullReq, - hi: &HolderInfo{ - FragmentInfo: make(map[string]FragmentInfo), - }, - } - err = h.Process(ctx, inspector) - sort.Strings(inspector.hi.FragmentNames) - return inspector.hi, err -} - // Open initializes the root data directory for the holder. func (h *Holder) Open() error { h.opening = true @@ -734,16 +447,15 @@ func (h *Holder) maybeSpool(msg Message) bool { return true } -// Activate runs the background tasks relevant to keeping a holder in a stable -// state, such as scanning it for needed snapshots, or flushing caches. This -// is separate from opening because, while a server would nearly always want -// to do this, other use cases (like consistency checks of a data directory) +// Activate runs the background tasks relevant to keeping a holder in +// a stable state, such as flushing caches. This is separate from +// opening because, while a server would nearly always want to do +// this, other use cases (like consistency checks of a data directory) // need to avoid it even getting started. func (h *Holder) Activate() { // Periodically flush cache. - h.wg.Add(2) + h.wg.Add(1) go func() { defer h.wg.Done(); h.monitorCacheFlush() }() - go func() { defer h.wg.Done(); h.SnapshotQueue.ScanHolder(h, h.closing) }() } // checkForeignIndex is a check before applying a foreign @@ -777,7 +489,6 @@ func (h *Holder) processForeignIndexFields() error { // Close closes all open fragments. func (h *Holder) Close() error { - if h == nil { return nil } @@ -791,7 +502,6 @@ func (h *Holder) Close() error { // Notify goroutines of closing and wait for completion. close(h.closing) h.wg.Wait() - for _, index := range h.Indexes() { if err := index.Close(); err != nil { return errors.Wrap(err, "closing index") @@ -809,10 +519,6 @@ func (h *Holder) Close() error { h.opened.mu.Lock() h.opened.ch = make(chan struct{}) h.opened.mu.Unlock() - if h.SnapshotQueue != nil { - h.SnapshotQueue.Stop() - h.SnapshotQueue = nil - } if h.lookupDB != nil { err := h.lookupDB.Close() @@ -827,13 +533,6 @@ func (h *Holder) Close() error { return nil } -func (h *Holder) NeedsSnapshot() bool { - h.mu.RLock() - defer h.mu.RUnlock() - - return h.txf.NeedsSnapshot() -} - // HasData returns true if Holder contains at least one index. // This is used to determine if the rebalancing of data is necessary // when a node joins the cluster. @@ -1967,130 +1666,6 @@ func uint64InSlice(i uint64, s []uint64) bool { return false } -// Process loops through a holder based on the Check functions in op, calling -// the Process functions in op when indicated. -func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) { - var fieldNames, viewNames []string - var fragNums []uint64 - - indexes := h.Indexes() - for _, idx := range indexes { - if err = ctx.Err(); err != nil { - return err - } - if idx == nil { - continue - } - indexName := idx.name - process, recurse := op.CheckIndex(indexName) - if !process && !recurse { - continue - } - - if err = ctx.Err(); err != nil { - return err - } - if process { - err = op.ProcessIndex(idx) - if err != nil { - return err - } - } - if !recurse { - continue - } - fieldNames = fieldNames[:0] - idx.mu.Lock() - for fieldName := range idx.fields { - fieldNames = append(fieldNames, fieldName) - } - idx.mu.Unlock() - for _, fieldName := range fieldNames { - if err = ctx.Err(); err != nil { - return err - } - process, recurse := op.CheckField(idx.name, fieldName) - if !process && !recurse { - continue - } - idx.mu.Lock() - field := idx.fields[fieldName] - idx.mu.Unlock() - if field == nil { - continue - } - if err = ctx.Err(); err != nil { - return err - } - if process { - err = op.ProcessField(field) - if err != nil { - return err - } - } - if !recurse { - continue - } - viewNames = viewNames[:0] - field.mu.Lock() - for viewName := range field.viewMap { - viewNames = append(viewNames, viewName) - } - field.mu.Unlock() - for _, viewName := range viewNames { - if err = ctx.Err(); err != nil { - return err - } - process, recurse := op.CheckView(indexName, fieldName, viewName) - if !process && !recurse { - continue - } - field.mu.Lock() - view := field.viewMap[viewName] - field.mu.Unlock() - if view == nil { - continue - } - if err = ctx.Err(); err != nil { - return err - } - if process { - err = op.ProcessView(view) - if err != nil { - return err - } - } - if !recurse { - continue - } - fragNums = fragNums[:0] - view.mu.Lock() - for fragNum := range view.fragments { - fragNums = append(fragNums, fragNum) - } - view.mu.Unlock() - for _, fragNum := range fragNums { - if err = ctx.Err(); err != nil { - return err - } - process := op.CheckFragment(indexName, fieldName, viewName, fragNum) - if !process { - continue - } - view.mu.Lock() - frag := view.fragments[fragNum] - view.mu.Unlock() - err = op.ProcessFragment(frag) - if err != nil { - return err - } - } - } - } - } - return nil -} - // used by Index.openFields(), enabling Tx / Txf by telling // the holder about its own indexes. func (h *Holder) addIndex(idx *Index) { @@ -2111,34 +1686,6 @@ func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) { return h.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil } -func (h *Holder) HasRoaringData() (has bool, err error) { - idxs := h.Indexes() - for _, idx := range idxs { - paths, err := listFilesUnderDir(idx.path, false, "", true) - if err != nil { - return false, errors.Wrap(err, "HasRoaringData listFilesUnderDir") - } - index := idx.name - - for _, relpath := range paths { - field, view, shard, err := fragmentSpecFromRoaringPath(relpath) - if err != nil { - continue // ignore .meta paths - } - abspath := idx.path + sep + relpath - - hasData, err := roaringFragmentHasData(abspath, index, field, view, shard) - if err != nil { - return false, errors.Wrap(err, "HasRoaringData roaringFragmentHasData") - } - if hasData { - return true, nil - } - } - } - return -} - func decodeCreateIndexMessage(ser Serializer, b []byte) (*CreateIndexMessage, error) { var cim CreateIndexMessage if err := ser.Unmarshal(b, &cim); err != nil { diff --git a/holder_internal_test.go b/holder_internal_test.go index b1455aa5e..e18704ac8 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -2,189 +2,12 @@ package pilosa import ( - "context" - "fmt" - "os" - "testing" - "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/testhook" ) -var _ = fmt.Printf - -type testHolderOperator struct { - indexSeen, indexProcessed int - fieldSeen, fieldProcessed int - viewSeen, viewProcessed int - fragmentSeen, fragmentProcessed int - waitHere chan struct{} -} - -func (t *testHolderOperator) CheckIndex(string) (bool, bool) { - t.indexSeen++ - return true, true -} - -func (t *testHolderOperator) CheckField(string, string) (bool, bool) { - t.fieldSeen++ - return true, true -} - -func (t *testHolderOperator) CheckView(string, string, string) (bool, bool) { - t.viewSeen++ - return true, true -} - -func (t *testHolderOperator) CheckFragment(string, string, string, uint64) bool { - t.fragmentSeen++ - return true -} - -func (t *testHolderOperator) ProcessIndex(*Index) error { - t.indexProcessed++ - return nil -} - -func (t *testHolderOperator) ProcessField(*Field) error { - t.fieldProcessed++ - return nil -} - -func (t *testHolderOperator) ProcessView(*view) error { - t.viewProcessed++ - return nil -} - -func (t *testHolderOperator) ProcessFragment(*fragment) error { - if t.waitHere != nil { - <-t.waitHere - } - t.fragmentProcessed++ - return nil -} - -func makeHolder(tb testing.TB, backend string) (*Holder, string, error) { - path, err := testhook.TempDir(tb, "pilosa-") - if err != nil { - return nil, "", err - } - cfg := mustHolderConfig() - if backend != "" { - cfg.StorageConfig.Backend = backend - cfg.StorageConfig.FsyncEnabled = false - } - h := NewHolder(path, cfg) - return h, path, h.Open() -} - -func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { - - idx, err := h.CreateIndexIfNotExists(index, IndexOptions{}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - - f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault()) - if err != nil { - t.Fatalf("setting bit: %v", err) - } - _, err = f.SetBit(nil, rowID, columnID, nil) - if err != nil { - t.Fatalf("setting bit: %v", err) - } -} - -func TestHolderOperatorProcess(t *testing.T) { - h, path, err := makeHolder(t, "") - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - defer h.Close() - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - testOp := testHolderOperator{} - ctx := context.Background() - err = h.Process(ctx, &testOp) - if err != nil { - t.Fatalf("processing holder: %v", err) - } - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp != expected { - t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) - } -} - -func TestHolderOperatorCancel(t *testing.T) { - h, path, err := makeHolder(t, "") - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - defer h.Close() - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - // Here, we want to ensure that the operation gets cancelled - // successfully. In practice we expect it to process one fragment, then - // end up blocked on the waitHere, then get cancelled... But the - // waitHere blockage isn't really something holder.Process can do - // anything about, so we close the channel, so two fragments are - // processed. But in theory you could end up with only one fragment - // processed if this goroutine managed to cancel before the processor - // gets to the next fragment. Point is, it shouldn't hit all three, - // because the checks against the cancellation should fire before it - // gets there. - testOp := testHolderOperator{waitHere: make(chan struct{})} - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - err = h.Process(ctx, &testOp) - close(done) - }() - testOp.waitHere <- struct{}{} - cancel() - close(testOp.waitHere) - <-done - if err != context.Canceled { - t.Fatalf("processing holder: expected context.Canceled, got %v", err) - } - testOp.waitHere = nil - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp == expected { - t.Fatalf("holder processor did not cancel. expected something other than %#v", expected) - } -} - -// mustHolderConfig is meant to help minimize the number of places in the code -// where we're reading the PILOSA_STORAGE_BACKEND environment variable for -// testing purposes. Ideally we would handle this differently, but this is a -// first attempt at improving things. Note: the actual os.Getenv() call was -// moved to the CurrentBackend() function. +// mustHolderConfig sets up a default holder config for tests. func mustHolderConfig() *HolderConfig { cfg := DefaultHolderConfig() - if backend := CurrentBackend(); backend != "" { - _ = MustBackendToTxtype(backend) - cfg.StorageConfig.Backend = backend - } cfg.StorageConfig.FsyncEnabled = false cfg.RBFConfig.FsyncEnabled = false cfg.Schemator = disco.InMemSchemator diff --git a/holder_test.go b/holder_test.go index cff3cf7da..1485f59fc 100644 --- a/holder_test.go +++ b/holder_test.go @@ -5,13 +5,12 @@ import ( "context" "math" "os" - "path/filepath" "reflect" "strings" "testing" "time" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/disco" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/test" @@ -21,10 +20,7 @@ import ( // mustHolderConfig provides a default test-friendly holder config. func mustHolderConfig() *pilosa.HolderConfig { cfg := pilosa.DefaultHolderConfig() - if backend := pilosa.CurrentBackend(); backend != "" { - _ = pilosa.MustBackendToTxtype(backend) - cfg.StorageConfig.Backend = backend - } + cfg.StorageConfig.Backend = "rbf" cfg.StorageConfig.FsyncEnabled = false cfg.RBFConfig.FsyncEnabled = false cfg.Schemator = disco.InMemSchemator @@ -55,109 +51,6 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %v", err) } }) - t.Run("ErrFragmentStoragePermission", func(t *testing.T) { - roaringOnlyTest(t) - - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := test.MustOpenHolder(t) - defer h.Close() - - var idx *pilosa.Index - var err error - if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - - var shard uint64 - tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) - defer tx.Rollback() - - if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := field.SetBit(tx, 0, 0, nil); err != nil { - t.Fatal(err) - } else if err := tx.Commit(); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil { - t.Fatal(err) - } - defer func() { - _ = os.Chmod(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 0644) - }() - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) { - roaringOnlyTest(t) - - h := test.MustOpenHolder(t) - defer h.Close() - - var idx *pilosa.Index - var err error - if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - - var shard uint64 - tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback() - - if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := field.SetBit(tx, 0, 0, nil); err != nil { - t.Fatal(err) - } else if err := tx.Commit(); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil { - t.Fatal(err) - } - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: shard=0, err=opening storage: unmarshal storage") { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) { - roaringOnlyTest(t) - - h := test.MustOpenHolder(t) - defer h.Close() - - idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - var shard uint64 - tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) - defer tx.Rollback() - - if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := field.SetBit(tx, 0, 0, nil); err != nil { - t.Fatal(err) - } else if err := tx.Commit(); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Truncate(filepath.Join(h.IndexesPath(), "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil { - t.Fatal(err) - } - - if err := h.Reopen(); err != nil { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ForeignIndex", func(t *testing.T) { t.Run("ErrForeignIndexNotFound", func(t *testing.T) { h := test.MustOpenHolder(t) diff --git a/http/error.go b/http/error.go deleted file mode 100644 index 733f3f655..000000000 --- a/http/error.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package http - -// Error defines a standard application error. -type Error struct { - // Human-readable message. - Message string `json:"message"` -} - -// Error returns the string representation of the error message. -func (e *Error) Error() string { - return e.Message -} diff --git a/http/handler.go b/http_handler.go similarity index 90% rename from http/handler.go rename to http_handler.go index 2439fa539..44663ef14 100644 --- a/http/handler.go +++ b/http_handler.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http +package pilosa import ( "bytes" @@ -29,14 +29,13 @@ import ( "github.com/felixge/fgprof" "github.com/gorilla/handlers" "github.com/gorilla/mux" - pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/authz" - "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/rbf" + "github.com/molecula/featurebase/v3/storage" "github.com/molecula/featurebase/v3/topology" "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" @@ -50,7 +49,7 @@ import ( type Handler struct { Handler http.Handler - fileSystem pilosa.FileSystem + fileSystem FileSystem logger logger.Logger @@ -59,7 +58,7 @@ type Handler struct { // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec - api *pilosa.API + api *API ln net.Listener // url is used to hold the advertise bind address for printing a log during startup. @@ -67,6 +66,9 @@ type Handler struct { closeTimeout time.Duration + serializer Serializer + roaringSerializer Serializer + server *http.Server middleware []func(http.Handler) http.Handler @@ -95,7 +97,7 @@ type errorResponse struct { Error string `json:"error"` } -// handlerOption is a functional option type for pilosa.Handler +// handlerOption is a functional option type for Handler type handlerOption func(s *Handler) error func OptHandlerMiddleware(middleware func(http.Handler) http.Handler) handlerOption { @@ -115,7 +117,7 @@ func OptHandlerAllowedOrigins(origins []string) handlerOption { } } -func OptHandlerAPI(api *pilosa.API) handlerOption { +func OptHandlerAPI(api *API) handlerOption { return func(h *Handler) error { h.api = api return nil @@ -136,7 +138,7 @@ func OptHandlerAuthZ(gp *authz.GroupPermissions) handlerOption { } } -func OptHandlerFileSystem(fs pilosa.FileSystem) handlerOption { +func OptHandlerFileSystem(fs FileSystem) handlerOption { return func(h *Handler) error { h.fileSystem = fs return nil @@ -157,6 +159,20 @@ func OptHandlerQueryLogger(logger logger.Logger) handlerOption { } } +func OptHandlerSerializer(s Serializer) handlerOption { + return func(h *Handler) error { + h.serializer = s + return nil + } +} + +func OptHandlerRoaringSerializer(s Serializer) handlerOption { + return func(h *Handler) error { + h.roaringSerializer = s + return nil + } +} + // OptHandlerListener set the listener that will be used by the HTTP server. // Url must be the advertised URL. It will be used to show a log to the user // about where the Web UI is. This option is mandatory. @@ -182,15 +198,8 @@ var importOk []byte // NewHandler returns a new instance of Handler with a default logger. func NewHandler(opts ...handlerOption) (*Handler, error) { - makeImportOk.Do(func() { - var err error - importOk, err = proto.DefaultSerializer.Marshal(&pilosa.ImportResponse{Err: ""}) - if err != nil { - panic(fmt.Sprintf("trying to cache import-OK response: %v", err)) - } - }) handler := &Handler{ - fileSystem: pilosa.NopFileSystem, + fileSystem: NopFileSystem, logger: logger.NopLogger, closeTimeout: time.Second * 30, } @@ -201,6 +210,16 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { return nil, errors.Wrap(err, "applying option") } } + if handler.serializer == nil || handler.roaringSerializer == nil { + return nil, errors.New("must use serializer options when creating handler") + } + makeImportOk.Do(func() { + var err error + importOk, err = handler.serializer.Marshal(&ImportResponse{Err: ""}) + if err != nil { + panic(fmt.Sprintf("trying to cache import-OK response: %v", err)) + } + }) // if OptHandlerFileSystem is used, it must be before newRouter is called handler.Handler = newRouter(handler) @@ -349,7 +368,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { queryRequest := r.Context().Value(contextKeyQueryRequest) var queryString string - if req, ok := queryRequest.(*pilosa.QueryRequest); ok { + if req, ok := queryRequest.(*QueryRequest); ok { queryString = req.Query } @@ -377,7 +396,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { stats := h.api.StatsWithTags(statsTags) if stats != nil { - stats.Timing(pilosa.MetricHTTPRequest, dur, 0.1) + stats.Timing(MetricHTTPRequest, dur, 0.1) } }) } @@ -561,10 +580,13 @@ func (h *Handler) chkInternal(handler http.HandlerFunc) http.HandlerFunc { func (h *Handler) chkAuthN(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if h.auth != nil { - if _, err := h.auth.Authenticate(getToken(r)); err != nil { + uinfo, err := h.auth.Authenticate(r.Context(), getToken(r)) + if err != nil { http.Error(w, errors.Wrap(err, "authenticating").Error(), http.StatusUnauthorized) return } + // just in case it got refreshed + h.auth.SetCookie(w, uinfo.Token, uinfo.Expiry) } ctx := context.WithValue(r.Context(), "token", r.Header["Authorization"]) handler.ServeHTTP(w, r.WithContext(ctx)) @@ -583,11 +605,13 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http lperm := perm // check if the user is authenticated - uinfo, err := h.auth.Authenticate(getToken(r)) + uinfo, err := h.auth.Authenticate(r.Context(), getToken(r)) if err != nil { http.Error(w, errors.Wrap(err, "authenticating").Error(), http.StatusForbidden) return } + // just in case it got refreshed + h.auth.SetCookie(w, uinfo.Token, uinfo.Expiry) // put the user's groups in the context ctx := context.WithValue(r.Context(), contextKeyGroupMembership, uinfo.Groups) @@ -603,7 +627,7 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http // figure out what the user is querying for queryString := "" queryRequest := r.Context().Value(contextKeyQueryRequest) - if req, ok := queryRequest.(*pilosa.QueryRequest); ok { + if req, ok := queryRequest.(*QueryRequest); ok { queryString = req.Query q, err := pql.ParseString(queryString) @@ -733,10 +757,21 @@ func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { h *Handler - Success bool `json:"success"` - Name string `json:"name,omitempty"` - CreatedAt int64 `json:"createdAt,omitempty"` - Error *Error `json:"error,omitempty"` + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + Error *HTTPError `json:"error,omitempty"` +} + +// Error defines a standard application error. +type HTTPError struct { + // Human-readable message. + Message string `json:"message"` +} + +// Error returns the string representation of the error message. +func (e *HTTPError) Error() string { + return e.Message } // check determines success or failure based on the error. @@ -751,18 +786,18 @@ func (r *successResponse) check(err error) (statusCode int) { // Determine HTTP status code based on the error type. switch cause.(type) { - case pilosa.BadRequestError: + case BadRequestError: statusCode = http.StatusBadRequest - case pilosa.ConflictError: + case ConflictError: statusCode = http.StatusConflict - case pilosa.NotFoundError: + case NotFoundError: statusCode = http.StatusNotFound default: statusCode = http.StatusInternalServerError } r.Success = false - r.Error = &Error{Message: err.Error()} + r.Error = &HTTPError{Message: err.Error()} return statusCode } @@ -872,7 +907,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { return } if !h.permissions.IsAdmin(g.([]authn.Group)) { - var filtered []*pilosa.IndexInfo + var filtered []*IndexInfo allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read) for _, s := range schema { for _, index := range allowed { @@ -886,7 +921,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } } - if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { + if err := json.NewEncoder(w).Encode(Schema{Indexes: schema}); err != nil { h.logger.Errorf("write schema response error: %s", err) } } @@ -913,7 +948,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) return } if !h.permissions.IsAdmin(g.([]authn.Group)) { - var filtered []*pilosa.IndexInfo + var filtered []*IndexInfo allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read) for _, s := range schema { for _, index := range allowed { @@ -926,7 +961,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) schema = filtered } } - if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { + if err := json.NewEncoder(w).Encode(Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } @@ -939,7 +974,7 @@ func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { remote = true } - schema := &pilosa.Schema{} + schema := &Schema{} if err := json.NewDecoder(r.Body).Decode(schema); err != nil { http.Error(w, fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err), http.StatusBadRequest) return @@ -980,12 +1015,12 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) { } if !h.permissions.IsAdmin(g.([]authn.Group)) { allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read) - filteredNodeUsages := map[string]pilosa.NodeUsage{} + filteredNodeUsages := map[string]NodeUsage{} for nodeId, nodeUsage := range nodeUsages { - filteredIndexUsage := pilosa.NodeUsage{ - Disk: pilosa.DiskUsage{ - IndexUsage: map[string]pilosa.IndexUsage{}, + filteredIndexUsage := NodeUsage{ + Disk: DiskUsage{ + IndexUsage: map[string]IndexUsage{}, }, } for index, idxUsage := range nodeUsage.Disk.IndexUsage { @@ -1057,7 +1092,7 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { } type getSchemaResponse struct { - Indexes []*pilosa.IndexInfo `json:"indexes"` + Indexes []*IndexInfo `json:"indexes"` } type getStatusResponse struct { @@ -1067,8 +1102,7 @@ type getStatusResponse struct { ClusterName string `json:"clusterName"` } -func hash(s string) string { - +func httpHash(s string) string { hasher := blake3.New() _, _ = hasher.Write([]byte(s)) var buf [16]byte @@ -1084,11 +1118,11 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Read previouly parsed request from context qreq := r.Context().Value(contextKeyQueryRequest) qerr := r.Context().Value(contextKeyQueryError) - req, ok := qreq.(*pilosa.QueryRequest) + req, ok := qreq.(*QueryRequest) if DoPerQueryProfiling { - backend := pilosa.CurrentBackend() - reqHash := hash(req.Query) + backend := storage.DefaultBackend + reqHash := httpHash(req.Query) qlen := len(req.Query) if qlen > 100 { @@ -1111,7 +1145,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { if err != nil || !ok { w.WriteHeader(http.StatusBadRequest) - e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &QueryResponse{Err: err}) if e != nil { h.logger.Errorf("write query response error: %v (while trying to write another error: %v)", e, err) } @@ -1123,9 +1157,9 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp, err := h.api.Query(r.Context(), req) if err != nil { switch errors.Cause(err) { - case pilosa.ErrTooManyWrites: + case ErrTooManyWrites: w.WriteHeader(http.StatusRequestEntityTooLarge) - case pilosa.ErrTranslateStoreReadOnly: + case ErrTranslateStoreReadOnly: u := h.api.PrimaryReplicaNodeURL() u.Path, u.RawQuery = r.URL.Path, r.URL.RawQuery http.Redirect(w, r, u.String(), http.StatusFound) @@ -1133,7 +1167,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { default: w.WriteHeader(http.StatusBadRequest) } - e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &QueryResponse{Err: err}) if e != nil { h.logger.Errorf("write query response error: %v (while trying to write another error: %v)", e, err) } @@ -1145,7 +1179,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // doing nothing right now. if resp.Err != nil { switch errors.Cause(resp.Err) { - case pilosa.ErrTooManyWrites: + case ErrTooManyWrites: w.WriteHeader(http.StatusRequestEntityTooLarge) default: w.WriteHeader(http.StatusBadRequest) @@ -1281,7 +1315,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { } type postIndexRequest struct { - Options pilosa.IndexOptions `json:"options"` + Options IndexOptions `json:"options"` } //_postIndexRequest is necessary to avoid recursion while decoding. @@ -1296,14 +1330,14 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error { return errors.Wrap(err, "unmarshalling unexpected values") } - validIndexOptions := getValidOptions(pilosa.IndexOptions{}) + validIndexOptions := getValidOptions(IndexOptions{}) err := validateOptions(m, validIndexOptions) if err != nil { return err } // Unmarshal expected values. _p := _postIndexRequest{ - Options: pilosa.IndexOptions{ + Options: IndexOptions{ Keys: false, TrackExistence: true, }, @@ -1388,7 +1422,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { // Decode request. req := postIndexRequest{ - Options: pilosa.IndexOptions{ + Options: IndexOptions{ Keys: false, TrackExistence: true, }, @@ -1402,7 +1436,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { if index != nil { resp.CreatedAt = index.CreatedAt() - } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + } else if _, ok = errors.Cause(err).(ConflictError); ok { if index, _ = h.api.Index(r.Context(), indexName); index != nil { resp.CreatedAt = index.CreatedAt() } @@ -1477,13 +1511,13 @@ func (h *Handler) handleGetPastQueries(w http.ResponseWriter, r *http.Request) { } -func fieldOptionsToFunctionalOpts(opt fieldOptions) []pilosa.FieldOption { +func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption { // Convert json options into functional options. - var fos []pilosa.FieldOption + var fos []FieldOption switch opt.Type { - case pilosa.FieldTypeSet: - fos = append(fos, pilosa.OptFieldTypeSet(*opt.CacheType, *opt.CacheSize)) - case pilosa.FieldTypeInt: + case FieldTypeSet: + fos = append(fos, OptFieldTypeSet(*opt.CacheType, *opt.CacheSize)) + case FieldTypeInt: if opt.Min == nil { min := pql.NewDecimal(int64(math.MinInt64), 0) opt.Min = &min @@ -1492,8 +1526,8 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []pilosa.FieldOption { max := pql.NewDecimal(int64(math.MaxInt64), 0) opt.Max = &max } - fos = append(fos, pilosa.OptFieldTypeInt(opt.Min.ToInt64(0), opt.Max.ToInt64(0))) - case pilosa.FieldTypeDecimal: + fos = append(fos, OptFieldTypeInt(opt.Min.ToInt64(0), opt.Max.ToInt64(0))) + case FieldTypeDecimal: scale := int64(0) if opt.Scale != nil { scale = *opt.Scale @@ -1515,27 +1549,27 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []pilosa.FieldOption { minmax = append(minmax, *opt.Max) } } - fos = append(fos, pilosa.OptFieldTypeDecimal(scale, minmax...)) - case pilosa.FieldTypeTimestamp: + fos = append(fos, OptFieldTypeDecimal(scale, minmax...)) + case FieldTypeTimestamp: if opt.Epoch == nil { - epoch := pilosa.DefaultEpoch + epoch := DefaultEpoch opt.Epoch = &epoch } - fos = append(fos, pilosa.OptFieldTypeTimestamp(opt.Epoch.UTC(), *opt.TimeUnit)) - case pilosa.FieldTypeTime: - fos = append(fos, pilosa.OptFieldTypeTime(*opt.TimeQuantum, opt.NoStandardView)) - case pilosa.FieldTypeMutex: - fos = append(fos, pilosa.OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize)) - case pilosa.FieldTypeBool: - fos = append(fos, pilosa.OptFieldTypeBool()) + fos = append(fos, OptFieldTypeTimestamp(opt.Epoch.UTC(), *opt.TimeUnit)) + case FieldTypeTime: + fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, opt.NoStandardView)) + case FieldTypeMutex: + fos = append(fos, OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize)) + case FieldTypeBool: + fos = append(fos, OptFieldTypeBool()) } if opt.Keys != nil { if *opt.Keys { - fos = append(fos, pilosa.OptFieldKeys()) + fos = append(fos, OptFieldKeys()) } } if opt.ForeignIndex != nil { - fos = append(fos, pilosa.OptFieldForeignIndex(*opt.ForeignIndex)) + fos = append(fos, OptFieldForeignIndex(*opt.ForeignIndex)) } return fos } @@ -1579,13 +1613,13 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos := fieldOptionsToFunctionalOpts(req.Options) field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...) - if _, ok = err.(pilosa.BadRequestError); ok { + if _, ok = err.(BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) return } if field != nil { resp.CreatedAt = field.CreatedAt() - } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + } else if _, ok = errors.Cause(err).(ConflictError); ok { if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { resp.CreatedAt = field.CreatedAt() } @@ -1675,7 +1709,7 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { opt.Epoch = fSpec.FieldOptions.Epoch opt.TimeUnit = fSpec.FieldOptions.Unit if fSpec.FieldOptions.TimeQuantum != nil { - timeQuantumVal := pilosa.TimeQuantum(*fSpec.FieldOptions.TimeQuantum) + timeQuantumVal := TimeQuantum(*fSpec.FieldOptions.TimeQuantum) opt.TimeQuantum = &timeQuantumVal } @@ -1693,7 +1727,7 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { // a later error, but if the list of fields is empty, the entire index was new, // and should be cleaned up, in which case there's no need to track or delete // the specific fields separately. -func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *pilosa.Index, returnedFields []string, err error) { +func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) { // create index indexName := schema.IndexName var createdFields []string @@ -1706,7 +1740,7 @@ func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) default: return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType) } - opts := pilosa.IndexOptions{ + opts := IndexOptions{ Keys: useKeys, TrackExistence: true, } @@ -1730,7 +1764,7 @@ func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) case "ensure", "require": index, err = h.api.Index(ctx, indexName) if err != nil { - if _, ok := err.(pilosa.NotFoundError); !ok { + if _, ok := err.(NotFoundError); !ok { return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err) } else { err = nil @@ -1790,7 +1824,7 @@ func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) field, schemaErr := h.api.Field(ctx, indexName, fieldName) if schemaErr != nil { // NotFoundError is fine - if _, ok := schemaErr.(pilosa.NotFoundError); !ok { + if _, ok := schemaErr.(NotFoundError); !ok { return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err) } } @@ -1902,35 +1936,35 @@ type postFieldRequest struct { Options fieldOptions `json:"options"` } -// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, +// fieldOptions tracks FieldOptions. It is made up of pointers to values, // and used for input validation. type fieldOptions struct { - Type string `json:"type,omitempty"` - CacheType *string `json:"cacheType,omitempty"` - CacheSize *uint32 `json:"cacheSize,omitempty"` - Min *pql.Decimal `json:"min,omitempty"` - Max *pql.Decimal `json:"max,omitempty"` - Scale *int64 `json:"scale,omitempty"` - Epoch *time.Time `json:"epoch,omitempty"` - TimeUnit *string `json:"timeUnit,omitempty"` - TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` - Keys *bool `json:"keys,omitempty"` - NoStandardView bool `json:"noStandardView,omitempty"` - ForeignIndex *string `json:"foreignIndex,omitempty"` + Type string `json:"type,omitempty"` + CacheType *string `json:"cacheType,omitempty"` + CacheSize *uint32 `json:"cacheSize,omitempty"` + Min *pql.Decimal `json:"min,omitempty"` + Max *pql.Decimal `json:"max,omitempty"` + Scale *int64 `json:"scale,omitempty"` + Epoch *time.Time `json:"epoch,omitempty"` + TimeUnit *string `json:"timeUnit,omitempty"` + TimeQuantum *TimeQuantum `json:"timeQuantum,omitempty"` + Keys *bool `json:"keys,omitempty"` + NoStandardView bool `json:"noStandardView,omitempty"` + ForeignIndex *string `json:"foreignIndex,omitempty"` } func (o *fieldOptions) validate() error { // Pointers to default values. - defaultCacheType := pilosa.DefaultCacheType - defaultCacheSize := uint32(pilosa.DefaultCacheSize) + defaultCacheType := DefaultCacheType + defaultCacheSize := uint32(DefaultCacheSize) switch o.Type { - case pilosa.FieldTypeSet, "": + case FieldTypeSet, "": // Because FieldTypeSet is the default, its arguments are // not required. Instead, the defaults are applied whenever // a value does not exist. if o.Type == "" { - o.Type = pilosa.FieldTypeSet + o.Type = FieldTypeSet } if o.CacheType == nil { o.CacheType = &defaultCacheType @@ -1939,59 +1973,59 @@ func (o *fieldOptions) validate() error { o.CacheSize = &defaultCacheSize } if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) + return NewBadRequestError(errors.New("min does not apply to field type set")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) + return NewBadRequestError(errors.New("max does not apply to field type set")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) } - case pilosa.FieldTypeInt: + case FieldTypeInt: if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + return NewBadRequestError(errors.New("cacheType does not apply to field type int")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) } - case pilosa.FieldTypeDecimal: + case FieldTypeDecimal: if o.Scale == nil { - return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) + return NewBadRequestError(errors.New("decimal field requires a scale argument")) } else if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + return NewBadRequestError(errors.New("cacheType does not apply to field type int")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == FieldTypeDecimal { + return NewBadRequestError(errors.New("decimal field cannot be a foreign key")) } - case pilosa.FieldTypeTimestamp: + case FieldTypeTimestamp: if o.TimeUnit == nil { - return pilosa.NewBadRequestError(errors.New("timestamp field requires a timeUnit argument")) - } else if !pilosa.IsValidTimeUnit(*o.TimeUnit) { - return pilosa.NewBadRequestError(errors.New("invalid timeUnit argument")) + return NewBadRequestError(errors.New("timestamp field requires a timeUnit argument")) + } else if !IsValidTimeUnit(*o.TimeUnit) { + return NewBadRequestError(errors.New("invalid timeUnit argument")) } else if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type timestamp")) + return NewBadRequestError(errors.New("cacheType does not apply to field type timestamp")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type timestamp")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type timestamp")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type timestamp")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type timestamp")) } else if o.ForeignIndex != nil { - return pilosa.NewBadRequestError(errors.New("timestamp field cannot be a foreign key")) + return NewBadRequestError(errors.New("timestamp field cannot be a foreign key")) } - case pilosa.FieldTypeTime: + case FieldTypeTime: if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) + return NewBadRequestError(errors.New("cacheType does not apply to field type time")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type time")) } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) + return NewBadRequestError(errors.New("min does not apply to field type time")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) + return NewBadRequestError(errors.New("max does not apply to field type time")) } else if o.TimeQuantum == nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) + return NewBadRequestError(errors.New("timeQuantum is required for field type time")) } - case pilosa.FieldTypeMutex: + case FieldTypeMutex: if o.CacheType == nil { o.CacheType = &defaultCacheType } @@ -1999,27 +2033,27 @@ func (o *fieldOptions) validate() error { o.CacheSize = &defaultCacheSize } if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) + return NewBadRequestError(errors.New("min does not apply to field type mutex")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) + return NewBadRequestError(errors.New("max does not apply to field type mutex")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) } - case pilosa.FieldTypeBool: + case FieldTypeBool: if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + return NewBadRequestError(errors.New("cacheType does not apply to field type bool")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + return NewBadRequestError(errors.New("min does not apply to field type bool")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + return NewBadRequestError(errors.New("max does not apply to field type bool")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) } else if o.Keys != nil { - return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + return NewBadRequestError(errors.New("keys does not apply to field type bool")) } else if o.ForeignIndex != nil { - return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) + return NewBadRequestError(errors.New("bool field cannot be a foreign key")) } default: return errors.Errorf("invalid field type: %s", o.Type) @@ -2050,7 +2084,7 @@ func (h *Handler) handleGetTransactionList(w http.ResponseWriter, r *http.Reques trnsMap, err := h.api.Transactions(r.Context()) if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary: + case ErrNodeNotPrimary: http.Error(w, err.Error(), http.StatusBadRequest) default: http.Error(w, "problem getting transactions: "+err.Error(), http.StatusInternalServerError) @@ -2059,7 +2093,7 @@ func (h *Handler) handleGetTransactionList(w http.ResponseWriter, r *http.Reques } // Convert the map of transactions to a slice. - trnsList := make([]*pilosa.Transaction, len(trnsMap)) + trnsList := make([]*Transaction, len(trnsMap)) var i int for _, v := range trnsMap { trnsList[i] = v @@ -2085,7 +2119,7 @@ func (h *Handler) handleGetTransactions(w http.ResponseWriter, r *http.Request) trnsMap, err := h.api.Transactions(r.Context()) if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary: + case ErrNodeNotPrimary: http.Error(w, err.Error(), http.StatusBadRequest) default: http.Error(w, "problem getting transactions: "+err.Error(), http.StatusInternalServerError) @@ -2100,18 +2134,18 @@ func (h *Handler) handleGetTransactions(w http.ResponseWriter, r *http.Request) } type TransactionResponse struct { - Transaction *pilosa.Transaction `json:"transaction,omitempty"` - Error string `json:"error,omitempty"` + Transaction *Transaction `json:"transaction,omitempty"` + Error string `json:"error,omitempty"` } -func (h *Handler) doTransactionResponse(w http.ResponseWriter, err error, trns *pilosa.Transaction) { +func (h *Handler) doTransactionResponse(w http.ResponseWriter, err error, trns *Transaction) { if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary, pilosa.ErrTransactionExists: + case ErrNodeNotPrimary, ErrTransactionExists: w.WriteHeader(http.StatusBadRequest) - case pilosa.ErrTransactionExclusive: + case ErrTransactionExclusive: w.WriteHeader(http.StatusConflict) - case pilosa.ErrTransactionNotFound: + case ErrTransactionNotFound: w.WriteHeader(http.StatusNotFound) default: w.WriteHeader(http.StatusInternalServerError) @@ -2146,7 +2180,7 @@ func (h *Handler) handlePostTransaction(w http.ResponseWriter, r *http.Request) http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - reqTrns := &pilosa.Transaction{} + reqTrns := &Transaction{} if err := json.NewDecoder(r.Body).Decode(reqTrns); err != nil || reqTrns.Timeout == 0 { if err == nil { http.Error(w, "timeout is required and cannot be 0", http.StatusBadRequest) @@ -2209,7 +2243,7 @@ func (h *Handler) handleGetIndexShardSnapshot(w http.ResponseWriter, r *http.Req rc, err := h.api.IndexShardSnapshot(r.Context(), indexName, shard) if err != nil { switch errors.Cause(err) { - case pilosa.ErrIndexNotFound: + case ErrIndexNotFound: http.Error(w, err.Error(), http.StatusNotFound) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2226,7 +2260,7 @@ func (h *Handler) handleGetIndexShardSnapshot(w http.ResponseWriter, r *http.Req } // readQueryRequest parses an query parameters from r. -func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { +func (h *Handler) readQueryRequest(r *http.Request) (*QueryRequest, error) { switch r.Header.Get("Content-Type") { case "application/x-protobuf": return h.readProtobufQueryRequest(r) @@ -2246,15 +2280,15 @@ func (w *passthroughWriter) Write(p []byte) (int, error) { } // readProtobufQueryRequest parses query parameters in protobuf from r. -func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { +func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, error) { // Slurp the body. body, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } - qreq := &pilosa.QueryRequest{} - err = proto.DefaultSerializer.Unmarshal(body, qreq) + qreq := &QueryRequest{} + err = h.serializer.Unmarshal(body, qreq) if err != nil { return nil, errors.Wrap(err, "unmarshalling query request") } @@ -2262,7 +2296,7 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques } // readURLQueryRequest parses query parameters from URL parameters from r. -func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { +func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { q := r.URL.Query() // Parse query string. @@ -2288,7 +2322,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er } } - return &pilosa.QueryRequest{ + return &QueryRequest{ Query: query, Shards: shards, Profile: profile, @@ -2296,7 +2330,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er } // writeQueryResponse writes the response from the executor to w. -func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error { +func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error { if !validHeaderAcceptJSON(r.Header) { w.Header().Set("Content-Type", "application/protobuf") return h.writeProtobufQueryResponse(w, resp, headerAcceptRoaringRow(r.Header)) @@ -2306,10 +2340,10 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res } // writeProtobufQueryResponse writes the response from the executor to w as protobuf. -func (h *Handler) writeProtobufQueryResponse(w io.Writer, resp *pilosa.QueryResponse, writeRoaring bool) error { - serializer := proto.DefaultSerializer +func (h *Handler) writeProtobufQueryResponse(w io.Writer, resp *QueryResponse, writeRoaring bool) error { + serializer := h.serializer if writeRoaring { - serializer = proto.RoaringSerializer + serializer = h.roaringSerializer } if buf, err := serializer.Marshal(resp); err != nil { return errors.Wrap(err, "marshalling") @@ -2320,7 +2354,7 @@ func (h *Handler) writeProtobufQueryResponse(w io.Writer, resp *pilosa.QueryResp } // writeJSONQueryResponse writes the response from the executor to w as JSON. -func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse) error { +func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *QueryResponse) error { return json.NewEncoder(w).Encode(resp) } @@ -2412,9 +2446,9 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { if err = h.api.ExportCSV(r.Context(), index, field, shard, w); err != nil { switch errors.Cause(err) { - case pilosa.ErrFragmentNotFound: + case ErrFragmentNotFound: break - case pilosa.ErrClusterDoesNotOwnShard: + case ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2503,9 +2537,9 @@ func (h *Handler) handleGetNodes(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { buf, err := h.api.FragmentBlockData(r.Context(), r.Body) if err != nil { - if _, ok := err.(pilosa.BadRequestError); ok { + if _, ok := err.(BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) - } else if errors.Cause(err) == pilosa.ErrFragmentNotFound { + } else if errors.Cause(err) == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2538,7 +2572,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request blocks, err := h.api.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), q.Get("view"), shard) if err != nil { - if errors.Cause(err) == pilosa.ErrFragmentNotFound { + if errors.Cause(err) == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2556,7 +2590,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request } type getFragmentBlocksResponse struct { - Blocks []pilosa.FragmentBlock `json:"blocks"` + Blocks []FragmentBlock `json:"blocks"` } // handleGetFragmentData handles GET /internal/fragment/data requests. @@ -2608,7 +2642,7 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) // Retrieve partition data from holder. p, err := h.api.TranslateData(r.Context(), q.Get("index"), int(partition)) - if redir, ok := err.(pilosa.RedirectError); ok { + if redir, ok := err.(RedirectError); ok { newURL := *r.URL newURL.Host = redir.HostPort http.Redirect(w, r, newURL.String(), http.StatusSeeOther) @@ -2684,7 +2718,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht removeNode, err := h.api.RemoveNode(req.ID) if err != nil { - if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + if errors.Cause(err) == ErrNodeIDNotExists { http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) } else { http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) @@ -2720,10 +2754,10 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re var msg string if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary: + case ErrNodeNotPrimary: http.Error(w, err.Error(), http.StatusBadRequest) return - case pilosa.ErrResizeNotRunning: + case ErrResizeNotRunning: msg = err.Error() default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2766,7 +2800,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques err := h.api.ClusterMessage(r.Context(), r.Body) if err != nil { switch err := err.(type) { - case pilosa.MessageProcessingError: + case MessageProcessingError: http.Error(w, err.Error(), http.StatusInternalServerError) default: http.Error(w, err.Error(), http.StatusBadRequest) @@ -2784,14 +2818,14 @@ type defaultClusterMessageResponse struct{} func (h *Handler) handlePostTranslateData(w http.ResponseWriter, r *http.Request) { // Parse offsets for all indexes and fields from POST body. - offsets := make(pilosa.TranslateOffsetMap) + offsets := make(TranslateOffsetMap) if err := json.NewDecoder(r.Body).Decode(&offsets); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Stream all translation data. rd, err := h.api.GetTranslateEntryReader(r.Context(), offsets) - if errors.Cause(err) == pilosa.ErrNotImplemented { + if errors.Cause(err) == ErrNotImplemented { http.Error(w, err.Error(), http.StatusNotImplemented) return } else if err != nil { @@ -2808,7 +2842,7 @@ func (h *Handler) handlePostTranslateData(w http.ResponseWriter, r *http.Request enc := json.NewEncoder(w) for { // Read from store. - var entry pilosa.TranslateEntry + var entry TranslateEntry if err := rd.ReadEntry(&entry); err == io.EOF { return } else if err != nil { @@ -2864,10 +2898,24 @@ func (s queryValidationSpec) validate(query url.Values) error { type ClientOption func(client *http.Client, dialer *net.Dialer) *http.Client +func ClientResponseHeaderTimeoutOption(dur time.Duration) ClientOption { + return func(client *http.Client, dialer *net.Dialer) *http.Client { + client.Transport.(*http.Transport).ResponseHeaderTimeout = dur + return client + } +} + +func ClientDialTimeoutOption(dur time.Duration) ClientOption { + return func(client *http.Client, dialer *net.Dialer) *http.Client { + dialer.Timeout = dur + return client + } +} + func GetHTTPClient(t *tls.Config, opts ...ClientOption) *http.Client { dialer := &net.Dialer{ Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, + KeepAlive: 15 * time.Second, DualStack: true, } transport := &http.Transport{ @@ -2875,7 +2923,7 @@ func GetHTTPClient(t *tls.Config, opts ...ClientOption) *http.Client { DialContext: dialer.DialContext, MaxIdleConns: 1000, MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, + IdleConnTimeout: 20 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } @@ -2917,13 +2965,13 @@ func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Re http.Error(w, err.Error(), http.StatusBadRequest) } } - opt := func(o *pilosa.ImportOptions) error { + opt := func(o *ImportOptions) error { o.SimPowerLossAfter = loss return nil } - req := &pilosa.AtomicRecord{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + req := &AtomicRecord{} + if err := h.serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -2937,7 +2985,7 @@ func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Re } if err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + case ErrClusterDoesNotOwnShard, ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2965,7 +3013,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] index, err := h.api.Index(r.Context(), indexName) if err != nil { - if errors.Cause(err) == pilosa.ErrIndexNotFound { + if errors.Cause(err) == ErrIndexNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2975,7 +3023,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { fieldName := mux.Vars(r)["field"] field := index.Field(fieldName) if field == nil { - http.Error(w, pilosa.ErrFieldNotFound.Error(), http.StatusNotFound) + http.Error(w, ErrFieldNotFound.Error(), http.StatusNotFound) return } @@ -2984,9 +3032,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { doClear := q.Get("clear") == "true" doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - opts := []pilosa.ImportOption{ - pilosa.OptImportOptionsClear(doClear), - pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + opts := []ImportOption{ + OptImportOptionsClear(doClear), + OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), } // Read entire body. @@ -2996,11 +3044,11 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } // Unmarshal request based on field type. - if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal || field.Type() == pilosa.FieldTypeTimestamp { + if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal || field.Type() == FieldTypeTimestamp { // Field type: Int // Marshal into request object. - req := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + req := &ImportValueRequest{} + if err := h.serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -3010,7 +3058,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.ImportValue(r.Context(), qcx, req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + case ErrClusterDoesNotOwnShard, ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -3025,8 +3073,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } else { // Field type: set, time, mutex // Marshal into request object. - req := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + req := &ImportRequest{} + if err := h.serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -3036,7 +3084,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.Import(r.Context(), qcx, req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + case ErrClusterDoesNotOwnShard, ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -3163,9 +3211,9 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request return } - req := &pilosa.ImportRoaringRequest{} + req := &ImportRoaringRequest{} span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") - err = proto.DefaultSerializer.Unmarshal(body, req) + err = h.serializer.Unmarshal(body, req) span.Finish() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -3178,16 +3226,16 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest) return } - resp := &pilosa.ImportResponse{} + resp := &ImportResponse{} // TODO give meaningful stats for import err = h.api.ImportRoaring(ctx, indexName, fieldName, shard, remote, req) if err != nil { resp.Err = err.Error() - if _, ok := err.(pilosa.BadRequestError); ok { + if _, ok := err.(BadRequestError); ok { w.WriteHeader(http.StatusBadRequest) - } else if _, ok := err.(pilosa.NotFoundError); ok { + } else if _, ok := err.(NotFoundError); ok { w.WriteHeader(http.StatusNotFound) - } else if _, ok := err.(pilosa.PreconditionFailedError); ok { + } else if _, ok := err.(PreconditionFailedError); ok { w.WriteHeader(http.StatusPreconditionFailed) } else { w.WriteHeader(http.StatusInternalServerError) @@ -3195,7 +3243,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request } // Marshal response object. - buf, err := proto.DefaultSerializer.Marshal(resp) + buf, err := h.serializer.Marshal(resp) if err != nil { http.Error(w, fmt.Sprintf("marshal import-roaring response: %v", err), http.StatusInternalServerError) return @@ -3232,7 +3280,7 @@ func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { req := &ingest.ShardedRequest{} span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") - err = proto.DefaultSerializer.Unmarshal(body, req) + err = h.serializer.Unmarshal(body, req) span.Finish() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -3273,10 +3321,10 @@ func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request h.logger.Errorf("writing translate keys response: %v", err) } - case pilosa.ErrTranslatingKeyNotFound: + case ErrTranslatingKeyNotFound: http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusNotFound) - case pilosa.ErrTranslateStoreReadOnly: + case ErrTranslateStoreReadOnly: http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusPreconditionFailed) default: @@ -3513,7 +3561,7 @@ func (h *Handler) handleReserveIDs(w http.ResponseWriter, r *http.Request) { return } - var req pilosa.IDAllocReserveRequest + var req IDAllocReserveRequest req.Offset = ^uint64(0) err = json.Unmarshal(bd, &req) if err != nil { @@ -3523,12 +3571,12 @@ func (h *Handler) handleReserveIDs(w http.ResponseWriter, r *http.Request) { ids, err := h.api.ReserveIDs(req.Key, req.Session, req.Offset, req.Count) if err != nil { - var esync pilosa.ErrIDOffsetDesync + var esync ErrIDOffsetDesync if errors.As(err, &esync) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusConflict) err = json.NewEncoder(w).Encode(struct { - pilosa.ErrIDOffsetDesync + ErrIDOffsetDesync Err string `json:"error"` }{ ErrIDOffsetDesync: esync, @@ -3567,7 +3615,7 @@ func (h *Handler) handleCommitIDs(w http.ResponseWriter, r *http.Request) { return } - var req pilosa.IDAllocCommitRequest + var req IDAllocCommitRequest err = json.Unmarshal(bd, &req) if err != nil { http.Error(w, "failed to decode request", http.StatusBadRequest) @@ -3675,17 +3723,19 @@ func (h *Handler) handleCheckAuthentication(w http.ResponseWriter, r *http.Reque http.Error(w, "", http.StatusNoContent) return } - uinfo, err := h.auth.Authenticate(getToken(r)) + uinfo, err := h.auth.Authenticate(r.Context(), getToken(r)) if uinfo == nil || err != nil { w.Header().Add("Content-Type", "text/plain") http.Error(w, err.Error(), http.StatusUnauthorized) return } + // just in case it got refreshed + h.auth.SetCookie(w, uinfo.Token, uinfo.Expiry) + w.Header().Add("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) //nolint:errcheck - } func (h *Handler) handleUserInfo(w http.ResponseWriter, r *http.Request) { @@ -3697,12 +3747,14 @@ func (h *Handler) handleUserInfo(w http.ResponseWriter, r *http.Request) { http.Error(w, "", http.StatusNoContent) return } - uinfo, err := h.auth.Authenticate(getToken(r)) + uinfo, err := h.auth.Authenticate(r.Context(), getToken(r)) if err != nil { h.logger.Errorf("error authenticating: %v", err) http.Error(w, err.Error(), http.StatusForbidden) return } + // just in case it got refreshed + h.auth.SetCookie(w, uinfo.Token, uinfo.Expiry) if err := json.NewEncoder(w).Encode(uinfo); err != nil { h.logger.Errorf("writing user info: %s", err) diff --git a/http/handler_internal_test.go b/http_handler_internal_test.go similarity index 91% rename from http/handler_internal_test.go rename to http_handler_internal_test.go index 25173ede6..455a40c64 100644 --- a/http/handler_internal_test.go +++ b/http_handler_internal_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http +package pilosa import ( "bytes" @@ -7,7 +7,6 @@ import ( "encoding/json" "io/ioutil" "net/http" - gohttp "net/http" "net/http/httptest" "net/url" "os" @@ -17,7 +16,6 @@ import ( "time" "github.com/golang-jwt/jwt" - pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "golang.org/x/oauth2" @@ -33,9 +31,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}}, - {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}}, - {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{TrackExistence: true}}}, + {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: IndexOptions{TrackExistence: false}}}, + {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: IndexOptions{Keys: true, TrackExistence: true}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "unknown key: badKey:test"}, @@ -107,8 +105,8 @@ func decimalPtr(d pql.Decimal) *pql.Decimal { // Test fieldOption validation. func TestFieldOptionValidation(t *testing.T) { - timeQuantum := pilosa.TimeQuantum("YMD") - defaultCacheSize := uint32(pilosa.DefaultCacheSize) + timeQuantum := TimeQuantum("YMD") + defaultCacheSize := uint32(DefaultCacheSize) tests := []struct { json string expected postFieldRequest @@ -116,17 +114,17 @@ func TestFieldOptionValidation(t *testing.T) { }{ // FieldType: Set {json: `{"options": {}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeSet, - CacheType: stringPtr(pilosa.DefaultCacheType), + Type: FieldTypeSet, + CacheType: stringPtr(DefaultCacheType), CacheSize: &defaultCacheSize, }}}, {json: `{"options": {"type": "set"}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeSet, - CacheType: stringPtr(pilosa.DefaultCacheType), + Type: FieldTypeSet, + CacheType: stringPtr(DefaultCacheType), CacheSize: &defaultCacheSize, }}}, {json: `{"options": {"type": "set", "cacheType": "lru"}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeSet, + Type: FieldTypeSet, CacheType: stringPtr("lru"), CacheSize: &defaultCacheSize, }}}, @@ -138,7 +136,7 @@ func TestFieldOptionValidation(t *testing.T) { {json: `{"options": {"type": "int"}}`, err: "min is required for field type int"}, {json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"}, {json: `{"options": {"type": "int", "min": 0, "max": 1001}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeInt, + Type: FieldTypeInt, Min: decimalPtr(pql.NewDecimal(0, 0)), Max: decimalPtr(pql.NewDecimal(1001, 0)), }}}, @@ -149,7 +147,7 @@ func TestFieldOptionValidation(t *testing.T) { // FieldType: Time {json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"}, {json: `{"options": {"type": "time", "timeQuantum": "YMD"}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeTime, + Type: FieldTypeTime, TimeQuantum: &timeQuantum, }}}, {json: `{"options": {"type": "time", "timeQuantum": "YMD", "min": 0}}`, err: "min does not apply to field type time"}, @@ -189,7 +187,7 @@ func readResponse(w *httptest.ResponseRecorder) ([]byte, error) { func TestAuthentication(t *testing.T) { type evaluate func(w *httptest.ResponseRecorder, data []byte) - type endpoint func(w gohttp.ResponseWriter, r *gohttp.Request) + type endpoint func(w http.ResponseWriter, r *http.Request) var ( ClientId = "e9088663-eb08-41d7-8f65-efb5f54bbb71" ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" @@ -230,8 +228,6 @@ func TestAuthentication(t *testing.T) { // make a valid token tkn := jwt.New(jwt.SigningMethodHS256) claims := tkn.Claims.(jwt.MapClaims) - groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "thing", GroupName: "whatever"}}) - claims["molecula-idp-groups"] = groupString claims["oid"] = "42" claims["name"] = "todd" validToken, err := tkn.SignedString([]byte(secretKey)) @@ -255,7 +251,7 @@ func TestAuthentication(t *testing.T) { } expiredToken = "Bearer " + expiredToken - validCookie := &gohttp.Cookie{ + validCookie := &http.Cookie{ Name: "molecula-chip", Value: token.AccessToken, Path: "/", @@ -276,7 +272,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` method string yamlData string token string - cookie *gohttp.Cookie + cookie *http.Cookie handler endpoint fn evaluate }{ @@ -337,7 +333,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` handler: h.handleCheckAuthentication, fn: func(w *httptest.ResponseRecorder, data []byte) { // no valid token in header == Unauthorized - if w.Result().StatusCode != gohttp.StatusUnauthorized { + if w.Result().StatusCode != http.StatusUnauthorized { t.Errorf("expected http code 401, got: %+v", w.Result().StatusCode) } }, @@ -379,7 +375,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` token: "", handler: h.handleUserInfo, fn: func(w *httptest.ResponseRecorder, data []byte) { - if got := w.Result().StatusCode; got != gohttp.StatusForbidden { + if got := w.Result().StatusCode; got != http.StatusForbidden { t.Errorf("expected 403, got %v", got) } }, @@ -487,7 +483,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` path: "/index/{index}/query", kind: "middleware", cookie: validCookie, - handler: func(w gohttp.ResponseWriter, r *gohttp.Request) { + handler: func(w http.ResponseWriter, r *http.Request) { f := hOff.chkAuthZ(hOff.handlePostQuery, authz.Admin) f(w, r) }, @@ -501,9 +497,9 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` name: "MW-CreateIndexInsufficientPerms", path: "/index/abcd", kind: "bearer", - method: gohttp.MethodPost, + method: http.MethodPost, token: validToken, - handler: func(w gohttp.ResponseWriter, r *gohttp.Request) { + handler: func(w http.ResponseWriter, r *http.Request) { h := h var p authz.GroupPermissions if err := p.ReadPermissionsFile(strings.NewReader(permissions1)); err != nil { @@ -515,7 +511,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` f(w, r) }, fn: func(w *httptest.ResponseRecorder, data []byte) { - if got, want := w.Result().StatusCode, gohttp.StatusForbidden; got != want { + if got, want := w.Result().StatusCode, http.StatusForbidden; got != want { t.Errorf("expected %v, got %v", want, got) } }, @@ -527,13 +523,13 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` path: "/index/{index}/query", kind: "bearer", token: validToken, - handler: func(w gohttp.ResponseWriter, r *gohttp.Request) { + handler: func(w http.ResponseWriter, r *http.Request) { h := h f := h.chkAuthZ(h.handlePostQuery, authz.Write) f(w, r) }, fn: func(w *httptest.ResponseRecorder, data []byte) { - if got, want := w.Result().StatusCode, gohttp.StatusInternalServerError; got != want { + if got, want := w.Result().StatusCode, http.StatusInternalServerError; got != want { t.Errorf("expected %v, got %v", want, got) } }, @@ -543,7 +539,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` path: "/index/{index}/query", kind: "bearer", token: validToken, - handler: func(w gohttp.ResponseWriter, r *gohttp.Request) { + handler: func(w http.ResponseWriter, r *http.Request) { h := h var p authz.GroupPermissions if err := p.ReadPermissionsFile(strings.NewReader(permissions1)); err != nil { @@ -554,7 +550,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` f(w, r) }, fn: func(w *httptest.ResponseRecorder, data []byte) { - if got, want := w.Result().StatusCode, gohttp.StatusBadRequest; got != want { + if got, want := w.Result().StatusCode, http.StatusBadRequest; got != want { t.Errorf("expected %v, got: %+v", want, got) } }, @@ -565,7 +561,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` switch test.kind { case "type1", "middleware": t.Run(test.name, func(t *testing.T) { - r := httptest.NewRequest(gohttp.MethodGet, test.path, nil) + r := httptest.NewRequest(http.MethodGet, test.path, nil) w := httptest.NewRecorder() if test.cookie != nil { r.AddCookie(test.cookie) @@ -579,7 +575,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` }) case "type2": t.Run(test.name, func(t *testing.T) { - r := httptest.NewRequest(gohttp.MethodGet, test.path, nil) + r := httptest.NewRequest(http.MethodGet, test.path, nil) w := httptest.NewRecorder() r.Form = url.Values{} r.Header.Set("Content-Type", "application/x-www-form-urlencoded") @@ -597,7 +593,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` case "bearer": t.Run(test.name, func(t *testing.T) { if test.method == "" { - test.method = gohttp.MethodGet + test.method = http.MethodGet } r := httptest.NewRequest(test.method, test.path, nil) w := httptest.NewRecorder() @@ -628,8 +624,6 @@ func TestChkAuthN(t *testing.T) { // make a valid token tkn := jwt.New(jwt.SigningMethodHS256) claims := tkn.Claims.(jwt.MapClaims) - groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "thing", GroupName: "whatever"}}) - claims["molecula-idp-groups"] = groupString claims["oid"] = "42" claims["name"] = "A. Token" validToken, err := tkn.SignedString(a.SecretKey()) @@ -639,15 +633,7 @@ func TestChkAuthN(t *testing.T) { validToken = "Bearer " + validToken // make an invalid token - invalidKey, err := hex.DecodeString("DEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEEDDEADBEED") - if err != nil { - t.Fatal(err) - } - invalidToken, err := tkn.SignedString(invalidKey) - if err != nil { - t.Fatal(err) - } - invalidToken = "Bearer " + invalidToken + invalidToken := "Bearer " + "thisis.a.bad.token" // make an expired token claims["exp"] = "1" diff --git a/http/handler_test.go b/http_handler_test.go similarity index 90% rename from http/handler_test.go rename to http_handler_test.go index c58a4bd3e..a692a8435 100644 --- a/http/handler_test.go +++ b/http_handler_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http_test +package pilosa_test import ( "encoding/json" @@ -10,17 +10,17 @@ import ( "testing" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" ) func TestHandlerOptions(t *testing.T) { - _, err := http.NewHandler() + _, err := pilosa.NewHandler() if err == nil { t.Fatalf("expected error making handler without options, got nil") } - _, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{})) + _, err = pilosa.NewHandler(pilosa.OptHandlerAPI(&pilosa.API{})) if err == nil { t.Fatalf("expected error making handler without options, got nil") } @@ -30,24 +30,30 @@ func TestHandlerOptions(t *testing.T) { t.Fatalf("creating listener: %v", err) } - _, err = http.NewHandler(http.OptHandlerListener(ln, ln.Addr().String())) + _, err = pilosa.NewHandler(pilosa.OptHandlerListener(ln, ln.Addr().String())) if err == nil { t.Fatalf("expected error making handler without options, got nil") } + + _, err = pilosa.NewHandler(pilosa.OptHandlerListener(ln, ln.Addr().String()), pilosa.OptHandlerSerializer(proto.Serializer{}), pilosa.OptHandlerSerializer(proto.RoaringSerializer)) + if err == nil { + t.Fatalf("expected error making handler without enough options, got nil") + } + } func TestMarshalUnmarshalTransactionResponse(t *testing.T) { tests := []struct { name string - tr *http.TransactionResponse + tr *pilosa.TransactionResponse }{ { name: "nil transaction", - tr: &http.TransactionResponse{}, + tr: &pilosa.TransactionResponse{}, }, { name: "empty transaction", - tr: &http.TransactionResponse{Transaction: &pilosa.Transaction{}}, + tr: &pilosa.TransactionResponse{Transaction: &pilosa.Transaction{}}, }, } @@ -58,7 +64,7 @@ func TestMarshalUnmarshalTransactionResponse(t *testing.T) { t.Fatalf("marshaling: %v", err) } - mytr := &http.TransactionResponse{} + mytr := &pilosa.TransactionResponse{} err = json.Unmarshal(data, mytr) if err != nil { t.Fatalf("unmarshalling: %v", err) diff --git a/http/translator.go b/http_translator.go similarity index 75% rename from http/translator.go rename to http_translator.go index 4c161d590..0a28315b7 100644 --- a/http/translator.go +++ b/http_translator.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http +package pilosa import ( "bytes" @@ -12,25 +12,24 @@ import ( "reflect" "sync" - "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/logger" ) -func GetOpenTranslateReaderFunc(client *http.Client) pilosa.OpenTranslateReaderFunc { +func GetOpenTranslateReaderFunc(client *http.Client) OpenTranslateReaderFunc { return GetOpenTranslateReaderWithLockerFunc(client, nopLocker{}) } -func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) pilosa.OpenTranslateReaderFunc { +func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) OpenTranslateReaderFunc { lockType := reflect.TypeOf(locker) if lockType.Kind() == reflect.Ptr { lockType = lockType.Elem() } - return func(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) { + return func(ctx context.Context, nodeURL string, offsets TranslateOffsetMap) (TranslateEntryReader, error) { return openTranslateReader(ctx, nodeURL, offsets, client, reflect.New(lockType).Interface().(sync.Locker)) } } -func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap, client *http.Client, locker sync.Locker) (pilosa.TranslateEntryReader, error) { +func openTranslateReader(ctx context.Context, nodeURL string, offsets TranslateOffsetMap, client *http.Client, locker sync.Locker) (TranslateEntryReader, error) { r := NewTranslateEntryReader(ctx, client) r.locker = locker @@ -47,9 +46,9 @@ type nopLocker struct{} func (nopLocker) Lock() {} func (nopLocker) Unlock() {} -// TranslateEntryReader represents an implementation of pilosa.TranslateEntryReader. +// TranslateEntryReader represents an implementation of TranslateEntryReader. // It consolidates all index & field translate entries into a single reader. -type TranslateEntryReader struct { +type HTTPTranslateEntryReader struct { locker sync.Locker ctx context.Context @@ -60,7 +59,7 @@ type TranslateEntryReader struct { // Lookup of offsets for each index & field. // Must be set before calling Open(). - Offsets pilosa.TranslateOffsetMap + Offsets TranslateOffsetMap // URL to stream entries from. // Must be set before calling Open(). @@ -72,17 +71,17 @@ type TranslateEntryReader struct { } // NewTranslateEntryReader returns a new instance of TranslateEntryReader. -func NewTranslateEntryReader(ctx context.Context, client *http.Client) *TranslateEntryReader { +func NewTranslateEntryReader(ctx context.Context, client *http.Client) *HTTPTranslateEntryReader { if client == nil { client = http.DefaultClient } - r := &TranslateEntryReader{locker: nopLocker{}, HTTPClient: client, Logger: logger.NopLogger} + r := &HTTPTranslateEntryReader{locker: nopLocker{}, HTTPClient: client, Logger: logger.NopLogger} r.ctx, r.cancel = context.WithCancel(ctx) return r } // Open initiates the reader. -func (r *TranslateEntryReader) Open() error { +func (r *HTTPTranslateEntryReader) Open() error { // Serialize map of offsets to request body. requestBody, err := json.Marshal(r.Offsets) if err != nil { @@ -107,7 +106,7 @@ func (r *TranslateEntryReader) Open() error { // Handle error codes. if resp.StatusCode == http.StatusNotImplemented { r.body.Close() - return pilosa.ErrNotImplemented + return ErrNotImplemented } else if resp.StatusCode != http.StatusOK { body, _ := ioutil.ReadAll(resp.Body) r.body.Close() @@ -117,7 +116,7 @@ func (r *TranslateEntryReader) Open() error { } // Close stops the reader. -func (r *TranslateEntryReader) Close() error { +func (r *HTTPTranslateEntryReader) Close() error { if r.cancel != nil { r.cancel() } @@ -132,7 +131,7 @@ func (r *TranslateEntryReader) Close() error { // ReadEntry reads the next entry from the stream into entry. // Returns io.EOF at the end of the stream. -func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { +func (r *HTTPTranslateEntryReader) ReadEntry(entry *TranslateEntry) error { r.locker.Lock() defer r.locker.Unlock() diff --git a/http/translator_test.go b/http_translator_test.go similarity index 92% rename from http/translator_test.go rename to http_translator_test.go index 74a579f9c..8df8d86d0 100644 --- a/http/translator_test.go +++ b/http_translator_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http_test +package pilosa_test import ( "context" @@ -8,8 +8,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/test" ) @@ -41,7 +40,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { } // Connect to server and stream all available data. - r := http.NewTranslateEntryReader(context.Background(), nil) + r := pilosa.NewTranslateEntryReader(context.Background(), nil) r.URL = primary.URL() // Wait to ensure writes make it to translate store @@ -123,7 +122,7 @@ func BenchmarkReadEntryNoMutex(b *testing.B) { defer teardown() for n := 0; n < b.N; n++ { - r, err := http.GetOpenTranslateReaderFunc(nil)(ctx, url, offset) + r, err := pilosa.GetOpenTranslateReaderFunc(nil)(ctx, url, offset) if err != nil { b.Fatalf("opening translate reader: %+v", err) } @@ -138,7 +137,7 @@ func BenchmarkReadEntryWithMutex(b *testing.B) { defer teardown() for n := 0; n < b.N; n++ { - r, err := http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})(ctx, url, offset) + r, err := pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})(ctx, url, offset) if err != nil { b.Fatalf("opening translate reader: %+v", err) } diff --git a/index.go b/index.go index 864e71346..ba3901d67 100644 --- a/index.go +++ b/index.go @@ -93,10 +93,6 @@ func (i *Index) NewTx(txo Txo) Tx { return i.holder.txf.NewTx(txo) } -func (i *Index) NeedsSnapshot() bool { - return i.holder.txf.NeedsSnapshot() -} - // CreatedAt is an timestamp for a specific version of an index. func (i *Index) CreatedAt() int64 { i.mu.RLock() diff --git a/index_internal_test.go b/index_internal_test.go index a36a2475a..ca953d458 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -28,14 +28,3 @@ func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index { return index } - -// reopen closes the index and reopens it. -func (i *Index) reopen() error { - if err := i.Close(); err != nil { - return err - } - if err := i.Open(); err != nil { - return err - } - return nil -} diff --git a/internal/authclustertests/docker-compose.yml b/internal/authclustertests/docker-compose.yml deleted file mode 100644 index 8590fabe4..000000000 --- a/internal/authclustertests/docker-compose.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: '2' -services: - pilosa1: - build: - context: ../.. - dockerfile: Dockerfile-clustertests - image: ptest - environment: - - PILOSA_NAME=pilosa1 - - PILOSA_ETCD_DIR=/root/.etcd - - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 - - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201 - - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 - - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301 - - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 - - PILOSA_CLUSTER_REPLICAS=3 - networks: - - pilosanet - command: - - "/featurebase server --bind pilosa1:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf" - pilosa2: - build: - context: ../.. - dockerfile: Dockerfile-clustertests - image: ptest - environment: - - PILOSA_NAME=pilosa2 - - PILOSA_ETCD_DIR=/root/.etcd - - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 - - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201 - - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 - - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301 - - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 - - PILOSA_CLUSTER_REPLICAS=3 - networks: - - pilosanet - command: - - "/featurebase server --bind pilosa2:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf" - pilosa3: - build: - context: ../.. - dockerfile: Dockerfile-clustertests - image: ptest - environment: - - PILOSA_NAME=pilosa3 - - PILOSA_ETCD_DIR=/root/.etcd - - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 - - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201 - - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 - - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301 - - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 - - PILOSA_CLUSTER_REPLICAS=3 - networks: - - pilosanet - command: - - "/featurebase server --bind pilosa3:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf" - client1: - build: - context: . - dockerfile: ../clustertests/Dockerfile - depends_on: - - "pilosa1" - - "pilosa2" - - "pilosa3" - environment: - - ENABLE_PILOSA_CLUSTER_TESTS=1 - - GO111MODULE=on - - PROJECT=authclustertests - - ENABLE_AUTH=1 - networks: - - pilosanet - volumes: - - /var/run/docker.sock:/var/run/docker.sock - command: - - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests" -networks: - pilosanet: diff --git a/internal/clustertests/Dockerfile b/internal/clustertests/Dockerfile deleted file mode 100644 index 8d2b5d0e2..000000000 --- a/internal/clustertests/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM ptest - -COPY . /go/src/github.com/molecula/featurebase/internal/clustertests diff --git a/internal/clustertests/Dockerfile-fakeIDP b/internal/clustertests/Dockerfile-fakeIDP new file mode 100644 index 000000000..b53d4d2c2 --- /dev/null +++ b/internal/clustertests/Dockerfile-fakeIDP @@ -0,0 +1,7 @@ +FROM golang:latest + +WORKDIR / +COPY fakeidp ./ +RUN go build . + +ENTRYPOINT ["/fakeidp"] diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index 4ad532e8a..a54f6256b 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -2,12 +2,14 @@ package clustertest import ( + "bytes" "context" "fmt" "io" "net/http" "os" "os/exec" + "strings" "testing" "time" @@ -15,37 +17,36 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/disco" - picli "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/logger" + "github.com/pkg/errors" ) -// container turns a docker-compose service name into a container name -// assuming the project name is set in the enviroment as PROJECT. This -// refers to the "-p" argument to docker-compose. NOTE: this assumes -// docker-compose joins the project name with a separating -// underscore... this may not always be true as I've seen a dash used -// as well, but I think it is true in recent versions. -func container(svc string) string { +// container turns a docker-compose service name into a container ID +// by calling "docker-compose ps" +func container(t *testing.T, svc string) string { project := "clustertests" - if os.Getenv("ENABLE_AUTH") == "1" { - project = "authclustertests" - } - if p := os.Getenv("PROJECT"); p != "" { project = p } - return project + "_" + svc + "_1" + stdout, stderr, err := runCmd("docker-compose", "-p", project, "ps", "-q", svc) + if err != nil { + t.Fatalf("couldn't construct container name, err: %v, stderr:\n%s\nstdout:\n%s", err, stderr, stdout) + } + name := strings.Trim(stdout, "\n") + return name } func GetAuthToken(t *testing.T) string { t.Helper() + var ( ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71" ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" - AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize" - TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token" - GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true" - LogoutURL = "https://login.microsoftonline.com/common/oauth2/v2.0/logout" + AuthorizeURL = "fakeidp:10101/authorize" + TokenURL = "fakeidp:10101/token" + GroupEndpointURL = "fakeidp:10101/groups" + LogoutURL = "fakeidp:10101/logout" Scopes = []string{"https://graph.microsoft.com/.default", "offline_access"} Key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" ) @@ -62,12 +63,13 @@ func GetAuthToken(t *testing.T) string { ClientSecret, Key, ) + if err != nil { + t.Fatalf("NewAuth: %v", err) + } // make a valid token tkn := jwt.New(jwt.SigningMethodHS256) claims := tkn.Claims.(jwt.MapClaims) - groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}}) - claims["molecula-idp-groups"] = groupString claims["oid"] = "42" claims["name"] = "valid" token, err := tkn.SignedString([]byte(a.SecretKey())) @@ -87,15 +89,15 @@ func TestClusterStuff(t *testing.T) { auth = true } - cli1, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil)) + cli1, err := pilosa.NewInternalClient("pilosa1:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { t.Fatalf("getting client: %v", err) } - cli2, err := picli.NewInternalClient("pilosa2:10101", picli.GetHTTPClient(nil)) + cli2, err := pilosa.NewInternalClient("pilosa2:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { t.Fatalf("getting client: %v", err) } - cli3, err := picli.NewInternalClient("pilosa3:10101", picli.GetHTTPClient(nil)) + cli3, err := pilosa.NewInternalClient("pilosa3:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { t.Fatalf("getting client: %v", err) } @@ -134,7 +136,7 @@ func TestClusterStuff(t *testing.T) { } // Check query results from each node. - for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { + for i, cli := range []*pilosa.InternalClient{cli1, cli2, cli3} { r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) if err != nil { t.Fatalf("count querying pilosa%d: %v", i, err) @@ -144,24 +146,20 @@ func TestClusterStuff(t *testing.T) { } } t.Run("long pause", func(t *testing.T) { - pcmd := exec.Command("/pumba", "pause", container("pilosa3"), "--duration", "10s") - pcmd.Stdout = os.Stdout - pcmd.Stderr = os.Stderr + if err := sendCmd("docker", "pause", container(t, "pilosa3")); err != nil { + t.Fatalf("sending pause: %v", err) + } t.Log("pausing pilosa3 for 10s") - - if err := pcmd.Start(); err != nil { - t.Fatalf("starting pumba command: %v", err) + time.Sleep(time.Second * 10) + if err := sendCmd("docker", "unpause", container(t, "pilosa3")); err != nil { + t.Fatalf("sending unpause: %v", err) } - if err := pcmd.Wait(); err != nil { - t.Fatalf("waiting on pumba pause cmd: %v", err) - } - t.Log("done with pause, waiting for stability") waitForStatus(t, cli1.Status, string(disco.ClusterStateNormal), 30, time.Second, ctx) t.Log("done waiting for stability") // Check query results from each node. - for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { + for i, cli := range []*pilosa.InternalClient{cli1, cli2, cli3} { r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) if err != nil { t.Fatalf("count querying pilosa%d: %v", i, err) @@ -174,7 +172,7 @@ func TestClusterStuff(t *testing.T) { t.Run("backup", func(t *testing.T) { // do backup with node 1 down, but restart it after a few seconds - if err := sendCmd("docker", "stop", container("pilosa1")); err != nil { + if err := sendCmd("docker", "stop", container(t, "pilosa1")); err != nil { t.Fatalf("sending stop command: %v", err) } var backupCmd *exec.Cmd @@ -192,7 +190,7 @@ func TestClusterStuff(t *testing.T) { } } time.Sleep(time.Second * 5) - if err = sendCmd("docker", "start", container("pilosa1")); err != nil { + if err = sendCmd("docker", "start", container(t, "pilosa1")); err != nil { t.Fatalf("sending start command: %v", err) } @@ -228,18 +226,27 @@ func TestClusterStuff(t *testing.T) { } } time.Sleep(time.Millisecond * 50) - if err = sendCmd("docker", "stop", container("pilosa2")); err != nil { + if err = sendCmd("docker", "stop", container(t, "pilosa2")); err != nil { t.Fatalf("sending stop command: %v", err) } time.Sleep(time.Second * 10) - if err = sendCmd("docker", "start", container("pilosa2")); err != nil { + if err = sendCmd("docker", "start", container(t, "pilosa2")); err != nil { t.Fatalf("sending stop command: %v", err) } if err := restoreCmd.Wait(); err != nil { t.Fatalf("restore failed: %v", err) } + if err = sendCmd("docker", "pause", container(t, "pilosa1")); err != nil { + t.Fatalf("sending pause command: %v", err) + } + if err = sendCmd("docker", "pause", container(t, "pilosa2")); err != nil { + t.Fatalf("sending pause command: %v", err) + } + if err = sendCmd("docker", "pause", container(t, "pilosa3")); err != nil { + t.Fatalf("sending pause command: %v", err) + } // now do backup with all nodes down and too short a timeout // so it fails. Has be to be all 3 because the cluster has // replicas=3 and the backup command will retry on replicas. @@ -254,27 +261,19 @@ func TestClusterStuff(t *testing.T) { t.Fatalf("sending second backup command: %v", err) } } - time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail - if err = sendCmd("docker", "stop", container("pilosa1")); err != nil { - t.Fatalf("sending stop command: %v", err) - } - if err = sendCmd("docker", "stop", container("pilosa2")); err != nil { - t.Fatalf("sending stop command: %v", err) - } - if err = sendCmd("docker", "stop", container("pilosa3")); err != nil { - t.Fatalf("sending stop command: %v", err) - } - time.Sleep(time.Second * 5) + t.Logf("sleeping 8s") + time.Sleep(time.Second * 8) + t.Logf("restarting FB nodes") - if err = sendCmd("docker", "start", container("pilosa1")); err != nil { - t.Fatalf("sending start command: %v", err) + if err = sendCmd("docker", "unpause", container(t, "pilosa1")); err != nil { + t.Fatalf("sending unpause command: %v", err) } - if err = sendCmd("docker", "start", container("pilosa2")); err != nil { - t.Fatalf("sending start command: %v", err) + if err = sendCmd("docker", "unpause", container(t, "pilosa2")); err != nil { + t.Fatalf("sending unpause command: %v", err) } - if err = sendCmd("docker", "start", container("pilosa3")); err != nil { - t.Fatalf("sending start command: %v", err) + if err = sendCmd("docker", "unpause", container(t, "pilosa3")); err != nil { + t.Fatalf("sending unpause command: %v", err) } if err = backupCmd.Wait(); err == nil { t.Fatal("backup command should have errored but didn't") @@ -308,3 +307,14 @@ func waitForStatus(t *testing.T, stator func(context.Context) (string, error), s t.Fatalf("waited %s for status: %s, got: %s", waited.String(), status, s) } } + +// runCmd is a helper which uses os.Exec to run a command and returns +// stdout and stderr as separate strings, and any error returned from +// Command.Run +func runCmd(name string, args ...string) (sout, serr string, err error) { + cmd := exec.Command(name, args...) + stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} + cmd.Stdout, cmd.Stderr = stdout, stderr + err = cmd.Run() + return stdout.String(), stderr.String(), errors.Wrap(err, "running command") +} diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 0454035c9..42476bb33 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -4,7 +4,6 @@ services: build: context: ../.. dockerfile: Dockerfile-clustertests - image: ptest environment: - PILOSA_NAME=pilosa1 - PILOSA_ETCD_DIR=/root/.etcd @@ -17,12 +16,11 @@ services: networks: - pilosanet command: - - "/featurebase server --bind pilosa1:10101" + - "/featurebase server --bind pilosa1:10101 ${CLUSTERTESTS_FB_ARGS}" pilosa2: build: context: ../.. dockerfile: Dockerfile-clustertests - image: ptest environment: - PILOSA_NAME=pilosa2 - PILOSA_ETCD_DIR=/root/.etcd @@ -35,12 +33,11 @@ services: networks: - pilosanet command: - - "/featurebase server --bind pilosa2:10101" + - "/featurebase server --bind pilosa2:10101 ${CLUSTERTESTS_FB_ARGS}" pilosa3: build: context: ../.. dockerfile: Dockerfile-clustertests - image: ptest environment: - PILOSA_NAME=pilosa3 - PILOSA_ETCD_DIR=/root/.etcd @@ -53,25 +50,32 @@ services: networks: - pilosanet command: - - "/featurebase server --bind pilosa3:10101" + - "/featurebase server --bind pilosa3:10101 ${CLUSTERTESTS_FB_ARGS}" client1: build: - context: . + context: ../.. + dockerfile: Dockerfile-clustertests-client depends_on: - "pilosa1" - "pilosa2" - "pilosa3" + - "fakeidp" environment: - ENABLE_PILOSA_CLUSTER_TESTS=1 - GO111MODULE=on - PROJECT=${PROJECT} - - ENABLE_AUTH=0 + - ENABLE_AUTH=${ENABLE_AUTH} networks: - pilosanet volumes: - /var/run/docker.sock:/var/run/docker.sock command: - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests" - + fakeidp: + build: + context: . + dockerfile: Dockerfile-fakeIDP + networks: + - pilosanet networks: pilosanet: diff --git a/internal/clustertests/fakeidp/go.mod b/internal/clustertests/fakeidp/go.mod new file mode 100644 index 000000000..7d0a51cee --- /dev/null +++ b/internal/clustertests/fakeidp/go.mod @@ -0,0 +1,5 @@ +module fakeidp + +go 1.17 + +require github.com/golang-jwt/jwt v3.2.2+incompatible diff --git a/internal/clustertests/fakeidp/go.sum b/internal/clustertests/fakeidp/go.sum new file mode 100644 index 000000000..efdb2a9a1 --- /dev/null +++ b/internal/clustertests/fakeidp/go.sum @@ -0,0 +1,2 @@ +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= diff --git a/internal/clustertests/fakeidp/server.go b/internal/clustertests/fakeidp/server.go new file mode 100644 index 000000000..5a8a9ab83 --- /dev/null +++ b/internal/clustertests/fakeidp/server.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/hex" + "log" + "net/http" + "strconv" + "time" + + "github.com/golang-jwt/jwt" +) + +func groups(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"value":[{"id":"group-id-test","displayName":"group-id-test"}]}`)) +} + +func token(w http.ResponseWriter, req *http.Request) { + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = "42" + claims["name"] = "valid" + expiresIn := 2 * time.Hour + claims["exp"] = strconv.Itoa(int(time.Now().Add(expiresIn).Unix())) + k, err := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + if err != nil { + log.Fatalf("i am not equipped to handle this!!! %v", err) + } + fresh, err := tkn.SignedString(k) + if err != nil { + log.Fatalf("i am not equipped to handle this!!! %v", err) + } + body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}` + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + w.Write([]byte(body)) +} + +func main() { + http.HandleFunc("/groups", groups) + http.HandleFunc("/token", token) + log.Println("FAKEIDP SERVER UP AND RUNNING") + log.Fatal(http.ListenAndServe(":10101", nil)) +} diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index af053a6bf..164765f26 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -17,7 +17,7 @@ import ( pilosa "github.com/molecula/featurebase/v3" boltdb "github.com/molecula/featurebase/v3/boltdb" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/topology" "github.com/pkg/errors" @@ -43,18 +43,18 @@ func sendCmd(cmd string, args ...string) error { return nil } -func unpauseNode(node string) error { - unpauseArgs := []string{"container", "unpause", container(node)} +func unpauseNode(t *testing.T, node string) error { + unpauseArgs := []string{"container", "unpause", container(t, node)} return sendCmd("docker", unpauseArgs...) } -func pauseNode(node string) error { - pauseArgs := []string{"container", "pause", container(node)} +func pauseNode(t *testing.T, node string) error { + pauseArgs := []string{"container", "pause", container(t, node)} return sendCmd("docker", pauseArgs...) } type keyInserter struct { - client *http.InternalClient + client *pilosa.InternalClient uri *net.URI index string keys []string @@ -69,10 +69,10 @@ func getAddress(node string) string { return node + ":10101" } -func getClients(addrs []string) ([]*http.InternalClient, error) { - clients := make([]*http.InternalClient, 0, len(addrs)) +func getClients(addrs []string) ([]*pilosa.InternalClient, error) { + clients := make([]*pilosa.InternalClient, 0, len(addrs)) for _, addr := range addrs { - c, err := http.NewInternalClient(addr, http.GetHTTPClient(nil)) + c, err := pilosa.NewInternalClient(addr, pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { return nil, err } @@ -93,7 +93,7 @@ func getURIsFromAddresses(addrs []string) ([]*net.URI, error) { return uris, nil } -func readIndexTranslateData(ctx context.Context, client *http.InternalClient, dirPath, index string, partition int) error { +func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient, dirPath, index string, partition int) error { // read translateStore contents from endpoint r, err := client.IndexTranslateDataReader(ctx, index, partition) if err != nil { @@ -177,7 +177,7 @@ var errOpRetriable = errors.New("If operation failed on this error, it can be re func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, keys []string) error { // get client that's connected to node address := getAddress(node) - client, err := http.NewInternalClient(address, http.GetHTTPClient(nil)) + client, err := pilosa.NewInternalClient(address, pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { return err } @@ -346,7 +346,7 @@ func TestPauseReplica(t *testing.T) { // pause node t.Logf("pause %s", nodeToPause) - err = pauseNode(nodeToPause) + err = pauseNode(t, nodeToPause) if err != nil { t.Fatalf("error on pause node %s: %v", nodeToPause, err) } @@ -369,7 +369,7 @@ func TestPauseReplica(t *testing.T) { // wait for cluster status to get back to normal t.Logf("unpause %s", nodeToPause) - err = unpauseNode(nodeToPause) + err = unpauseNode(t, nodeToPause) if err != nil { t.Fatalf("error on unpause node %s: %v", nodeToPause, err) } diff --git a/internal/authclustertests/testdata/certs/README.md b/internal/clustertests/testdata/certs/README.md similarity index 100% rename from internal/authclustertests/testdata/certs/README.md rename to internal/clustertests/testdata/certs/README.md diff --git a/internal/authclustertests/testdata/certs/localhost.crt b/internal/clustertests/testdata/certs/localhost.crt similarity index 100% rename from internal/authclustertests/testdata/certs/localhost.crt rename to internal/clustertests/testdata/certs/localhost.crt diff --git a/internal/authclustertests/testdata/certs/localhost.csr b/internal/clustertests/testdata/certs/localhost.csr similarity index 100% rename from internal/authclustertests/testdata/certs/localhost.csr rename to internal/clustertests/testdata/certs/localhost.csr diff --git a/internal/authclustertests/testdata/certs/localhost.key b/internal/clustertests/testdata/certs/localhost.key similarity index 100% rename from internal/authclustertests/testdata/certs/localhost.key rename to internal/clustertests/testdata/certs/localhost.key diff --git a/internal/authclustertests/testdata/certs/pilosa-ca.crl b/internal/clustertests/testdata/certs/pilosa-ca.crl similarity index 100% rename from internal/authclustertests/testdata/certs/pilosa-ca.crl rename to internal/clustertests/testdata/certs/pilosa-ca.crl diff --git a/internal/authclustertests/testdata/certs/pilosa-ca.crt b/internal/clustertests/testdata/certs/pilosa-ca.crt similarity index 100% rename from internal/authclustertests/testdata/certs/pilosa-ca.crt rename to internal/clustertests/testdata/certs/pilosa-ca.crt diff --git a/internal/authclustertests/testdata/certs/pilosa-ca.key b/internal/clustertests/testdata/certs/pilosa-ca.key similarity index 100% rename from internal/authclustertests/testdata/certs/pilosa-ca.key rename to internal/clustertests/testdata/certs/pilosa-ca.key diff --git a/internal/authclustertests/testdata/featurebase.conf b/internal/clustertests/testdata/featurebase.conf similarity index 97% rename from internal/authclustertests/testdata/featurebase.conf rename to internal/clustertests/testdata/featurebase.conf index 8661df8cf..eb587fbcb 100644 --- a/internal/authclustertests/testdata/featurebase.conf +++ b/internal/clustertests/testdata/featurebase.conf @@ -298,8 +298,8 @@ # Suffix should contain .crt or .pem [tls] - certificate = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.crt" - key = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.key" + certificate = "/go/src/github.com/molecula/featurebase/internal/clustertests/testdata/certs/localhost.crt" + key = "/go/src/github.com/molecula/featurebase/internal/clustertests/testdata/certs/localhost.key" # ============================================================================== # Tracing Section @@ -373,11 +373,11 @@ client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71" client-secret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" authorize-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/authorize" - token-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/token" - group-endpoint-url = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true" + token-url="http://fakeidp:10101/token" + group-endpoint-url = "http://fakeidp:10101/groups" logout-url = "https://login.microsoftonline.com/common/oauth2/v2.0/logout" scopes = ["https://graph.microsoft.com/.default", "offline_access"] secret-key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" - permissions = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/permissions.yaml" + permissions = "/go/src/github.com/molecula/featurebase/internal/clustertests/testdata/permissions.yaml" query-log-path = "query-log-test.log" redirect-base-url = "https://localhost:10101" diff --git a/internal/authclustertests/testdata/permissions.yaml b/internal/clustertests/testdata/permissions.yaml similarity index 100% rename from internal/authclustertests/testdata/permissions.yaml rename to internal/clustertests/testdata/permissions.yaml diff --git a/http/client.go b/internal_client.go similarity index 90% rename from http/client.go rename to internal_client.go index 362ac053c..fa1bc6653 100644 --- a/http/client.go +++ b/internal_client.go @@ -1,5 +1,5 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package http +// Copyright 2022 Molecula Corp. All rights reserved. +package pilosa import ( "bytes" @@ -20,9 +20,7 @@ import ( "time" "github.com/hashicorp/go-retryablehttp" - pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" - "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" @@ -34,7 +32,7 @@ import ( // InternalClient represents a client to the Pilosa cluster. type InternalClient struct { defaultURI *pnet.URI - serializer pilosa.Serializer + serializer Serializer log logger.Logger @@ -42,7 +40,7 @@ type InternalClient struct { httpClient *http.Client retryableClient *retryablehttp.Client // the local node's API, used for operations that we can short-circuit that way - api *pilosa.API + api *API // secret Key for auth across nodes secretKey string @@ -53,7 +51,7 @@ type InternalClient struct { // of going through http. func NewInternalClient(host string, remoteClient *http.Client, opts ...InternalClientOption) (*InternalClient, error) { if host == "" { - return nil, pilosa.ErrHostRequired + return nil, ErrHostRequired } uri, err := pnet.NewURIFromAddress(host) @@ -67,6 +65,12 @@ func NewInternalClient(host string, remoteClient *http.Client, opts ...InternalC type InternalClientOption func(c *InternalClient) +func WithSerializer(s Serializer) InternalClientOption { + return func(c *InternalClient) { + c.serializer = s + } +} + // WithSecretKey adds the secretKey used for inter-node communication when auth // is enabled func WithSecretKey(secretKey string) InternalClientOption { @@ -94,7 +98,6 @@ func WithClientRetryPeriod(period time.Duration) InternalClientOption { rc.RetryWaitMin = min rc.RetryMax = int(attempts) rc.CheckRetry = retryWith400Policy - rc.Logger = logger.NopLogger c.retryableClient = rc } } @@ -123,7 +126,6 @@ func retryWith400Policy(ctx context.Context, resp *http.Response, err error) (bo func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client, opts ...InternalClientOption) *InternalClient { ic := &InternalClient{ defaultURI: defaultURI, - serializer: proto.Serializer{}, httpClient: remoteClient, log: logger.NewStandardLogger(os.Stderr), } @@ -168,7 +170,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64 return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -201,7 +203,7 @@ func (c *InternalClient) AvailableShards(ctx context.Context, indexName string) return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -221,7 +223,7 @@ func (c *InternalClient) AvailableShards(ctx context.Context, indexName string) // SchemaNode returns all index and field schema information from the specified // node. -func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*pilosa.IndexInfo, error) { +func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema") defer span.Finish() @@ -235,7 +237,7 @@ func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bo return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -254,7 +256,7 @@ func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bo } // Schema returns all index and field schema information. -func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { +func (c *InternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema") defer span.Finish() @@ -267,7 +269,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -305,7 +307,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf [] req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -321,7 +323,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf [] var msg string // try to decode a JSON response var sr successResponse - qr := &pilosa.QueryResponse{} + qr := &QueryResponse{} if err = json.Unmarshal(buf, &sr); err == nil { msg = sr.Error.Error() } else if err := c.serializer.Unmarshal(buf, qr); err == nil { @@ -356,7 +358,7 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -389,7 +391,7 @@ func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -418,7 +420,7 @@ func (c *InternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, indexNam return nil, errors.Wrap(err, "creating request") } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -435,7 +437,7 @@ func (c *InternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, indexNam return out, err } -func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilosa.Schema, remote bool) error { +func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error { u := uri.Path(fmt.Sprintf("/schema?remote=%v", remote)) buf, err := json.Marshal(s) if err != nil { @@ -449,7 +451,7 @@ func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilos req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -464,7 +466,7 @@ func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilos } // CreateIndex creates a new index on the server. -func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { +func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndex") defer span.Finish() @@ -496,14 +498,14 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusConflict { - return pilosa.ErrIndexExists + return ErrIndexExists } return err } @@ -525,7 +527,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -557,7 +559,7 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -576,21 +578,21 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { } // Query executes query against the index. -func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Query") defer span.Finish() return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } // QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { +func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "QueryNode") defer span.Finish() if index == "" { - return nil, pilosa.ErrIndexRequired + return nil, ErrIndexRequired } else if queryRequest.Query == "" { - return nil, pilosa.ErrQueryRequired + return nil, ErrQueryRequired } buf, err := c.serializer.Marshal(queryRequest) if err != nil { @@ -616,7 +618,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -631,7 +633,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str return nil, errors.Wrap(err, "reading") } - qresp := &pilosa.QueryResponse{} + qresp := &QueryResponse{} if err := c.serializer.Unmarshal(body, qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } else if qresp.Err != nil { @@ -650,12 +652,12 @@ func getPrimaryNode(nodes []*topology.Node) *topology.Node { return nil } -func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { +func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureIndex") defer span.Finish() err := c.CreateIndex(ctx, name, options) - if err == nil || errors.Cause(err) == pilosa.ErrIndexExists { + if err == nil || errors.Cause(err) == ErrIndexExists { return nil } return err @@ -664,21 +666,21 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureField") defer span.Finish() - return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{}) + return c.EnsureFieldWithOptions(ctx, indexName, fieldName, FieldOptions{}) } -func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error { +func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt FieldOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureFieldWithOptions") defer span.Finish() err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt) - if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { + if err == nil || errors.Cause(err) == ErrFieldExists { return nil } return err } // importNode sends a pre-marshaled import request to a node. -func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error { +func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, index, field string, buf []byte, opts *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.importNode") defer span.Finish() @@ -703,7 +705,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -719,7 +721,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in return errors.Wrap(err, "reading") } - var isresp pilosa.ImportResponse + var isresp ImportResponse if err := c.serializer.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) } else if s := isresp.Err; s != "" { @@ -737,7 +739,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in // that in here with a type switch seems messy. Similarly, index/field/shard // exist because we can't access those members of the two slightly different // structs. -func (c *InternalClient) importHelper(ctx context.Context, req pilosa.Message, process func() error, index string, field string, shard uint64, options *pilosa.ImportOptions) error { +func (c *InternalClient) importHelper(ctx context.Context, req Message, process func() error, index string, field string, shard uint64, options *ImportOptions) error { // If we don't actually know what shards we're sending to, and we have // a local API and a qcx, we'll have a process function that uses the local // API. Otherwise, even if we have an API @@ -847,7 +849,7 @@ func (c *InternalClient) importHelper(ctx context.Context, req pilosa.Message, p // // If we get a non-nil qcx, and have an associated API, we'll use that API // directly for the local shard. -func (c *InternalClient) Import(ctx context.Context, qcx *pilosa.Qcx, req *pilosa.ImportRequest, options *pilosa.ImportOptions) error { +func (c *InternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import") defer span.Finish() @@ -875,7 +877,7 @@ func (c *InternalClient) Import(ctx context.Context, qcx *pilosa.Qcx, req *pilos // // If we get a non-nil qcx, and have an associated API, we'll use that API // directly for the local shard. -func (c *InternalClient) ImportValue(ctx context.Context, qcx *pilosa.Qcx, req *pilosa.ImportValueRequest, options *pilosa.ImportOptions) error { +func (c *InternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import") defer span.Finish() @@ -893,14 +895,14 @@ func (c *InternalClient) ImportValue(ctx context.Context, qcx *pilosa.Qcx, req * // ImportRoaring does fast import of raw bits in roaring format (pilosa or // official format, see API.ImportRoaring). -func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error { +func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring") defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } else if field == "" { - return pilosa.ErrFieldRequired + return ErrFieldRequired } if uri == nil { uri = c.defaultURI @@ -924,7 +926,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") httpReq.Header.Set("X-Pilosa-Row", "roaring") - httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("User-Agent", "pilosa/"+Version) httpReq = AddAuthToken(ctx, httpReq) // Execute request against the host. @@ -935,7 +937,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index defer resp.Body.Close() dec := json.NewDecoder(resp.Body) - rbody := &pilosa.ImportResponse{} + rbody := &ImportResponse{} err = dec.Decode(rbody) // Decode can return EOF when no error occurred. helpful! if err != nil && err != io.EOF { @@ -953,9 +955,9 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sha defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } else if field == "" { - return pilosa.ErrFieldRequired + return ErrFieldRequired } // Retrieve a list of nodes that own the shard. @@ -999,7 +1001,7 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *topology.Node, return errors.Wrap(err, "creating request") } req.Header.Set("Accept", "text/csv") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1042,14 +1044,14 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, pilosa.ErrFragmentNotFound + return nil, ErrFragmentNotFound } return nil, err } @@ -1060,19 +1062,19 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateField") defer span.Finish() - return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{}) + return c.CreateFieldWithOptions(ctx, index, field, FieldOptions{}) } // CreateFieldWithOptions creates a new field on the server. -func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { +func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldWithOptions") defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } - // convert pilosa.FieldOptions to fieldOptions + // convert FieldOptions to fieldOptions // // TODO this kind of sucks because it's one more place that needs // changes when we change anything with field options (and there @@ -1083,23 +1085,23 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel Type: opt.Type, } switch fieldOpt.Type { - case pilosa.FieldTypeSet, pilosa.FieldTypeMutex: + case FieldTypeSet, FieldTypeMutex: fieldOpt.CacheType = &opt.CacheType fieldOpt.CacheSize = &opt.CacheSize fieldOpt.Keys = &opt.Keys - case pilosa.FieldTypeInt: + case FieldTypeInt: fieldOpt.Min = &opt.Min fieldOpt.Max = &opt.Max - case pilosa.FieldTypeTime: + case FieldTypeTime: fieldOpt.TimeQuantum = &opt.TimeQuantum - case pilosa.FieldTypeBool: + case FieldTypeBool: // pass - case pilosa.FieldTypeDecimal: + case FieldTypeDecimal: fieldOpt.Min = &opt.Min fieldOpt.Max = &opt.Max fieldOpt.Scale = &opt.Scale default: - fieldOpt.Type = pilosa.DefaultFieldType + fieldOpt.Type = DefaultFieldType fieldOpt.Keys = &opt.Keys } @@ -1132,14 +1134,14 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusConflict { - return pilosa.ErrFieldExists + return ErrFieldExists } return err } @@ -1149,7 +1151,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) { +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentBlocks") defer span.Finish() @@ -1170,7 +1172,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, inde return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -1179,7 +1181,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, inde if err != nil { // Return the appropriate error. if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, pilosa.ErrFragmentNotFound + return nil, ErrFragmentNotFound } return nil, err } @@ -1201,7 +1203,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi if uri == nil { panic("need to pass a URI to BlockData") } - buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{ + buf, err := c.serializer.Marshal(&BlockDataRequest{ Index: index, Field: field, View: view, @@ -1221,7 +1223,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Accept", "application/protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1234,7 +1236,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi defer resp.Body.Close() // Decode response object. - var rsp pilosa.BlockDataResponse + var rsp BlockDataResponse if body, err := ioutil.ReadAll(resp.Body); err != nil { return nil, nil, errors.Wrap(err, "reading") } else if err := c.serializer.Unmarshal(body, &rsp); err != nil { @@ -1255,7 +1257,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []b } req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req.Header.Set("Connection", "keep-alive") if c.secretKey != "" { @@ -1273,16 +1275,16 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []b } // TranslateKeysNode function is mainly called to translate keys from primary node. -// If primary node returns 404 error the function wraps it with pilosa.ErrTranslatingKeyNotFound. +// If primary node returns 404 error the function wraps it with ErrTranslatingKeyNotFound. func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode") defer span.Finish() if index == "" { - return nil, pilosa.ErrIndexRequired + return nil, ErrIndexRequired } - buf, err := c.serializer.Marshal(&pilosa.TranslateKeysRequest{ + buf, err := c.serializer.Marshal(&TranslateKeysRequest{ Index: index, Field: field, Keys: keys, @@ -1303,14 +1305,14 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, errors.Wrap(pilosa.ErrTranslatingKeyNotFound, err.Error()) + return nil, errors.Wrap(ErrTranslatingKeyNotFound, err.Error()) } return nil, err } @@ -1322,7 +1324,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i return nil, errors.Wrap(err, "reading") } - tkresp := &pilosa.TranslateKeysResponse{} + tkresp := &TranslateKeysResponse{} if err := c.serializer.Unmarshal(body, tkresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1335,10 +1337,10 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in defer span.Finish() if index == "" { - return nil, pilosa.ErrIndexRequired + return nil, ErrIndexRequired } - buf, err := c.serializer.Marshal(&pilosa.TranslateIDsRequest{ + buf, err := c.serializer.Marshal(&TranslateIDsRequest{ Index: index, Field: field, IDs: ids, @@ -1358,7 +1360,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1374,7 +1376,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in return nil, errors.Wrap(err, "reading") } - tkresp := &pilosa.TranslateIDsResponse{} + tkresp := &TranslateIDsResponse{} if err := c.serializer.Unmarshal(body, tkresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1382,7 +1384,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in } // GetNodeUsage retrieves the size-on-disk information for the specified node. -func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]pilosa.NodeUsage, error) { +func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) { u := uri.Path("/ui/usage?remote=true") req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -1390,7 +1392,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1406,7 +1408,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s return nil, errors.Wrap(err, "reading") } - nodeUsages := make(map[string]pilosa.NodeUsage) // map of size 1 + nodeUsages := make(map[string]NodeUsage) // map of size 1 if err := json.Unmarshal(body, &nodeUsages); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1414,7 +1416,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s } // GetPastQueries retrieves the query history log for the specified node. -func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]pilosa.PastQueryStatus, error) { +func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) { u := uri.Path("/query-history?remote=true") req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -1422,7 +1424,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]p } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1438,7 +1440,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]p return nil, errors.Wrap(err, "reading") } - queries := make([]pilosa.PastQueryStatus, 100) + queries := make([]PastQueryStatus, 100) if err := json.Unmarshal(body, &queries); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1464,7 +1466,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1513,7 +1515,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1563,7 +1565,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1616,7 +1618,7 @@ func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1661,7 +1663,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, // Apply headers. req.Header.Set("Content-Length", strconv.Itoa(len(like))) req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1691,7 +1693,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, return matches, nil } -func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.Transaction, error) { +func (c *InternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Transactions") defer span.Finish() @@ -1701,7 +1703,7 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.T return nil, errors.Wrap(err, "creating transactions request") } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1712,15 +1714,15 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.T _, _ = io.Copy(ioutil.Discard, resp.Body) _ = resp.Body.Close() }() - trnsMap := make(map[string]*pilosa.Transaction) + trnsMap := make(map[string]*Transaction) err = json.NewDecoder(resp.Body).Decode(&trnsMap) return trnsMap, errors.Wrap(err, "json decoding") } -func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*pilosa.Transaction, error) { +func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.StartTransaction") defer span.Finish() - buf, err := json.Marshal(&pilosa.Transaction{ + buf, err := json.Marshal(&Transaction{ ID: id, Timeout: timeout, Exclusive: exclusive, @@ -1740,7 +1742,7 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -1757,14 +1759,14 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou return nil, errors.Wrap(err, "decoding response") } if resp.StatusCode == 409 { - err = pilosa.ErrTransactionExclusive + err = ErrTransactionExclusive } else if tr.Error != "" { err = errors.New(tr.Error) } return tr.Transaction, err } -func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) { +func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FinishTransaction") defer span.Finish() @@ -1775,7 +1777,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pil } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -1798,7 +1800,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pil return tr.Transaction, err } -func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) { +func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.GetTransaction") defer span.Finish() @@ -1812,7 +1814,7 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa return nil, errors.Wrap(err, "creating get transaction request") } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -1856,14 +1858,6 @@ func forwardAuthHeader(b bool) executeRequestOption { } } -type nopCloser struct { - *bytes.Reader -} - -func (n nopCloser) Close() error { - return nil -} - // executeRequest executes the given request and checks the Response. For // responses with non-2XX status, the body is read and closed, and an error is // returned. If the error is nil, the caller must ensure that the response body @@ -1923,7 +1917,7 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp var msg string // try to decode a JSON response var sr successResponse - qr := &pilosa.QueryResponse{} + qr := &QueryResponse{} if err = json.Unmarshal(buf, &sr); err == nil { msg = sr.Error.Error() } else if err := c.serializer.Unmarshal(buf, qr); err == nil { @@ -1936,8 +1930,18 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp return resp, nil } +// Bit represents the intersection of a row and a column. It can be specified by +// integer ids or string keys. +type Bit struct { + RowID uint64 + ColumnID uint64 + RowKey string + ColumnKey string + Timestamp int64 +} + // Bits is a slice of Bit. -type Bits []pilosa.Bit +type Bits []Bit func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Bits) Len() int { return len(p) } @@ -2030,10 +2034,10 @@ func (p Bits) Timestamps() []int64 { } // GroupByShard returns a map of bits by shard. -func (p Bits) GroupByShard() map[uint64][]pilosa.Bit { - m := make(map[uint64][]pilosa.Bit) +func (p Bits) GroupByShard() map[uint64][]Bit { + m := make(map[uint64][]Bit) for _, bit := range p { - shard := bit.ColumnID / pilosa.ShardWidth + shard := bit.ColumnID / ShardWidth m[shard] = append(m[shard], bit) } @@ -2045,8 +2049,16 @@ func (p Bits) GroupByShard() map[uint64][]pilosa.Bit { return m } +// FieldValue represents the value for a column within a +// range-encoded field. +type FieldValue struct { + ColumnID uint64 + ColumnKey string + Value int64 +} + // FieldValues represents a slice of field values. -type FieldValues []pilosa.FieldValue +type FieldValues []FieldValue func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p FieldValues) Len() int { return len(p) } @@ -2099,10 +2111,10 @@ func (p FieldValues) Values() []int64 { } // GroupByShard returns a map of field values by shard. -func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue { - m := make(map[uint64][]pilosa.FieldValue) +func (p FieldValues) GroupByShard() map[uint64][]FieldValue { + m := make(map[uint64][]FieldValue) for _, val := range p { - shard := val.ColumnID / pilosa.ShardWidth + shard := val.ColumnID / ShardWidth m[shard] = append(m[shard], val) } @@ -2115,7 +2127,7 @@ func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue { } // BitsByPos is a slice of bits sorted row then column. -type BitsByPos []pilosa.Bit +type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } @@ -2127,11 +2139,6 @@ func (p BitsByPos) Less(i, j int) bool { return p0 < p1 } -// pos returns the row position of a row/column pair. -func pos(rowID, columnID uint64) uint64 { - return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth) -} - func uriPathToURL(uri *pnet.URI, path string) url.URL { return url.URL{ Scheme: uri.Scheme, @@ -2171,14 +2178,14 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, pilosa.ErrFragmentNotFound + return nil, ErrFragmentNotFound } return nil, err } @@ -2190,7 +2197,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } if uri == nil { @@ -2206,7 +2213,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind if err != nil { return errors.Wrap(err, "creating request") } - httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("User-Agent", "pilosa/"+Version) token, ok := ctx.Value("token").(string) if ok && token != "" { httpReq.Header.Set("Authorization", token) @@ -2226,7 +2233,7 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } if uri == nil { @@ -2242,7 +2249,7 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind if err != nil { return errors.Wrap(err, "creating request") } - httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("User-Agent", "pilosa/"+Version) token, ok := ctx.Value("token").(string) if ok && token != "" { @@ -2272,7 +2279,7 @@ func (c *InternalClient) ShardReader(ctx context.Context, index string, shard ui return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2295,7 +2302,7 @@ func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2319,7 +2326,7 @@ func (c *InternalClient) IDAllocDataWriter(ctx context.Context, f io.Reader, pri return errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2346,7 +2353,7 @@ func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index str return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2354,7 +2361,7 @@ func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index str resp, err := c.executeRequest(req.WithContext(ctx), forwardAuthHeader(true)) if resp != nil && resp.StatusCode == http.StatusNotFound { resp.Body.Close() - return nil, pilosa.ErrTranslateStoreNotFound + return nil, ErrTranslateStoreNotFound } else if err != nil { return nil, err } @@ -2376,7 +2383,7 @@ func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, fi return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2384,16 +2391,14 @@ func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, fi resp, err := c.executeRequest(req.WithContext(ctx)) if resp != nil && resp.StatusCode == http.StatusNotFound { resp.Body.Close() - return nil, pilosa.ErrTranslateStoreNotFound + return nil, ErrTranslateStoreNotFound } else if err != nil { return nil, err } return resp.Body, nil } -// Status function is just a public function for this particular implementation of InternalClient. -// It's not require by pilosa.InternalClient interface. -// The function returns pilosa cluster state as a string ("NORMAL", "DEGRADED", "DOWN", "RESIZING", ...) +// Status returns pilosa cluster state as a string ("NORMAL", "DEGRADED", "DOWN", "RESIZING", ...) func (c *InternalClient) Status(ctx context.Context) (string, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Status") defer span.Finish() @@ -2407,7 +2412,7 @@ func (c *InternalClient) Status(ctx context.Context) (string, error) { return "", errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -2439,7 +2444,7 @@ func (c *InternalClient) PartitionNodes(ctx context.Context, partitionID int) ([ return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -2457,6 +2462,6 @@ func (c *InternalClient) PartitionNodes(ctx context.Context, partitionID int) ([ return a, nil } -func (c *InternalClient) SetInternalAPI(api *pilosa.API) { +func (c *InternalClient) SetInternalAPI(api *API) { c.api = api } diff --git a/http/client_test.go b/internal_client_test.go similarity index 97% rename from http/client_test.go rename to internal_client_test.go index 8625c1e2e..8d1531137 100644 --- a/http/client_test.go +++ b/internal_client_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http_test +package pilosa_test import ( "bufio" @@ -15,7 +15,7 @@ import ( "github.com/davecgh/go-spew/spew" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" @@ -122,9 +122,9 @@ func TestClient_MultiNode(t *testing.T) { // Connect to each node to compare results. client := make([]*Client, 3) - client[0] = MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil)) - client[1] = MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil)) - client[2] = MustNewClient(c.GetNode(2).URL(), http.GetHTTPClient(nil)) + client[0] = MustNewClient(c.GetNode(0).URL(), pilosa.GetHTTPClient(nil)) + client[1] = MustNewClient(c.GetNode(1).URL(), pilosa.GetHTTPClient(nil)) + client[2] = MustNewClient(c.GetNode(2).URL(), pilosa.GetHTTPClient(nil)) topN := 4 queryRequest := &pilosa.QueryRequest{ @@ -188,7 +188,7 @@ func TestClient_Export(t *testing.T) { cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys()) cmd.MustCreateField(t, "unkeyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)) - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) data := []pilosa.Bit{ {RowID: 1, ColumnID: 100, RowKey: "row1", ColumnKey: "col100"}, {RowID: 1, ColumnID: 101, RowKey: "row1", ColumnKey: "col101"}, @@ -376,7 +376,7 @@ func TestClient_Import(t *testing.T) { recIDs := []uint64{0, 3, 7} valueIDs := []uint64{0, 3, 7} - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) // set API to point at the local node c.SetInternalAPI(cmd.API) @@ -532,7 +532,7 @@ func TestClient_ImportRoaring(t *testing.T) { // Send import request. host := cluster.GetNode(0).URL() - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { @@ -656,7 +656,7 @@ func TestClient_ImportRoaring_MultiView(t *testing.T) { // Send import request. host := cluster.GetNode(0).URL() - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportRoaringRequest{Views: map[string][]byte{}} req.Views["a"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100") req.Views["b"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100") @@ -681,7 +681,7 @@ func TestClient_ImportKeys(t *testing.T) { cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys()) // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) baseReq := &pilosa.ImportRequest{ Index: "keyed", Field: "keyedf", @@ -774,8 +774,8 @@ func TestClient_ImportKeys(t *testing.T) { cmd0.MustCreateField(t, "keyed", "keyedf1", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys()) // Send import request. - c0 := MustNewClient(host0, http.GetHTTPClient(nil)) - c1 := MustNewClient(host1, http.GetHTTPClient(nil)) + c0 := MustNewClient(host0, pilosa.GetHTTPClient(nil)) + c1 := MustNewClient(host1, pilosa.GetHTTPClient(nil)) // Import to node0. t.Run("Import node0", func(t *testing.T) { @@ -852,7 +852,7 @@ func TestClient_ImportKeys(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: "i", Field: "f", @@ -931,7 +931,7 @@ func TestClient_ImportIDs(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: idxName, Field: fldName, @@ -999,7 +999,7 @@ func TestClient_ImportValue(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: "i", Field: "f", @@ -1078,7 +1078,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportRequest{ Index: "iset", Field: "fset", @@ -1114,7 +1114,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: "iint", Field: "fint", @@ -1155,7 +1155,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Set a bit on a different shard. hldr.SetBit("i", "f", 0, 1) - c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil)) + c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil)) blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", "standard", 0) if err != nil { t.Fatal(err) @@ -1180,7 +1180,7 @@ func TestClient_CreateDecimalField(t *testing.T) { defer cluster.Close() cmd := cluster.GetNode(0) - c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil)) + c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil)) index := "cdf" err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{}) @@ -1290,8 +1290,8 @@ func TestClientTransactions(t *testing.T) { coord := c.GetPrimary() other := c.GetNonPrimary() - client0 := MustNewClient(coord.URL(), http.GetHTTPClient(nil)) - client1 := MustNewClient(other.URL(), http.GetHTTPClient(nil)) + client0 := MustNewClient(coord.URL(), pilosa.GetHTTPClient(nil)) + client1 := MustNewClient(other.URL(), pilosa.GetHTTPClient(nil)) // can create, list, get, and finish a transaction var expDeadline time.Time @@ -1444,12 +1444,12 @@ func TestClientTransactions(t *testing.T) { // Client represents a test wrapper for pilosa.Client. type Client struct { - *http.InternalClient + *pilosa.InternalClient } // MustNewClient returns a new instance of Client. Panic on error. func MustNewClient(host string, h *gohttp.Client) *Client { - c, err := http.NewInternalClient(host, h) + c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) if err != nil { panic(err) } @@ -1497,7 +1497,7 @@ func TestClient_ImportRoaringExists(t *testing.T) { } // Send import request. host := node.URL() - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") diff --git a/mmap_test.go b/mmap_test.go deleted file mode 100644 index d1905eb9e..000000000 --- a/mmap_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "math/rand" - "runtime" - "testing" - - "github.com/molecula/featurebase/v3/logger" -) - -type cv struct { - cols []uint64 - vals []int64 -} - -func forceSnapshotsCheckMapping(t *testing.T) { - depth := uint64(6) - f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewStandard, 0) - tx.Rollback() - f.Logger = logger.NewLogfLogger(t) - defer f.Clean(t) - - tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() - - for i := 0; i < f.MaxOpN; i++ { - _, _ = f.setBit(tx, 0, uint64(32*i)) - } - // force snapshot so we get a mmapped row... - err := f.Snapshot() - if err != nil { - t.Fatalf("initial snapshot error: %v", err) - } - - values := make([]cv, 1024) - for i := range values { - cols := make([]uint64, 128) - vals := make([]int64, 128) - for j := range cols { - // pick values in the first 16 cols of each of the 16 - // shards in a default shardwidth, so each set will - // probably change some values from the previous one. - cols[j] = uint64(((rand.Int63n(16) & int64(i>>2)) << 16) + rand.Int63n(16)) - vals[j] = int64(rand.Int63n(1 << depth)) - } - values[i] = cv{cols, vals} - } - - // modify the original bitmap, until it causes a snapshot, which - // then invalidates the other map... - for i := 0; i < 32; i++ { - cv := values[i%len(values)] - // periodically force gc, so if we have a small pool of maps - // we'll go in and out of mapping mode - if i%5 == 0 { - runtime.GC() - } - err := f.importValue(tx, cv.cols, cv.vals, depth, (i%3 == 1)) - if err != nil { - t.Fatalf("importValue[%d]: %v", i, err) - } - err = f.Snapshot() - if err != nil { - t.Fatalf("snapshot[%d]: %v", i, err) - } - } -} diff --git a/pilosa.go b/pilosa.go index 354c435ff..9cf4f715f 100644 --- a/pilosa.go +++ b/pilosa.go @@ -2,13 +2,11 @@ package pilosa import ( - "os" "regexp" "time" "github.com/molecula/featurebase/v3/disco" pnet "github.com/molecula/featurebase/v3/net" - "github.com/molecula/featurebase/v3/storage" "github.com/pkg/errors" ) @@ -157,20 +155,3 @@ func AddressWithDefaults(addr string) (*pnet.URI, error) { } return pnet.NewURIFromAddress(addr) } - -// CurrentBackend is one step in an attempt to centralize (and either minimize -// or completely remove), the calls to environment variables throughout the -// tests. Ideally we could get rid of this and rely completely on the -// configuration parameters. -func CurrentBackend() string { - return os.Getenv("PILOSA_STORAGE_BACKEND") -} - -// CurrentBackendOrDefault tries the environment variable first, but falls back -// to the default backend if the environment variable is empty. -func CurrentBackendOrDefault() string { - if backend := os.Getenv("PILOSA_STORAGE_BACKEND"); backend != "" { - return backend - } - return storage.DefaultBackend -} diff --git a/pprof.go b/pprof.go index 07f87b1b2..400b48af7 100644 --- a/pprof.go +++ b/pprof.go @@ -19,10 +19,7 @@ import ( // commented out—in holder.go. func CPUProfileForDur(dur time.Duration, outpath string) { // per-query pprof output: - backend := CurrentBackend() - if backend == "" { - backend = storage.DefaultBackend - } + backend := storage.DefaultBackend path := outpath + "." + backend f, err := os.Create(path) vprint.PanicOn(err) @@ -45,10 +42,7 @@ func CPUProfileForDur(dur time.Duration, outpath string) { // commented out—in holder.go. func MemProfileForDur(dur time.Duration, outpath string) { // per-query pprof output: - backend := CurrentBackend() - if backend == "" { - backend = storage.DefaultBackend - } + backend := storage.DefaultBackend path := outpath + "." + backend f, err := os.Create(path) vprint.PanicOn(err) diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh index 086bf0874..37cc0a930 100644 --- a/qa/scripts/runSamsungGauntlet.sh +++ b/qa/scripts/runSamsungGauntlet.sh @@ -2,9 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh index 96866fd6d..6b598995e 100755 --- a/qa/scripts/runSmokeTest.sh +++ b/qa/scripts/runSmokeTest.sh @@ -2,8 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index 1229c0044..e7929ea9f 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -3,9 +3,6 @@ # To run script: ./setupSamsungGauntlet.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index fac0d7111..1059e5f29 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -3,9 +3,6 @@ # To run script: ./setupSmokeTest.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/teardownSamsungGauntlet.sh b/qa/scripts/teardownSamsungGauntlet.sh index ae614d99b..0e5598569 100755 --- a/qa/scripts/teardownSamsungGauntlet.sh +++ b/qa/scripts/teardownSamsungGauntlet.sh @@ -1,7 +1,6 @@ #!/bin/bash # To run script: ./teardownSamsungGauntlet.sh -# requires TF_VAR_gitlab_token env var to be set cd qa/tf/gauntlet/samsung export TF_IN_AUTOMATION=1 diff --git a/qa/scripts/teardownSmokeTest.sh b/qa/scripts/teardownSmokeTest.sh index 6215d419e..76eeb564b 100755 --- a/qa/scripts/teardownSmokeTest.sh +++ b/qa/scripts/teardownSmokeTest.sh @@ -1,8 +1,6 @@ #!/bin/bash # To run script: ./teardownSmokeTest.sh -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index 31ef6f49d..b6ba34e7f 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -1,8 +1,5 @@ #!/bin/bash -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index e460f1f48..629a63aeb 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -125,15 +125,14 @@ executeGeneralNodeConfigCommands() { ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data/featurebase" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /data/featurebase" - # TODO handle different archs - echo "Getting featurebase binary (https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64)..." - ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "curl --fail --header 'PRIVATE-TOKEN: ${TF_VAR_gitlab_token}' -o /home/ec2-user/featurebase_linux_arm64 https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64" + scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase_linux_arm64 ec2-user@${NODEIP}: if (( $? != 0 )) then - echo "Unable to get featurebase binary" + echo "featurebase binary copy failed" exit 1 fi + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase" diff --git a/qa/tf/ci/smoketest/variables.tf b/qa/tf/ci/smoketest/variables.tf index bab877497..578305341 100644 --- a/qa/tf/ci/smoketest/variables.tf +++ b/qa/tf/ci/smoketest/variables.tf @@ -8,11 +8,6 @@ variable "profile" { type = string } -variable "gitlab_token" { - description = "The API token for taking to Gitlab API - expected to come from an env variable." - type = string -} - variable "cluster_prefix" { type = string description = "This is a identifier that will be prefixed to created resources" diff --git a/qa/tf/gauntlet/samsung/variables.tf b/qa/tf/gauntlet/samsung/variables.tf index bab877497..578305341 100644 --- a/qa/tf/gauntlet/samsung/variables.tf +++ b/qa/tf/gauntlet/samsung/variables.tf @@ -8,11 +8,6 @@ variable "profile" { type = string } -variable "gitlab_token" { - description = "The API token for taking to Gitlab API - expected to come from an env variable." - type = string -} - variable "cluster_prefix" { type = string description = "This is a identifier that will be prefixed to created resources" diff --git a/rbf/db.go b/rbf/db.go index f27598467..a6c248e5b 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -704,7 +704,6 @@ func (db *DB) afterCurrentTx(callback func()) { defer db.mu.Unlock() txw.callback() }() - return } // removeTx removes an active transaction from the database. it obtains diff --git a/roaring/filter.go b/roaring/filter.go index 0f080cbc3..513f8e8f0 100644 --- a/roaring/filter.go +++ b/roaring/filter.go @@ -579,7 +579,6 @@ func (b *BitmapRowFilterMultiFilter) ConsiderData(key FilterKey, data *Container // offsets the input bitmap's containers have, it matches them against // corresponding keys. type BitmapBitmapFilter struct { - filter *Bitmap // We don't use this while iterating, but in ludicrous edge cases it might be holding a generation we need. TODO @seebs I don't understand why this mentions generations containers []*Container nextOffsets []uint64 callback func(uint64) error @@ -629,7 +628,6 @@ func (b *BitmapBitmapFilter) ConsiderData(key FilterKey, data *Container) Filter // because offset-within-row is what we care about. func NewBitmapBitmapFilter(filter *Bitmap, callback func(uint64) error) *BitmapBitmapFilter { b := &BitmapBitmapFilter{ - filter: filter, callback: callback, containers: make([]*Container, rowWidth), nextOffsets: make([]uint64, rowWidth), diff --git a/roaring/roaring.go b/roaring/roaring.go index cdcf25b89..f632aaca4 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -167,7 +167,6 @@ type ContainerIterator interface { // Bitmap represents a roaring bitmap. type Bitmap struct { Containers Containers - Source Source // User-defined flags. Flags byte @@ -248,7 +247,6 @@ func (b *Bitmap) Freeze() *Bitmap { // Create a copy of the bitmap structure. other := &Bitmap{ Containers: b.Containers.Freeze(), - Source: b.Source, } return other @@ -609,20 +607,13 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { hi0, hi1 := highbits(start), highbits(end) citer, _ := b.Containers.Iterator(hi0) other := NewSliceBitmap() - mappedAny := false for citer.Next() { k, c := citer.Value() if k >= hi1 { break } - if c.Mapped() { - mappedAny = true - } other.Containers.Put(off+(k-hi0), c.Freeze()) } - if b.Source != nil && mappedAny { - other.Source = b.Source - } return other } @@ -661,7 +652,6 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 { // Intersect returns the intersection of b and other. func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { output := NewBitmap() - usedB, usedOther := false, false iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() @@ -676,26 +666,12 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { kj, cj = jiter.Value() } else { // ki == kj newC := intersect(ci, cj) - if newC == ci { - usedB = true - } - if newC == cj { - usedOther = true - } output.Containers.Put(ki, newC) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() } } - switch { - case usedB && usedOther: - output.Source = MergeSources(b.Source, other.Source) - case usedB: - output.Source = b.Source - case usedOther: - output.Source = other.Source - } return output } @@ -1192,43 +1168,26 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) { func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) - usedB, usedOther := false, false i, j := iiter.Next(), jiter.Next() ki, ci := iiter.Value() kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { target.Containers.Put(ki, ci.Freeze()) - usedB = true i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { target.Containers.Put(kj, cj.Freeze()) - usedOther = true j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj newC := union(ci, cj) target.Containers.Put(ki, newC) - if newC == ci { - usedB = true - } - if newC == cj { - usedOther = true - } i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() } } - switch { - case usedB && usedOther: - target.Source = MergeSources(b.Source, other.Source) - case usedB: - target.Source = b.Source - case usedOther: - target.Source = other.Source - } } // unionInPlace stores the union of b and others into b. The others will @@ -1324,14 +1283,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { bitmapIters = make(handledIters, 0, requiredSliceSize) } - var sources []Source - if b.Source != nil { - sources = append(sources, b.Source) - } for _, other := range others { - if other.Source != nil { - sources = append(sources, other.Source) - } otherIter, _ := other.Containers.Iterator(0) if otherIter.Next() { bitmapIters = append(bitmapIters, handledIter{ @@ -1341,8 +1293,6 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { }) } } - // new bitmap might have containers from any of those bitmaps in it - b.Source = MergeSources(sources...) // Loop until we've exhausted every iter. hasNext := true @@ -1505,9 +1455,6 @@ func (b *Bitmap) singleDifference(other *Bitmap) *Bitmap { // Xor returns the bitwise exclusive or of b and other. func (b *Bitmap) Xor(other *Bitmap) *Bitmap { output := NewBitmap() - // Xor can end up with containers from either parent if the other - // had no container or an empty container. - output.Source = MergeSources(b.Source, other.Source) iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) diff --git a/roaring/source.go b/roaring/source.go deleted file mode 100644 index e2be583e2..000000000 --- a/roaring/source.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package roaring - -import ( - "strings" -) - -// A Source represents the source a given bitmap gets its data from, -// such as a memory-mapped file. When combining bitmaps, we might -// track them together in a single combined-source of some sort. -type Source interface { - ID() string - Dead() bool -} - -// MergeSources combines sources. If you have two bitmaps, and you're -// combining them, then the combination's source is a combination of -// those two sources. -func MergeSources(sources ...Source) Source { - sourceCount := 0 - totalCount := 0 - var lastSource Source - for _, s := range sources { - if s == nil { - continue - } - lastSource = s - if s, ok := s.(combinedSource); ok { - sourceCount++ - totalCount += len(s) - } else { - sourceCount++ - totalCount++ - } - } - // if there's no sources (this includes all sources being - // empty combinedSources), we don't have a source. - if totalCount == 0 { - return nil - } - // if there's exactly one source, combined or otherwise, that's - // fine, we'll just return it. - if sourceCount == 1 { - return lastSource - } - // make a new combinedSource, flattening any combinedSources - // already present. - newSources := make([]Source, 0, totalCount) - for _, s := range sources { - if s == nil { - continue - } - if s, ok := s.(combinedSource); ok { - newSources = append(newSources, s...) - } else { - newSources = append(newSources, s) - } - } - return combinedSource(newSources) -} - -// SetSource tells the bitmap what source to associate with new things it -// creates. This is possibly logically incorrect. -func (b *Bitmap) SetSource(s Source) { - b.Source = s -} - -type combinedSource []Source - -func (c combinedSource) ID() string { - ids := make([]string, len(c)) - for i := range c { - ids[i] = c[i].ID() - } - return strings.Join(ids, ",") -} - -func (c combinedSource) Dead() bool { - for i := range c { - if c[i].Dead() { - return true - } - } - return false -} diff --git a/rrtx.go b/rrtx.go deleted file mode 100644 index 0411e9c80..000000000 --- a/rrtx.go +++ /dev/null @@ -1,662 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - - "github.com/molecula/featurebase/v3/roaring" - txkey "github.com/molecula/featurebase/v3/short_txkey" - "github.com/molecula/featurebase/v3/storage" - - "github.com/molecula/featurebase/v3/vprint" - "github.com/pkg/errors" -) - -// RoaringTx represents a fake transaction object for Roaring storage. -type RoaringTx struct { - write bool - Index *Index - Field *Field - fragment *fragment - o Txo - sn int64 // serial number - - done bool - mu sync.Mutex // protect done as it changes state - - w *RoaringWrapper -} - -func (tx *RoaringTx) Type() string { - return RoaringTxn -} - -// based on view.openFragments() -func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err error) { - - shardMap = make(map[uint64]bool) - - path := filepath.Join(optionalViewPath, "fragments") - file, err := os.Open(path) - if os.IsNotExist(err) { - return - } else if err != nil { - return nil, errors.Wrap(err, "opening fragments directory") - } - defer file.Close() - - fis, err := file.Readdir(0) - if err != nil { - return nil, errors.Wrap(err, "reading fragments directory") - } - - for _, fi := range fis { - //vv("rrtx next fi = '%v'", fi.Name()) - if fi.IsDir() { - continue - } - name := fi.Name() - if strings.HasSuffix(name, ".cache") { - continue - } - - // Parse filename into integer. - shard, err := strconv.ParseUint(filepath.Base(name), 10, 64) - if err != nil { - //vv("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name()) - //panic(fmt.Sprintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())) - //tx.Index.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name()) - continue - } - shardMap[shard] = true - } - return -} - -// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE -// the transaction Commits or Rollsback. -func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - b, err := tx.bitmap(index, field, view, shard) - vprint.PanicOn(err) - return b.Iterator() -} - -// ImportRoaringBits return values changed and rowSet will be inaccurate if -// the data []byte is supplied. This mimics the traditional roaring-per-file -// and should be faster. -func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { - f, err := tx.getFragment(index, field, view, shard) - if err != nil { - return 0, nil, err - } - - changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize) - return -} - -func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} - -// Rollback -func (tx *RoaringTx) Rollback() { - tx.w.CleanupTx(tx) -} - -// Commit -func (tx *RoaringTx) Commit() error { - tx.w.CleanupTx(tx) - return nil -} - -func (tx *RoaringTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - return tx.bitmap(index, field, view, shard) -} - -func (tx *RoaringTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return nil, err - } - return b.Containers.Get(key), nil -} - -func (tx *RoaringTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return err - } - b.Containers.Put(key, c) - return nil -} - -func (tx *RoaringTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return err - } - b.Containers.Remove(key) - return nil -} - -func (tx *RoaringTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - //vv("RoaringTx.Add(index='%v', shard='%v') stack=\n%v", index, shard, stack()) - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return 0, err - } - // Note: do not replace b.AddN() with b.DirectAddN(). - // DirectAddN() does not do op-log operations inside roaring, so the - // on-disk representation no longer matches the in-memory operations. - count, err := b.AddN(a...) - return count, err -} - -func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return 0, err - } - return b.RemoveN(a...) -} - -func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return false, err - } - return b.Contains(v), nil -} - -func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return nil, false, errors.Wrap(err, "getting bitmap") - } - //vv("b bitmap back from bitmap(index='%v', field='%v', view='%v', shard='%v')='%#v'", index, field, view, shard, b.Slice()) - citer, found = b.Containers.Iterator(key) - return citer, found, nil -} - -func (tx *RoaringTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return err - } - return b.ForEach(fn) -} - -func (tx *RoaringTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return err - } - return b.ForEachRange(start, end, fn) -} - -func (tx *RoaringTx) Count(index, field, view string, shard uint64) (uint64, error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return 0, err - } - return b.Count(), nil -} - -func (tx *RoaringTx) Max(index, field, view string, shard uint64) (uint64, error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return 0, err - } - return b.Max(), nil -} - -func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return 0, false, err - } - v, ok := b.Min() - return v, ok, nil -} - -func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return 0, err - } - return b.CountRange(start, end), nil -} - -func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return nil, err - } - return b.OffsetRange(offset, start, end), nil -} - -// getFragment is used by IncrementOpN() and by bitmap() -func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) { - - // If a fragment is attached, always use it. Since it was set at Tx creation, - // it is highly likely to be correct. - if tx.fragment != nil { - // but still a basic sanity check. - if tx.fragment.index() != index || - tx.fragment.field() != field || - tx.fragment.view() != view || - tx.fragment.shard != shard { - - // still insist that index and shard match, since that is the current scope of all Tx. - if tx.fragment.index() != index || - tx.fragment.shard != shard { - panic(fmt.Sprintf("different fragment cached vs requested. index='%v', field='%v'; view='%v'; shard='%v'; tx.fragment='%#v'", index, field, view, shard, tx.fragment)) - } - // cannot use this fragment. - tx.fragment = nil - - } else { - return tx.fragment, nil - } - } - - // If a field is attached, start from there. - // Otherwise look up the field from the index. - f := tx.Field - - if f == nil { - // we cannot assume that the tx.Index that we "started" on is the same - // as the index we are being queried; it might be foreign: TestExecutor_ForeignIndex - // So go through the holder - idx := tx.Index.holder.Index(index) - if idx == nil { - // only thing we can try is the cached index, and hope we aren't being asked for a foreign index. - f = tx.Index.Field(field) - if f == nil { - return nil, newNotFoundError(ErrFieldNotFound, field) - } - } else { - if f = idx.Field(field); f == nil { - return nil, newNotFoundError(ErrFieldNotFound, field) - } - } - } - // INVAR: f is not nil. - - v := f.view(view) - if v == nil { - return nil, errors.Wrapf(ViewNotFound, "getting %s", view) - } - - frag := v.Fragment(shard) - - if frag == nil { - return nil, errors.Wrapf(FragmentNotFound, "field:%q, view:%q, shard:%d", field, view, shard) - } - - // Note: we cannot cache frag into tx.fragment. - // Empirically, it breaks 245 top-level pilosa tests. - // tx.fragment = frag // breaks the world. - - return frag, nil -} - -const ViewNotFound = Error("view not found") -const FragmentNotFound = Error("fragment not found") - -func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - frag, err := tx.getFragment(index, field, view, shard) - if err != nil { - return nil, errors.Wrap(err, "getFragment") - } - return frag.storage, nil -} - -func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) { - vs = NewFieldView2Shards() - - // A) open the index directory - f, err := os.Open(idx.FieldsPath()) - if err != nil { - return nil, errors.Wrap(err, "opening directory") - } - defer f.Close() - - fieldFIs, err := f.Readdir(0) - if err != nil { - return nil, errors.Wrap(err, "reading directory") - } - - //vv("roaringGetFieldView2Shards A) opened index path '%v'", idx.path) - - // B) read the name of each field under the index - for _, loopFieldFi := range fieldFIs { - fieldFI := loopFieldFi - if !fieldFI.IsDir() { - continue - } - field := fieldFI.Name() - - //vv("roaringGetFieldView2Shards B) on field '%v'", field) - - fieldPath := filepath.Join(idx.FieldsPath(), field) - - viewsDir := filepath.Join(fieldPath, "views") - file, err := os.Open(viewsDir) - if os.IsNotExist(err) { - //return nil - continue - } else if err != nil { - return nil, errors.Wrapf(err, "opening view directory '%v'", viewsDir) - } - defer file.Close() - - // C) read the name of each view under the field - - viewFIs, err := file.Readdir(0) - if err != nil { - return nil, errors.Wrapf(err, "reading views directory '%v'", viewsDir) - } - for _, viewFI := range viewFIs { - - if !viewFI.IsDir() { - continue - } - view := viewFI.Name() - roaringViewPath := filepath.Join(viewsDir, view) - - shardMap, err := roaringMapOfShards(roaringViewPath) - if err != nil { - return nil, errors.Wrapf(err, "reading view path directory '%v'", roaringViewPath) - } - if len(shardMap) == 0 { - //vv("roaringGetFieldView2Shards C) SAVED SPACE! field '%v' view '%v' had no shards", field, view) - continue - } - - ss := newShardSetFromMap(shardMap) - fv := txkey.FieldView{Field: field, View: view} - vs.addViewShardSet(fv, ss) - - //vv("roaringGetFieldView2Shards C) added field '%v' view '%v' with shards '%#v'", field, view, ss.shards) - } - } - return -} - -// inefficient for roaring. Instead use the roaringGetFieldView2Shards() above. -func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { - - // A) open the index directory - f, err := os.Open(idx.FieldsPath()) - if err != nil { - return nil, errors.Wrap(err, "opening directory") - } - defer f.Close() - - fieldFIs, err := f.Readdir(0) - if err != nil { - return nil, errors.Wrap(err, "reading directory") - } - - //vv("A) shard %v, opened index path '%v'", shard, idx.path) - - // B) read the name of each field under the index - for _, loopFieldFi := range fieldFIs { - fieldFI := loopFieldFi - if !fieldFI.IsDir() { - continue - } - field := fieldFI.Name() - - //vv("B) on field '%v'", field) - - fieldPath := filepath.Join(idx.FieldsPath(), field) - - viewsDir := filepath.Join(fieldPath, "views") - file, err := os.Open(viewsDir) - if os.IsNotExist(err) { - //return nil - continue - } else if err != nil { - return nil, errors.Wrapf(err, "opening view directory '%v'", viewsDir) - } - defer file.Close() - - // C) read the name of each view under the field - - viewFIs, err := file.Readdir(0) - if err != nil { - return nil, errors.Wrapf(err, "reading views directory '%v'", viewsDir) - } - for _, viewFI := range viewFIs { - - if !viewFI.IsDir() { - continue - } - view := viewFI.Name() - roaringViewPath := filepath.Join(viewsDir, view) - - shardMap, err := roaringMapOfShards(roaringViewPath) - if err != nil { - return nil, errors.Wrapf(err, "reading view path directory '%v'", roaringViewPath) - } - if len(shardMap) == 0 { - continue - } - - // once we know we have data for this shard! - if shardMap[shard] { - fv := txkey.FieldView{Field: field, View: view} - //vv("C) adding fv '%#v'", fv) - fvs = append(fvs, fv) - } - } - } - // directory stuff isn't returned in sorted order, we must sort. - sort.Slice(fvs, func(i, j int) bool { - if fvs[i].Field < fvs[j].Field { - return true - } - if fvs[i].Field > fvs[j].Field { - return false - } - return fvs[i].View < fvs[j].View - }) - return -} - -func (tx *RoaringTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} - -//////// registrar and wrapper machinery - -// roaringRegistrar mirrors the machinery expected -// for all backends for the roaring files approach. -// -type roaringRegistrar struct { - mu sync.Mutex - mp map[*RoaringWrapper]bool - - path2db map[string]*RoaringWrapper -} - -func (r *roaringRegistrar) Size() int { - r.mu.Lock() - defer r.mu.Unlock() - nmp := len(r.mp) - npa := len(r.path2db) - if nmp != npa { - panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) - } - return nmp -} - -var globalRoaringReg *roaringRegistrar = newRoaringRegistrar() - -func newRoaringRegistrar() *roaringRegistrar { - return &roaringRegistrar{ - mp: make(map[*RoaringWrapper]bool), - path2db: make(map[string]*RoaringWrapper), - } -} - -func (r *roaringRegistrar) unprotectedRegister(w *RoaringWrapper) { - r.mp[w] = true - r.path2db[w.path] = w -} - -// unregister removes w from r -func (r *roaringRegistrar) unregister(w *RoaringWrapper) { - r.mu.Lock() - delete(r.mp, w) - delete(r.path2db, w.path) - r.mu.Unlock() -} - -// openRoaringDB will check the registry and make a new instance only -// if one does not exist for its path0. Otherwise it returns -// the existing instance. -func (r *roaringRegistrar) OpenDBWrapper(path string, doAllocZero bool, _ *storage.Config) (DBWrapper, error) { - r.mu.Lock() - defer r.mu.Unlock() - w, ok := r.path2db[path] - if ok { - return w, nil - } - // otherwise, make a new roaring and store it in globalRoaringReg - w = &RoaringWrapper{ - reg: r, - path: path, - } - r.unprotectedRegister(w) - - return w, nil -} - -func (w *RoaringWrapper) SetHolder(h *Holder) { - w.h = h -} - -func (w *RoaringWrapper) Path() string { - return w.path -} - -func (w *RoaringWrapper) HasData() (has bool, err error) { - return w.h.HasRoaringData() -} - -func (w *RoaringWrapper) CleanupTx(tx Tx) { - r := tx.(*RoaringTx) - r.mu.Lock() - defer r.mu.Unlock() - if r.done { - return - } - r.done = true -} - -func (w *RoaringWrapper) OpenListString() (r string) { - return "RoaringWrapper.OpenListString() not yet implemented" -} - -func (w *RoaringWrapper) CloseDB() error { - return errors.New("CloseDB not supported in roaring") -} -func (w *RoaringWrapper) OpenDB() error { - return errors.New("OpenDB not supported in roaring") -} - -// statically confirm that RoaringTx satisfies the Tx interface. -var _ Tx = (*RoaringTx)(nil) - -// RoaringWrapper provides the NewTx() method. -type RoaringWrapper struct { - muDb sync.Mutex - - path string - - h *Holder - - reg *roaringRegistrar - - // make RoaringWrapper.Close() idempotent, avoiding panic on double Close() - closed bool -} - -var globalNextTxSnRoaring int64 - -func (w *RoaringWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - sn := atomic.AddInt64(&globalNextTxSnRoaring, 1) - return &RoaringTx{ - write: o.Write, - Field: o.Field, - Index: o.Index, - fragment: o.Fragment, - o: o, - sn: sn, - w: w, - }, nil -} - -// Close shuts down the Roaring database. -func (w *RoaringWrapper) Close() (err error) { - w.muDb.Lock() - defer w.muDb.Unlock() - if !w.closed { - w.reg.unregister(w) - w.closed = true - } - return nil -} - -func (w *RoaringWrapper) IsClosed() (closed bool) { - w.muDb.Lock() - closed = w.closed - w.muDb.Unlock() - return -} - -func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error { - //vv("RoaringWrapper.DeleteField(index = '%v', field = '%v', fieldPath = '%v'", index, field, fieldPath) - - // match txn sn count vs lmdb/etc. - atomic.AddInt64(&globalNextTxSnRoaring, 1) - - err := os.RemoveAll(fieldPath) - if err != nil { - return errors.Wrap(err, "removing directory") - } - return nil -} - -func (w *RoaringWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - - // match txn sn count vs lmdb/etc. - atomic.AddInt64(&globalNextTxSnRoaring, 1) - - fragment, ok := frag.(*fragment) - if !ok { - return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag) - } - - // Delete fragment file. - if err := os.Remove(fragment.path()); err != nil { - return errors.Wrap(err, "deleting fragment file") - } - - // Delete fragment cache file. - if err := os.Remove(fragment.cachePath()); err != nil { - return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard)) - } - return nil -} diff --git a/server.go b/server.go index 076142a9a..3475b670a 100644 --- a/server.go +++ b/server.go @@ -64,11 +64,10 @@ type Server struct { // nolint: maligned schemator disco.Schemator // External - systemInfo SystemInfo - gcNotifier GCNotifier - logger logger.Logger - queryLogger logger.Logger - snapshotQueue SnapshotQueue + systemInfo SystemInfo + gcNotifier GCNotifier + logger logger.Logger + queryLogger logger.Logger nodeID string uri pnet.URI @@ -87,7 +86,7 @@ type Server struct { // nolint: maligned // HolderConfig stashes server options that are really Holder options. holderConfig *HolderConfig - defaultClient InternalClient + defaultClient *InternalClient dataDir string // Threshold for logging long-running queries @@ -194,7 +193,7 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { // OptServerInternalClient is a functional option on Server // used to set the implementation of InternalClient. -func OptServerInternalClient(c InternalClient) ServerOption { +func OptServerInternalClient(c *InternalClient) ServerOption { return func(s *Server) error { s.defaultClient = c s.cluster.InternalClient = c @@ -406,7 +405,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { cluster: cluster, diagnostics: newDiagnosticsCollector(defaultDiagnosticServer), systemInfo: newNopSystemInfo(), - defaultClient: nopInternalClient{}, + defaultClient: &InternalClient{}, // TODO may need to make this a valid thing gcNotifier: NopGCNotifier, @@ -512,7 +511,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { return s, nil } -func (s *Server) InternalClient() InternalClient { +func (s *Server) InternalClient() *InternalClient { return s.defaultClient } @@ -544,13 +543,6 @@ func (s *Server) UpAndDown() error { func (s *Server) Open() error { s.logger.Infof("open server. PID %v", os.Getpid()) - if s.holder.NeedsSnapshot() { - // Start background monitoring. - s.snapshotQueue = newSnapshotQueue(10, 2, s.logger) - } else { - s.snapshotQueue = defaultSnapshotQueue //TODO (twg) rethink this - } - // Log startup err := s.holder.logStartup() if err != nil { @@ -612,7 +604,6 @@ func (s *Server) Open() error { return errors.Wrap(err, "opening Holder") } // bring up the background tasks for the holder. - s.holder.SnapshotQueue = s.snapshotQueue s.holder.Activate() // if we joined existing cluster then broadcast "resize on add" message if initState == disco.InitialClusterStateExisting { @@ -743,11 +734,6 @@ func (s *Server) Close() error { if s.holder != nil { errh = s.holder.Close() } - if s.snapshotQueue != nil { - s.holder.SnapshotQueue = nil - s.snapshotQueue.Stop() - s.snapshotQueue = nil - } // prefer to return holder error over cluster // error. This order is somewhat arbitrary. It would be better if we had diff --git a/server/grpc.go b/server/grpc.go index bd72dda9e..c05cd5561 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -1598,6 +1598,12 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { return nil, err } LogQuery(ctx, info.FullMethod, req, server.logger) + + // reset the molecula-chip cookie just in case the token was refreshed + md, ok := metadata.FromIncomingContext(ctx) + if uinfo, yeah := ctx.Value("userinfo").(*authn.UserInfo); ok && yeah { + server.auth.SetGRPCMetadata(ctx, md, uinfo.Token) + } return handler(ctx, req) }, )) @@ -1607,6 +1613,11 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { if err != nil { return err } + // reset the molecula-chip cookie just in case the token was refreshed + md, ok := metadata.FromIncomingContext(ctx) + if uinfo, yeah := ctx.Value("userinfo").(*authn.UserInfo); ok && yeah { + server.auth.SetGRPCMetadata(ctx, md, uinfo.Token) + } return handler(srv, &wrappedStream{ss, ctx}) }, )) @@ -1645,7 +1656,7 @@ func LogQuery(ctx context.Context, method string, req interface{}, logger logger } switch r := req.(type) { case *pb.QueryPQLRequest: - logger.Infof("GRPC: %v, %v, %v, %v, %v, %s", ip, ua, method, uinfo.UserID, uinfo.UserName, r.Pql) + logger.Infof("GRPC: %v, %v, %v, %v, %v, [%s]%s", ip, ua, method, uinfo.UserID, uinfo.UserName, r.Index, r.Pql) case *pb.QuerySQLRequest: logger.Infof("GRPC: %v, %v, %v, %v, %v, %s", ip, ua, method, uinfo.UserID, uinfo.UserName, r.Sql) default: @@ -1697,7 +1708,7 @@ func Valid(ctx context.Context, auth *authn.Auth) (context.Context, error) { } token := strings.TrimPrefix(authorization[0], "Bearer ") - uinfo, err := auth.Authenticate(token) + uinfo, err := auth.Authenticate(ctx, token) if err != nil { return ctx, status.Errorf(codes.Unauthenticated, err.Error()) } diff --git a/server/grpc_test.go b/server/grpc_test.go index 376b44ff2..e2ecae03f 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -1055,15 +1055,13 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` // make a valid token tkn := jwt.New(jwt.SigningMethodHS256) claims := tkn.Claims.(jwt.MapClaims) - groupString, _ := authn.ToGob64(groups) - claims["molecula-idp-groups"] = groupString claims["oid"] = "42" claims["name"] = name secretKey, _ := hex.DecodeString(auth.SecretKey) validToken, err := tkn.SignedString(secretKey) if err != nil { - panic(err) + t.Fatalf("unexpected error creating token %v", err) } validToken = "Bearer " + validToken @@ -1453,8 +1451,8 @@ func TestLogQuery(t *testing.T) { }, { name: "QueryPQLReq", - req: &pb.QueryPQLRequest{Pql: "Count(All())"}, - expected: fmt.Sprintf("GRPC: %v, %v, %v, %v, %v, %v\n", "", []string{}, "test!", uinfo.UserID, uinfo.UserName, "Count(All())"), + req: &pb.QueryPQLRequest{Index: "index", Pql: "Count(All())"}, + expected: fmt.Sprintf("GRPC: %v, %v, %v, %v, %v, %v\n", "", []string{}, "test!", uinfo.UserID, uinfo.UserName, "[index]Count(All())"), }, } for _, test := range cases { diff --git a/server/handler_test.go b/server/handler_test.go index 48dddde78..15712f414 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -21,7 +21,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" "github.com/molecula/featurebase/v3/encoding/proto" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" @@ -31,7 +30,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() cmd := cluster.GetNode(0) - h := cmd.Handler.(*http.Handler).Handler + h := cmd.Handler.(*pilosa.Handler).Handler t.Run("PostSchema", func(t *testing.T) { w := httptest.NewRecorder() @@ -70,7 +69,7 @@ func TestHandler_Endpoints(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() cmd := cluster.GetNode(0) - h := cmd.Handler.(*http.Handler).Handler + h := cmd.Handler.(*pilosa.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -1120,7 +1119,7 @@ func TestHandler_Endpoints(t *testing.T) { clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) defer clus.Close() w = httptest.NewRecorder() - h1 := clus.GetNode(0).Handler.(*http.Handler).Handler + h1 := clus.GetNode(0).Handler.(*pilosa.Handler).Handler h1.ServeHTTP(w, req) result = w.Result() @@ -1383,7 +1382,7 @@ func TestHandler_Endpoints(t *testing.T) { clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) defer clus.Close() w = httptest.NewRecorder() - h := clus.GetNode(0).Handler.(*http.Handler).Handler + h := clus.GetNode(0).Handler.(*pilosa.Handler).Handler h.ServeHTTP(w, req) result = w.Result() @@ -1402,11 +1401,11 @@ func TestCluster_TranslateStore(t *testing.T) { cluster.Nodes[0] = test.NewCommandNode(t, server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), ), ) - if err := cluster.GetIdleNode(0).Start(); err != nil { + if err := cluster.Start(); err != nil { t.Fatalf("starting node 0: %v", err) } defer cluster.GetIdleNode(0).Close() // nolint: errcheck @@ -1423,7 +1422,7 @@ func TestClusterTranslator(t *testing.T) { []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), )}, []server.CommandOption{ server.OptCommandServerOptions( @@ -1487,7 +1486,7 @@ func TestClusterTranslator(t *testing.T) { // defer cluster.Close() // cmd := cluster.GetNode(0) -// h := cmd.Handler.(*http.Handler).Handler +// h := cmd.Handler.(*pilosa.Handler).Handler // w := httptest.NewRecorder() diff --git a/server/server.go b/server/server.go index 8212f6816..697816f16 100644 --- a/server/server.go +++ b/server/server.go @@ -36,7 +36,6 @@ import ( petcd "github.com/molecula/featurebase/v3/etcd" "github.com/molecula/featurebase/v3/gcnotify" "github.com/molecula/featurebase/v3/gopsutil" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/prometheus" @@ -74,7 +73,7 @@ type Command struct { logger loggerLogger queryLogger loggerLogger - Handler pilosa.Handler + Handler pilosa.HandlerI grpcServer *grpcServer grpcLn net.Listener API *pilosa.API @@ -405,7 +404,7 @@ func (m *Command) SetupServer() error { // Save listenURI for later reference. m.listenURI = uri - c := http.GetHTTPClient(m.tlsConfig) + c := pilosa.GetHTTPClient(m.tlsConfig) // Get advertise address as uri. advertiseURI, err := pilosa.AddressWithDefaults(m.Config.Advertise) @@ -473,7 +472,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerDiagnosticsInterval(diagnosticsInterval), pilosa.OptServerExecutorPoolSize(m.Config.WorkerPoolSize), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})), pilosa.OptServerOpenIDAllocator(pilosa.OpenIDAllocator), pilosa.OptServerLogger(m.logger), pilosa.OptServerQueryLogger(m.queryLogger), @@ -498,9 +497,9 @@ func (m *Command) SetupServer() error { serverOptions = append(serverOptions, m.serverOptions...) if m.Config.Auth.Enable { - serverOptions = append(serverOptions, pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c, http.WithSecretKey(m.Config.Auth.SecretKey)))) + serverOptions = append(serverOptions, pilosa.OptServerInternalClient(pilosa.NewInternalClientFromURI(uri, c, pilosa.WithSecretKey(m.Config.Auth.SecretKey), pilosa.WithSerializer(proto.Serializer{})))) } else { - serverOptions = append(serverOptions, pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c))) + serverOptions = append(serverOptions, pilosa.OptServerInternalClient(pilosa.NewInternalClientFromURI(uri, c, pilosa.WithSerializer(proto.Serializer{})))) } m.Server, err = pilosa.NewServer(serverOptions...) @@ -548,7 +547,7 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "setting up queryLogger") } - m.queryLogger.Infof("Featurebase Server Started") + m.queryLogger.Infof("Starting Featurebase...") m.queryLogger.Infof("Group with admin level access: %v", p.Admin) m.queryLogger.Infof("Permissions: %+v", p.Permissions) @@ -572,18 +571,23 @@ func (m *Command) SetupServer() error { OptGRPCServerPerm(&p), OptGRPCServerQueryLogger(m.queryLogger), ) + if err != nil { + return errors.Wrap(err, "getting grpcServer") + } - m.Handler, err = http.NewHandler( - http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), - http.OptHandlerAPI(m.API), - http.OptHandlerLogger(m.logger), - http.OptHandlerQueryLogger(m.queryLogger), - http.OptHandlerFileSystem(&statik.FileSystem{}), - http.OptHandlerListener(m.ln, m.Config.Advertise), - http.OptHandlerCloseTimeout(m.closeTimeout), - http.OptHandlerMiddleware(m.grpcServer.middleware(m.Config.Handler.AllowedOrigins)), - http.OptHandlerAuthN(m.auth), - http.OptHandlerAuthZ(&p), + m.Handler, err = pilosa.NewHandler( + pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), + pilosa.OptHandlerAPI(m.API), + pilosa.OptHandlerLogger(m.logger), + pilosa.OptHandlerQueryLogger(m.queryLogger), + pilosa.OptHandlerFileSystem(&statik.FileSystem{}), + pilosa.OptHandlerListener(m.ln, m.Config.Advertise), + pilosa.OptHandlerCloseTimeout(m.closeTimeout), + pilosa.OptHandlerMiddleware(m.grpcServer.middleware(m.Config.Handler.AllowedOrigins)), + pilosa.OptHandlerAuthN(m.auth), + pilosa.OptHandlerAuthZ(&p), + pilosa.OptHandlerSerializer(proto.Serializer{}), + pilosa.OptHandlerRoaringSerializer(proto.RoaringSerializer), ) return errors.Wrap(err, "new handler") } diff --git a/server/server_test.go b/server/server_test.go index 3dfd92728..c1efbc8a7 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -19,7 +19,7 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/server" @@ -54,7 +54,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) + client, err := pilosa.NewInternalClient(m.API.Node().URI.HostPort(), pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) client.SetInternalAPI(m.API) if err != nil { t.Fatal(err) @@ -904,7 +904,7 @@ func TestQueryingWithQuotesAndStuff(t *testing.T) { m := test.RunCommand(t) defer m.Close() - client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) + client, err := pilosa.NewInternalClient(m.API.Node().URI.HostPort(), pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) client.SetInternalAPI(m.API) if err != nil { t.Fatal(err) diff --git a/server_internal_test.go b/server_internal_test.go index 9859e950d..da6d57578 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -2,7 +2,6 @@ package pilosa import ( - "runtime" "testing" "time" @@ -10,23 +9,6 @@ import ( "github.com/molecula/featurebase/v3/testhook" ) -// Ensure the file handle count is working -func TestCountOpenFiles(t *testing.T) { - roaringOnlyTest(t) - - // Windows is not supported yet - if runtime.GOOS == "windows" { - t.Skip("Skipping unsupported countOpenFiles test on Windows.") - } - count, err := countOpenFiles() - if err != nil { - t.Errorf("countOpenFiles failed: %s", err) - } - if count == 0 { - t.Error("countOpenFiles returned invalid value 0.") - } -} - func TestMonitorAntiEntropyZero(t *testing.T) { td, err := testhook.TempDirInDir(t, *TempDir, "") diff --git a/snapshotqueue.go b/snapshotqueue.go deleted file mode 100644 index ef0bda24c..000000000 --- a/snapshotqueue.go +++ /dev/null @@ -1,495 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa - -import ( - "context" - "fmt" - "io" - "math/bits" - "os" - "sync" - "sync/atomic" - "time" - - "github.com/molecula/featurebase/v3/logger" - "github.com/molecula/featurebase/v3/testhook" - "github.com/pkg/errors" -) - -// snapshotQueue is a thing which can handle enqueuing snapshots. A snapshot -// queue distinguishes between high-priority requests, which get satisfied -// by the next available worker, and regular requests, which get enqueued -// if there's space in the queue, and otherwise dropped. There's also a -// separate background task to scan a holder for fragments which may need -// snapshots, but which is processed only when the queue is empty, and only -// slowly. "Await" awaits an existing snapshot if one is already enqueued. -// "Immediate" tries to do one right away. (If one's already enqueued, this -// can leave it in the queue, which will ignore anything that shows up with -// the request flag cleared.) -// -// Await, Enqueue, and Immediate should be called only with the fragment lock -// held. -// -// If you create a queue, it should get stopped at some point. The -// atomicSnapshotQueue implementation used as defaultSnapshotQueue has -// a Start function which will tell you whether it actually started a -// queue. This logic exists because in a normal server case, you probably -// want the queue to be shut down as part of server shutdown, but if you're -// running cluster tests, you probably want to start and shop the queue as -// part of the test, not stop it when any server terminates. -// -// It's less likely to be desireable to start/stop individual queues, -// because fragments use the defaultSnapshotQueue anyway. This design -// needs revisiting. -type SnapshotQueue interface { - Immediate(*fragment) error - Enqueue(*fragment) - Await(*fragment) error - ScanHolder(*Holder, chan struct{}) - Stop() -} - -// queuelessSnapshotQueue isn't a snapshot queue, but it satisfies the -// interface. -type queuelessSnapshotQueue struct{} - -func (q *queuelessSnapshotQueue) Enqueue(f *fragment) { - // We don't actually try to enqueue the snapshot; it breaks things - // if a snapshot gets caused during a transaction. -} - -func (q *queuelessSnapshotQueue) Await(f *fragment) error { - return nil -} - -func (q *queuelessSnapshotQueue) Immediate(f *fragment) error { - return f.snapshot() -} - -func (q *queuelessSnapshotQueue) ScanHolder(h *Holder, done chan struct{}) { -} - -func (q *queuelessSnapshotQueue) Stop() { -} - -var defaultSnapshotQueue = &queuelessSnapshotQueue{} - -// newSnapshotQueue makes a new snapshot queue, of depth N, with -// w worker threads. -func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue { - ctx, cancel := context.WithCancel(context.Background()) - sq := &prioritySnapshotQueue{ - normal: make(chan snapshotRequest, n), - urgent: make(chan snapshotRequest), - background: make(chan snapshotRequest), - ctx: ctx, - cancel: cancel, - maxOpN: 10000, - logger: l, - } - if sq.logger == nil { - sq.logger = logger.NewStandardLogger(os.Stderr) - } - _ = testhook.Opened(NewAuditor(), sq, nil) - sq.spawnWorkers(w) - return sq -} - -type snapshotRequest struct { - frag *fragment - when time.Time -} - -// prioritySnapshotQueue gives preference to "immediate" requests, and -// dispreference to "background" requests from ScanHolder. It timestamps -// requests, so it can discard a request if the most recent snapshot is -// newer than the request. The snapshotPending flag in the fragment is -// used to track that a given fragment thinks it has been successfully -// enqueued. Background requests are not considered enqueued, since -// they'll never get processed if there's anything else. In normal workloads, -// immediate/urgent snapshots should be rare, but we'll happily drop -// most requests on the floor; the scanner should pick them up once things -// are quiet. -type prioritySnapshotQueue struct { - logger logger.Logger - urgent chan snapshotRequest - normal chan snapshotRequest - background chan snapshotRequest - ctx context.Context - cancel context.CancelFunc - mu sync.RWMutex - scanWG, workerWG sync.WaitGroup - maxOpN int - observedOpN [16]uint32 - stats struct { - enqueued uint32 - skipped uint32 - } - stopped bool -} - -func (sq *prioritySnapshotQueue) spawnWorkers(w int) { - sq.mu.Lock() - defer sq.mu.Unlock() - if sq.ctx.Err() != nil { - sq.logger.Infof("prioritySnapshotQueue worker: already done") - return - } - sq.workerWG.Add(w) - for i := 0; i < w; i++ { - go sq.worker(sq.ctx, sq.urgent, sq.normal, sq.background) - } -} - -func (sq *prioritySnapshotQueue) worker(ctx context.Context, urgent, normal, background chan snapshotRequest) { - defer sq.workerWG.Done() - done := ctx.Done() - ok := true - var req snapshotRequest - for ok { - req.frag = nil - select { - case _, ok = <-done: - case req, ok = <-urgent: - default: - select { - case _, ok = <-done: - case req, ok = <-urgent: - case req, ok = <-normal: - default: - select { - case _, ok = <-done: - case req, ok = <-urgent: - case req, ok = <-normal: - case req, ok = <-background: - } - } - } - if req.frag != nil { - sq.process(req) - } - } -} - -// process actually runs a fragment. it will do this if either the fragment -// has a pending snapshot, or the force flag is set. -func (sq *prioritySnapshotQueue) process(req snapshotRequest) { - f := req.frag - f.mu.Lock() - defer f.mu.Unlock() - if f.snapshotStamp.Before(req.when) { - f.snapshotErr = f.snapshot() - if f.snapshotErr != nil { - fmt.Printf("ERROR: snapshot error: %v\n", f.snapshotErr) - sq.logger.Errorf("snapshot error: %v", f.snapshotErr) - } - f.snapshotPending = false - f.snapshotCond.Broadcast() - } -} - -// Stop shuts down the snapshot queue. It first marks it as done, causing -// the background scanner(s), if any, to shut down, then waits for them, then -// closes and nils the queues. The background scanner has to get stopped -// because otherwise it might try to write to those closed queues. -func (sq *prioritySnapshotQueue) Stop() { - sq.mu.Lock() - defer sq.mu.Unlock() - if sq.stopped { - return - } - sq.stopped = true - sq.cancel() - // scanners need to be done before we close the other channels. - sq.scanWG.Wait() - close(sq.normal) - sq.normal = nil - close(sq.urgent) - sq.urgent = nil - close(sq.background) - sq.background = nil - _ = testhook.Closed(NewAuditor(), sq, nil) - enqueued := atomic.LoadUint32(&sq.stats.enqueued) - skipped := atomic.LoadUint32(&sq.stats.skipped) - if skipped > 0 || enqueued > 1 { - sq.logger.Infof("snapshot queue: enqueued %d, skipped %d\n", sq.stats.enqueued, sq.stats.skipped) - } -} - -// Enqueue tries to add a fragment to the queue, if the fragment is not already -// enqueued. You should hold a lock on the fragment when calling this. -func (sq *prioritySnapshotQueue) Enqueue(f *fragment) { - if f.snapshotPending { - return - } - sq.observeOpN(uint32(f.opN)) - sq.mu.RLock() - defer sq.mu.RUnlock() - if sq.normal == nil { - sq.logger.Infof("requested snapshot after snapshot queue was closed") - return - } - // we have to set this before enqueing, because it's - // otherwise possible that we're at the head of the queue, - // and the recipient gets the fragment before we execute the - // line after the send. - f.snapshotPending = true - // try to enqueue snapshot - select { - case sq.normal <- snapshotRequest{frag: f, when: time.Now()}: - atomic.AddUint32(&sq.stats.enqueued, 1) - return - default: - atomic.AddUint32(&sq.stats.skipped, 1) - f.snapshotPending = false - return - } -} - -// Await returns when f is not pending a snapshot. Call with the fragment lock -// held. Await waits on a condition variable inside f, associated with the -// fragment's lock, so this does not conflict with the lock being used for -// snapshots. -// -// Note that workers don't stop just because the queue's been stopped; only -// the background scanner is stopped. So an Await shouldn't block forever -// even if the queue gets shut down. If you're reading this, possibly that -// analysis is incorrect. -func (sq *prioritySnapshotQueue) Await(f *fragment) (err error) { - for f.snapshotPending { - f.snapshotCond.Wait() - } - err, f.snapshotErr = f.snapshotErr, nil - return err -} - -// Immediate forces an immediate snapshot of the given fragment. Call with -// the fragment locked. If the queue is already closing, the fragment does -// not get snapshotted. -func (sq *prioritySnapshotQueue) Immediate(f *fragment) error { - sq.mu.RLock() - // no deferred unlock, because we want to unlock this before calling Await. - // Not because that needs this lock, but because once we're that far, we - // *don't* need this lock anymore so someone else should have it. - if sq.urgent == nil { - sq.mu.RUnlock() - sq.logger.Errorf("requested immediate snapshot after snapshot queue was closed") - return errors.New("requested immediate snapshot after snapshot queue was closed") - } - f.snapshotPending = true - sq.observeOpN(uint32(f.opN)) - req := snapshotRequest{frag: f, when: time.Now()} - // if the fragment was already in the work queue, it's *possible* - // that the only available worker just picked it off the queue, and - // is now waiting on getting the fragment's lock, so it can run - // a snapshot. So we let go of the lock on the fragment, send the - // request, then request the fragment lock again, because Await will - // be sleeping on the condition variable associated with the lock, - // which means it needs to hold the lock so it can let it go during - // the wait... No, really, this made sense. - f.mu.Unlock() - sq.urgent <- req - sq.mu.RUnlock() - f.mu.Lock() - return sq.Await(f) -} - -// ScanHolder spawns a goroutine which iterates through the holder's -// indexes/fields/views/fragments, looking for fragments which have OpN -// high enough to justify a snapshot but don't seem to have one pending. -// It then dumps these in the low priority background queue. -func (sq *prioritySnapshotQueue) ScanHolder(h *Holder, done chan struct{}) { - sq.mu.Lock() - sq.scanWG.Add(1) - go sq.scanHolderWorker(h, sq.background, done) - sq.mu.Unlock() -} - -// observeOpN reports that a given value of opN was "observed", meaning, -// we encountered a fragment which had that value. This happens for every -// enqueue/immediate, including enqueue attempts which fail to actually -// enter the queue, and it also happens for fragments noticed by the background -// scan but which don't have high enough opN to trigger a snapshot. -func (sq *prioritySnapshotQueue) observeOpN(n uint32) { - // aka "log2(n) + 1", or 0 for n==0 - pow2 := 32 - bits.LeadingZeros32(n) - // 15 == 16384. Our usual fragment maxOpN is 10k, so most fragments - // should end up in the 8k-16k bucket, rather than the 16k+ bucket, - // unless we've got a lot of ingests with large batches going on, - // in which case the 16k bucket will win. - if pow2 > 15 { - pow2 = 15 - } - // store in inverse order so the lowest slot in the array is the - // highest cardinality - atomic.AddUint32(&sq.observedOpN[15-pow2], 1) -} - -// computeMaxOpN tries to pick a reasonable new maxOpN for the background -// scan to use. On a quiet system, we want to gradually lower opN, picking -// the fragments with the highest opN values first, because those offer the -// largest benefit. So, whenever we check a fragment in the background, if we -// *don't* snapshot it, we'll "observe" its OpN value, and then we pick a -// value which picks up at least 1/4 of them. -// -// If there's ingest activity, the Immediate and Enqueue operations will -// "observe" the OpN of fragments submitted to them. This can drive OpN back -// up, if those fragments frequently have very high opN values, which reflects -// the fact that we have enough of that activity that we don't need the -// background scanner adding more. -// -// If we have enough ingest activity that the background scanner never actually -// gets to submit work, we'll rarely get here, because the background scanner -// will block until there's no snapshots pending for the normal workload. -// When we do, we'll probably pick a MaxOpN which is dominated by the ingest -// workload's opN values. So for instance, if everything coming in from the -// ingest workload has 10k or more items, because that's the default fragment -// maxOpN, that will probably set the background snapshot queue value to 8k. -func (sq *prioritySnapshotQueue) computeMaxOpN() { - sq.logger.Debugf("observedOpN by power of 2: %d\n", sq.observedOpN[:]) - total := uint32(0) - for i := range sq.observedOpN { - total += atomic.LoadUint32(&sq.observedOpN[i]) - } - target := (total / 4) + 1 - subTotal := uint32(0) - for i := range sq.observedOpN { - v := atomic.LoadUint32(&sq.observedOpN[i]) - subTotal += v - if subTotal >= target { - prevMaxOpN := sq.maxOpN - sq.maxOpN = (1 << (15 - uint(i))) / 2 - if sq.maxOpN > 0 { - sq.maxOpN-- - } - if prevMaxOpN != sq.maxOpN { - sq.logger.Infof("background scan: %d/%d fragments considered have opN %d or higher\n", - subTotal, total, sq.maxOpN) - } - break - } - } - // It's conceptually possible that we'll miss a couple of observations - // here but that's not really important. This is all pretty approximate. - for i := range sq.observedOpN { - atomic.StoreUint32(&sq.observedOpN[i], 0) - } -} - -// prioritySnapshotQueueScanner is the data type that implements HolderOperator -// and represents a single scan of a holder, with a given maxOpN. -type prioritySnapshotQueueScanner struct { - HolderFilterAll - HolderProcessNone - sq *prioritySnapshotQueue - holder *Holder - queue chan snapshotRequest - ctx context.Context - maxOpN int - seen, hits, counter int -} - -func (s *prioritySnapshotQueueScanner) ProcessFragment(f *fragment) error { - if f == nil { - return nil - } - s.seen++ - // we can't defer this reasonably, because otherwise we'll keep - // the fragment locked forever if we end up trying to send it - // to the queue, but the workers are busy on other fragments. - f.mu.Lock() - open := f.open - snapshotPending, opN := f.snapshotPending, f.opN - f.mu.Unlock() - - // a pending snapshot is one that is either in the normal or - // immediate queue, or is trying to get into the normal queue - // and about to fail, but either way, it already got observed - // there, so we don't need to observe it here. A closed fragment - // doesn't matter to us -- it should be a transient state that - // happens during a shutdown, or shouldn't happen, but we don't - // care about it. - if snapshotPending || !open { - return nil - } - if opN <= s.maxOpN { - // observe the value but don't do a snapshot - s.sq.observeOpN(uint32(opN)) - s.counter++ - if s.counter == 1000 { - select { - case <-time.After(1 * time.Second): - case <-s.ctx.Done(): - return io.EOF - } - s.counter = 0 - } - return nil - } - // we don't observe values when we decide to trigger a snapshot, - // because those values will be changing anyway. we could also - // observe them as zero, but that's also sort of wrong. - s.hits++ - select { - case s.queue <- snapshotRequest{frag: f, when: time.Now()}: - s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path()) - case <-s.ctx.Done(): - return io.EOF - } - return nil - -} - -func contextMergedWithStructChan(ctx context.Context, ch chan struct{}) (context.Context, context.CancelFunc) { - canCancel, cancel := context.WithCancel(ctx) - go func() { - select { - case <-ctx.Done(): - cancel() - case <-ch: - cancel() - case <-canCancel.Done(): - // don't need to cancel, but do need to exit this - // function - } - }() - return canCancel, cancel -} - -// scanHolderWorker is a background task that scans a holder looking for -// fragments which need snapshots taken. It's the cleanup task for snapshots -// that would have been requested by Enqueue, but the queue was full. -func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) { - defer sq.scanWG.Done() - ctx, cancel := contextMergedWithStructChan(sq.ctx, done) - defer cancel() - scanner := &prioritySnapshotQueueScanner{ - sq: sq, - holder: h, - queue: background, - ctx: sq.ctx, - maxOpN: sq.maxOpN, - } - for { - err := h.Process(ctx, scanner) - if err != nil { - return - } - - if scanner.hits > 0 { - sq.logger.Infof("background scan: %d/%d fragments needed snapshots\n", scanner.hits, scanner.seen) - scanner.hits = 0 - } else { - sq.logger.Debugf("background scan: no fragments needed snapshots, waiting\n") - // No reason to be active if we're not finding anything. - select { - case <-time.After(60 * time.Second): - case <-ctx.Done(): - return - } - } - scanner.seen = 0 - sq.computeMaxOpN() - scanner.maxOpN = sq.maxOpN - } -} diff --git a/stats/stats_test.go b/stats/stats_test.go index b2867d184..81b62a5f2 100644 --- a/stats/stats_test.go +++ b/stats/stats_test.go @@ -9,8 +9,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/test" @@ -143,7 +142,7 @@ func TestStatsCount_APICalls(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() cmd := cluster.GetNode(0) - h := cmd.Handler.(*http.Handler).Handler + h := cmd.Handler.(*pilosa.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/stattx.go b/stattx.go index 8b34ec582..4780f70bc 100644 --- a/stattx.go +++ b/stattx.go @@ -12,6 +12,7 @@ import ( "github.com/molecula/featurebase/v3/debugstats" "github.com/molecula/featurebase/v3/roaring" txkey "github.com/molecula/featurebase/v3/short_txkey" + "github.com/molecula/featurebase/v3/storage" "github.com/molecula/featurebase/v3/vprint" ) @@ -56,7 +57,7 @@ func (w *callStats) reset() { } func (c *callStats) report() (r string) { - backend := CurrentBackend() + backend := storage.DefaultBackend r = fmt.Sprintf("callStats: (%v)\n", backend) c.mu.Lock() defer c.mu.Unlock() diff --git a/storage/config.go b/storage/config.go index f1307943e..efb44d9d7 100644 --- a/storage/config.go +++ b/storage/config.go @@ -3,9 +3,7 @@ package storage // public strings that pilosa/server/config.go can reference const ( - RoaringBackend string = "roaring" - RBFBackend string = "rbf" - BoltBackend string = "bolt" + RBFBackend string = "rbf" ) // DefaultBackend is set here. pilosa/server/config.go references it diff --git a/test/cluster.go b/test/cluster.go index 5aea1147f..d66da2c7b 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -593,7 +593,7 @@ func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption { pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore), pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond), pilosa.OptServerStorageConfig(&storage.Config{ - Backend: pilosa.CurrentBackendOrDefault(), + Backend: storage.DefaultBackend, FsyncEnabled: false, }), ), diff --git a/test/pilosa.go b/test/pilosa.go index 83041998c..d6663afd8 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -16,7 +16,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/disco" "github.com/molecula/featurebase/v3/encoding/proto" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/testhook" ) @@ -165,8 +164,8 @@ func (m *Command) IsPrimary() bool { } // Client returns a client to connect to the program. -func (m *Command) Client() *http.InternalClient { - return m.Server.InternalClient().(*http.InternalClient) +func (m *Command) Client() *pilosa.InternalClient { + return m.Server.InternalClient() } // Query executes a query against the program through the HTTP API. diff --git a/tournament.sh b/tournament.sh deleted file mode 100755 index 1729ef167..000000000 --- a/tournament.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -## tournament.sh runs a sequence of duels between greens and blues. -## Each test run changes the PILOSA_STORAGE_BACKEND and runs either -## one or two backends through the rigors of make testv-race. -## logs are saved to the tourna.log.${i} files. - -for i in rbf roaring bolt rbf_roaring roaring_rbf roaring_bolt; do - echo "$(date) starting ${i}, output to tourna.log.${i}" - echo "***=== ${i} ====================*** $(date)" &> tourna.log.${i} - PILOSA_STORAGE_BACKEND=${i} make testv-race 2>&1 > tourna.log.${i} -done - diff --git a/translator_test.go b/translator_test.go index 38cc47c51..5df16b4a9 100644 --- a/translator_test.go +++ b/translator_test.go @@ -13,7 +13,6 @@ import ( "github.com/google/go-cmp/cmp" pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/mock" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" @@ -156,25 +155,25 @@ func TestTranslation_KeyNotFound(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node3"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -312,19 +311,19 @@ func TestTranslation_Primary(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -388,25 +387,25 @@ func TestTranslation_TranslateIDsOnCluster(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node3"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() diff --git a/tx_test.go b/tx_test.go index 6f1815c8e..80d72b51c 100644 --- a/tx_test.go +++ b/tx_test.go @@ -4,13 +4,10 @@ package pilosa_test import ( "context" "fmt" - "strings" "testing" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" - "github.com/molecula/featurebase/v3/storage" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck ) @@ -47,21 +44,13 @@ func queryBalances(m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, in return } -func skipForRoaring(t *testing.T) { - src := pilosa.CurrentBackend() - if (storage.DefaultBackend == pilosa.RoaringTxn) || strings.Contains(src, "roaring") { - t.Skip("skip if roaring pseudo-txn involved -- won't show transactional rollback") - } -} - func TestAPI_ImportAtomicRecord(t *testing.T) { - skipForRoaring(t) c := test.MustRunCluster(t, 1, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() diff --git a/txfactory.go b/txfactory.go index c16208532..58134a6bb 100644 --- a/txfactory.go +++ b/txfactory.go @@ -5,8 +5,6 @@ import ( "fmt" "os" "path" - "path/filepath" - "strconv" "strings" "sync" @@ -17,8 +15,7 @@ import ( // public strings that pilosa/server/config.go can reference const ( - RoaringTxn string = "roaring" - RBFTxn string = "rbf" + RBFTxn string = "rbf" ) // DetectMemAccessPastTx true helps us catch places in api and executor @@ -377,9 +374,8 @@ type TxFactory struct { type txtype int const ( - noneTxn txtype = 0 - roaringTxn txtype = 1 // these don't really have any transactions - rbfTxn txtype = 2 + noneTxn txtype = 0 + rbfTxn txtype = 2 ) // DirectoryName just returns a string version of the transaction type. We @@ -388,8 +384,6 @@ const ( // replaced/removed) during that refactor. func (ty txtype) DirectoryName() string { switch ty { - case roaringTxn: - return "roaring" case rbfTxn: return "rbf" } @@ -397,18 +391,12 @@ func (ty txtype) DirectoryName() string { return "" } -func (txf *TxFactory) NeedsSnapshot() (b bool) { - return txf.typ == roaringTxn -} - func MustBackendToTxtype(backend string) (typ txtype) { if strings.Contains(backend, "_") { panic("blue-green comparisons removed") } switch backend { - case RoaringTxn: // "roaring" - return roaringTxn case RBFTxn: // "rbf" return rbfTxn } @@ -839,8 +827,6 @@ func (ty txtype) String() string { switch ty { case noneTxn: return "noneTxn" - case roaringTxn: - return "roaring" case rbfTxn: return "rbf" } @@ -848,73 +834,6 @@ func (ty txtype) String() string { return "" } -// fragmentSpecFromRoaringPath takes a path releative to the -// index directory, not including the name of the index itself. -// The path should not start with the path separator sep ('/' or '\\') rune. -func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, err error) { - if len(path) == 0 { - err = fmt.Errorf("fragmentSpecFromRoaringPath error: path '%v' too short", path) - return - } - if path[:1] == sep { - err = fmt.Errorf("fragmentSpecFromRoaringPath error: path '%v' cannot start with separator '%v'; must be relative to the index base directory", path, sep) - return - } - - // sample path: - // field view shard - // fields/myfield/views/standard/fragments/0 - s := strings.Split(path, "/") - n := len(s) - if n != 6 { - err = fmt.Errorf("len(s)=%v, but expected 5. path='%v'", n, path) - return - } - field = s[1] - view = s[3] - shard, err = strconv.ParseUint(s[5], 10, 64) - if err != nil { - err = fmt.Errorf("fragmentSpecFromRoaringPath(path='%v') could not parse shard '%v' as uint: '%v'", path, s[5], err) - } - return -} - -// listFilesUnderDir returns the paths of files found under directory root. -// If includeRoot is true, it returns the full path, otherwise paths are relative to root. -// If requriedSuffix is supplied, the returned file paths will end in that, -// and any other files found during the walk of the directory tree will be ignored. -// If ignoreEmpty is true, files of size 0 will be excluded. -func listFilesUnderDir(root string, includeRoot bool, requiredSuffix string, ignoreEmpty bool) (files []string, err error) { - if !dirExists(root) { - return nil, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root) - } - n := len(root) + 1 - if includeRoot { - n = 0 - } - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if len(path) < n { - // ignore - } else { - if info == nil { - vprint.PanicOn(fmt.Sprintf("info was nil for path = '%v'", path)) - } - if info.IsDir() { - // skip directories. - } else { - if ignoreEmpty && info.Size() == 0 { - return nil - } - if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) { - files = append(files, path[n:]) - } - } - } - return nil - }) - return -} - func dirExists(name string) bool { fi, err := os.Stat(name) if err != nil { @@ -937,25 +856,13 @@ func fileSize(name string) (int64, error) { var _ = anyGlobalDBWrappersStillOpen // happy linter func anyGlobalDBWrappersStillOpen() bool { - if globalRoaringReg.Size() != 0 { - return true - } - if globalRbfDBReg.Size() != 0 { - return true - } - return false -} - -func (f *TxFactory) hasRoaring() bool { - return f.typ == roaringTxn + return globalRbfDBReg.Size() != 0 } func (f *TxFactory) hasRBF() bool { return f.typ == rbfTxn } -var _ = (&TxFactory{}).hasRoaring // happy linter - func (f *TxFactory) GetDBShardPath(index string, shard uint64, idx *Index, ty txtype, write bool) (shardPath string, err error) { dbs, err := f.dbPerShard.GetDBShard(index, shard, idx) if err != nil { diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index 46f8c918b..b32887605 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -8,8 +8,8 @@ import ( func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) { // txtype.String() method MUST return strings that match // our const definitions at the top of txfactory.go. - check := []txtype{roaringTxn, rbfTxn} - expect := []string{RoaringTxn, RBFTxn} + check := []txtype{rbfTxn} + expect := []string{RBFTxn} for i, chk := range check { obs := chk.String() if obs != expect[i] { diff --git a/util.go b/util.go index ad7397688..eb9f958ce 100644 --- a/util.go +++ b/util.go @@ -4,13 +4,8 @@ package pilosa // util.go: a place for generic, reusable utilities. import ( - "os" "reflect" - "syscall" "time" - - "github.com/molecula/featurebase/v3/roaring" - "github.com/pkg/errors" ) // LeftShifted16MaxContainerKey is 0xffffffffffff0000. It is similar @@ -43,65 +38,6 @@ func NilInside(iface interface{}) bool { func highbits(v uint64) uint64 { return v >> 16 } func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } -// called by Holder.hasRoaringData() -func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) { - - var info roaring.BitmapInfo - _ = info - var f *os.File - f, err = os.Open(path) - if err != nil { - return - } - - var fi os.FileInfo - fi, err = f.Stat() - if err != nil { - return - } - - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - err = errors.Wrap(err, "mmapping") - return - } - defer func() { - err = syscall.Munmap(data) - if err != nil { - err = errors.Wrap(err, "roaringFragmentHasData: munmap failed") - } - err = f.Close() - if err != nil { - err = errors.Wrap(err, "roaringFragmentHasData f.Close() in defer") - } - }() - - // Attach the mmap file to the bitmap. - var rbm *roaring.Bitmap - rbm, _, err = roaring.InspectBinary(data, true, &info) - if err != nil { - err = errors.Wrap(err, "inspecting") - return - } - - if info.ContainerCount > 0 { - return true, nil - } - if info.Ops > 0 { - return true, nil - } - - citer, found := rbm.Containers.Iterator(0) - _ = found - - for citer.Next() { - return true, nil - } - - return -} - // GetLoopProgress returns the estimated remaining time to iterate through some // items as well as the loop completion percentage with the following // parameters: