Merge branch 'master' into cicd-smoketest

This commit is contained in:
pokeeffe-molecula 2022-01-07 11:58:10 -06:00
commit eb60fb301f
12 changed files with 577 additions and 257 deletions

View file

@ -266,6 +266,7 @@ smoke test:
- ./qa/scripts/teardownSmokeTest.sh
needs:
- job: build for linux arm64
allow_failure: true
artifacts:
when: always
paths:
@ -273,49 +274,49 @@ smoke test:
reports:
junit: report.xml
gauntlet:
stage: gauntlet
timeout: 4h
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
PROFILE: "default"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
TF_VAR_cluster_prefix: ""
TF_VAR_branch: ""
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
- aws configure set region "us-east-2"
- aws configure set aws_profile $PROFILE
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq wget
- wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
- tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
- TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
- TF_VAR_branch=$CI_COMMIT_BRANCH
- echo "Branch --> $TF_VAR_branch"
script:
- ./qa/scripts/setupSamsungGauntlet.sh
- ./qa/scripts/testSamsungGauntlet.sh
after_script:
- ./qa/scripts/teardownSamsungGauntlet.sh
# gauntlet:
# stage: gauntlet
# timeout: 4h
# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
# variables:
# PROFILE: "default"
# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
# AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
# TF_VAR_cluster_prefix: ""
# TF_VAR_branch: ""
# rules:
# - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
# before_script:
# - apt-get update && apt-get install -y gnupg software-properties-common curl git
# - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
# - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
# - apt-get update && apt-get install terraform
# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
# - aws configure set region "us-east-2"
# - aws configure set aws_profile $PROFILE
# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
# - chmod 400 gitlab-featurebase-ci.pem
# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
# - eval $(ssh-agent -s)
# - mkdir -p ~/.ssh
# - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
# - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
# - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
# - chmod 700 /root/.ssh
# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
# - apt update && apt -y install jq wget
# - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
# - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
# - export PATH=$PATH:/usr/local/go/bin
# - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
# - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
# - TF_VAR_branch=$CI_COMMIT_BRANCH
# - echo "Branch --> $TF_VAR_branch"
# script:
# - ./qa/scripts/setupSamsungGauntlet.sh
# - ./qa/scripts/testSamsungGauntlet.sh
# after_script:
# - ./qa/scripts/teardownSamsungGauntlet.sh

View file

@ -8,7 +8,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"time"
@ -19,7 +19,7 @@ import (
"golang.org/x/oauth2"
)
// Auth holds state and helper methods needed for authentication
// Auth holds state, configuration, and utilities needed for authentication.
type Auth struct {
logger logger.Logger
cookieName string
@ -29,7 +29,7 @@ type Auth struct {
secure *securecookie.SecureCookie
groupEndpoint string
logoutEndpoint string
fbURL string
fbURL string // fbURL is the domain FB is hosted on, used for post logout redirection
oAuthConfig *oauth2.Config
}
@ -38,7 +38,7 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenUR
auth := &Auth{
logger: logger,
cookieName: "molecula-chip",
refreshWithin: time.Minute * time.Duration(15),
refreshWithin: 15 * time.Minute,
groupEndpoint: groupEndpoint,
logoutEndpoint: logout,
fbURL: url,
@ -67,8 +67,8 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenUR
return auth, nil
}
// CookieValue holds the value of an authenticated user's cookie
type CookieValue struct {
// AuthContext holds the value of an authenticated user's cookie
type AuthContext struct {
UserID string
UserName string
GroupMembership []Group
@ -82,6 +82,11 @@ type Group struct {
GroupName string `json:"displayName"`
}
// Groups holds a slice of Group informations for marshalling from Json
type Groups struct {
Groups []Group `json:"value"`
}
// UserInfo holds user information for an authenticated user
type UserInfo struct {
UserID string `json:"userid"`
@ -116,17 +121,15 @@ func (a *Auth) Authenticate(w http.ResponseWriter, r *http.Request) ([]Group, er
}
// Login redirects a user to login to their configured oAuth login endpoint
// Login redirects a user to login to their configured oAuth authorize endpoint
func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
}
// Logout sets the molecula-chip cookie to an empty cookie and redirects the
// user to a configured "logged out" endpoint
// Logout clears out user cookie and redirects user to IdP's logout endpoint
func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
newCookie := a.getEmptyCookie()
http.SetCookie(w, newCookie)
http.SetCookie(w, a.getEmptyCookie())
redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL)
http.Redirect(w, r, redirect, http.StatusTemporaryRedirect)
}
@ -135,14 +138,16 @@ func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
// the identity provider and sets a secure cookie holding the user information.
func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
code := r.FormValue("code")
token, err := a.getToken(code)
token, err := a.getToken(r, code)
if err != nil {
a.logger.Warnf("getting token from IdP: %+v", err)
http.Error(w, "Bad Request: 400", http.StatusBadRequest)
return
}
cv, err := a.newCookieValue(token)
cv, err := a.newAuthContext(token)
if err != nil || cv == nil {
a.logger.Warnf("creating cookie: %+v", err)
http.Error(w, "Bad Request: 400", http.StatusBadRequest)
return
}
@ -151,37 +156,46 @@ func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}
// GetUserInfo gets and returns user info from a request
// GetUserInfo reads user's cookie and returns their username and userId
func (a *Auth) GetUserInfo(w http.ResponseWriter, r *http.Request) *UserInfo {
var resp UserInfo
cookie, err := a.readCookie(w, r)
if err != nil {
//add logging
a.logger.Warnf("was not able to read cookie for req: %+v", r)
return &resp
}
resp.UserID = cookie.UserID
resp.UserName = cookie.UserName
return &resp
return &UserInfo{
UserID: cookie.UserID,
UserName: cookie.UserName,
}
}
func (a *Auth) getToken(code string) (*oauth2.Token, error) {
token, err := a.oAuthConfig.Exchange(context.Background(), code)
// getToken exhanges authorization code for an oAuth2 token
func (a *Auth) getToken(r *http.Request, code string) (*oauth2.Token, error) {
token, err := a.oAuthConfig.Exchange(r.Context(), code)
if err != nil {
return nil, errors.Wrap(err, "exchanging auth code for token")
}
return token, nil
}
func (a *Auth) newCookieValue(token *oauth2.Token) (*CookieValue, error) {
// newAuthContext parses a jwt `token` and returns relevant information in a cookie value struct
func (a *Auth) newAuthContext(token *oauth2.Token) (*AuthContext, error) {
if token == nil {
return nil, errors.New("baking cookie due to nil token")
}
if token.AccessToken == "" {
return nil, errors.New("no access token provided")
}
accessParsed, err := jwt.Parse(token.AccessToken, nil)
if accessParsed == nil || accessParsed.Claims == nil {
return nil, errors.Wrap(err, "parsing jwt claims from access tokens")
// We are using ParseUnverified here because we're using the OAuth2.0 authZ code flow
// which assumes that the IdP gives good responses. This means that if the IdP is
// insecure, then we are too. But that's the way OAuth works, unfortunately.
// Also, we assume the jwt is not tampered with bc we communicate with the IdP over HTTPS only.
accessParsed, _, err := new(jwt.Parser).ParseUnverified(token.AccessToken, jwt.MapClaims{})
if accessParsed == nil || accessParsed.Claims == nil || err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("%v parsing jwt claims from access tokens", accessParsed))
}
claims := accessParsed.Claims.(jwt.MapClaims)
@ -191,31 +205,31 @@ func (a *Auth) newCookieValue(token *oauth2.Token) (*CookieValue, error) {
}
// not needed at this point in the logic and makes the encoded cookie too large
token.AccessToken = ""
return &CookieValue{
return &AuthContext{
UserID: claims["oid"].(string),
UserName: claims["name"].(string),
GroupMembership: groups,
GroupMembership: groups.Groups,
Token: token,
}, nil
}
func (a *Auth) getGroupMembership(token *oauth2.Token) ([]Group, error) {
var groups []Group
var bearer = fmt.Sprintf("Bearer %s", token.AccessToken)
// getGroupMembership uses a oauth2 token to retrieve group membership information from IdP
func (a *Auth) getGroupMembership(token *oauth2.Token) (Groups, error) {
var groups Groups
req, err := http.NewRequest("GET", a.groupEndpoint, nil)
if err != nil {
return groups, errors.Wrap(err, "creating new request to group endpoint")
}
req.Header.Add("Authorization", bearer)
client := &http.Client{}
response, err := client.Do(req)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken))
response, err := http.DefaultClient.Do(req)
if err != nil {
return groups, errors.Wrap(err, "getting group membership info")
}
defer response.Body.Close()
rawGroups, err := ioutil.ReadAll(response.Body)
rawGroups, err := io.ReadAll(response.Body)
if err != nil {
return groups, errors.Wrap(err, "failed reading group membership response")
}
@ -227,30 +241,30 @@ func (a *Auth) getGroupMembership(token *oauth2.Token) ([]Group, error) {
return groups, nil
}
func (a *Auth) readCookie(w http.ResponseWriter, r *http.Request) (*CookieValue, error) {
// readCookie decodes an encrypted and signed cookie and returns the contained info
func (a *Auth) readCookie(w http.ResponseWriter, r *http.Request) (*AuthContext, error) {
cookie, err := r.Cookie(a.cookieName)
if err != nil {
return nil, errors.Wrap(err, "cookie not found")
}
var value CookieValue
var value AuthContext
err = a.secure.Decode(a.cookieName, cookie.Value, &value)
if err != nil {
newCookie := a.getEmptyCookie()
http.SetCookie(w, newCookie)
http.SetCookie(w, a.getEmptyCookie())
return nil, errors.Wrap(err, "decoding cookie")
}
return &value, nil
}
func (a *Auth) setCookie(w http.ResponseWriter, cookie *CookieValue) error {
func (a *Auth) setCookie(w http.ResponseWriter, cookie *AuthContext) error {
encoded, err := a.secure.Encode(a.cookieName, cookie)
if err != nil {
return errors.Wrap(err, "encoding CookieValue")
return errors.Wrap(err, "encoding AuthContext")
}
newCookie := &http.Cookie{
http.SetCookie(w, &http.Cookie{
Name: a.cookieName,
Value: encoded,
Path: "/",
@ -258,12 +272,11 @@ func (a *Auth) setCookie(w http.ResponseWriter, cookie *CookieValue) error {
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: cookie.Token.Expiry,
}
http.SetCookie(w, newCookie)
})
return nil
}
func (a *Auth) refreshToken(w http.ResponseWriter, cookie *CookieValue) error {
func (a *Auth) refreshToken(w http.ResponseWriter, cookie *AuthContext) error {
if cookie.Token.RefreshToken == "" {
return errors.New("no refresh token found, check auth scopes to see if refresh tokens are being provided by your IdP")
}
@ -274,9 +287,9 @@ func (a *Auth) refreshToken(w http.ResponseWriter, cookie *CookieValue) error {
}
if newToken.Expiry != cookie.Token.Expiry {
cv, err := a.newCookieValue(newToken)
cv, err := a.newAuthContext(newToken)
if err != nil {
errors.Wrap(err, "setting cookie")
return errors.Wrap(err, "creating cookie value from token")
}
a.setCookie(w, cv)

View file

@ -56,7 +56,7 @@ func TestAuth(t *testing.T) {
GroupID: "abcd123-A",
GroupName: "Romantic Painters",
}
validCV := CookieValue{
validCV := AuthContext{
UserID: "snowstorm",
UserName: "J.M.W. Turner",
GroupMembership: []Group{grp},
@ -67,20 +67,22 @@ func TestAuth(t *testing.T) {
w := httptest.NewRecorder()
err := a.setCookie(w, &validCV)
if err != nil {
t.Errorf("expected no errors, got: %v", err)
t.Fatalf("expected no errors, got: %v", err)
}
if w.Result().Cookies()[0].Value == "" {
t.Errorf("expected some value, got: %+v", w.Result().Cookies()[0].Value)
t.Errorf("expected something, got empty string")
}
if w.Result().Cookies()[0].Path != "/" {
t.Errorf("expected path to be /, got: %+v", w.Result().Cookies()[0].Path)
if got, want := w.Result().Cookies()[0].Path, "/"; got != want {
t.Fatalf("path=%s, want %s", got, want)
}
})
t.Run("GetEmptyCookie", func(t *testing.T) {
c := a.getEmptyCookie()
if c.Value != "" {
t.Errorf("expected empty cookie, got: %+v", c.Value)
t.Fatalf("expected empty cookie, got: %+v", c.Value)
}
})
t.Run("KeyLength", func(t *testing.T) {
@ -98,20 +100,20 @@ func TestAuth(t *testing.T) {
ShortKey,
)
if err == nil || !strings.Contains(err.Error(), "decoding block key") {
t.Errorf("expected error decoding block key got: %v", err)
t.Fatalf("expected error decoding block key got: %v", err)
}
})
t.Run("NewCookieValue-BadAccessToken", func(t *testing.T) {
_, err := a.newCookieValue(&tokenAT)
t.Run("NewAuthContext-BadAccessToken", func(t *testing.T) {
_, err := a.newAuthContext(&tokenAT)
if err == nil || !strings.Contains(err.Error(), "jwt claims") {
t.Errorf("expected failure regarding jwt claims, got: %v", err)
t.Fatalf("expected failure regarding jwt claims, got: %v", err)
}
})
t.Run("CookieValue-NoAccessToken", func(t *testing.T) {
_, err := a.newCookieValue(&tokenNoAT)
t.Run("AuthContext-NoAccessToken", func(t *testing.T) {
_, err := a.newAuthContext(&tokenNoAT)
if err == nil || !strings.Contains(err.Error(), "access token") {
t.Errorf("expected failure regarding access token, got: %v", err)
t.Fatalf("expected failure regarding access token, got: %v", err)
}
})

View file

@ -25,8 +25,32 @@ import (
)
type GroupPermissions struct {
Permissions map[string]map[string]string `yaml:"user-groups"`
Admin string `yaml:"admin"`
Permissions map[string]map[string]Permission `yaml:"user-groups"`
Admin string `yaml:"admin"`
}
type Permission string
const (
None Permission = ""
Read Permission = "read"
Write Permission = "write"
Admin Permission = "admin"
)
// Satisfies returns whether `p` satisfies the permissions required by `b`
func (p Permission) Satisfies(b Permission) bool {
switch p {
case "":
return b == ""
case "read":
return b == "" || b == "read"
case "write":
return b == "" || b == "read" || b == "write"
case "admin":
return b == "" || b == "read" || b == "write" || b == "admin"
}
return false
}
func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
@ -44,19 +68,18 @@ func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error)
return
}
func (p *GroupPermissions) GetPermissions(groups []authn.Group, index string) (permission string, errors error) {
func (p *GroupPermissions) GetPermissions(groups []authn.Group, index string) (permission Permission, errors error) {
if admin := p.IsAdmin(groups); admin {
return "admin", nil
return Admin, nil
}
allPermissions := map[string]bool{
"write": false,
"read": false,
allPermissions := map[Permission]bool{
Write: false,
Read: false,
}
if len(groups) == 0 {
return "", fmt.Errorf("user is not part of any groups in identity provider")
return None, fmt.Errorf("user is not part of any groups in identity provider")
}
var groupsDenied []string
@ -65,7 +88,7 @@ func (p *GroupPermissions) GetPermissions(groups []authn.Group, index string) (p
if perm, ok := p.Permissions[group.GroupID][index]; ok {
allPermissions[perm] = true
} else {
return "", fmt.Errorf("user %s does not have permission to index %s", group.UserID, index)
return None, fmt.Errorf("user %s does not have permission to index %s", group.UserID, index)
}
} else {
groupsDenied = append(groupsDenied, group.GroupID)
@ -73,15 +96,15 @@ func (p *GroupPermissions) GetPermissions(groups []authn.Group, index string) (p
}
if len(groupsDenied) == len(groups) {
return "", fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied)
return None, fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied)
}
if allPermissions["write"] {
return "write", nil
} else if allPermissions["read"] {
return "read", nil
if allPermissions[Write] {
return Write, nil
} else if allPermissions[Read] {
return Read, nil
} else {
return "", fmt.Errorf("no permissions found")
return None, fmt.Errorf("no permissions found")
}
}
@ -94,7 +117,7 @@ func (p *GroupPermissions) IsAdmin(groups []authn.Group) bool {
return false
}
func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission string) (indexList []string) {
func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission Permission) (indexList []string) {
// if user is admin, find all indexes in permissions file and return them
if admin := p.IsAdmin(groups); admin {
for groupId := range p.Permissions {
@ -108,9 +131,7 @@ func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredP
for _, group := range groups {
if _, ok := p.Permissions[group.GroupID]; ok {
for index, permission := range p.Permissions[group.GroupID] {
if permission == desiredPermission {
indexList = append(indexList, index)
} else if permission == "write" && desiredPermission == "read" {
if permission >= desiredPermission {
indexList = append(indexList, index)
}
}

View file

@ -40,16 +40,16 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
singlePermission := authz.GroupPermissions{
Permissions: map[string]map[string]string{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": "read"},
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
multiPermission := authz.GroupPermissions{
Permissions: map[string]map[string]string{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": "read", "test2": "write"},
"dca35310-ecda-4f23-86cd-876aee559900": {"test": "write"}},
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read, "test2": authz.Write},
"dca35310-ecda-4f23-86cd-876aee559900": {"test": authz.Write}},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
@ -123,56 +123,56 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
yamlData string
groups []authn.Group
index string
userAccess string
userAccess authz.Permission
err string
}{
{
permissions1,
groupsList1,
"test",
"",
authz.None,
"user is not part of any groups in identity provider",
},
{
permissions1,
groupsList3,
"test1",
"",
authz.None,
"does not have permission to index",
},
{
permissions2,
groupsList2,
"test",
"",
authz.None,
"does not have permission to FeatureBase",
},
{
permissions1,
groupsList3,
"test",
"read",
authz.Read,
"",
},
{
permissions2,
groupsList3,
"test",
"write",
authz.Write,
"",
},
{
permissions3,
groupsList4,
"test",
"admin",
authz.Admin,
"",
},
{
permissions4,
groupsList3,
"test",
"",
authz.None,
"no permissions found",
},
}
@ -214,8 +214,8 @@ func TestAuth_IsAdmin(t *testing.T) {
}
groupPermissions := authz.GroupPermissions{
Permissions: map[string]map[string]string{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": "write"},
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Write},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
@ -259,13 +259,13 @@ func TestAuth_GetAuthorizedIndexList(t *testing.T) {
}
p := authz.GroupPermissions{
Permissions: map[string]map[string]string{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {
"test1": "read",
"test2": "write",
"test1": authz.Read,
"test2": authz.Write,
},
"dca35310-ecda-4f23-86cd-876aee559900": {
"test3": "read",
"test3": authz.Read,
},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
@ -273,32 +273,32 @@ func TestAuth_GetAuthorizedIndexList(t *testing.T) {
tests := []struct {
groups []authn.Group
permission string
permission authz.Permission
output []string
}{
{
group1,
"read",
authz.Read,
[]string{"test1", "test2"},
},
{
group1,
"write",
authz.Write,
[]string{"test2"},
},
{
group3,
"write",
authz.Write,
nil,
},
{
group2,
"read",
authz.Read,
[]string{"test1", "test2", "test3"},
},
{
group2,
"write",
authz.Write,
[]string{"test1", "test2", "test3"},
},
}

View file

@ -121,5 +121,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.StringVar(&srv.Config.Auth.HashKey, "auth.hash-key", srv.Config.Auth.HashKey, "First Secret for Auth.")
flags.StringVar(&srv.Config.Auth.BlockKey, "auth.block-key", srv.Config.Auth.BlockKey, "Second Secret for Auth.")
flags.StringVar(&srv.Config.Auth.PermissionsFile, "auth.permissions", srv.Config.Auth.PermissionsFile, "Permissions' file with group authorization.")
flags.StringVar(&srv.Config.Auth.QueryLogPath, "auth.query-log-path", srv.Config.Auth.QueryLogPath, "Path to log user queries")
}

View file

@ -30,6 +30,7 @@ import (
"github.com/gorilla/mux"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/authn"
"github.com/molecula/featurebase/v2/authz"
"github.com/molecula/featurebase/v2/encoding/proto"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/logger"
@ -52,6 +53,8 @@ type Handler struct {
logger logger.Logger
querylogger logger.Logger
// Keeps the query argument validators for each handler
validators map[string]*queryValidationSpec
@ -70,6 +73,8 @@ type Handler struct {
pprofCPUProfileBuffer *bytes.Buffer
auth *authn.Auth
permissions *authz.GroupPermissions
}
// externalPrefixFlag denotes endpoints that are intended to be exposed to clients.
@ -116,9 +121,16 @@ func OptHandlerAPI(api *pilosa.API) handlerOption {
}
}
func OptHandlerAuth(auth *authn.Auth) handlerOption {
func OptHandlerAuthN(authn *authn.Auth) handlerOption {
return func(h *Handler) error {
h.auth = auth
h.auth = authn
return nil
}
}
func OptHandlerAuthZ(gp *authz.GroupPermissions) handlerOption {
return func(h *Handler) error {
h.permissions = gp
return nil
}
}
@ -137,6 +149,13 @@ func OptHandlerLogger(logger logger.Logger) handlerOption {
}
}
func OptHandlerQueryLogger(logger logger.Logger) handlerOption {
return func(h *Handler) error {
h.querylogger = logger
return nil
}
}
// OptHandlerListener set the listener that will be used by the HTTP server.
// Url must be the advertised URL. It will be used to show a log to the user
// about where the Web UI is. This option is mandatory.
@ -264,8 +283,14 @@ type contextKeyQuery int
const (
contextKeyQueryRequest contextKeyQuery = iota
contextKeyQueryError
contextKeyGroupMembership
contextKeyPermission
)
func GetContextKeyPermission() contextKeyQuery {
return contextKeyPermission
}
// addQueryContext puts the results of handler.readQueryRequest into the Context for use by
// both other middleware and any handlers.
func (h *Handler) addQueryContext(next http.Handler) http.Handler {
@ -368,97 +393,100 @@ var latticeRoutes = []string{"/tables", "/query", "/querybuilder", "/signin"} //
// newRouter creates a new mux http router.
func newRouter(handler *Handler) http.Handler {
router := mux.NewRouter()
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort")
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode")
router.HandleFunc("/cluster/resize/abort", handler.chkAuthZ(handler.handlePostClusterResizeAbort, authz.Admin)).Methods("POST").Name("PostClusterResizeAbort")
router.HandleFunc("/cluster/resize/remove-node", handler.chkAuthZ(handler.handlePostClusterResizeRemoveNode, authz.Admin)).Methods("POST").Name("PostClusterResizeRemoveNode")
// TODO: figure out how to protect these if needed
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler()).Methods("GET")
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
router.Handle("/metrics", promhttp.Handler())
router.HandleFunc("/metrics.json", handler.handleGetMetricsJSON).Methods("GET").Name("GetMetricsJSON")
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
router.HandleFunc("/import-atomic-record", handler.handlePostImportAtomicRecord).Methods("POST").Name("PostImportAtomicRecord")
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes")
router.HandleFunc("/index", handler.handlePostIndex).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/", handler.handlePostIndex).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex")
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex")
//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/field", handler.handlePostField).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField")
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport")
router.HandleFunc("/index/{index}/field/{field}/mutex-check", handler.handleGetMutexCheck).Methods("GET").Name("GetMutexCheck")
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
router.HandleFunc("/schema/details", handler.handleGetSchemaDetails).Methods("GET").Name("GetSchemaDetails")
router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema")
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus")
router.HandleFunc("/transaction", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
router.HandleFunc("/transaction/", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
router.HandleFunc("/transaction/{id}", handler.handleGetTransaction).Methods("GET").Name("GetTransaction")
router.HandleFunc("/transaction/{id}", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
router.HandleFunc("/transaction/{id}/finish", handler.handlePostFinishTransaction).Methods("POST").Name("PostFinishTransaction")
router.HandleFunc("/transactions", handler.handleGetTransactions).Methods("GET").Name("GetTransactions")
router.HandleFunc("/queries", handler.handleGetActiveQueries).Methods("GET").Name("GetActiveQueries")
router.HandleFunc("/query-history", handler.handleGetPastQueries).Methods("GET").Name("GetPastQueries")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
router.HandleFunc("/metrics.json", handler.chkAuthZ(handler.handleGetMetricsJSON, authz.Admin)).Methods("GET").Name("GetMetricsJSON")
router.HandleFunc("/export", handler.chkAuthZ(handler.handleGetExport, authz.Read)).Methods("GET").Name("GetExport")
router.HandleFunc("/import-atomic-record", handler.chkAuthZ(handler.handlePostImportAtomicRecord, authz.Admin)).Methods("POST").Name("PostImportAtomicRecord")
router.HandleFunc("/index", handler.chkAuthZ(handler.handleGetIndexes, authz.Read)).Methods("GET").Name("GetIndexes")
router.HandleFunc("/index", handler.chkAuthZ(handler.handlePostIndex, authz.Admin)).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/", handler.chkAuthZ(handler.handlePostIndex, authz.Admin)).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.chkAuthZ(handler.handleGetIndex, authz.Read)).Methods("GET").Name("GetIndex")
router.HandleFunc("/index/{index}", handler.chkAuthZ(handler.handlePostIndex, authz.Admin)).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.chkAuthZ(handler.handleDeleteIndex, authz.Admin)).Methods("DELETE").Name("DeleteIndex")
//router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handleGetFields, authz.Read)).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handleDeleteField, authz.Write)).Methods("DELETE").Name("DeleteField")
router.HandleFunc("/index/{index}/field/{field}/import", handler.chkAuthZ(handler.handlePostImport, authz.Read)).Methods("POST").Name("PostImport")
router.HandleFunc("/index/{index}/field/{field}/mutex-check", handler.chkAuthZ(handler.handleGetMutexCheck, authz.Read)).Methods("GET").Name("GetMutexCheck")
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.chkAuthZ(handler.handlePostImportRoaring, authz.Read)).Methods("POST").Name("PostImportRoaring")
router.HandleFunc("/index/{index}/query", handler.chkAuthZ(handler.handlePostQuery, authz.Read)).Methods("POST").Name("PostQuery")
router.HandleFunc("/info", handler.chkAuthZ(handler.handleGetInfo, authz.Admin)).Methods("GET").Name("GetInfo")
router.HandleFunc("/recalculate-caches", handler.chkAuthZ(handler.handleRecalculateCaches, authz.Admin)).Methods("POST").Name("RecalculateCaches")
router.HandleFunc("/schema", handler.chkAuthZ(handler.handleGetSchema, authz.Read)).Methods("GET").Name("GetSchema")
router.HandleFunc("/schema/details", handler.chkAuthZ(handler.handleGetSchemaDetails, authz.Read)).Methods("GET").Name("GetSchemaDetails")
router.HandleFunc("/schema", handler.chkAuthZ(handler.handlePostSchema, authz.Admin)).Methods("POST").Name("PostSchema")
router.HandleFunc("/status", handler.chkAuthZ(handler.handleGetStatus, authz.Read)).Methods("GET").Name("GetStatus")
router.HandleFunc("/transaction", handler.chkAuthZ(handler.handlePostTransaction, authz.Read)).Methods("POST").Name("PostTransaction")
router.HandleFunc("/transaction/", handler.chkAuthZ(handler.handlePostTransaction, authz.Read)).Methods("POST").Name("PostTransaction")
router.HandleFunc("/transaction/{id}", handler.chkAuthZ(handler.handleGetTransaction, authz.Read)).Methods("GET").Name("GetTransaction")
router.HandleFunc("/transaction/{id}", handler.chkAuthZ(handler.handlePostTransaction, authz.Read)).Methods("POST").Name("PostTransaction")
router.HandleFunc("/transaction/{id}/finish", handler.chkAuthZ(handler.handlePostFinishTransaction, authz.Read)).Methods("POST").Name("PostFinishTransaction")
router.HandleFunc("/transactions", handler.chkAuthZ(handler.handleGetTransactions, authz.Read)).Methods("GET").Name("GetTransactions")
router.HandleFunc("/queries", handler.chkAuthZ(handler.handleGetActiveQueries, authz.Read)).Methods("GET").Name("GetActiveQueries")
router.HandleFunc("/query-history", handler.chkAuthZ(handler.handleGetPastQueries, authz.Read)).Methods("GET").Name("GetPastQueries")
router.HandleFunc("/version", handler.chkAuthZ(handler.handleGetVersion, authz.Read)).Methods("GET").Name("GetVersion")
// /ui endpoints are for UI use; they may change at any time.
router.HandleFunc("/ui/usage", handler.handleGetUsage).Methods("GET").Name("GetUsage")
router.HandleFunc("/ui/transaction", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList")
router.HandleFunc("/ui/transaction/", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList")
router.HandleFunc("/ui/shard-distribution", handler.handleGetShardDistribution).Methods("GET").Name("GetShardDistribution")
router.HandleFunc("/ui/usage", handler.chkAuthZ(handler.handleGetUsage, authz.Read)).Methods("GET").Name("GetUsage")
router.HandleFunc("/ui/transaction", handler.chkAuthZ(handler.handleGetTransactionList, authz.Read)).Methods("GET").Name("GetTransactionList")
router.HandleFunc("/ui/transaction/", handler.chkAuthZ(handler.handleGetTransactionList, authz.Read)).Methods("GET").Name("GetTransactionList")
router.HandleFunc("/ui/shard-distribution", handler.chkAuthZ(handler.handleGetShardDistribution, authz.Read)).Methods("GET").Name("GetShardDistribution")
// /internal endpoints are for internal use only; they may change at any time.
// DO NOT rely on these for external applications!
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage")
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData")
router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/internal/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/internal/partition/nodes", handler.handleGetPartitionNodes).Methods("GET").Name("GetPartitionNodes")
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
router.HandleFunc("/internal/translate/data", handler.handlePostTranslateData).Methods("POST").Name("PostTranslateData")
router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys")
router.HandleFunc("/internal/translate/ids", handler.handlePostTranslateIDs).Methods("POST").Name("PostTranslateIDs")
router.HandleFunc("/internal/index/{index}/field/{field}/mutex-check", handler.handleInternalGetMutexCheck).Methods("GET").Name("InternalGetMutexCheck")
router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE")
router.HandleFunc("/internal/index/{index}/shard/{shard}/snapshot", handler.handleGetIndexShardSnapshot).Methods("GET").Name("GetIndexShardSnapshot")
router.HandleFunc("/internal/index/{index}/shards", handler.handleGetIndexAvailableShards).Methods("GET").Name("GetIndexAvailableShards")
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/ingest/{index}", handler.handlePostIngestData).Methods("POST").Name("PostIngestData")
router.HandleFunc("/internal/ingest/{index}/node", handler.handlePostIngestNode).Methods("POST").Name("PostIngestNode")
router.HandleFunc("/internal/cluster/message", handler.chkAuthN(handler.handlePostClusterMessage)).Methods("POST").Name("PostClusterMessage")
router.HandleFunc("/internal/fragment/block/data", handler.chkAuthN(handler.handleGetFragmentBlockData)).Methods("GET").Name("GetFragmentBlockData")
router.HandleFunc("/internal/fragment/blocks", handler.chkAuthN(handler.handleGetFragmentBlocks)).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/internal/fragment/data", handler.chkAuthN(handler.handleGetFragmentData)).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/internal/fragment/nodes", handler.chkAuthN(handler.handleGetFragmentNodes)).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/internal/partition/nodes", handler.chkAuthN(handler.handleGetPartitionNodes)).Methods("GET").Name("GetPartitionNodes")
router.HandleFunc("/internal/translate/data", handler.chkAuthN(handler.handleGetTranslateData)).Methods("GET").Name("GetTranslateData")
router.HandleFunc("/internal/translate/data", handler.chkAuthN(handler.handlePostTranslateData)).Methods("POST").Name("PostTranslateData")
router.HandleFunc("/internal/translate/keys", handler.chkAuthN(handler.handlePostTranslateKeys)).Methods("POST").Name("PostTranslateKeys")
router.HandleFunc("/internal/translate/ids", handler.chkAuthN(handler.handlePostTranslateIDs)).Methods("POST").Name("PostTranslateIDs")
router.HandleFunc("/internal/index/{index}/field/{field}/mutex-check", handler.chkAuthN(handler.handleInternalGetMutexCheck)).Methods("GET").Name("InternalGetMutexCheck")
router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.chkAuthN(handler.handleDeleteRemoteAvailableShard)).Methods("DELETE")
router.HandleFunc("/internal/index/{index}/shard/{shard}/snapshot", handler.chkAuthN(handler.handleGetIndexShardSnapshot)).Methods("GET").Name("GetIndexShardSnapshot")
router.HandleFunc("/internal/index/{index}/shards", handler.chkAuthN(handler.handleGetIndexAvailableShards)).Methods("GET").Name("GetIndexAvailableShards")
router.HandleFunc("/internal/nodes", handler.chkAuthN(handler.handleGetNodes)).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.chkAuthN(handler.handleGetShardsMax)).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/ingest/{index}", handler.chkAuthN(handler.handlePostIngestData)).Methods("POST").Name("PostIngestData")
router.HandleFunc("/internal/ingest/{index}/node", handler.chkAuthN(handler.handlePostIngestNode)).Methods("POST").Name("PostIngestNode")
router.HandleFunc("/internal/schema", handler.handleIngestSchema).Methods("POST").Name("PostIngestSchema")
router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.handleFindIndexKeys).Methods("POST").Name("FindIndexKeys")
router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.handleCreateIndexKeys).Methods("POST").Name("CreateIndexKeys")
router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.handlePostTranslateIndexDB).Methods("POST").Name("PostTranslateIndexDB")
router.HandleFunc("/internal/translate/field/{index}/{field}", handler.handlePostTranslateFieldDB).Methods("POST").Name("PostTranslateFieldDB")
router.HandleFunc("/internal/translate/field/{index}/{field}/keys/find", handler.handleFindFieldKeys).Methods("POST").Name("FindFieldKeys")
router.HandleFunc("/internal/translate/field/{index}/{field}/keys/create", handler.handleCreateFieldKeys).Methods("POST").Name("CreateFieldKeys")
router.HandleFunc("/internal/translate/field/{index}/{field}/keys/like", handler.handleMatchField).Methods("POST").Name("MatchFieldKeys")
router.HandleFunc("/internal/schema", handler.chkAuthN(handler.handleIngestSchema)).Methods("POST").Name("PostIngestSchema")
router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.chkAuthN(handler.handleFindIndexKeys)).Methods("POST").Name("FindIndexKeys")
router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.chkAuthN(handler.handleCreateIndexKeys)).Methods("POST").Name("CreateIndexKeys")
router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.chkAuthN(handler.handlePostTranslateIndexDB)).Methods("POST").Name("PostTranslateIndexDB")
router.HandleFunc("/internal/translate/field/{index}/{field}", handler.chkAuthN(handler.handlePostTranslateFieldDB)).Methods("POST").Name("PostTranslateFieldDB")
router.HandleFunc("/internal/translate/field/{index}/{field}/keys/find", handler.chkAuthN(handler.handleFindFieldKeys)).Methods("POST").Name("FindFieldKeys")
router.HandleFunc("/internal/translate/field/{index}/{field}/keys/create", handler.chkAuthN(handler.handleCreateFieldKeys)).Methods("POST").Name("CreateFieldKeys")
router.HandleFunc("/internal/translate/field/{index}/{field}/keys/like", handler.chkAuthN(handler.handleMatchField)).Methods("POST").Name("MatchFieldKeys")
router.HandleFunc("/internal/idalloc/reserve", handler.handleReserveIDs).Methods("POST").Name("ReserveIDs")
router.HandleFunc("/internal/idalloc/commit", handler.handleCommitIDs).Methods("POST").Name("CommitIDs")
router.HandleFunc("/internal/idalloc/restore", handler.handleRestoreIDAlloc).Methods("POST").Name("RestoreIDAllocData")
router.HandleFunc("/internal/idalloc/reset/{index}", handler.handleResetIDAlloc).Methods("POST").Name("ResetIDAlloc")
router.HandleFunc("/internal/idalloc/data", handler.handleIDAllocData).Methods("GET").Name("IDAllocData")
router.HandleFunc("/internal/idalloc/reserve", handler.chkAuthN(handler.handleReserveIDs)).Methods("POST").Name("ReserveIDs")
router.HandleFunc("/internal/idalloc/commit", handler.chkAuthN(handler.handleCommitIDs)).Methods("POST").Name("CommitIDs")
router.HandleFunc("/internal/idalloc/restore", handler.chkAuthN(handler.handleRestoreIDAlloc)).Methods("POST").Name("RestoreIDAllocData")
router.HandleFunc("/internal/idalloc/reset/{index}", handler.chkAuthN(handler.handleResetIDAlloc)).Methods("POST").Name("ResetIDAlloc")
router.HandleFunc("/internal/idalloc/data", handler.chkAuthN(handler.handleIDAllocData)).Methods("GET").Name("IDAllocData")
router.HandleFunc("/internal/restore/{index}/{shardID}", handler.handlePostRestore).Methods("POST").Name("Restore")
router.HandleFunc("/internal/restore/{index}/{shardID}", handler.chkAuthN(handler.handlePostRestore)).Methods("POST").Name("Restore")
router.HandleFunc("/internal/debug/rbf", handler.handleGetInternalDebugRBFJSON).Methods("GET").Name("GetInternalDebugRBFJSON")
router.HandleFunc("/internal/debug/rbf", handler.chkAuthN(handler.handleGetInternalDebugRBFJSON)).Methods("GET").Name("GetInternalDebugRBFJSON")
// endpoints for collecting cpu profiles from a chosen begin point to
// when the client wants to stop. Used for profiling imports that
// could be long or short.
router.HandleFunc("/cpu-profile/start", handler.handleCPUProfileStart).Methods("GET").Name("CPUProfileStart")
router.HandleFunc("/cpu-profile/stop", handler.handleCPUProfileStop).Methods("GET").Name("CPUProfileStop")
router.HandleFunc("/cpu-profile/start", handler.chkAuthZ(handler.handleCPUProfileStart, authz.Admin)).Methods("GET").Name("CPUProfileStart")
router.HandleFunc("/cpu-profile/stop", handler.chkAuthZ(handler.handleCPUProfileStop, authz.Admin)).Methods("GET").Name("CPUProfileStop")
router.HandleFunc("/login", handler.handleLogin).Methods("GET").Name("Login")
router.HandleFunc("/logout", handler.handleLogout).Methods("GET").Name("Logout")
@ -512,6 +540,77 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.Handler.ServeHTTP(w, r)
}
func (h *Handler) chkAuthN(handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if h.auth != nil {
if _, err := h.auth.Authenticate(w, r); err != nil {
http.Error(w, errors.Wrap(err, "authenticating").Error(), http.StatusBadRequest)
return
}
}
handler.ServeHTTP(w, r)
}
}
func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
lperm := perm
if h.auth != nil {
groups, err := h.auth.Authenticate(w, r)
if err != nil {
http.Error(w, errors.Wrap(err, "authenticating").Error(), http.StatusBadRequest)
return
}
if h.permissions == nil {
h.logger.Errorf("authentication is turned on without authorization permissions set")
http.Error(w, "authorizing", http.StatusInternalServerError)
return
}
uinfo := h.auth.GetUserInfo(w, r)
var queryString string
queryRequest := r.Context().Value(contextKeyQueryRequest)
if req, ok := queryRequest.(*pilosa.QueryRequest); ok {
queryString = req.Query
q, err := pql.ParseString(queryString)
if err != nil {
http.Error(w, errors.Wrap(err, "parsing query string").Error(), http.StatusBadRequest)
return
}
if q.WriteCallN() > 0 {
lperm = authz.Write
}
}
queryString = strings.Replace(queryString, "\n", "", -1)
if r.Method == "POST" {
h.querylogger.Infof("User ID: %s, User Name: %s, Endpoint: %s, Index: %s, Query: %s, Err: %v", uinfo.UserID, uinfo.UserName, r.URL.Path, "indexName", queryString, err)
}
ctx := context.WithValue(r.Context(), contextKeyGroupMembership, groups)
indexName, ok := mux.Vars(r)["index"]
if ok {
p, err := h.permissions.GetPermissions(groups, indexName)
ctx = context.WithValue(r.Context(), contextKeyPermission, p)
if err != nil || !p.Satisfies(lperm) {
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusForbidden)
return
}
}
handler.ServeHTTP(w, r.WithContext(ctx))
} else {
handler.ServeHTTP(w, r)
}
}
}
// statikHandler implements the http.Handler interface, and responds to
// requests for static assets with the appropriate file contents embedded
// in a statik filesystem.
@ -680,6 +779,30 @@ func headerAcceptRoaringRow(header http.Header) bool {
return false
}
func (h *Handler) filterResponse(w http.ResponseWriter, r *http.Request, schema []*pilosa.IndexInfo) []*pilosa.IndexInfo {
if h.auth != nil {
g := r.Context().Value(contextKeyGroupMembership)
if g == nil {
http.Error(w, "Forbidden", http.StatusForbidden)
return nil
}
indexes := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read)
var new []*pilosa.IndexInfo
for _, s := range schema {
for _, index := range indexes {
if s.Name == index {
new = append(new, s)
}
}
}
return new
}
return schema
}
// handleGetSchema handles GET /schema requests.
func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
@ -696,6 +819,8 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
h.logger.Printf("getting schema error: %s", err)
}
schema = h.filterResponse(w, r, schema)
if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil {
h.logger.Errorf("write schema response error: %s", err)
}
@ -714,6 +839,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request)
h.logger.Printf("error getting detailed schema: %s", err)
return
}
schema = h.filterResponse(w, r, schema)
if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil {
h.logger.Printf("write schema response error: %s", err)
}
@ -800,6 +926,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
}
func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
@ -2422,6 +2549,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
// Decode request.
var req removeNodeRequest
err := json.NewDecoder(r.Body).Decode(&req)
@ -2459,6 +2587,7 @@ type removeNodeResponse struct {
// handlePostClusterResizeAbort handles POST /cluster/resize/abort request.
func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
@ -3398,9 +3527,7 @@ func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleLogin(w http.ResponseWriter, r *http.Request) {
if h.auth == nil {
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusNoContent)
w.Write([]byte("Auth Off")) //nolint:errcheck
http.Error(w, "", http.StatusNoContent)
return
}
@ -3409,9 +3536,7 @@ func (h *Handler) handleLogin(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleRedirect(w http.ResponseWriter, r *http.Request) {
if h.auth == nil {
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusNoContent)
w.Write([]byte("Auth Off")) //nolint:errcheck
http.Error(w, "", http.StatusNoContent)
return
}
h.auth.Redirect(w, r)
@ -3423,9 +3548,7 @@ func (h *Handler) handleCheckAuthentication(w http.ResponseWriter, r *http.Reque
return
}
if h.auth == nil {
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusNoContent)
w.Write([]byte("Auth Off")) //nolint:errcheck
http.Error(w, "", http.StatusNoContent)
return
}
groups, err := h.auth.Authenticate(w, r)
@ -3446,9 +3569,7 @@ func (h *Handler) handleUserInfo(w http.ResponseWriter, r *http.Request) {
return
}
if h.auth == nil {
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusNoContent)
w.Write([]byte("Auth Off")) //nolint:errcheck
http.Error(w, "", http.StatusNoContent)
return
}
if err := json.NewEncoder(w).Encode(h.auth.GetUserInfo(w, r)); err != nil {
@ -3458,9 +3579,7 @@ func (h *Handler) handleUserInfo(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleLogout(w http.ResponseWriter, r *http.Request) {
if h.auth == nil {
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusNoContent)
w.Write([]byte("Auth Off")) //nolint:errcheck
http.Error(w, "", http.StatusNoContent)
return
}
h.auth.Logout(w, r)

View file

@ -18,6 +18,9 @@ import (
"github.com/gorilla/securecookie"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/authn"
"github.com/stretchr/testify/assert"
"github.com/molecula/featurebase/v2/authz"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/pql"
"golang.org/x/oauth2"
@ -184,7 +187,7 @@ func readResponse(w *httptest.ResponseRecorder) ([]byte, error) {
return ioutil.ReadAll(res.Body)
}
func TestHandlerAuth(t *testing.T) {
func TestAuthentication(t *testing.T) {
type evaluate func(w *httptest.ResponseRecorder, data []byte)
type endpoint func(w gohttp.ResponseWriter, r *gohttp.Request)
var (
@ -243,23 +246,23 @@ func TestHandlerAuth(t *testing.T) {
GroupName: "Romantic Painters",
}
validCV := authn.CookieValue{
validCV := authn.AuthContext{
UserID: "snowstorm",
UserName: "J.M.W. Turner",
GroupMembership: []authn.Group{grp},
Token: &token,
}
emptyCV := authn.CookieValue{
emptyCV := authn.AuthContext{
UserID: "narcissus",
UserName: "Caravaggio",
GroupMembership: []authn.Group{},
Token: &token,
}
expiredCV := authn.CookieValue{
expiredCV := authn.AuthContext{
UserID: "narcissus",
UserName: "Caravaggio",
GroupMembership: []authn.Group{},
GroupMembership: []authn.Group{grp},
Token: &expiredToken,
}
@ -309,13 +312,19 @@ func TestHandlerAuth(t *testing.T) {
Expires: token.Expiry,
}
permissions1 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee559900":
"test": "write"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
tests := []struct {
name string
path string
kind string
cookie *gohttp.Cookie
handler endpoint
fn evaluate
name string
path string
kind string
yamlData string
cookie *gohttp.Cookie
handler endpoint
fn evaluate
}{
{
name: "Login",
@ -538,6 +547,88 @@ func TestHandlerAuth(t *testing.T) {
}
},
},
{
name: "MW-AuthOff",
path: "/index/{index}/query",
kind: "middleware",
cookie: validCookie,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
f := hOff.chkAuthZ(hOff.handlePostQuery, authz.Admin)
f(w, r)
},
fn: func(w *httptest.ResponseRecorder, data []byte) {
if w.Result().StatusCode != 400 {
t.Errorf("expected http code 400, got: %+v", w.Result().StatusCode)
}
},
},
{
name: "MW-ExpiredAuth",
path: "/index/{index}/query",
kind: "middleware",
cookie: expiredCookie,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
f := h.chkAuthN(h.handlePostQuery)
f(w, r)
},
fn: func(w *httptest.ResponseRecorder, data []byte) {
if w.Result().StatusCode != 307 {
t.Errorf("expected http code 307, got: %+v", w.Result().StatusCode)
}
},
},
{
name: "MW-ExpiredAuth2",
path: "/index/{index}/query",
kind: "middleware",
cookie: expiredCookie,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
f := h.chkAuthZ(h.handlePostQuery, authz.Admin)
f(w, r)
},
fn: func(w *httptest.ResponseRecorder, data []byte) {
if w.Result().StatusCode != 307 {
t.Errorf("expected http code 307, got: %+v", w.Result().StatusCode)
}
},
},
{
name: "MW-NoPermissions",
path: "/index/{index}/query",
kind: "middleware",
cookie: validCookie,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
h := h
f := h.chkAuthZ(h.handlePostQuery, authz.Write)
assert.Panics(t, func() { f(w, r) }, "expected panic")
},
fn: func(w *httptest.ResponseRecorder, data []byte) {},
},
{
name: "MW-NoIndexNoAdmin",
path: "/index/{index}/query",
kind: "middleware",
cookie: validCookie,
handler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
h := h
permFile := strings.NewReader(permissions1)
var p authz.GroupPermissions
if err := p.ReadPermissionsFile(permFile); err != nil {
t.Errorf("Error: %s", err)
}
h.permissions = &p
f := h.chkAuthZ(h.handlePostQuery, authz.Write)
f(w, r)
},
fn: func(w *httptest.ResponseRecorder, data []byte) {
if w.Result().StatusCode != 400 {
t.Errorf("expected http code 400, got: %+v", w.Result().StatusCode)
}
},
},
}
for _, test := range tests {
@ -546,7 +637,9 @@ func TestHandlerAuth(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
r := httptest.NewRequest(gohttp.MethodGet, test.path, nil)
w := httptest.NewRecorder()
r.AddCookie(test.cookie)
if test.cookie != nil {
r.AddCookie(test.cookie)
}
test.handler(w, r)
data, err := readResponse(w)
if err != nil {
@ -571,6 +664,22 @@ func TestHandlerAuth(t *testing.T) {
test.fn(w, data)
})
case "middleware":
t.Run(test.name, func(t *testing.T) {
r := httptest.NewRequest(gohttp.MethodGet, test.path, nil)
w := httptest.NewRecorder()
if test.cookie != nil {
r.AddCookie(test.cookie)
}
test.handler(w, r)
data, err := readResponse(w)
if err != nil {
t.Errorf("expected no errors reading response, got: %+v", err)
}
test.fn(w, data)
})
}
}

View file

@ -386,3 +386,4 @@ log-path = "/var/log/molecula/featurebase.log"
# hash-key = ""
# block-key = ""
# permissions = ""
# query-log-path = ""

View file

@ -67,6 +67,7 @@ type Server struct { // nolint: maligned
systemInfo SystemInfo
gcNotifier GCNotifier
logger logger.Logger
querylogger logger.Logger
snapshotQueue SnapshotQueue
nodeID string
@ -112,6 +113,13 @@ func OptServerLogger(l logger.Logger) ServerOption {
}
}
func OptServerQueryLogger(l logger.Logger) ServerOption {
return func(s *Server) error {
s.querylogger = l
return nil
}
}
// OptServerReplicaN is a functional option on Server
// used to set the number of replicas.
func OptServerReplicaN(n int) ServerOption {

View file

@ -250,6 +250,7 @@ type Auth struct {
HashKey string `toml:"hash-key"`
BlockKey string `toml:"block-key"`
PermissionsFile string `toml:"permissions"`
QueryLogPath string `toml:"query-log-path"`
}
// Namespace returns the namespace to use based on the Future flag.

View file

@ -69,8 +69,10 @@ type Command struct {
// done will be closed when Command.Close() is called
done chan struct{}
logOutput io.Writer
logger loggerLogger
logOutput io.Writer
querylogOutput io.Writer
logger loggerLogger
querylogger loggerLogger
Handler pilosa.Handler
grpcServer *grpcServer
@ -473,6 +475,7 @@ func (m *Command) SetupServer() error {
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})),
pilosa.OptServerOpenIDAllocator(pilosa.OpenIDAllocator),
pilosa.OptServerLogger(m.logger),
pilosa.OptServerQueryLogger(m.querylogger),
pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
pilosa.OptServerStatsClient(statsClient),
@ -523,6 +526,7 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "new grpc server")
}
var p authz.GroupPermissions
if m.Config.Auth.Enable {
m.Config.MustValidateAuth()
permsFile, err := os.Open(m.Config.Auth.PermissionsFile)
@ -531,7 +535,6 @@ func (m *Command) SetupServer() error {
}
defer permsFile.Close()
var p authz.GroupPermissions
if err = p.ReadPermissionsFile(permsFile); err != nil {
return err
}
@ -542,11 +545,19 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "instantiating authN object")
}
err = m.setupQueryLogger()
if err != nil {
return errors.Wrap(err, "setting up querylogger")
}
m.querylogger.Infof("Group with admin level access: %v", p.Admin)
m.querylogger.Infof("Permissions: %+v", p.Permissions)
// disable postgres binding if auth is enabled
m.Config.Postgres.Bind = ""
// TLS must be enabled if auth is
if m.Config.TLS.CertificatePath == "" || m.Config.TLS.CertificateKeyPath == "" || m.Config.TLS.CACertPath == "" {
if m.Config.TLS.CertificatePath == "" || m.Config.TLS.CertificateKeyPath == "" {
return fmt.Errorf("transport layer security (TLS) is not configured properly. TLS is required when AuthN/Z is enabled, current configuration: %v", m.Config.TLS)
}
@ -556,11 +567,13 @@ func (m *Command) SetupServer() error {
http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
http.OptHandlerAPI(m.API),
http.OptHandlerLogger(m.logger),
http.OptHandlerQueryLogger(m.querylogger),
http.OptHandlerFileSystem(&statik.FileSystem{}),
http.OptHandlerListener(m.ln, m.Config.Advertise),
http.OptHandlerCloseTimeout(m.closeTimeout),
http.OptHandlerMiddleware(m.grpcServer.middleware(m.Config.Handler.AllowedOrigins)),
http.OptHandlerAuth(m.auth),
http.OptHandlerAuthN(m.auth),
http.OptHandlerAuthZ(&p),
)
return errors.Wrap(err, "new handler")
}
@ -606,6 +619,37 @@ func (m *Command) setupLogger() error {
return nil
}
func (m *Command) setupQueryLogger() error {
var f *logger.FileWriter
var err error
if m.Config.Auth.QueryLogPath == "" {
f, err = logger.NewFileWriterMode("queries/query.log", 0600)
if err != nil {
return errors.Wrap(err, "opening file")
}
} else {
f, err = logger.NewFileWriterMode(m.Config.Auth.QueryLogPath, 0600)
if err != nil {
return errors.Wrap(err, "opening file")
}
}
m.querylogOutput = f
m.querylogger = logger.NewStandardLogger(m.querylogOutput)
sighup := make(chan os.Signal, 1)
signal.Notify(sighup, syscall.SIGHUP)
go func() {
for range sighup {
if err := f.Reopen(); err != nil {
m.querylogger.Infof("reopen: %s\n", err.Error())
}
}
}()
return nil
}
// Close shuts down the server.
func (m *Command) Close() error {
select {