scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #55 from seebs/pluginfix

Browse source 
Pluginfix
This commit is contained in:
Matthew Jaffee 2019-12-20 12:53:09 -06:00 • committed by GitHub
commit d86a3c3f2f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 71 additions and 440 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Dockerfile-clustertests
View file

@ -8,10 +8,7 @@ LABEL maintainer "dev@pilosa.com"
COPY . /go/src/github.com/pilosa/pilosa/
RUN cd /go/src/github.com/pilosa/pilosa \
&& GO111MODULE=on make vendor
RUN cd /go/src/github.com/pilosa/pilosa \
&& CGO_ENABLED=0 make install FLAGS="-a"
&& CGO_ENABLED=0 make install FLAGS="-a -mod=vendor"
# download pumba for fault injection
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba

8
Makefile
View file

@ -16,12 +16,16 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE))
BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
define LICENSE_HASH_CODE
head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
endef
LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
PLUGINS=distinct
export GO111MODULE=on
export GOPRIVATE=github.com/molecula
export PLUGINS
# Run tests and compile Pilosa
default: test build
@ -85,14 +89,14 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
# running. This will catch changes to internal/clustertests/*.go, but if you
# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
# pilosa image.
clustertests:
clustertests: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) build client1
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
# Like clustertests, but rebuilds all images.
clustertests-build:
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build

2
executor.go
View file

@ -23,7 +23,7 @@ import (
"sync"
"time"
"github.com/pilosa/pilosa/v2/ext"
"github.com/molecula/ext"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"

238
ext/ext.go
View file

@ -1,238 +0,0 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for
// plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The
// replacement design is already in process, but it needs more refinement
// to address issues. This one has those issues, and more.
//
// In the current design, plugins will be loaded at runtime using the
// go `plugin` package, so they should be built as a main package using
// the plugin build mode.
//
// Plugins should not import other packages from Pilosa.
//
// To advertise their functionality, plugins define one or more of a
// handful of symbols which will be checked for at plugin load and used
// to register their functionality.
//
// The plugin interface will check for the following function(s). If the
// functions exist, they must have the given signatures. If they return
// a non-nil error, no ops are registered, and the error message will
// be reported in the Pilosa server's logs.
//
// BitmapOps() ([]BitmapOp, error)
//
// These functions may be absent, and may return nil slices; in either
// case, no ops are registered.
package ext
// The Bitmap type represents a Pilosa bitmap, and is used for bitmap
// operations.
type Bitmap interface {
// AddN and RemoveN can be used to add or remove values from a bitmap.
AddN(a ...uint64) (int, error)
RemoveN(a ...uint64) (int, error)
// Lookups
Max() uint64
Min() (uint64, bool)
Count() uint64
Any() bool
Contains(uint64) bool
Slice() []uint64
SliceRange(uint64, uint64) []uint64
// ContainerBits stores the next 1<<16 bits, starting at the provided
// bit index. It may use a provided []uint64 to store them, or may
// provide its own. Don't write to those bits. Offset must be a multiple
// of 1<<16.
ContainerBits(uint64, []uint64) []uint64
// These operators provide existing implemented binary ops.
Intersect(Bitmap) Bitmap
Union(Bitmap) Bitmap
IntersectionCount(Bitmap) uint64
Difference(Bitmap) Bitmap
Xor(Bitmap) Bitmap
Shift(int) (Bitmap, error)
Flip(uint64, uint64) Bitmap
// New() is an atrocity: it creates a new bitmap, unrelated to the
// existing bitmap. This lets you create a new bitmap without having
// imported any of the packages that have bitmap creation tools, because
// the bitmap wrapper type has to give you one.
New() Bitmap
}
// SignedBitmap represents a bitmap that can contain both positive and negative
// values.
type SignedBitmap struct {
Pos, Neg Bitmap
}
// A BitmapOp represents a new bitmap operation that should be exposed
// in PQL.
type BitmapOpInput byte
type BitmapOpOutput byte
type BitmapOpArity byte
type BitmapOpPrecall byte
type BitmapOpType struct {
Input BitmapOpInput
Arity BitmapOpArity
Output BitmapOpOutput
Precall BitmapOpPrecall
}
const (
OpArityUnary = BitmapOpArity(iota)
OpArityBinary
OpArityNary
)
const (
// Unary: Exactly one bitmap.
OpInputBitmap = BitmapOpInput(iota)
// The really weird special case used for BSI, where we end up
// needing to do BSI computations. Arguments will be a
// single BitmapBSI, and a []Bitmap for other operands if any.
OpInputNaryBSI
)
const (
OpOutputCount = BitmapOpOutput(iota)
OpOutputBitmap
OpOutputSignedBitmap
)
const (
OpPrecallNone = BitmapOpPrecall(iota)
OpPrecallGlobal
OpPrecallLocal // unimplemented
)
// Regardless of arity, non-BSI functions should always take []Bitmap.
type BitmapOpFunc interface {
BitmapOpType() BitmapOpType
}
// BitmapOpBitmap should actually always be func([]Bitmap) Bitmap, but
// might be different kinds.
type BitmapOpBitmap interface {
BitmapOpArity() BitmapOpArity
BitmapOpFunc() GenericBitmapOpBitmap
}
// the common underlying type of the other BitmapOpBitmap functions
type GenericBitmapOpBitmap func([]Bitmap, map[string]interface{}) Bitmap
// BitmapBSI represents the way a single BSI field is passed into a function
// which takes a BSI field.
type BitmapBSI struct {
FieldData Bitmap
ShardWidth uint64
Offset int64
Depth uint
}
type BitmapOpBSIBitmap func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
func (b BitmapOpBSIBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Output: OpOutputSignedBitmap}
}
type BitmapOpBSIBitmapPrecall func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap
func (b BitmapOpBSIBitmapPrecall) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Precall: OpPrecallGlobal, Output: OpOutputSignedBitmap}
}
type BitmapOpUnaryCount func([]Bitmap, map[string]interface{}) int64
func (b BitmapOpUnaryCount) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputCount}
}
type BitmapOpUnaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpUnaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputBitmap}
}
func (b BitmapOpUnaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityUnary
}
func (b BitmapOpUnaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
type BitmapOpBinaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpBinaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityBinary, Output: OpOutputBitmap}
}
func (b BitmapOpBinaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityBinary
}
func (b BitmapOpBinaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
type BitmapOpNaryBitmap func([]Bitmap, map[string]interface{}) Bitmap
func (b BitmapOpNaryBitmap) BitmapOpType() BitmapOpType {
return BitmapOpType{Input: OpInputBitmap, Arity: OpArityNary, Output: OpOutputBitmap}
}
func (b BitmapOpNaryBitmap) BitmapOpArity() BitmapOpArity {
return OpArityNary
}
func (b BitmapOpNaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap {
return GenericBitmapOpBitmap(b)
}
// BitmapOp represents an operation to be supported in PQL. Operations
// on bitmaps should always take []Bitmap. Operations on InputNaryBSI should
// take a []Bitmap, plus a Bitmap/shard-width/offset/depth.
//
// Reserved is a list of words to treat as reserved words in a prototype.
// This is not currently used but might be later, and I want to have the
// concept handy now.
type BitmapOp struct {
Name string
Func BitmapOpFunc
Reserved []string
}
// ExtensionInfo tells us about the extension. The ExtensionAPI string
// should be "v0". The version is a human-readable version, use something
// that seems meaningful. Name and Description are reasonably self-explanatory,
// I hope.
//
// Extensions should define a function:
// func ExtensionInfo(extensionAPI string) (*ExtensionInfo, error)
// which reports their extension info if they think they can coexist with that
// API string.
type ExtensionInfo struct {
Name string // Extension name.
Description string // Short description.
Version string // Human-readable version info for extension.
ExtensionAPI string // Extension API version. Should be v0 for now.
License string // License info.
BitmapOps []BitmapOp // List of provided ops.
}

1
ext/samples/.gitignore vendored
View file

@ -1 +0,0 @@
*/*.so

125
ext/samples/some/some.go
View file

@ -1,125 +0,0 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"math/bits"
"github.com/molecula/apophenia"
"github.com/pilosa/pilosa/v2/ext"
)
// This could be dynamically generated, but for now it's not.
// nolint:unused,deadcode
var extInfoTemplate = &ext.ExtensionInfo{
Name: "some",
Description: "some of the bits/all of the bits/none of the bits",
Version: "0.01",
ExtensionAPI: "v0",
License: "unreleased",
BitmapOps: []ext.BitmapOp{
{Name: "Some", Func: ext.BitmapOpUnaryBitmap(Some), Reserved: []string{"p", "seed"}},
},
}
// ExtensionInfo is the entry point used by the plugin code.
func ExtensionInfo(api string) (*ext.ExtensionInfo, error) { // nolint:unused,deadcode
return extInfoTemplate, nil
}
const batchSize = 1024
// Some returns some of the bits from its first input bitmap. Takes seed (int)
// and p (float) values. Seed defaults to 0.
func Some(inputs []ext.Bitmap, args map[string]interface{}) ext.Bitmap {
if len(inputs) == 0 || inputs[0] == nil {
return nil
}
input := inputs[0]
min, ok := input.Min()
// no bits found?
if !ok {
return nil
}
// start at multiple of 128 not greater than min.
min &^= 127
max := input.Max()
p, ok := args["p"].(float64)
if !ok {
return nil
}
// no bits or impossible probability range
if p <= 0 || p > 1 {
return nil
}
// every bit
if p == 1 {
return inputs[0]
}
// On failure, we default to 0.
seed, _ := args["seed"].(int64)
densityScale := uint64(256)
density := uint64(p * float64(densityScale))
for density == 0 {
densityScale <<= 1
density = uint64(p * float64(densityScale))
// too small
if densityScale > (1 << 32) {
return nil
}
}
w, err := apophenia.NewWeighted(apophenia.NewSequence(seed))
if err != nil {
return nil
}
someBits := input.New()
toAdd := make([]uint64, batchSize)
toAddN := 0
offset := apophenia.OffsetFor(apophenia.SequenceWeighted, 0, 0, 0)
for i := min; i < max; i += 128 {
offset.Lo = i
randomBits := w.Bits(offset, density, densityScale)
bit := uint64(0)
for randomBits.Lo != 0 {
next := uint64(bits.TrailingZeros64(randomBits.Lo) + 1)
randomBits.Lo >>= next
toAdd[toAddN] = next + bit + i
toAddN++
bit += next
}
bit = 64
for randomBits.Hi != 0 {
next := uint64(bits.TrailingZeros64(randomBits.Hi) + 1)
randomBits.Hi >>= next
toAdd[toAddN] = next + bit + i
toAddN++
bit += next
}
if toAddN > (batchSize - 128) {
// ignore error
_, _ = someBits.AddN(toAdd[:toAddN]...)
toAddN = 0
}
}
if toAddN > 0 {
_, _ = someBits.AddN(toAdd[:toAddN]...)
}
return input.Intersect(someBits)
}
func main() {
fmt.Printf("this is a plugin module only.\n")
}

2
extension.go
View file

@ -17,7 +17,7 @@ package pilosa
import (
"fmt"
"github.com/pilosa/pilosa/v2/ext"
"github.com/molecula/ext"
"github.com/pilosa/pilosa/v2/roaring"
)

21
extensions/distinct.go Normal file
View file

@ -0,0 +1,21 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build plugindistinct
package extensions
import (
_ "github.com/molecula/extensions/distinct"
)

18
extensions/dummy.go Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This package contains only things which are conditional on build
// tags.
package extensions

3
go.mod
View file

@ -18,7 +18,8 @@ require (
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0
github.com/pkg/errors v0.8.1

4
go.sum
View file

@ -89,6 +89,10 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4=
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4=
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=

2
internal/clustertests/docker-compose.yml
View file

@ -51,6 +51,6 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command:
- "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests"
- "cd /go/src/github.com/pilosa/pilosa/ && go test -mod=vendor -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests"
networks:
pilosanet:

2
pql/ast.go
View file

@ -23,7 +23,7 @@ import (
"strings"
"time"
"github.com/pilosa/pilosa/v2/ext"
"github.com/molecula/ext"
)
// Query represents a PQL query.

2
row.go
View file

@ -18,7 +18,7 @@ import (
"encoding/json"
"sort"
"github.com/pilosa/pilosa/v2/ext"
"github.com/molecula/ext"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)

78
server.go
View file

@ -17,19 +17,19 @@ package pilosa
import (
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"plugin"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/pilosa/pilosa/v2/ext"
"github.com/molecula/ext"
// extensions pulls in some extensions depending on build tags
_ "github.com/pilosa/pilosa/v2/extensions"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
@ -61,7 +61,6 @@ type Server struct { // nolint: maligned
hosts []string
clusterDisabled bool
serializer Serializer
extensionPath string
extensions []*ext.ExtensionInfo
// External
@ -341,8 +340,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
if err != nil {
return nil, err
}
s.extensionPath = filepath.Join(path, ".extensions")
s.holder.Path = path
// s.holder.translateFile.Path = filepath.Join(path, ".keys")
s.holder.Logger = s.logger
@ -383,7 +380,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.broadcaster = s
s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
s.holder.broadcaster = s
err = s.loadPlugins()
err = s.loadExtensions()
if err != nil {
s.logger.Printf("not all plugins loaded successfully")
}
@ -420,67 +417,20 @@ func (s *Server) InternalClient() InternalClient {
return s.defaultClient
}
func (s *Server) loadPlugins() error {
var anyError error
dir, err := os.Open(s.extensionPath)
if err != nil {
// don't complain about it not existing, that's fine.
if os.IsNotExist(err) {
s.logger.Printf("extension interface v0: no extensions directory.")
return nil
}
return errors.Wrap(err, "opening extension path:")
}
defer dir.Close()
for files, err := dir.Readdir(64); err != io.EOF; files, err = dir.Readdir(64) {
if err != nil {
return errors.Wrap(err, "searching extension directory:")
}
for _, file := range files {
name := file.Name()
// only .so files are likely plugins.
if !strings.HasSuffix(name, ".so") {
continue
}
// only regular files are candidates for loading.
mode := file.Mode()
if !mode.IsRegular() {
s.logger.Printf("extension file '%s' is not a regular file", name)
continue
}
err = s.loadPlugin(name)
if err != nil {
s.logger.Printf("loading extension %s: %v", name, err)
anyError = err
}
func (s *Server) loadExtensions() error {
exts := ext.NewExtensions()
var lastError error
for _, extension := range exts {
if err := s.loadExtension(extension); err != nil {
lastError = err
}
}
return anyError
return lastError
}
func (s *Server) loadPlugin(name string) error {
path := filepath.Join(s.extensionPath, name)
p, err := plugin.Open(path)
if err != nil {
return err
}
pluginExtInfo, err := p.Lookup("ExtensionInfo")
if err != nil {
return fmt.Errorf("%s: no ExtensionInfo found", name)
}
extInfoFunc, ok := pluginExtInfo.(func(string) (*ext.ExtensionInfo, error))
if !ok {
return fmt.Errorf("%s: unexpected %T instead of ExtensionInfo object", name, pluginExtInfo)
}
extInfo, err := extInfoFunc("v0")
if err != nil {
return errors.Wrap(err, name)
}
if extInfo == nil {
return fmt.Errorf("%s: nil ExtensionInfo", name)
}
func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error {
if extInfo.ExtensionAPI != "v0" {
return fmt.Errorf("%s: unsupported extension API %s", name, extInfo.ExtensionAPI)
return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI)
}
s.extensions = append(s.extensions, extInfo)
bitmapOps := extInfo.BitmapOps
@ -500,7 +450,7 @@ func (s *Server) loadPlugin(name string) error {
unknownOps++
}
}
err = s.executor.registerOps(bitmapOps)
err := s.executor.registerOps(bitmapOps)
if err != nil {
s.logger.Printf("warning: extension registration failed: %v", err)
} else {