scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

updated with master

Browse source
This commit is contained in:
Yuce Tekol 2017-09-27 16:28:27 +03:00
commit ce9033f828
No known key found for this signature in database
GPG key ID: CB59E46D2FB90573
15 changed files with 396 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
client.go
View file

@ -339,6 +339,22 @@ func (c *Client) Import(ctx context.Context, index, frame string, slice uint64,
return nil
}
func (c *Client) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
err := c.CreateIndex(ctx, name, options)
if err == nil || err == ErrIndexExists {
return nil
}
return err
}
func (c *Client) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error {
err := c.CreateFrame(ctx, indexName, frameName, options)
if err == nil || err == ErrFrameExists {
return nil
}
return err
}
// MarshalImportPayload marshalls the import parameters into a protobuf byte slice.
func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.

9
cmd/import.go
View file

@ -20,6 +20,7 @@ import (
"github.com/spf13/cobra"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/ctl"
)
@ -49,12 +50,20 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
return nil
},
}
flags := importCmd.Flags()
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.")
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame")
flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame")
flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
return importCmd
}

29
ctl/import.go
View file

@ -38,6 +38,13 @@ type ImportCommand struct {
Index string `json:"index"`
Frame string `json:"frame"`
// Options for index & frame to be created if they don't exist
IndexOptions pilosa.IndexOptions
FrameOptions pilosa.FrameOptions
// CreateSchema ensures the schema exists before import
CreateSchema bool
// Filenames to import from.
Paths []string `json:"paths"`
@ -57,8 +64,7 @@ type ImportCommand struct {
// NewImportCommand returns a new instance of ImportCommand.
func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand {
return &ImportCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
BufferSize: 10000000,
}
}
@ -83,6 +89,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
}
cmd.Client = client
if cmd.CreateSchema {
err := cmd.ensureSchema(ctx)
if err != nil {
return err
}
}
// Import each path and import by slice.
for _, path := range cmd.Paths {
// Parse path into bits.
@ -95,6 +108,18 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
return nil
}
func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions)
if err != nil {
return fmt.Errorf("Error Creating Index: %s", err)
}
err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions)
if err != nil {
return fmt.Errorf("Error Creating Frame: %s", err)
}
return nil
}
// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
a := make([]pilosa.Bit, 0, cmd.BufferSize)

4
ctl/import_test.go
View file

@ -71,11 +71,9 @@ func TestImportCommand_Run(t *testing.T) {
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
cm.Index = "i"
cm.Frame = "f"
cm.CreateSchema = true
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {

323
docs/client-libraries.md
View file

@ -13,225 +13,252 @@ nav = [
### Go
You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-client-pilosa). Check out its [README](https://github.com/pilosa/go-client-pilosa/blob/master/README.md) for more information and installation instructions.
You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
```go
package startrace
package main
import (
"fmt"
"fmt"
pilosa "github.com/pilosa/go-client-pilosa"
"github.com/pilosa/go-pilosa"
)
func main() {
// Let's create Index and Frame objects, which will contain the settings
// for the corresponding indexes and frames.
repository, _ := pilosa.NewIndex("repository", nil)
// We will just use the default client which assumes the server is at http://localhost:10101
client := pilosa.DefaultClient()
stargazerOptions := &pilosa.RowOptions{
TimeQuantum: pilosa.TimeQuantumYearMonthDay,
InverseEnabled: true,
}
stargazer, _ := repository.Frame("stargazer", stargazerOptions)
// Let's load the schema from the server.
// Note that, for this example the schema should be created beforehand
// and the stargazer data should be imported.
// See the Getting Started repository: https://github.com/pilosa/getting-started/
schema, err := client.Schema()
if err != nil {
// Most calls will return an error value.
// You should handle them appropriately.
// We will just terminate the program in this case.
// Error handling was left out for brevity in the rest of the code.
panic(err)
}
languageOptions := &pilosa.RowOptions{
InverseEnabled: true,
}
language, _ := repository.Frame("language", languageOptions)
// We need to refer to indexes and frames before we can use them in a query.
repository, _ := schema.Index("repository", nil)
stargazer, _ := repository.Frame("stargazer", nil)
language, _ := repository.Frame("language", nil)
// We will just use the default client which assumes the server is at http://localhost:10101
client := pilosa.DefaultClient()
var response *pilosa.QueryResponse
var response *pilosa.QueryResponse
var result *pilosa.QueryResult
// Which repositories did user 14 star:
response, _ = client.Query(stargazer.Bitmap(14), nil)
fmt.Println("User 14 starred: ", response.Result().Bitmap.Bits)
// Which repositories did user 8 star:
response, _ = client.Query(stargazer.Bitmap(8), nil)
result = response.Result()
if result != nil {
fmt.Println("User 8 starred: ", result.Bitmap.Bits)
}
// What are the top 5 languages in the sample data?
response, err = client.Query(language.TopN(5), nil)
languageIDs := []uint64{}
for _, item := range response.Result().CountItems {
languageIDs = append(languageIDs, item.ID)
}
fmt.Println("Top 5 languages: ", languageIDs)
// What are the top 5 languages in the sample data:
response, _ = client.Query(language.TopN(5), nil)
if result != nil {
fmt.Println("Top 5 languages: ", result.Bitmap.Bits)
}
// Which repositories were starred by both user 14 and 19:
response, _ = client.Query(
repository.Intersect(
stargazer.Bitmap(14),
stargazer.Bitmap(19)),
nil)
fmt.Println("Both user 14 and 19 starred:", response.Result().Bitmap.Bits)
// Which repositories were starred by user 8 and 18:
response, _ = client.Query(
repository.Intersect(
stargazer.Bitmap(8),
stargazer.Bitmap(18)),
nil)
result = response.Result()
if result != nil {
fmt.Println("Repositories starred by both user 8 and 18: ", result.Bitmap.Bits)
}
// Which repositories were starred by user 14 or 19:
response, _ = client.Query(
repository.Union(
stargazer.Bitmap(14),
stargazer.Bitmap(19)),
nil)
fmt.Println("User 14 or 19 starred:", response.Result().Bitmap.Bits)
// Which repositories were starred by user 8 and 18 and also were written in language 1
response, _ = client.Query(
repository.Intersect(
stargazer.Bitmap(8),
stargazer.Bitmap(18),
language.Bitmap(1)),
nil)
result = response.Result()
if result != nil {
fmt.Println("Repositories starred by both user 8 and 18 and are in language 1: ", result.Bitmap.Bits)
}
// Which repositories were starred by user 14 or 19 and were written in language 1:
response, _ = client.Query(
repository.Intersect(
repository.Union(
stargazer.Bitmap(14),
stargazer.Bitmap(19),
),
language.Bitmap(1),
), nil)
fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Bitmap.Bits)
// Set user 99999 as a stargazer for repository 77777:
_, err = client.Query(stargazer.SetBit(99999, 77777), nil)
if err != nil {
fmt.Println("Error setting bit: ", err)
}
// Set user 99999 as a stargazer for repository 77777?
client.Query(stargazer.SetBit(99999, 77777), nil)
}
```
### Python
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.rst) for more information and installation instructions.
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
```python
from pilosa import Index, Client, PilosaError, TimeQuantum
# Let's create Index and Frame objects, which will contain the settings
# for the corresponding indexes and frames.
repository = Index("repository")
stargazer = repository.frame("stargazer",
time_quantum=TimeQuantum.YEAR_MONTH_DAY,
inverse_enabled=True)
language = repository.frame("language", inverse_enabled=True)
# We will just use the default client which assumes the server is at http://localhost:10101
client = Client()
# Let's load the schema from the server.
# Note that, for this example the schema should be created beforehand
# and the stargazer data should be imported.
# See the Getting Started repository: https://github.com/pilosa/getting-started/
# Let's create Index and Frame objects, which will contain the settings
# for the corresponding indexes and frames.
try:
schema = client.schema()
except PilosaError as e:
# Most calls will raise an exception on errors.
# You should handle them appropriately.
# We will just terminate the program in this case.
raise SystemExit(e)
# We need to refer to indexes and frames before we can use them in a query.
repository = schema.index("repository")
stargazer = repository.frame("stargazer")
language = repository.frame("language")
# Which repositories did user 8 star:
response = client.query(stargazer.bitmap(8))
if response.result:
print("User 8 starred: ", result.bitmap.bits)
repository_ids = client.query(stargazer.bitmap(14)).result.bitmap.bits
print("User 8 starred: ", repository_ids)
# What are the top 5 languages in the sample data:
response = client.query(language.topn(5))
if response.result:
print("Top 5 languages: ", result.bitmap.bits)
top_languages = client.query(language.topn(5)).result.count_items
print("Top 5 languages: ", [item.id for item in top_languages])
# Which repositories were starred by user 8 and 18:
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18)))
if response.result:
print("Repositories starred by both user 8 and 18: ", result.bitmap.bits)
# Which repositories were starred by both user 14 and 19:
query = repository.intersect(
stargazer.bitmap(14),
stargazer.bitmap(19)
)
mutually_starred = client.query(query).result.bitmap.bits
print("Both user 14 and 19 starred:", mutually_starred)
# Which repositories were starred by user 8 and 18 and also were written in language 1
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18),
language.bitmap(1)))
if response.result:
print("Repositories starred by both user 8 and 18 and are in language 1: ", result.bitmap.bits)
# Which repositories were starred by user 14 or 19:
query = repository.union(
stargazer.bitmap(14),
stargazer.bitmap(19)
)
either_starred = client.query(query).result.bitmap.bits
print("User 14 or 19 starred:", either_starred)
# Which repositories were starred by user 14 or 19 and were written in language 1:
query = repository.intersect(
repository.union(
stargazer.bitmap(14),
stargazer.bitmap(19)
),
language.bitmap(1)
)
mutually_starred = client.query(query).result.bitmap.bits
print("User 14 or 19 starred, written in language 1:", mutually_starred)
# Set user 99999 as a stargazer for repository 77777
try:
client.query(stargazer.setbit(99999, 77777))
except PilosaError as ex:
print("Error setting bit: ", ex)
client.query(stargazer.setbit(99999, 77777))
```
### Java
You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
```java
import com.pilosa.client.*;
import com.pilosa.client.orm.*;
import com.pilosa.client.exceptions.PilosaException;
import java.util.ArrayList;
import java.util.List;
public class StarTrace {
public static void main(String[] args) {
// Let's create Index and Frame objects, which will contain the settings
// for the corresponding indexes and frames.
IndexOptions repositoryOptions = IndexOptions.withDefaults();
Index repository = Index.withName("repository", repositoryOptions);
FrameOptions stargazerOptions = FrameOptions.builder()
.setTimeQuantum(TimeQuantum.YEAR_MONTH_DAY)
.setInverseEnabled(true)
.build();
Frame stargazer = repository.frame("stargazer", stargazerOptions);
FrameOptions languageOptions = FrameOptions.builder()
.setInverseEnabled(true)
.build();
Frame language = repository.frame("language", languageOptions);
// We will just use the default client which assumes the server is at http://localhost:10101
PilosaClient client = PilosaClient.defaultClient();
// Let's load the schema from the server.
Schema schema;
try {
schema = client.readSchema();
}
catch (PilosaException ex) {
// Most calls will return an error value.
// You should handle them appropriately.
// We will just terminate the program in this case.
throw new RuntimeException(ex);
}
// We need to refer to indexes and frames before we can use them in a query.
Index repository = schema.index("repository");
Frame stargazer = repository.frame("stargazer");
Frame language = repository.frame("language");
QueryResponse response;
QueryResult result;
PqlQuery query;
List<Long> repositoryIDs;
// Which repositories did user 8 star:
response = client.query(stargazer.bitmap(8));
result = response.getResult();
if (result != null) {
System.out.println("User 8 starred: " + result.getBitmap().getBits());
}
// Which repositories did user 14 star:
response = client.query(stargazer.bitmap(14));
repositoryIDs = response.getResult().getBitmap().getBits();
System.out.println("User 14 starred: " + repositoryIDs);
// What are the top 5 languages in the sample data:
response = client.query(language.topN(5));
result = response.getResult();
if (result != null) {
System.out.println("Top 5 languages: " + result.getBitmap().getBits());
List<CountResultItem> top_languages = response.getResult().getCountItems();
List<Long> languageIDs = new ArrayList<>();
for (CountResultItem item : top_languages) {
languageIDs.add(item.getID());
}
// Which repositories were starred by user 8 and 18:
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18)));
result = response.getResult();
if (result != null) {
System.out.println("Repositories starred by both user 8 and 18: "
+ result.getBitmap().getBits());
}
System.out.println("Top Languages: " +languageIDs);
// Which repositories were starred by user 8 and 18 and also were written in language 1
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18),
language.bitmap(1)));
result = response.getResult();
if (result != null) {
System.out.println("Repositories starred by both user 8 and 18 and are in language 1: "
+ result.getBitmap().getBits());
}
// Which repositories were starred by both user 14 and 19:
query = repository.intersect(
stargazer.bitmap(14),
stargazer.bitmap(19)
);
response = client.query(query);
repositoryIDs = response.getResult().getBitmap().getBits();
System.out.println("Both user 14 and 19 starred: " + repositoryIDs);
// Which repositories were starred by user 14 or 19:
query = repository.union(
stargazer.bitmap(14),
stargazer.bitmap(19)
);
response = client.query(query);
repositoryIDs = response.getResult().getBitmap().getBits();
System.out.println("User 14 or 19 starred: " + repositoryIDs);
// Which repositories were starred by user 14 or 19 and were written in language 1:
query = repository.intersect(
repository.union(
stargazer.bitmap(14),
stargazer.bitmap(19)
),
language.bitmap(1)
);
response = client.query(query);
repositoryIDs = response.getResult().getBitmap().getBits();
System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs);
// Set user 99999 as a stargazer for repository 77777:
try {
client.query(stargazer.setBit(99999, 77777))
}
catch (PilosaException ex) {
System.out.println("Error setting bit: " + ex)
}
client.query(stargazer.setBit(99999, 77777));
}
}
```

12
executor.go
View file

@ -92,8 +92,12 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Determine slices and inverseSlices for use in e.executeCall().
if needsSlices {
// Round up the number of slices.
maxSlice := e.Holder.Index(index).MaxSlice()
maxInverseSlice := e.Holder.Index(index).MaxInverseSlice()
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
@ -108,10 +112,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
}
// Fetch column label from index.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
columnLabel = idx.ColumnLabel()
}
}

17
fragment.go
View file

@ -67,7 +67,7 @@ const (
// Fragment represents the intersection of a frame and slice in an index.
type Fragment struct {
mu sync.Mutex
mu sync.RWMutex
// Composite identifiers
index string
@ -187,8 +187,9 @@ func (f *Fragment) Open() error {
// openStorage opens the storage bitmap.
func (f *Fragment) openStorage() error {
// Create a roaring bitmap to serve as storage for the slice.
f.storage = roaring.NewBitmap()
if f.storage == nil {
f.storage = roaring.NewBitmap()
}
// Open the data file to be mmap'd and used as an ops log.
file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
@ -312,7 +313,8 @@ func (f *Fragment) close() error {
func (f *Fragment) closeStorage() error {
// Clear the storage bitmap so it doesn't access the closed mmap.
f.storage = roaring.NewBitmap()
//f.storage = roaring.NewBitmap()
// Unmap the file.
if f.storageData != nil {
@ -543,11 +545,8 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
// FieldSum returns the sum of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) {
f.mu.Lock()
defer f.mu.Unlock()
// Compute count based on the existance bit.
row := f.row(uint64(bitDepth), true, true)
row := f.Row(uint64(bitDepth))
if filter != nil {
row = row.Intersect(filter)
}
@ -561,7 +560,7 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e
// 10*(2^0) + 4*(2^1) + 3*(2^2) = 30
//
for i := uint(0); i < bitDepth; i++ {
row := f.row(uint64(i), true, true)
row := f.Row(uint64(i))
if filter != nil {
row = row.Intersect(filter)
}

26
frame.go
View file

@ -43,7 +43,7 @@ const (
// Frame represents a container for views.
type Frame struct {
mu sync.Mutex
mu sync.RWMutex
path string
index string
name string
@ -113,8 +113,8 @@ func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore }
// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
f.mu.Lock()
defer f.mu.Unlock()
f.mu.RLock()
defer f.mu.RUnlock()
var max uint64
for _, view := range f.views {
@ -129,8 +129,8 @@ func (f *Frame) MaxSlice() uint64 {
// MaxInverseSlice returns the max inverse slice in the frame.
func (f *Frame) MaxInverseSlice() uint64 {
f.mu.Lock()
defer f.mu.Unlock()
f.mu.RLock()
defer f.mu.RUnlock()
view := f.views[ViewInverse]
if view == nil {
@ -166,9 +166,9 @@ func (f *Frame) SetRowLabel(v string) error {
// RowLabel returns the row label.
func (f *Frame) RowLabel() string {
f.mu.Lock()
f.mu.RLock()
v := f.rowLabel
f.mu.Unlock()
f.mu.RUnlock()
return v
}
@ -217,8 +217,8 @@ func (f *Frame) CacheSize() uint32 {
// Options returns all options for this frame.
func (f *Frame) Options() FrameOptions {
f.mu.Lock()
defer f.mu.Unlock()
f.mu.RLock()
defer f.mu.RUnlock()
return f.options()
}
@ -404,8 +404,8 @@ func (f *Frame) Close() error {
// Schema returns the frame's current schema.
func (f *Frame) Schema() *FrameSchema {
f.mu.Lock()
defer f.mu.Unlock()
f.mu.RLock()
defer f.mu.RUnlock()
return f.schema
}
@ -506,8 +506,8 @@ func (f *Frame) ViewPath(name string) string {
// View returns a view in the frame by name.
func (f *Frame) View(name string) *View {
f.mu.Lock()
defer f.mu.Unlock()
f.mu.RLock()
defer f.mu.RUnlock()
return f.view(name)
}

6
holder.go
View file

@ -38,7 +38,7 @@ const (
// Holder represents a container for indexes.
type Holder struct {
mu sync.Mutex
mu sync.RWMutex
// Indexes by name.
indexes map[string]*Index
@ -183,8 +183,8 @@ func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, na
// Index returns the index by name.
func (h *Holder) Index(name string) *Index {
h.mu.Lock()
defer h.mu.Unlock()
h.mu.RLock()
defer h.mu.RUnlock()
return h.index(name)
}

40
index.go
View file

@ -37,7 +37,7 @@ const (
// Index represents a container for frames.
type Index struct {
mu sync.Mutex
mu sync.RWMutex
path string
name string
@ -129,12 +129,26 @@ func (i *Index) SetColumnLabel(v string) error {
// ColumnLabel returns the column label.
func (i *Index) ColumnLabel() string {
i.mu.Lock()
i.mu.RLock()
v := i.columnLabel
i.mu.Unlock()
i.mu.RUnlock()
return v
}
// Options returns all options for this index.
func (i *Index) Options() IndexOptions {
i.mu.RLock()
defer i.mu.RUnlock()
return i.options()
}
func (i *Index) options() IndexOptions {
return IndexOptions{
ColumnLabel: i.columnLabel,
TimeQuantum: i.timeQuantum,
}
}
// Open opens and initializes the index.
func (i *Index) Open() error {
// Ensure the path exists.
@ -262,8 +276,8 @@ func (i *Index) MaxSlice() uint64 {
if i == nil {
return 0
}
i.mu.Lock()
defer i.mu.Unlock()
i.mu.RLock()
defer i.mu.RUnlock()
max := i.remoteMaxSlice
for _, f := range i.frames {
@ -288,8 +302,8 @@ func (i *Index) MaxInverseSlice() uint64 {
if i == nil {
return 0
}
i.mu.Lock()
defer i.mu.Unlock()
i.mu.RLock()
defer i.mu.RUnlock()
max := i.remoteMaxInverseSlice
for _, f := range i.frames {
@ -309,8 +323,8 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
// TimeQuantum returns the default time quantum for the index.
func (i *Index) TimeQuantum() TimeQuantum {
i.mu.Lock()
defer i.mu.Unlock()
i.mu.RLock()
defer i.mu.RUnlock()
return i.timeQuantum
}
@ -345,8 +359,8 @@ func (i *Index) InputDefinitionPath() string {
// Frame returns a frame in the index by name.
func (i *Index) Frame(name string) *Frame {
i.mu.Lock()
defer i.mu.Unlock()
i.mu.RLock()
defer i.mu.RUnlock()
return i.frame(name)
}
@ -366,8 +380,8 @@ func (i *Index) inputDefinition(name string) *InputDefinition { return i.inputDe
// Frames returns a list of all frames in the index.
func (i *Index) Frames() []*Frame {
i.mu.Lock()
defer i.mu.Unlock()
i.mu.RLock()
defer i.mu.RUnlock()
a := make([]*Frame, 0, len(i.frames))
for _, f := range i.frames {

18
input_definition_test.go
View file

@ -203,14 +203,14 @@ func TestHandleAction(t *testing.T) {
name string
value interface{}
expected uint64
err string
err string
}{
{name: "integer single-row-bool", action: pilosa.InputSingleRowBool, value: 1, err: "single-row-boolean value"},
{name: "string single-row-bool", action: pilosa.InputSingleRowBool,value: "1", err: "single-row-boolean value 1 must equate to a Bool"},
{name: "string value-to-row", action: pilosa.InputValueToRow,value: "25", err: "value-to-row value must equate to an integer"},
{name: "string mapping", action: pilosa.InputMapping,value: "test", err: "Value test does not exist in definition map"},
{name: "int mapping", action: pilosa.InputMapping,value: 25, err: "Mapping value must be a string"},
{name: "invalid action", action: "test",value: true, err: "Unrecognized Value Destination"},
{name: "string single-row-bool", action: pilosa.InputSingleRowBool, value: "1", err: "single-row-boolean value 1 must equate to a Bool"},
{name: "string value-to-row", action: pilosa.InputValueToRow, value: "25", err: "value-to-row value must equate to an integer"},
{name: "string mapping", action: pilosa.InputMapping, value: "test", err: "Value test does not exist in definition map"},
{name: "int mapping", action: pilosa.InputMapping, value: 25, err: "Mapping value must be a string"},
{name: "invalid action", action: "test", value: true, err: "Unrecognized Value Destination"},
}
for _, r := range tests {
t.Run(r.name, func(t *testing.T) {
@ -226,6 +226,9 @@ func TestHandleAction(t *testing.T) {
value = true
action.ValueDestination = pilosa.InputSingleRowBool
b, err := pilosa.HandleAction(action, value, colID, timestamp)
if err != nil {
t.Fatalf("err with HandleAction: %v", err)
}
if b != nil {
if b.ColumnID != 0 {
t.Fatalf("Unexpected ColumnID %v", b.ColumnID)
@ -248,6 +251,9 @@ func TestHandleAction(t *testing.T) {
action.ValueDestination = pilosa.InputSetTimestamp
t.Run("nil bit", func(t *testing.T) {
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if err != nil {
t.Fatalf("err with HandleAction: %v", err)
}
if b != nil {
t.Fatalf("Expected nil bit is set")
}

53
roaring/roaring.go
View file

@ -617,18 +617,38 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
// Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)).
keyN := binary.LittleEndian.Uint32(data[4:8])
b.keys = make([]uint64, keyN)
b.containers = make([]*container, keyN)
if len(b.keys) == 0 {
b.keys = make([]uint64, 0, keyN)
b.containers = make([]*container, 0, keyN)
} else if int(keyN) < len(b.keys) { //shrink
// nil out to allow to be GCed
for i := range b.containers[keyN:] {
b.containers[int(keyN)+i] = nil
}
b.keys = b.keys[:keyN]
b.containers = b.containers[:keyN]
}
headerSize := headerBaseSize
// Descriptive header section: Read container keys and cardinalities.
for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
b.containers[i] = &container{
container_type: byte(binary.LittleEndian.Uint16(buf[8:10])),
n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1,
mapped: true,
// Reuse memory if possible
if i >= len(b.keys) {
b.keys = append(b.keys, binary.LittleEndian.Uint64(buf[0:8]))
b.containers = append(b.containers, &container{
container_type: byte(binary.LittleEndian.Uint16(buf[8:10])),
n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1,
mapped: true,
})
} else {
b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
c := b.containers[i]
c.container_type = byte(binary.LittleEndian.Uint16(buf[8:10]))
c.n = int(binary.LittleEndian.Uint16(buf[10:12])) + 1
c.mapped = true
}
}
opsOffset := headerSize + int(keyN)*12
@ -945,7 +965,7 @@ const RunMaxSize = 2048
// an array or RLE container is used, depending on the contents. For containers
// with more than 4,096 values, the values are encoded into bitmaps.
type container struct {
container_type byte // number of integers in container
container_type byte // array, bitmap, or run
n int // number of integers in container
array []uint16 // used for array containers
bitmap []uint64 // used for bitmap containers
@ -1605,6 +1625,19 @@ func (c *container) clone() *container {
return other
}
// flipBitmap returns a new bitmap containter containing the inverse of all
// bits in c.
func (c *container) flipBitmap() *container {
other := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
for i, bitmap := range c.bitmap {
other.bitmap[i] = ^bitmap
}
other.n = other.count()
return other
}
// WriteTo writes c to w.
func (c *container) WriteTo(w io.Writer) (n int64, err error) {
if c.isArray() {
@ -2513,6 +2546,10 @@ func differenceRunBitmap(a, b *container) *container {
if a.n == 0 || b.n == 0 {
return a.clone()
}
// If a is full, difference is the flip of b.
if a.runs[0].start == 0 && a.runs[0].last == 65535 {
return b.flipBitmap()
}
itr := newBufBitmapIterator(newBitmapIterator(b.bitmap))
return differenceRunIterator(a, itr)
}

32
roaring/roaring_internal_test.go
View file

@ -1869,6 +1869,38 @@ func TestXorRunRun(t *testing.T) {
}
}
func TestBitmapFlip(t *testing.T) {
c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
ttable := []struct {
original uint64
flipped uint64
}{
{0x0000000000000000, 0xFFFFFFFFFFFFFFFF},
{0xFFFFFFFFFFFFFFFF, 0x0000000000000000},
{0xFFFFFFFFFFFFFFF0, 0x000000000000000F},
{0xFFFFFFEFFFFFFFFF, 0x0000001000000000},
{0x0000001000000000, 0xFFFFFFEFFFFFFFFF},
}
expectedN := int(65536)
for i, tt := range ttable {
c.bitmap[i] = tt.original
expectedN -= int(popcount(tt.original))
}
o := c.flipBitmap()
for i, tt := range ttable {
if o.bitmap[i] != tt.flipped {
t.Fatalf("bitmapFlip calculation. expected %v, got %v", tt.flipped, o.bitmap[i])
}
}
if o.n != expectedN {
t.Fatalf("bitmapFlip calculation. expected count %v, got %v", expectedN, o.n)
}
}
func TestBitmapXorRange(t *testing.T) {
c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
tests := []struct {

17
time.go
View file

@ -53,6 +53,23 @@ func (q TimeQuantum) Valid() bool {
}
}
// The following methods are required to implement pflag Value interface.
// Set sets the time quantum value.
func (q *TimeQuantum) Set(value string) error {
*q = TimeQuantum(value)
return nil
}
func (q TimeQuantum) String() string {
return string(q)
}
// Type returns the type of a time quantum value.
func (q TimeQuantum) Type() string {
return "TimeQuantum"
}
// ParseTimeQuantum parses v into a time quantum.
func ParseTimeQuantum(v string) (TimeQuantum, error) {
q := TimeQuantum(strings.ToUpper(v))

10
view.go
View file

@ -43,7 +43,7 @@ func IsValidView(name string) bool {
// View represents a container for frame data.
type View struct {
mu sync.Mutex
mu sync.RWMutex
path string
index string
frame string
@ -174,8 +174,8 @@ func (v *View) Close() error {
// MaxSlice returns the max slice in the view.
func (v *View) MaxSlice() uint64 {
v.mu.Lock()
defer v.mu.Unlock()
v.mu.RLock()
defer v.mu.RUnlock()
var max uint64
for slice := range v.fragments {
@ -194,8 +194,8 @@ func (v *View) FragmentPath(slice uint64) string {
// Fragment returns a fragment in the view by slice.
func (v *View) Fragment(slice uint64) *Fragment {
v.mu.Lock()
defer v.mu.Unlock()
v.mu.RLock()
defer v.mu.RUnlock()
return v.fragment(slice)
}