diff --git a/client.go b/client.go index 799b6bf4d..01a0d5bd9 100644 --- a/client.go +++ b/client.go @@ -339,6 +339,22 @@ func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, return nil } +func (c *Client) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { + err := c.CreateIndex(ctx, name, options) + if err == nil || err == ErrIndexExists { + return nil + } + return err +} + +func (c *Client) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error { + err := c.CreateFrame(ctx, indexName, frameName, options) + if err == nil || err == ErrFrameExists { + return nil + } + return err +} + // MarshalImportPayload marshalls the import parameters into a protobuf byte slice. func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. diff --git a/cmd/import.go b/cmd/import.go index 944b8b9e6..a4276efa3 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/cobra" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/ctl" ) @@ -49,12 +50,20 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. return nil }, } + flags := importCmd.Flags() flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") + flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") + flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index") + flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") + flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame") + flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame") + flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") + flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") return importCmd } diff --git a/ctl/import.go b/ctl/import.go index a12cb0d06..68fbff691 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -38,6 +38,13 @@ type ImportCommand struct { Index string `json:"index"` Frame string `json:"frame"` + // Options for index & frame to be created if they don't exist + IndexOptions pilosa.IndexOptions + FrameOptions pilosa.FrameOptions + + // CreateSchema ensures the schema exists before import + CreateSchema bool + // Filenames to import from. Paths []string `json:"paths"` @@ -57,8 +64,7 @@ type ImportCommand struct { // NewImportCommand returns a new instance of ImportCommand. func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand { return &ImportCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - + CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), BufferSize: 10000000, } } @@ -83,6 +89,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } cmd.Client = client + if cmd.CreateSchema { + err := cmd.ensureSchema(ctx) + if err != nil { + return err + } + } + // Import each path and import by slice. for _, path := range cmd.Paths { // Parse path into bits. @@ -95,6 +108,18 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { return nil } +func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { + err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions) + if err != nil { + return fmt.Errorf("Error Creating Index: %s", err) + } + err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions) + if err != nil { + return fmt.Errorf("Error Creating Frame: %s", err) + } + return nil +} + // importPath parses a path into bits and imports it to the server. func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { a := make([]pilosa.Bit, 0, cmd.BufferSize) diff --git a/ctl/import_test.go b/ctl/import_test.go index dd9eba675..0023ad79a 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -71,11 +71,9 @@ func TestImportCommand_Run(t *testing.T) { s.Handler.Holder = hldr.Holder cm.Host = s.Host() - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) - cm.Index = "i" cm.Frame = "f" + cm.CreateSchema = true cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { diff --git a/docs/client-libraries.md b/docs/client-libraries.md index 4c776a04a..cad26a18f 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -13,225 +13,252 @@ nav = [ ### Go -You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-client-pilosa). Check out its [README](https://github.com/pilosa/go-client-pilosa/blob/master/README.md) for more information and installation instructions. +You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. ```go -package startrace +package main import ( - "fmt" + "fmt" - pilosa "github.com/pilosa/go-client-pilosa" + "github.com/pilosa/go-pilosa" ) func main() { - // Let's create Index and Frame objects, which will contain the settings - // for the corresponding indexes and frames. - repository, _ := pilosa.NewIndex("repository", nil) + // We will just use the default client which assumes the server is at http://localhost:10101 + client := pilosa.DefaultClient() - stargazerOptions := &pilosa.RowOptions{ - TimeQuantum: pilosa.TimeQuantumYearMonthDay, - InverseEnabled: true, - } - stargazer, _ := repository.Frame("stargazer", stargazerOptions) + // Let's load the schema from the server. + // Note that, for this example the schema should be created beforehand + // and the stargazer data should be imported. + // See the Getting Started repository: https://github.com/pilosa/getting-started/ + schema, err := client.Schema() + if err != nil { + // Most calls will return an error value. + // You should handle them appropriately. + // We will just terminate the program in this case. + // Error handling was left out for brevity in the rest of the code. + panic(err) + } - languageOptions := &pilosa.RowOptions{ - InverseEnabled: true, - } - language, _ := repository.Frame("language", languageOptions) + // We need to refer to indexes and frames before we can use them in a query. + repository, _ := schema.Index("repository", nil) + stargazer, _ := repository.Frame("stargazer", nil) + language, _ := repository.Frame("language", nil) - // We will just use the default client which assumes the server is at http://localhost:10101 - client := pilosa.DefaultClient() + var response *pilosa.QueryResponse - var response *pilosa.QueryResponse - var result *pilosa.QueryResult + // Which repositories did user 14 star: + response, _ = client.Query(stargazer.Bitmap(14), nil) + fmt.Println("User 14 starred: ", response.Result().Bitmap.Bits) - // Which repositories did user 8 star: - response, _ = client.Query(stargazer.Bitmap(8), nil) - result = response.Result() - if result != nil { - fmt.Println("User 8 starred: ", result.Bitmap.Bits) - } + // What are the top 5 languages in the sample data? + response, err = client.Query(language.TopN(5), nil) + languageIDs := []uint64{} + for _, item := range response.Result().CountItems { + languageIDs = append(languageIDs, item.ID) + } + fmt.Println("Top 5 languages: ", languageIDs) - // What are the top 5 languages in the sample data: - response, _ = client.Query(language.TopN(5), nil) - if result != nil { - fmt.Println("Top 5 languages: ", result.Bitmap.Bits) - } + // Which repositories were starred by both user 14 and 19: + response, _ = client.Query( + repository.Intersect( + stargazer.Bitmap(14), + stargazer.Bitmap(19)), + nil) + fmt.Println("Both user 14 and 19 starred:", response.Result().Bitmap.Bits) - // Which repositories were starred by user 8 and 18: - response, _ = client.Query( - repository.Intersect( - stargazer.Bitmap(8), - stargazer.Bitmap(18)), - nil) - result = response.Result() - if result != nil { - fmt.Println("Repositories starred by both user 8 and 18: ", result.Bitmap.Bits) - } + // Which repositories were starred by user 14 or 19: + response, _ = client.Query( + repository.Union( + stargazer.Bitmap(14), + stargazer.Bitmap(19)), + nil) + fmt.Println("User 14 or 19 starred:", response.Result().Bitmap.Bits) - // Which repositories were starred by user 8 and 18 and also were written in language 1 - response, _ = client.Query( - repository.Intersect( - stargazer.Bitmap(8), - stargazer.Bitmap(18), - language.Bitmap(1)), - nil) - result = response.Result() - if result != nil { - fmt.Println("Repositories starred by both user 8 and 18 and are in language 1: ", result.Bitmap.Bits) - } + // Which repositories were starred by user 14 or 19 and were written in language 1: + response, _ = client.Query( + repository.Intersect( + repository.Union( + stargazer.Bitmap(14), + stargazer.Bitmap(19), + ), + language.Bitmap(1), + ), nil) + fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Bitmap.Bits) - // Set user 99999 as a stargazer for repository 77777: - _, err = client.Query(stargazer.SetBit(99999, 77777), nil) - if err != nil { - fmt.Println("Error setting bit: ", err) - } + // Set user 99999 as a stargazer for repository 77777? + client.Query(stargazer.SetBit(99999, 77777), nil) } ``` ### Python -You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.rst) for more information and installation instructions. +You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. ```python from pilosa import Index, Client, PilosaError, TimeQuantum -# Let's create Index and Frame objects, which will contain the settings -# for the corresponding indexes and frames. -repository = Index("repository") -stargazer = repository.frame("stargazer", - time_quantum=TimeQuantum.YEAR_MONTH_DAY, - inverse_enabled=True) -language = repository.frame("language", inverse_enabled=True) - # We will just use the default client which assumes the server is at http://localhost:10101 client = Client() +# Let's load the schema from the server. +# Note that, for this example the schema should be created beforehand +# and the stargazer data should be imported. +# See the Getting Started repository: https://github.com/pilosa/getting-started/ + +# Let's create Index and Frame objects, which will contain the settings +# for the corresponding indexes and frames. +try: + schema = client.schema() +except PilosaError as e: + # Most calls will raise an exception on errors. + # You should handle them appropriately. + # We will just terminate the program in this case. + raise SystemExit(e) + +# We need to refer to indexes and frames before we can use them in a query. +repository = schema.index("repository") +stargazer = repository.frame("stargazer") +language = repository.frame("language") + # Which repositories did user 8 star: -response = client.query(stargazer.bitmap(8)) -if response.result: - print("User 8 starred: ", result.bitmap.bits) +repository_ids = client.query(stargazer.bitmap(14)).result.bitmap.bits +print("User 8 starred: ", repository_ids) # What are the top 5 languages in the sample data: -response = client.query(language.topn(5)) -if response.result: - print("Top 5 languages: ", result.bitmap.bits) +top_languages = client.query(language.topn(5)).result.count_items +print("Top 5 languages: ", [item.id for item in top_languages]) -# Which repositories were starred by user 8 and 18: -response = client.query( - repository.intersect( - stargazer.bitmap(8), - stargazer.bitmap(18))) -if response.result: - print("Repositories starred by both user 8 and 18: ", result.bitmap.bits) +# Which repositories were starred by both user 14 and 19: +query = repository.intersect( + stargazer.bitmap(14), + stargazer.bitmap(19) +) +mutually_starred = client.query(query).result.bitmap.bits +print("Both user 14 and 19 starred:", mutually_starred) -# Which repositories were starred by user 8 and 18 and also were written in language 1 -response = client.query( - repository.intersect( - stargazer.bitmap(8), - stargazer.bitmap(18), - language.bitmap(1))) -if response.result: - print("Repositories starred by both user 8 and 18 and are in language 1: ", result.bitmap.bits) +# Which repositories were starred by user 14 or 19: +query = repository.union( + stargazer.bitmap(14), + stargazer.bitmap(19) +) +either_starred = client.query(query).result.bitmap.bits +print("User 14 or 19 starred:", either_starred) + +# Which repositories were starred by user 14 or 19 and were written in language 1: +query = repository.intersect( + repository.union( + stargazer.bitmap(14), + stargazer.bitmap(19) + ), + language.bitmap(1) +) +mutually_starred = client.query(query).result.bitmap.bits +print("User 14 or 19 starred, written in language 1:", mutually_starred) # Set user 99999 as a stargazer for repository 77777 -try: - client.query(stargazer.setbit(99999, 77777)) -except PilosaError as ex: - print("Error setting bit: ", ex) - +client.query(stargazer.setbit(99999, 77777)) ``` ### Java You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. ```java import com.pilosa.client.*; import com.pilosa.client.orm.*; +import com.pilosa.client.exceptions.PilosaException; + +import java.util.ArrayList; +import java.util.List; public class StarTrace { public static void main(String[] args) { - // Let's create Index and Frame objects, which will contain the settings - // for the corresponding indexes and frames. - IndexOptions repositoryOptions = IndexOptions.withDefaults(); - Index repository = Index.withName("repository", repositoryOptions); - - FrameOptions stargazerOptions = FrameOptions.builder() - .setTimeQuantum(TimeQuantum.YEAR_MONTH_DAY) - .setInverseEnabled(true) - .build(); - Frame stargazer = repository.frame("stargazer", stargazerOptions); - - FrameOptions languageOptions = FrameOptions.builder() - .setInverseEnabled(true) - .build(); - Frame language = repository.frame("language", languageOptions); - // We will just use the default client which assumes the server is at http://localhost:10101 PilosaClient client = PilosaClient.defaultClient(); + // Let's load the schema from the server. + Schema schema; + try { + schema = client.readSchema(); + } + catch (PilosaException ex) { + // Most calls will return an error value. + // You should handle them appropriately. + // We will just terminate the program in this case. + throw new RuntimeException(ex); + } + + // We need to refer to indexes and frames before we can use them in a query. + Index repository = schema.index("repository"); + Frame stargazer = repository.frame("stargazer"); + Frame language = repository.frame("language"); + QueryResponse response; QueryResult result; + PqlQuery query; + List repositoryIDs; - // Which repositories did user 8 star: - response = client.query(stargazer.bitmap(8)); - result = response.getResult(); - if (result != null) { - System.out.println("User 8 starred: " + result.getBitmap().getBits()); - } + // Which repositories did user 14 star: + response = client.query(stargazer.bitmap(14)); + repositoryIDs = response.getResult().getBitmap().getBits(); + System.out.println("User 14 starred: " + repositoryIDs); // What are the top 5 languages in the sample data: response = client.query(language.topN(5)); - result = response.getResult(); - if (result != null) { - System.out.println("Top 5 languages: " + result.getBitmap().getBits()); + List top_languages = response.getResult().getCountItems(); + List languageIDs = new ArrayList<>(); + for (CountResultItem item : top_languages) { + languageIDs.add(item.getID()); } - // Which repositories were starred by user 8 and 18: - response = client.query( - repository.intersect( - stargazer.bitmap(8), - stargazer.bitmap(18))); - result = response.getResult(); - if (result != null) { - System.out.println("Repositories starred by both user 8 and 18: " - + result.getBitmap().getBits()); - } + System.out.println("Top Languages: " +languageIDs); - // Which repositories were starred by user 8 and 18 and also were written in language 1 - response = client.query( - repository.intersect( - stargazer.bitmap(8), - stargazer.bitmap(18), - language.bitmap(1))); - result = response.getResult(); - if (result != null) { - System.out.println("Repositories starred by both user 8 and 18 and are in language 1: " - + result.getBitmap().getBits()); - } + // Which repositories were starred by both user 14 and 19: + query = repository.intersect( + stargazer.bitmap(14), + stargazer.bitmap(19) + ); + response = client.query(query); + repositoryIDs = response.getResult().getBitmap().getBits(); + System.out.println("Both user 14 and 19 starred: " + repositoryIDs); + + // Which repositories were starred by user 14 or 19: + query = repository.union( + stargazer.bitmap(14), + stargazer.bitmap(19) + ); + response = client.query(query); + repositoryIDs = response.getResult().getBitmap().getBits(); + System.out.println("User 14 or 19 starred: " + repositoryIDs); + + // Which repositories were starred by user 14 or 19 and were written in language 1: + query = repository.intersect( + repository.union( + stargazer.bitmap(14), + stargazer.bitmap(19) + ), + language.bitmap(1) + ); + response = client.query(query); + repositoryIDs = response.getResult().getBitmap().getBits(); + System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs); // Set user 99999 as a stargazer for repository 77777: - try { - client.query(stargazer.setBit(99999, 77777)) - } - catch (PilosaException ex) { - System.out.println("Error setting bit: " + ex) - } - + client.query(stargazer.setBit(99999, 77777)); } } ``` diff --git a/executor.go b/executor.go index 1507c3050..d9a9b395b 100644 --- a/executor.go +++ b/executor.go @@ -92,8 +92,12 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Determine slices and inverseSlices for use in e.executeCall(). if needsSlices { // Round up the number of slices. - maxSlice := e.Holder.Index(index).MaxSlice() - maxInverseSlice := e.Holder.Index(index).MaxInverseSlice() + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + maxSlice := idx.MaxSlice() + maxInverseSlice := idx.MaxInverseSlice() // Generate a slices of all slices. slices = make([]uint64, maxSlice+1) @@ -108,10 +112,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } // Fetch column label from index. - idx := e.Holder.Index(index) - if idx == nil { - return nil, ErrIndexNotFound - } columnLabel = idx.ColumnLabel() } } diff --git a/fragment.go b/fragment.go index 829b4e03b..18e2cf1b5 100644 --- a/fragment.go +++ b/fragment.go @@ -67,7 +67,7 @@ const ( // Fragment represents the intersection of a frame and slice in an index. type Fragment struct { - mu sync.Mutex + mu sync.RWMutex // Composite identifiers index string @@ -187,8 +187,9 @@ func (f *Fragment) Open() error { // openStorage opens the storage bitmap. func (f *Fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the slice. - f.storage = roaring.NewBitmap() - + if f.storage == nil { + f.storage = roaring.NewBitmap() + } // Open the data file to be mmap'd and used as an ops log. file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { @@ -312,7 +313,8 @@ func (f *Fragment) close() error { func (f *Fragment) closeStorage() error { // Clear the storage bitmap so it doesn't access the closed mmap. - f.storage = roaring.NewBitmap() + + //f.storage = roaring.NewBitmap() // Unmap the file. if f.storageData != nil { @@ -543,11 +545,8 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) ( // FieldSum returns the sum of a given field as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) { - f.mu.Lock() - defer f.mu.Unlock() - // Compute count based on the existance bit. - row := f.row(uint64(bitDepth), true, true) + row := f.Row(uint64(bitDepth)) if filter != nil { row = row.Intersect(filter) } @@ -561,7 +560,7 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e // 10*(2^0) + 4*(2^1) + 3*(2^2) = 30 // for i := uint(0); i < bitDepth; i++ { - row := f.row(uint64(i), true, true) + row := f.Row(uint64(i)) if filter != nil { row = row.Intersect(filter) } diff --git a/frame.go b/frame.go index f9580cc6b..1d6867746 100644 --- a/frame.go +++ b/frame.go @@ -43,7 +43,7 @@ const ( // Frame represents a container for views. type Frame struct { - mu sync.Mutex + mu sync.RWMutex path string index string name string @@ -113,8 +113,8 @@ func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore } // MaxSlice returns the max slice in the frame. func (f *Frame) MaxSlice() uint64 { - f.mu.Lock() - defer f.mu.Unlock() + f.mu.RLock() + defer f.mu.RUnlock() var max uint64 for _, view := range f.views { @@ -129,8 +129,8 @@ func (f *Frame) MaxSlice() uint64 { // MaxInverseSlice returns the max inverse slice in the frame. func (f *Frame) MaxInverseSlice() uint64 { - f.mu.Lock() - defer f.mu.Unlock() + f.mu.RLock() + defer f.mu.RUnlock() view := f.views[ViewInverse] if view == nil { @@ -166,9 +166,9 @@ func (f *Frame) SetRowLabel(v string) error { // RowLabel returns the row label. func (f *Frame) RowLabel() string { - f.mu.Lock() + f.mu.RLock() v := f.rowLabel - f.mu.Unlock() + f.mu.RUnlock() return v } @@ -217,8 +217,8 @@ func (f *Frame) CacheSize() uint32 { // Options returns all options for this frame. func (f *Frame) Options() FrameOptions { - f.mu.Lock() - defer f.mu.Unlock() + f.mu.RLock() + defer f.mu.RUnlock() return f.options() } @@ -404,8 +404,8 @@ func (f *Frame) Close() error { // Schema returns the frame's current schema. func (f *Frame) Schema() *FrameSchema { - f.mu.Lock() - defer f.mu.Unlock() + f.mu.RLock() + defer f.mu.RUnlock() return f.schema } @@ -506,8 +506,8 @@ func (f *Frame) ViewPath(name string) string { // View returns a view in the frame by name. func (f *Frame) View(name string) *View { - f.mu.Lock() - defer f.mu.Unlock() + f.mu.RLock() + defer f.mu.RUnlock() return f.view(name) } diff --git a/holder.go b/holder.go index 4270c2594..0cf5f6901 100644 --- a/holder.go +++ b/holder.go @@ -38,7 +38,7 @@ const ( // Holder represents a container for indexes. type Holder struct { - mu sync.Mutex + mu sync.RWMutex // Indexes by name. indexes map[string]*Index @@ -183,8 +183,8 @@ func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, na // Index returns the index by name. func (h *Holder) Index(name string) *Index { - h.mu.Lock() - defer h.mu.Unlock() + h.mu.RLock() + defer h.mu.RUnlock() return h.index(name) } diff --git a/index.go b/index.go index 345215690..256c44184 100644 --- a/index.go +++ b/index.go @@ -37,7 +37,7 @@ const ( // Index represents a container for frames. type Index struct { - mu sync.Mutex + mu sync.RWMutex path string name string @@ -129,12 +129,26 @@ func (i *Index) SetColumnLabel(v string) error { // ColumnLabel returns the column label. func (i *Index) ColumnLabel() string { - i.mu.Lock() + i.mu.RLock() v := i.columnLabel - i.mu.Unlock() + i.mu.RUnlock() return v } +// Options returns all options for this index. +func (i *Index) Options() IndexOptions { + i.mu.RLock() + defer i.mu.RUnlock() + return i.options() +} + +func (i *Index) options() IndexOptions { + return IndexOptions{ + ColumnLabel: i.columnLabel, + TimeQuantum: i.timeQuantum, + } +} + // Open opens and initializes the index. func (i *Index) Open() error { // Ensure the path exists. @@ -262,8 +276,8 @@ func (i *Index) MaxSlice() uint64 { if i == nil { return 0 } - i.mu.Lock() - defer i.mu.Unlock() + i.mu.RLock() + defer i.mu.RUnlock() max := i.remoteMaxSlice for _, f := range i.frames { @@ -288,8 +302,8 @@ func (i *Index) MaxInverseSlice() uint64 { if i == nil { return 0 } - i.mu.Lock() - defer i.mu.Unlock() + i.mu.RLock() + defer i.mu.RUnlock() max := i.remoteMaxInverseSlice for _, f := range i.frames { @@ -309,8 +323,8 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) { // TimeQuantum returns the default time quantum for the index. func (i *Index) TimeQuantum() TimeQuantum { - i.mu.Lock() - defer i.mu.Unlock() + i.mu.RLock() + defer i.mu.RUnlock() return i.timeQuantum } @@ -345,8 +359,8 @@ func (i *Index) InputDefinitionPath() string { // Frame returns a frame in the index by name. func (i *Index) Frame(name string) *Frame { - i.mu.Lock() - defer i.mu.Unlock() + i.mu.RLock() + defer i.mu.RUnlock() return i.frame(name) } @@ -366,8 +380,8 @@ func (i *Index) inputDefinition(name string) *InputDefinition { return i.inputDe // Frames returns a list of all frames in the index. func (i *Index) Frames() []*Frame { - i.mu.Lock() - defer i.mu.Unlock() + i.mu.RLock() + defer i.mu.RUnlock() a := make([]*Frame, 0, len(i.frames)) for _, f := range i.frames { diff --git a/input_definition_test.go b/input_definition_test.go index 6531ca69d..f2daade2c 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -203,14 +203,14 @@ func TestHandleAction(t *testing.T) { name string value interface{} expected uint64 - err string + err string }{ {name: "integer single-row-bool", action: pilosa.InputSingleRowBool, value: 1, err: "single-row-boolean value"}, - {name: "string single-row-bool", action: pilosa.InputSingleRowBool,value: "1", err: "single-row-boolean value 1 must equate to a Bool"}, - {name: "string value-to-row", action: pilosa.InputValueToRow,value: "25", err: "value-to-row value must equate to an integer"}, - {name: "string mapping", action: pilosa.InputMapping,value: "test", err: "Value test does not exist in definition map"}, - {name: "int mapping", action: pilosa.InputMapping,value: 25, err: "Mapping value must be a string"}, - {name: "invalid action", action: "test",value: true, err: "Unrecognized Value Destination"}, + {name: "string single-row-bool", action: pilosa.InputSingleRowBool, value: "1", err: "single-row-boolean value 1 must equate to a Bool"}, + {name: "string value-to-row", action: pilosa.InputValueToRow, value: "25", err: "value-to-row value must equate to an integer"}, + {name: "string mapping", action: pilosa.InputMapping, value: "test", err: "Value test does not exist in definition map"}, + {name: "int mapping", action: pilosa.InputMapping, value: 25, err: "Mapping value must be a string"}, + {name: "invalid action", action: "test", value: true, err: "Unrecognized Value Destination"}, } for _, r := range tests { t.Run(r.name, func(t *testing.T) { @@ -226,6 +226,9 @@ func TestHandleAction(t *testing.T) { value = true action.ValueDestination = pilosa.InputSingleRowBool b, err := pilosa.HandleAction(action, value, colID, timestamp) + if err != nil { + t.Fatalf("err with HandleAction: %v", err) + } if b != nil { if b.ColumnID != 0 { t.Fatalf("Unexpected ColumnID %v", b.ColumnID) @@ -248,6 +251,9 @@ func TestHandleAction(t *testing.T) { action.ValueDestination = pilosa.InputSetTimestamp t.Run("nil bit", func(t *testing.T) { b, err = pilosa.HandleAction(action, value, colID, timestamp) + if err != nil { + t.Fatalf("err with HandleAction: %v", err) + } if b != nil { t.Fatalf("Expected nil bit is set") } diff --git a/roaring/roaring.go b/roaring/roaring.go index 456c9fccc..602fb9c8b 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -617,18 +617,38 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)). keyN := binary.LittleEndian.Uint32(data[4:8]) - b.keys = make([]uint64, keyN) - b.containers = make([]*container, keyN) + + if len(b.keys) == 0 { + b.keys = make([]uint64, 0, keyN) + b.containers = make([]*container, 0, keyN) + } else if int(keyN) < len(b.keys) { //shrink + // nil out to allow to be GCed + for i := range b.containers[keyN:] { + b.containers[int(keyN)+i] = nil + } + b.keys = b.keys[:keyN] + b.containers = b.containers[:keyN] + } headerSize := headerBaseSize // Descriptive header section: Read container keys and cardinalities. for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] { - b.keys[i] = binary.LittleEndian.Uint64(buf[0:8]) - b.containers[i] = &container{ - container_type: byte(binary.LittleEndian.Uint16(buf[8:10])), - n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, - mapped: true, + // Reuse memory if possible + if i >= len(b.keys) { + b.keys = append(b.keys, binary.LittleEndian.Uint64(buf[0:8])) + b.containers = append(b.containers, &container{ + container_type: byte(binary.LittleEndian.Uint16(buf[8:10])), + n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, + mapped: true, + }) + } else { + b.keys[i] = binary.LittleEndian.Uint64(buf[0:8]) + c := b.containers[i] + c.container_type = byte(binary.LittleEndian.Uint16(buf[8:10])) + c.n = int(binary.LittleEndian.Uint16(buf[10:12])) + 1 + c.mapped = true + } } opsOffset := headerSize + int(keyN)*12 @@ -945,7 +965,7 @@ const RunMaxSize = 2048 // an array or RLE container is used, depending on the contents. For containers // with more than 4,096 values, the values are encoded into bitmaps. type container struct { - container_type byte // number of integers in container + container_type byte // array, bitmap, or run n int // number of integers in container array []uint16 // used for array containers bitmap []uint64 // used for bitmap containers @@ -1605,6 +1625,19 @@ func (c *container) clone() *container { return other } +// flipBitmap returns a new bitmap containter containing the inverse of all +// bits in c. +func (c *container) flipBitmap() *container { + other := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + + for i, bitmap := range c.bitmap { + other.bitmap[i] = ^bitmap + } + + other.n = other.count() + return other +} + // WriteTo writes c to w. func (c *container) WriteTo(w io.Writer) (n int64, err error) { if c.isArray() { @@ -2513,6 +2546,10 @@ func differenceRunBitmap(a, b *container) *container { if a.n == 0 || b.n == 0 { return a.clone() } + // If a is full, difference is the flip of b. + if a.runs[0].start == 0 && a.runs[0].last == 65535 { + return b.flipBitmap() + } itr := newBufBitmapIterator(newBitmapIterator(b.bitmap)) return differenceRunIterator(a, itr) } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 5d1a985d1..9339e6df9 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1869,6 +1869,38 @@ func TestXorRunRun(t *testing.T) { } } +func TestBitmapFlip(t *testing.T) { + c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + + ttable := []struct { + original uint64 + flipped uint64 + }{ + {0x0000000000000000, 0xFFFFFFFFFFFFFFFF}, + {0xFFFFFFFFFFFFFFFF, 0x0000000000000000}, + {0xFFFFFFFFFFFFFFF0, 0x000000000000000F}, + {0xFFFFFFEFFFFFFFFF, 0x0000001000000000}, + {0x0000001000000000, 0xFFFFFFEFFFFFFFFF}, + } + + expectedN := int(65536) + for i, tt := range ttable { + c.bitmap[i] = tt.original + expectedN -= int(popcount(tt.original)) + } + + o := c.flipBitmap() + + for i, tt := range ttable { + if o.bitmap[i] != tt.flipped { + t.Fatalf("bitmapFlip calculation. expected %v, got %v", tt.flipped, o.bitmap[i]) + } + } + if o.n != expectedN { + t.Fatalf("bitmapFlip calculation. expected count %v, got %v", expectedN, o.n) + } +} + func TestBitmapXorRange(t *testing.T) { c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} tests := []struct { diff --git a/time.go b/time.go index ed6f62bf0..517292071 100644 --- a/time.go +++ b/time.go @@ -53,6 +53,23 @@ func (q TimeQuantum) Valid() bool { } } +// The following methods are required to implement pflag Value interface. + +// Set sets the time quantum value. +func (q *TimeQuantum) Set(value string) error { + *q = TimeQuantum(value) + return nil +} + +func (q TimeQuantum) String() string { + return string(q) +} + +// Type returns the type of a time quantum value. +func (q TimeQuantum) Type() string { + return "TimeQuantum" +} + // ParseTimeQuantum parses v into a time quantum. func ParseTimeQuantum(v string) (TimeQuantum, error) { q := TimeQuantum(strings.ToUpper(v)) diff --git a/view.go b/view.go index 2d5b7d776..e9bbb9c5f 100644 --- a/view.go +++ b/view.go @@ -43,7 +43,7 @@ func IsValidView(name string) bool { // View represents a container for frame data. type View struct { - mu sync.Mutex + mu sync.RWMutex path string index string frame string @@ -174,8 +174,8 @@ func (v *View) Close() error { // MaxSlice returns the max slice in the view. func (v *View) MaxSlice() uint64 { - v.mu.Lock() - defer v.mu.Unlock() + v.mu.RLock() + defer v.mu.RUnlock() var max uint64 for slice := range v.fragments { @@ -194,8 +194,8 @@ func (v *View) FragmentPath(slice uint64) string { // Fragment returns a fragment in the view by slice. func (v *View) Fragment(slice uint64) *Fragment { - v.mu.Lock() - defer v.mu.Unlock() + v.mu.RLock() + defer v.mu.RUnlock() return v.fragment(slice) }