mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 02:44:59 +00:00
Merge branch 'master' into shardwidth22
This commit is contained in:
Cody Soyland
GitHub
commit
7ede65bf80
20 changed files with 1844 additions and 1679 deletions
34
cluster.go
34
cluster.go
|
|
@ -296,18 +296,18 @@ func (c *cluster) unprotectedIsCoordinator() bool {
|
|||
// nodes with its version of Cluster.Status.
|
||||
func (c *cluster) setCoordinator(n *Node) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
// Verify that the new Coordinator value matches
|
||||
// this node.
|
||||
if c.Node.ID != n.ID {
|
||||
c.mu.Unlock()
|
||||
return fmt.Errorf("coordinator node does not match this node")
|
||||
}
|
||||
|
||||
// Update IsCoordinator on all nodes (locally).
|
||||
_ = c.unprotectedUpdateCoordinator(n)
|
||||
c.mu.Unlock()
|
||||
|
||||
// Send the update coordinator message to all nodes.
|
||||
err := c.broadcaster.SendSync(
|
||||
err := c.unprotectedSendSync(
|
||||
&UpdateCoordinatorMessage{
|
||||
New: n,
|
||||
})
|
||||
|
|
@ -316,7 +316,25 @@ func (c *cluster) setCoordinator(n *Node) error {
|
|||
}
|
||||
|
||||
// Broadcast cluster status.
|
||||
return c.broadcaster.SendSync(c.status())
|
||||
return c.unprotectedSendSync(c.unprotectedStatus())
|
||||
}
|
||||
|
||||
// unprotectedSendSync is used in place of c.broadcaster.SendSync (which is
|
||||
// Server.SendSync) because Server.SendSync needs to obtain a cluster lock to
|
||||
// get the list of nodes. TODO: the reference loop from
|
||||
// Server->cluster->broadcaster(Server) will likely continue to cause confusion
|
||||
// and should be refactored.
|
||||
func (c *cluster) unprotectedSendSync(m Message) error {
|
||||
var eg errgroup.Group
|
||||
for _, node := range c.nodes {
|
||||
node := node
|
||||
// Don't send to myself.
|
||||
if node.ID == c.Node.ID {
|
||||
continue
|
||||
}
|
||||
eg.Go(func() error { return c.broadcaster.SendTo(node, m) })
|
||||
}
|
||||
return eg.Wait()
|
||||
}
|
||||
|
||||
// updateCoordinator updates this nodes Coordinator value as well as
|
||||
|
|
@ -533,12 +551,6 @@ func (c *cluster) determineClusterState() (clusterState string) {
|
|||
return ClusterStateStarting
|
||||
}
|
||||
|
||||
func (c *cluster) status() *ClusterStatus {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.unprotectedStatus()
|
||||
}
|
||||
|
||||
// unprotectedStatus returns the the cluster's status including what nodes it contains, its ID, and current state.
|
||||
func (c *cluster) unprotectedStatus() *ClusterStatus {
|
||||
return &ClusterStatus{
|
||||
|
|
@ -1083,7 +1095,7 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error {
|
|||
}
|
||||
// Broadcast cluster status changes to the cluster.
|
||||
status := c.unprotectedStatus()
|
||||
return c.broadcaster.SendSync(status) // TODO fix c.Status
|
||||
return c.unprotectedSendSync(status) // TODO fix c.Status
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -56,6 +56,11 @@ When importing large datasets remember it is much faster to pre sort the data by
|
|||
pilosa import --sort -i project -f stargazer project-stargazer.csv
|
||||
```
|
||||
|
||||
We recommend importing data using official Pilosa client libraries. You can find the corresponding documentation at:
|
||||
* [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md)
|
||||
* [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md)
|
||||
* [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md)
|
||||
|
||||
##### Importing Integer Values
|
||||
|
||||
If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`.
|
||||
|
|
@ -283,13 +288,13 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
|
|||
- **NumViews:** Number of views in the Cluster.
|
||||
- **OpenFiles:** Open file handle count.
|
||||
- **GoRoutines:** Go routine count.
|
||||
|
||||
|
||||
You can opt-out of the Pilosa diagnostics reporting by setting the command line configuration option `--metric.diagnostics=false`, the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option.
|
||||
|
||||
### Metrics
|
||||
|
||||
Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default.
|
||||
The metrics configuration options are:
|
||||
The metrics configuration options are:
|
||||
|
||||
- [Host](../configuration/#metric-host): specify host that receives metric events
|
||||
- [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics
|
||||
|
|
|
|||
|
|
@ -10,281 +10,9 @@ nav = [
|
|||
|
||||
## Client Libraries
|
||||
|
||||
This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is best to avoid using the output of a hash or randomly distributed ids with Pilosa.
|
||||
We have the following official client libraries. You can find more information in their repositories:
|
||||
* [Go client repository](https://github.com/pilosa/go-pilosa)
|
||||
* [Java client repository](https://github.com/pilosa/java-pilosa)
|
||||
* [Python client repository](https://github.com/pilosa/python-pilosa)
|
||||
|
||||
### Go
|
||||
|
||||
You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions.
|
||||
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
|
||||
Error handling has been omitted in the example below for brevity.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/pilosa/go-pilosa"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// We will just use the default client which assumes the server is at http://localhost:10101
|
||||
client := pilosa.DefaultClient()
|
||||
|
||||
// Let's load the schema from the server.
|
||||
// Note that, for this example the schema should be created beforehand
|
||||
// and the stargazer data should be imported.
|
||||
// See the Getting Started repository: https://github.com/pilosa/getting-started/
|
||||
schema, err := client.Schema()
|
||||
if err != nil {
|
||||
// Most calls will return an error value.
|
||||
// You should handle them appropriately.
|
||||
// We will just terminate the program in this case.
|
||||
// Error handling was left out for brevity in the rest of the code.
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// We need to refer to indexes and fields before we can use them in a query.
|
||||
repository := schema.Index("repository")
|
||||
stargazer := repository.Field("stargazer")
|
||||
language := repository.Field("language")
|
||||
|
||||
var response *pilosa.QueryResponse
|
||||
|
||||
// Which repositories did user 14 star:
|
||||
response, _ = client.Query(stargazer.Row(14))
|
||||
fmt.Println("User 14 starred: ", response.Result().Row().Columns)
|
||||
|
||||
// What are the top 5 languages in the sample data?
|
||||
response, err = client.Query(language.TopN(5))
|
||||
languageIDs := []uint64{}
|
||||
for _, item := range response.Result().CountItems() {
|
||||
languageIDs = append(languageIDs, item.ID)
|
||||
}
|
||||
fmt.Println("Top 5 languages: ", languageIDs)
|
||||
|
||||
// Which repositories were starred by both user 14 and 19:
|
||||
response, _ = client.Query(
|
||||
repository.Intersect(
|
||||
stargazer.Row(14),
|
||||
stargazer.Row(19)))
|
||||
fmt.Println("Both user 14 and 19 starred:", response.Result().Row().Columns)
|
||||
|
||||
// Which repositories were starred by user 14 or 19:
|
||||
response, _ = client.Query(
|
||||
repository.Union(
|
||||
stargazer.Row(14),
|
||||
stargazer.Row(19)))
|
||||
fmt.Println("User 14 or 19 starred:", response.Result().Row().Columns)
|
||||
|
||||
// Which repositories were starred by user 14 or 19 and were written in language 1:
|
||||
response, _ = client.Query(
|
||||
repository.Intersect(
|
||||
repository.Union(
|
||||
stargazer.Row(14),
|
||||
stargazer.Row(19),
|
||||
),
|
||||
language.Row(1)))
|
||||
fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns)
|
||||
|
||||
// Set user 99999 as a stargazer for repository 77777?
|
||||
client.Query(stargazer.Set(99999, 77777))
|
||||
}
|
||||
```
|
||||
|
||||
Running the above program should produce output like this:
|
||||
```
|
||||
User 14 starred: [1 2 3 362 368 391 396 409 416 430 436 450 454 460 461 464 466 469 470 483 484 486 490 491 503 504 514]
|
||||
Top 5 languages: [5 1 4 9 13]
|
||||
Both user 14 and 19 starred: [2 3 362 396 416 461 464 466 470 486]
|
||||
User 14 or 19 starred: [1 2 3 361 362 368 376 377 378 382 386 388 391 396 398 400 409 411 412 416 426 428 430 435 436 450 452 453 454 456 460 461 464 465 466 469 470 483 484 486 487 489 490 491 500 503 504 505 512 514]
|
||||
User 14 or 19 starred, written in language 1: [1 2 362 368 382 386 416 426 435 456 461 483 500 503 504 514]
|
||||
```
|
||||
|
||||
### Python
|
||||
|
||||
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions.
|
||||
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
|
||||
Error handling has been omitted in the example below for brevity.
|
||||
|
||||
```python
|
||||
from __future__ import print_function
|
||||
from pilosa import Index, Client, PilosaError, TimeQuantum
|
||||
|
||||
# We will just use the default client which assumes the server is at http://localhost:10101
|
||||
client = Client()
|
||||
|
||||
# Let's load the schema from the server.
|
||||
# Note that, for this example the schema should be created beforehand
|
||||
# and the stargazer data should be imported.
|
||||
# See the Getting Started repository: https://github.com/pilosa/getting-started/
|
||||
|
||||
# Let's create Index and Field objects, which will contain the settings
|
||||
# for the corresponding indexes and fields.
|
||||
try:
|
||||
schema = client.schema()
|
||||
except PilosaError as e:
|
||||
# Most calls will raise an exception on errors.
|
||||
# You should handle them appropriately.
|
||||
# We will just terminate the program in this case.
|
||||
raise SystemExit(e)
|
||||
|
||||
# We need to refer to indexes and fields before we can use them in a query.
|
||||
repository = schema.index("repository")
|
||||
stargazer = repository.field("stargazer")
|
||||
language = repository.field("language")
|
||||
|
||||
# Which repositories did user 8 star:
|
||||
repository_ids = client.query(stargazer.row(14)).result.row.columns
|
||||
print("User 8 starred: ", repository_ids)
|
||||
|
||||
# What are the top 5 languages in the sample data:
|
||||
top_languages = client.query(language.topn(5)).result.count_items
|
||||
print("Top 5 languages: ", [item.id for item in top_languages])
|
||||
|
||||
# Which repositories were starred by both user 14 and 19:
|
||||
query = repository.intersect(
|
||||
stargazer.row(14),
|
||||
stargazer.row(19)
|
||||
)
|
||||
mutually_starred = client.query(query).result.row.columns
|
||||
print("Both user 14 and 19 starred:", mutually_starred)
|
||||
|
||||
# Which repositories were starred by user 14 or 19:
|
||||
query = repository.union(
|
||||
stargazer.row(14),
|
||||
stargazer.row(19)
|
||||
)
|
||||
either_starred = client.query(query).result.row.columns
|
||||
print("User 14 or 19 starred:", either_starred)
|
||||
|
||||
# Which repositories were starred by user 14 or 19 and were written in language 1:
|
||||
query = repository.intersect(
|
||||
repository.union(
|
||||
stargazer.row(14),
|
||||
stargazer.row(19)
|
||||
),
|
||||
language.row(1)
|
||||
)
|
||||
mutually_starred = client.query(query).result.row.columns
|
||||
print("User 14 or 19 starred, written in language 1:", mutually_starred)
|
||||
|
||||
# Set user 99999 as a stargazer for repository 77777
|
||||
client.query(stargazer.set(99999, 77777))
|
||||
```
|
||||
|
||||
Running the above program should produce output like this:
|
||||
```
|
||||
('User 8 starred: ', [1L, 2L, 3L, 362L, 368L, 391L, 396L, 409L, 416L, 430L, 436L, 450L, 454L, 460L, 461L, 464L, 466L, 469L, 470L, 483L, 484L, 486L, 490L, 491L, 503L, 504L, 514L])
|
||||
('Top 5 languages: ', [5L, 1L, 4L, 9L, 13L])
|
||||
('Both user 14 and 19 starred:', [2L, 3L, 362L, 396L, 416L, 461L, 464L, 466L, 470L, 486L])
|
||||
('User 14 or 19 starred:', [1L, 2L, 3L, 361L, 362L, 368L, 376L, 377L, 378L, 382L, 386L, 388L, 391L, 396L, 398L, 400L, 409L, 411L, 412L, 416L, 426L, 428L, 430L, 435L, 436L, 450L, 452L, 453L, 454L, 456L, 460L, 461L, 464L, 465L, 466L, 469L, 470L, 483L, 484L, 486L, 487L, 489L, 490L, 491L, 500L, 503L, 504L, 505L, 512L, 514L])
|
||||
('User 14 or 19 starred, written in language 1:', [1L, 2L, 362L, 368L, 382L, 386L, 416L, 426L, 435L, 456L, 461L, 483L, 500L, 503L, 504L, 514L])
|
||||
```
|
||||
|
||||
### Java
|
||||
|
||||
You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions.
|
||||
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
|
||||
Error handling has been omitted in the example below for brevity.
|
||||
|
||||
```java
|
||||
import com.pilosa.client.*;
|
||||
import com.pilosa.client.orm.*;
|
||||
import com.pilosa.client.exceptions.PilosaException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class StarTrace {
|
||||
public static void main(String[] args) {
|
||||
// We will just use the default client which assumes the server is at http://localhost:10101
|
||||
PilosaClient client = PilosaClient.defaultClient();
|
||||
|
||||
// Let's load the schema from the server.
|
||||
Schema schema;
|
||||
try {
|
||||
schema = client.readSchema();
|
||||
}
|
||||
catch (PilosaException ex) {
|
||||
// Most calls will return an error value.
|
||||
// You should handle them appropriately.
|
||||
// We will just terminate the program in this case.
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
|
||||
// We need to refer to indexes and fields before we can use them in a query.
|
||||
Index repository = schema.index("repository");
|
||||
Field stargazer = repository.field("stargazer");
|
||||
Field language = repository.field("language");
|
||||
|
||||
QueryResponse response;
|
||||
QueryResult result;
|
||||
PqlQuery query;
|
||||
List<Long> repositoryIDs;
|
||||
|
||||
// Which repositories did user 14 star:
|
||||
response = client.query(stargazer.row(14));
|
||||
repositoryIDs = response.getResult().getRow().getColumns();
|
||||
System.out.println("User 14 starred: " + repositoryIDs);
|
||||
|
||||
// What are the top 5 languages in the sample data:
|
||||
response = client.query(language.topN(5));
|
||||
List<CountResultItem> top_languages = response.getResult().getCountItems();
|
||||
List<Long> languageIDs = new ArrayList<Long>();
|
||||
for (CountResultItem item : top_languages) {
|
||||
languageIDs.add(item.getID());
|
||||
}
|
||||
|
||||
System.out.println("Top Languages: " +languageIDs);
|
||||
|
||||
// Which repositories were starred by both user 14 and 19:
|
||||
query = repository.intersect(
|
||||
stargazer.row(14),
|
||||
stargazer.row(19)
|
||||
);
|
||||
response = client.query(query);
|
||||
repositoryIDs = response.getResult().getRow().getColumns();
|
||||
System.out.println("Both user 14 and 19 starred: " + repositoryIDs);
|
||||
|
||||
// Which repositories were starred by user 14 or 19:
|
||||
query = repository.union(
|
||||
stargazer.row(14),
|
||||
stargazer.row(19)
|
||||
);
|
||||
response = client.query(query);
|
||||
repositoryIDs = response.getResult().getRow().getColumns();
|
||||
System.out.println("User 14 or 19 starred: " + repositoryIDs);
|
||||
|
||||
// Which repositories were starred by user 14 or 19 and were written in language 1:
|
||||
query = repository.intersect(
|
||||
repository.union(
|
||||
stargazer.row(14),
|
||||
stargazer.row(19)
|
||||
),
|
||||
language.row(1)
|
||||
);
|
||||
response = client.query(query);
|
||||
repositoryIDs = response.getResult().getRow().getColumns();
|
||||
System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs);
|
||||
|
||||
// Set user 99999 as a stargazer for repository 77777:
|
||||
client.query(stargazer.set(99999, 77777));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Running the above program should produce output like this:
|
||||
```
|
||||
User 14 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514]
|
||||
Top Languages: [5, 1, 4, 9, 13]
|
||||
Both user 14 and 19 starred: [2, 3, 362, 396, 416, 461, 464, 466, 470, 486]
|
||||
User 14 or 19 starred: [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514]
|
||||
User 14 or 19 starred, written in language 1: [1, 2, 362, 368, 382, 386, 416, 426, 435, 456, 461, 483, 500, 503, 504, 514]
|
||||
```
|
||||
Check out our [Getting Started](https://github.com/pilosa/getting-started) repository for sample code for the official clients.
|
||||
|
|
|
|||
159
docs/examples.md
159
docs/examples.md
|
|
@ -83,10 +83,10 @@ lfm := pdk.LinearFloatMapper{
|
|||
|
||||
`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
|
||||
|
||||
This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Frame`). TODO update so this makes sense
|
||||
This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the ColumnMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Field`). <!-- TODO update so this makes sense -->
|
||||
```go
|
||||
pdk.BitMapper{
|
||||
Frame: "dist_miles",
|
||||
pdk.ColumnMapper{
|
||||
Field: "dist_miles",
|
||||
Mapper: lfm,
|
||||
Parsers: []pdk.Parser{pdk.FloatParser{}},
|
||||
Fields: []int{fields["trip_distance"]},
|
||||
|
|
@ -107,9 +107,9 @@ These same objects are represented in the JSON definition file:
|
|||
"Res": 3600
|
||||
}
|
||||
],
|
||||
"BitMappers": [
|
||||
"ColumnMappers": [
|
||||
{
|
||||
"Frame": "dist_miles",
|
||||
"Field": "dist_miles",
|
||||
"Mapper": {
|
||||
"Name": "lfm0"
|
||||
},
|
||||
|
|
@ -122,9 +122,9 @@ These same objects are represented in the JSON definition file:
|
|||
}
|
||||
```
|
||||
|
||||
Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names (in the source data) to column indices (in Pilosa). We use these names in the BitMapper definitions to keep things human-readable.
|
||||
Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of ColumnMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names (in the source data) to column indices (in Pilosa). We use these names in the ColumnMapper definitions to keep things human-readable.
|
||||
|
||||
**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one.
|
||||
**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The ColumnMapper definition is very similar to the previous one.
|
||||
|
||||
**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
|
||||
|
||||
|
|
@ -132,7 +132,7 @@ Here, we define a list of Mappers, each including a name, which we use to refer
|
|||
|
||||
When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
|
||||
|
||||
We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day".
|
||||
We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day".
|
||||
|
||||
We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day".
|
||||
|
||||
|
|
@ -189,16 +189,26 @@ TopN(pickup_grid_id)
|
|||
Average of `total_amount` per `passenger_count` can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average.
|
||||
|
||||
```python
|
||||
queries = ''
|
||||
import pilosa
|
||||
|
||||
client = pilosa.Client()
|
||||
schema = client.schema()
|
||||
taxi = schema.index("taxi")
|
||||
passenger_count = taxi.field("passenger_count")
|
||||
total_amount_dollars = taxi.field("total_amount_dollars")
|
||||
|
||||
queries = []
|
||||
pcounts = range(10)
|
||||
for i in pcounts:
|
||||
queries += "TopN(Row(passenger_count=%d), total_amount_dollars)" % i
|
||||
queries.append(total_amount_dollars.topn(passenger_count.row(i))
|
||||
query = taxi.batch_query(**queries)
|
||||
results = client.query(query)
|
||||
resp = requests.post(qurl, data=queries)
|
||||
|
||||
average_amounts = []
|
||||
for pcount, topn in zip(pcounts, resp.json()['results']):
|
||||
wsum = sum([r['count'] * r['key'] for r in topn])
|
||||
count = sum([r['count'] for r in topn])
|
||||
for pcount, result in zip(pcounts, resp.results):
|
||||
wsum = sum([r.count * r.id for r in result.count_items])
|
||||
count = sum([r.count for r in result.count_items])
|
||||
average_amounts.append(float(wsum)/count)
|
||||
```
|
||||
|
||||
|
|
@ -206,127 +216,6 @@ for pcount, topn in zip(pcounts, resp.json()['results']):
|
|||
Note that the <a href="../data-model/#bsi-range-encoding">BSI</a>-powered <a href="../query-language/#sum">Sum</a> query now provides an alternative approach to this kind of query.
|
||||
</div>
|
||||
|
||||
<!-- Disabled until we have the time to update the Jupyter notebook --YT
|
||||
For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
|
||||
|
||||
<!--
|
||||
|
||||
### Chemical similarity search
|
||||
|
||||
<div class="warning">
|
||||
This example uses the inverse frames feature, which is deprecated as of v0.9.0. The example will soon be updated to reflect the current Pilosa API.
|
||||
</div>
|
||||
|
||||
#### Overview
|
||||
|
||||
The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals.
|
||||
|
||||
We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. Others have tried to solve this chemical similarity search problem using databases (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
|
||||
|
||||
Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient.
|
||||
```
|
||||
T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B))
|
||||
```
|
||||
|
||||
A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical.
|
||||
|
||||
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a [Github repository](https://github.com/pilosa/chem-usecase).
|
||||
|
||||
#### Data model
|
||||
|
||||
We use the [latest ChEMBL release](ftp://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/) chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id.
|
||||
|
||||
Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions.
|
||||
|
||||
Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with:
|
||||
```
|
||||
SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23"
|
||||
threshold = 90
|
||||
```
|
||||
|
||||
return the set of molecules that have at least a 90% similarity with the given molecule.
|
||||
|
||||
#### Import process
|
||||
|
||||
To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key.
|
||||
|
||||
For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase:
|
||||
```
|
||||
python import_from_sdf.py -p <path_to_sdf_file> -file id_fingerprint.csv
|
||||
```
|
||||
|
||||
|
||||
First, follow the instruction in the [getting started](../getting-started/) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
|
||||
The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
|
||||
```
|
||||
curl localhost:10101/index/mole \
|
||||
-X POST
|
||||
|
||||
curl localhost:10101/index/mole/frame/fingerprint \
|
||||
-X POST \
|
||||
-d '{"options": {"inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
|
||||
|
||||
```
|
||||
|
||||
Run the following commands to import the csv data into the `mole` index:
|
||||
```
|
||||
pilosa import -d mole -f fingerprint id_fingerprint.csv
|
||||
```
|
||||
|
||||
#### Queries
|
||||
|
||||
Get chembl_id from a given SMILES:
|
||||
```
|
||||
python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23"
|
||||
```
|
||||
|
||||
Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps:
|
||||
|
||||
* Convert SMILES to fingerprint bit "on" positions
|
||||
|
||||
```python
|
||||
from rdkit import Chem
|
||||
from rdkit.Chem import AllChem
|
||||
mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23")
|
||||
fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits())
|
||||
```
|
||||
|
||||
* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist.
|
||||
|
||||
```python
|
||||
for m in mole_ids:
|
||||
mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(row=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
|
||||
existed_mol = False
|
||||
if len(mol) == len(fp):
|
||||
found = m
|
||||
existed_mol = True
|
||||
break
|
||||
```
|
||||
|
||||
Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70%
|
||||
```
|
||||
python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70
|
||||
```
|
||||
|
||||
Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps:
|
||||
|
||||
* Get chembl_id from a SMILES (steps discussed above)
|
||||
|
||||
* Query Pilosa’s TopN to get list of similarity chembl_id
|
||||
```python
|
||||
query_string = 'TopN(Bitmap(row=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
|
||||
topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string)
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command:
|
||||
```
|
||||
python benchmarks.py -id 6223
|
||||
```
|
||||
|
||||
As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa.
|
||||
|
||||
Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster
|
||||
|
||||
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1720,6 +1720,17 @@ func TestExecutor_Execute_Range_Deprecated(t *testing.T) {
|
|||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
})
|
||||
|
||||
rq2 := []string{
|
||||
`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`,
|
||||
}
|
||||
responses = runCallTest(t, writeQuery, rq2,
|
||||
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
|
||||
t.Run("OldRange", func(t *testing.T) {
|
||||
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
|
||||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("RowIDColumnKey", func(t *testing.T) {
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -1,5 +1,7 @@
|
|||
module github.com/pilosa/pilosa
|
||||
|
||||
replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108
|
||||
|
||||
require (
|
||||
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
|
||||
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -61,6 +61,12 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
|
|||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pilosa/memberlist v0.1.3 h1:6am86S+mnY3zKPmH5yHtTqdNpqH/KjxF6WSHk95Msyo=
|
||||
github.com/pilosa/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07 h1:f1Xp66+XJjfFAqnhX3T/4X3ItZK1H+r9neBnK+nV1ec=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 h1:6QjQrHgdgVR7nnbzPwJwZ1dliUdjYtFi6ma50GtLOwA=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
|
|
|
|||
21
handler.go
21
handler.go
|
|
@ -45,18 +45,19 @@ type QueryResponse struct {
|
|||
|
||||
// MarshalJSON marshals QueryResponse into a JSON-encoded byte slice
|
||||
func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
|
||||
var output struct {
|
||||
Results []interface{} `json:"results,omitempty"`
|
||||
ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
output.Results = resp.Results
|
||||
output.ColumnAttrSets = resp.ColumnAttrSets
|
||||
|
||||
if resp.Err != nil {
|
||||
output.Err = resp.Err.Error()
|
||||
return json.Marshal(struct {
|
||||
Err string `json:"error"`
|
||||
}{Err: resp.Err.Error()})
|
||||
}
|
||||
return json.Marshal(output)
|
||||
|
||||
return json.Marshal(struct {
|
||||
Results []interface{} `json:"results"`
|
||||
ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
|
||||
}{
|
||||
Results: resp.Results,
|
||||
ColumnAttrSets: resp.ColumnAttrSets,
|
||||
})
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
|
|
|
|||
|
|
@ -964,10 +964,12 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er
|
|||
}
|
||||
|
||||
// writeQueryResponse writes the response from the executor to w.
|
||||
func (h *Handler) writeQueryResponse(w io.Writer, r *http.Request, resp *pilosa.QueryResponse) error {
|
||||
func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error {
|
||||
if !validHeaderAcceptJSON(r.Header) {
|
||||
w.Header().Set("Content-Type", "application/protobuf")
|
||||
return h.writeProtobufQueryResponse(w, resp)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
return h.writeJSONQueryResponse(w, resp)
|
||||
}
|
||||
|
||||
|
|
|
|||
18
pql/ast.go
18
pql/ast.go
|
|
@ -103,7 +103,9 @@ func (q *Query) endConditional() {
|
|||
|
||||
func (q *Query) addField(field string) {
|
||||
elem := q.lastCallStackElem()
|
||||
if elem == nil || elem.lastField != "" {
|
||||
if elem == nil {
|
||||
panic(fmt.Sprintf("addField called with '%s' while element is nil", field))
|
||||
} else if elem.lastField != "" {
|
||||
panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, elem.lastField))
|
||||
}
|
||||
elem.lastField = field
|
||||
|
|
@ -112,6 +114,15 @@ func (q *Query) addField(field string) {
|
|||
}
|
||||
}
|
||||
|
||||
// validateArgField ensures that field does not already
|
||||
// exist as a key in the Args map before adding the new
|
||||
// key/value.
|
||||
func (q *Query) validateArgField(elem *callStackElem) {
|
||||
if _, exists := elem.call.Args[elem.lastField]; exists {
|
||||
panic(fmt.Sprintf("%s: %s", duplicateArgErrorMessage, elem.lastField))
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Query) addVal(val interface{}) {
|
||||
elem := q.lastCallStackElem()
|
||||
if elem == nil || elem.lastField == "" {
|
||||
|
|
@ -123,11 +134,13 @@ func (q *Query) addVal(val interface{}) {
|
|||
return
|
||||
}
|
||||
if elem.lastCond != ILLEGAL {
|
||||
q.validateArgField(elem) // case 1
|
||||
elem.call.Args[elem.lastField] = &Condition{
|
||||
Op: elem.lastCond,
|
||||
Value: val,
|
||||
}
|
||||
} else {
|
||||
q.validateArgField(elem) // case 2
|
||||
elem.call.Args[elem.lastField] = val
|
||||
}
|
||||
elem.lastField = ""
|
||||
|
|
@ -162,11 +175,13 @@ func (q *Query) addNumVal(val string) {
|
|||
}
|
||||
return
|
||||
} else if elem.lastCond != ILLEGAL {
|
||||
q.validateArgField(elem) // case 3
|
||||
elem.call.Args[elem.lastField] = &Condition{
|
||||
Op: elem.lastCond,
|
||||
Value: ival,
|
||||
}
|
||||
} else {
|
||||
q.validateArgField(elem) // case 4
|
||||
elem.call.Args[elem.lastField] = ival
|
||||
}
|
||||
elem.lastField = ""
|
||||
|
|
@ -175,6 +190,7 @@ func (q *Query) addNumVal(val string) {
|
|||
|
||||
func (q *Query) startList() {
|
||||
elem := q.lastCallStackElem()
|
||||
q.validateArgField(elem) // case 5
|
||||
if elem.lastCond != ILLEGAL {
|
||||
elem.call.Args[elem.lastField] = &Condition{
|
||||
Op: elem.lastCond,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
package pql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
|
|
@ -25,6 +26,9 @@ import (
|
|||
// timeFormat is the go-style time format used to parse string dates.
|
||||
const timeFormat = "2006-01-02T15:04"
|
||||
|
||||
// duplicateArgErrorMessage is used as an error string in the parser.
|
||||
const duplicateArgErrorMessage = "duplicate argument provided"
|
||||
|
||||
// parser represents a parser for the PQL language.
|
||||
type parser struct {
|
||||
r io.Reader
|
||||
|
|
@ -59,6 +63,20 @@ func (p *parser) Parse() (*Query, error) {
|
|||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parsing")
|
||||
}
|
||||
p.Execute()
|
||||
|
||||
// Handle specific panics from the parser and return them as errors.
|
||||
var v interface{}
|
||||
func() {
|
||||
defer func() { v = recover() }()
|
||||
p.Execute()
|
||||
}()
|
||||
if v != nil {
|
||||
if strings.HasPrefix(v.(string), duplicateArgErrorMessage) {
|
||||
return nil, fmt.Errorf("%s", v)
|
||||
} else {
|
||||
panic(v)
|
||||
}
|
||||
}
|
||||
|
||||
return &p.Query, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close
|
|||
/ 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
|
||||
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
|
||||
/ 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()}
|
||||
/ 'Range' {p.startCall("Range")} open field sp '=' sp value comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()}
|
||||
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
|
||||
allargs <- Call (comma Call)* (comma args)? / args / sp
|
||||
args <- arg (comma args)? sp
|
||||
|
|
|
|||
2622
pql/pql.peg.go
2622
pql/pql.peg.go
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,21 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package pql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
|
@ -267,6 +282,10 @@ func TestPEGWorking(t *testing.T) {
|
|||
my-frame
|
||||
=9)`,
|
||||
ncalls: 1},
|
||||
{
|
||||
name: "OldRange",
|
||||
input: "Range(blah=1, 2019-04-07T00:00, 2019-08-07T00:00)",
|
||||
ncalls: 1},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
|
|
@ -672,3 +691,47 @@ func TestPQLDeepEquality(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateArgError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
call string
|
||||
}{
|
||||
// case 1
|
||||
{
|
||||
name: "StringConditional",
|
||||
call: "Row(a==foo, a==bar)",
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
name: "StringValue",
|
||||
call: "Row(a=foo, a=bar)",
|
||||
},
|
||||
// case 3
|
||||
{
|
||||
name: "IntConditional",
|
||||
call: "Row(a>5, a>6)",
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
name: "IntValue",
|
||||
call: "Row(a=7, a=8)",
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
name: "List",
|
||||
call: "Row(a=[7], a=[7,8])",
|
||||
},
|
||||
}
|
||||
for i, test := range tests {
|
||||
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
|
||||
_, err := ParseString(test.call)
|
||||
expErr := fmt.Sprintf("%s: a", duplicateArgErrorMessage)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for duplicate argument: %s", test.call)
|
||||
} else if err.Error() != expErr {
|
||||
t.Fatalf("expected error: %s, but got: %v", expErr, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3667,15 +3667,9 @@ func (op *op) apply(b *Bitmap) (changed bool) {
|
|||
case opTypeRemove:
|
||||
return b.remove(op.value)
|
||||
case opTypeAddBatch:
|
||||
for _, v := range op.values {
|
||||
nc := b.DirectAdd(v)
|
||||
changed = nc || changed
|
||||
}
|
||||
changed = b.DirectAddN(op.values...) > 0
|
||||
case opTypeRemoveBatch:
|
||||
for _, v := range op.values {
|
||||
nc := b.remove(v)
|
||||
changed = nc || changed
|
||||
}
|
||||
changed = b.DirectRemoveN(op.values...) > 0
|
||||
default:
|
||||
panic(fmt.Sprintf("invalid op type: %d", op.typ))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -589,7 +589,8 @@ func (s *Server) SendSync(m Message) error {
|
|||
return fmt.Errorf("marshaling message: %v", err)
|
||||
}
|
||||
msg = append([]byte{getMessageType(m)}, msg...)
|
||||
for _, node := range s.cluster.nodes {
|
||||
|
||||
for _, node := range s.cluster.Nodes() {
|
||||
node := node
|
||||
// Don't forward the message to ourselves.
|
||||
if s.uri == node.URI {
|
||||
|
|
|
|||
|
|
@ -309,6 +309,189 @@ func TestClusterResize_AddNode(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// Ensure that adding a node correctly resizes the cluster.
|
||||
func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
|
||||
t.Run("WithIndex", func(t *testing.T) {
|
||||
// Configure node0
|
||||
m0 := test.MustRunCluster(t, 1)[0]
|
||||
defer m0.Close()
|
||||
|
||||
seed := m0.GossipAddress()
|
||||
|
||||
// Create a client for each node.
|
||||
client0 := m0.Client()
|
||||
|
||||
// Create indexes and fields on one node.
|
||||
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client0.CreateField(context.Background(), "i", "f"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
errc := make(chan error)
|
||||
go func() {
|
||||
_, err := m0.API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
|
||||
errc <- err
|
||||
}()
|
||||
|
||||
// Configure node1
|
||||
m1 := test.NewCommandNode(false)
|
||||
m1.Config.Gossip.Port = "0"
|
||||
m1.Config.Gossip.Seeds = []string{seed}
|
||||
err := m1.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("starting second main: %v", err)
|
||||
}
|
||||
defer m1.Close()
|
||||
|
||||
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
|
||||
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
|
||||
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
|
||||
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
|
||||
}
|
||||
|
||||
if err := <-errc; err != nil {
|
||||
t.Fatalf("error from index creation: %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("ContinuousShards", func(t *testing.T) {
|
||||
// Configure node0
|
||||
m0 := test.MustRunCluster(t, 1)[0]
|
||||
defer m0.Close()
|
||||
|
||||
seed := m0.GossipAddress()
|
||||
|
||||
// Create a client for each node.
|
||||
client0 := m0.Client()
|
||||
|
||||
// Create indexes and fields on one node.
|
||||
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client0.CreateField(context.Background(), "i", "f"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write data on first node.
|
||||
if _, err := m0.Query("i", "", `
|
||||
Set(1, f=1)
|
||||
Set(1300000, f=1)
|
||||
`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// exp is the expected result for the Row queries that follow.
|
||||
exp := `{"results":[{"attrs":{},"columns":[1,1300000]}]}` + "\n"
|
||||
|
||||
// Verify the data exists on the single node.
|
||||
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
|
||||
// Configure node1
|
||||
m1 := test.NewCommandNode(false)
|
||||
m1.Config.Gossip.Port = "0"
|
||||
m1.Config.Gossip.Seeds = []string{seed}
|
||||
err := m1.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("starting second main: %v", err)
|
||||
}
|
||||
errc := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := m0.API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
|
||||
errc <- err
|
||||
}()
|
||||
defer m1.Close()
|
||||
|
||||
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
|
||||
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
|
||||
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
|
||||
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
|
||||
}
|
||||
|
||||
// Verify the data exists on both nodes.
|
||||
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
if res, err := m1.Query("i", "", `Row(f=1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
})
|
||||
t.Run("SkippedShard", func(t *testing.T) {
|
||||
// Configure node0
|
||||
m0 := test.MustRunCluster(t, 1)[0]
|
||||
defer m0.Close()
|
||||
|
||||
seed := m0.GossipAddress()
|
||||
|
||||
// Create a client for each node.
|
||||
client0 := m0.Client()
|
||||
|
||||
// Create indexes and fields on one node.
|
||||
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client0.CreateField(context.Background(), "i", "f"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write data on first node. Note that no data is placed on shard 1.
|
||||
if _, err := m0.Query("i", "", `
|
||||
Set(1, f=1)
|
||||
Set(2400000, f=1)
|
||||
`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// exp is the expected result for the Row queries that follow.
|
||||
exp := `{"results":[{"attrs":{},"columns":[1,2400000]}]}` + "\n"
|
||||
|
||||
// Verify the data exists on the single node.
|
||||
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
|
||||
// Configure node1
|
||||
m1 := test.NewCommandNode(false)
|
||||
m1.Config.Gossip.Port = "0"
|
||||
m1.Config.Gossip.Seeds = []string{seed}
|
||||
errc := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := m0.API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
|
||||
errc <- err
|
||||
}()
|
||||
err := m1.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("starting second main: %v", err)
|
||||
}
|
||||
defer m1.Close()
|
||||
|
||||
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
|
||||
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
|
||||
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
|
||||
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
|
||||
}
|
||||
|
||||
// Verify the data exists on both nodes.
|
||||
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
if res, err := m1.Query("i", "", `Row(f=1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure that redundant gossip seeds are used
|
||||
func TestCluster_GossipMembership(t *testing.T) {
|
||||
t.Run("Node0Down", func(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -249,6 +249,8 @@ func TestHandler_Endpoints(t *testing.T) {
|
|||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"results":[2]}`+"\n" {
|
||||
t.Fatalf("unexpected body: %q", body)
|
||||
} else if w.Header().Get("Content-Type") != "application/json" {
|
||||
t.Fatalf("unexpected header: %q", w.Header().Get("Content-Type"))
|
||||
}
|
||||
|
||||
})
|
||||
|
|
@ -287,6 +289,8 @@ func TestHandler_Endpoints(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
} else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 {
|
||||
t.Fatalf("unexpected response type: %#v", resp.Results[0])
|
||||
} else if w.Header().Get("Content-Type") != "application/protobuf" {
|
||||
t.Fatalf("unexpected header: %q", w.Header().Get("Content-Type"))
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -447,6 +451,14 @@ func TestHandler_Endpoints(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("Query empty", func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("")))
|
||||
if body := w.Body.String(); body != `{"results":[]}`+"\n" {
|
||||
t.Fatalf("unexpected body: %q", body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Method not allowed", func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil))
|
||||
|
|
|
|||
|
|
@ -630,6 +630,53 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRemoveConcurrentIndexCreation(t *testing.T) {
|
||||
cluster := test.MustNewCluster(t, 3)
|
||||
for _, c := range cluster {
|
||||
c.Config.Cluster.ReplicaN = 2
|
||||
}
|
||||
err := cluster.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("starting cluster: %v", err)
|
||||
}
|
||||
|
||||
var wait = true
|
||||
for wait {
|
||||
wait = false
|
||||
for _, node := range cluster {
|
||||
if node.API.State() != pilosa.ClusterStateNormal {
|
||||
wait = true
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond * 1)
|
||||
}
|
||||
|
||||
errc := make(chan error)
|
||||
go func() {
|
||||
_, err := cluster[0].API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
|
||||
errc <- err
|
||||
}()
|
||||
|
||||
if _, err := cluster[0].API.RemoveNode(cluster[2].API.Node().ID); err != nil {
|
||||
t.Fatalf("removing node: %v", err)
|
||||
}
|
||||
|
||||
for i := 0; cluster[0].API.State() != pilosa.ClusterStateNormal; i++ {
|
||||
time.Sleep(time.Millisecond)
|
||||
if i > 10 {
|
||||
t.Fatalf("expected state to be DEGRADED, but got %s", cluster[0].API.State())
|
||||
}
|
||||
}
|
||||
|
||||
hosts := cluster[0].API.Hosts(context.Background())
|
||||
if len(hosts) != 2 {
|
||||
t.Fatalf("unexpected hosts: %v", hosts)
|
||||
}
|
||||
if err := <-errc; err != nil {
|
||||
t.Fatalf("error from index creation: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure program imports timestamps as UTC.
|
||||
func TestMain_ImportTimestamp(t *testing.T) {
|
||||
m := test.MustRunCommand()
|
||||
|
|
|
|||
|
|
@ -338,7 +338,7 @@ func (bcast) SendAsync(Message) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// SendTo is a test implemenetation of Broadcaster SendTo method.
|
||||
// SendTo is a test implementation of Broadcaster SendTo method.
|
||||
func (b bcast) SendTo(to *Node, m Message) error {
|
||||
switch obj := m.(type) {
|
||||
case *ResizeInstruction:
|
||||
|
|
@ -349,6 +349,20 @@ func (b bcast) SendTo(to *Node, m Message) error {
|
|||
case *ResizeInstructionComplete:
|
||||
coord := b.t.clusterByID(to.ID)
|
||||
go coord.markResizeInstructionComplete(obj)
|
||||
case *ClusterStatus:
|
||||
// Apply the send message to the node.
|
||||
for _, c := range b.t.Clusters {
|
||||
if c.Node.ID == to.ID {
|
||||
c.mergeClusterStatus(obj)
|
||||
}
|
||||
}
|
||||
b.t.mu.RLock()
|
||||
if obj.State == ClusterStateNormal && b.t.resizing {
|
||||
close(b.t.resizeDone)
|
||||
}
|
||||
b.t.mu.RUnlock()
|
||||
default:
|
||||
panic(fmt.Sprintf("message not handled:\n%#v\n", obj))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue