diff --git a/api_test.go b/api_test.go index 0c7c63287..684970c5e 100644 --- a/api_test.go +++ b/api_test.go @@ -22,7 +22,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/shardwidth" "github.com/molecula/featurebase/v3/test" @@ -36,21 +35,21 @@ func TestAPI_Import(t *testing.T) { pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -222,19 +221,19 @@ func TestAPI_ImportValue(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -529,7 +528,7 @@ func TestAPI_Ingest(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -648,7 +647,7 @@ func BenchmarkIngest(b *testing.B) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -709,7 +708,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -1430,7 +1429,7 @@ func TestAPI_RBFDebugInfo(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() diff --git a/client.go b/client.go index 4f0f40ecd..43929b952 100644 --- a/client.go +++ b/client.go @@ -3,12 +3,8 @@ package pilosa import ( "context" - "io" - "time" - "github.com/molecula/featurebase/v3/ingest" pnet "github.com/molecula/featurebase/v3/net" - "github.com/molecula/featurebase/v3/topology" ) // Bit represents the intersection of a row and a column. It can be specified by @@ -29,74 +25,6 @@ type FieldValue struct { Value int64 } -// InternalClient should be implemented by any struct that enables any transport between nodes -// TODO: Refactor -// Note from Travis: Typically an interface containing more than two or three methods is an indication that -// something hasn't been architected correctly. -// While I understand that putting the entire Client behind an interface might require this many methods, -// I don't want to let it go unquestioned. -// Another note from Travis: I think we eventually want to unify `InternalClient` with -// the `github.com/molecula/featurebase/v3/client` client. -// Doing that may obviate the need to refactor this. -type InternalClient interface { - InternalQueryClient - - AvailableShards(ctx context.Context, indexName string) ([]uint64, error) - MaxShardByIndex(ctx context.Context) (map[string]uint64, error) - Schema(ctx context.Context) ([]*IndexInfo, error) - PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error - CreateIndex(ctx context.Context, index string, opt IndexOptions) error - FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) - PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error) - Nodes(ctx context.Context) ([]*topology.Node, error) - Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) - Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error - EnsureIndex(ctx context.Context, name string, options IndexOptions) error - EnsureField(ctx context.Context, indexName string, fieldName string) error - EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error - ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error - ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error - CreateField(ctx context.Context, index, field string) error - CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error - FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) - SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error - RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) - RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) - ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error - ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) - MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) - IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error - - IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) - IDAllocDataWriter(ctx context.Context, f io.Reader, primary *topology.Node) error - IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) - FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) - - StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) - FinishTransaction(ctx context.Context, id string) (*Transaction, error) - Transactions(ctx context.Context) (map[string]*Transaction, error) - GetTransaction(ctx context.Context, id string) (*Transaction, error) - - GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) - GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) - - // ImportFieldKeys and ImportIndexKeys are mainly used when - // restoring a backup. They take a readerFunc which returns a - // reader rather than taking an io.Reader directly to allow for - // efficient retries (rather than reading the entire request body - // into a buffer and reusing it). Reader returned from the func - // must be properly closed by the implementation. - ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error - ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error - - // SetInternalAPI tells the client the API it should use for internal/loopback ops - // where applicable. - SetInternalAPI(api *API) -} - -//=============== - // InternalQueryClient is the internal interface for querying a node. type InternalQueryClient interface { SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) @@ -159,142 +87,3 @@ func newNopInternalQueryClient() nopInternalQueryClient { } var _ InternalQueryClient = newNopInternalQueryClient() - -//=============== - -type nopInternalClient struct{ nopInternalQueryClient } - -func newNopInternalClient() nopInternalClient { - return nopInternalClient{} -} - -var _ InternalClient = newNopInternalClient() - -func (n nopInternalClient) AvailableShards(ctx context.Context, indexName string) ([]uint64, error) { - return nil, nil -} - -func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) { - return nil, nil -} -func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } -func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error { - return nil -} - -func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { - return nil -} -func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) { - return nil, nil -} -func (n nopInternalClient) PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error) { - return nil, nil -} -func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { - return nil, nil -} -func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { - return nil, nil -} -func (n nopInternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error { - return nil -} -func (n nopInternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error { - return nil -} - -func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { - return nil -} - -func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { - return nil, nil -} - -func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { - return nil -} - -func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) IDAllocDataWriter(cctx context.Context, f io.Reader, primary *topology.Node) error { - return nil -} - -func (n nopInternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { - return nil -} -func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { - return nil -} -func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { - return nil -} -func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { - return nil -} -func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } -func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { - return nil -} -func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { - return nil, nil -} -func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { - return nil, nil, nil -} -func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error { - return nil -} -func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) { - return nil, nil -} -func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) { - return nil, nil -} - -func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) { - return nil, nil -} -func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) { - return nil, nil -} -func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) { - return nil, nil -} -func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) { - return nil, nil -} - -func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) { - return nil, nil -} - -func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) { - return nil, nil -} -func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error { - return nil -} - -func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error { - return nil -} - -func (c nopInternalClient) SetInternalAPI(api *API) { -} diff --git a/cluster.go b/cluster.go index 1f45d2c15..687c6ca8e 100644 --- a/cluster.go +++ b/cluster.go @@ -102,7 +102,7 @@ type cluster struct { // nolint: maligned logger logger.Logger - InternalClient InternalClient + InternalClient *InternalClient confirmDownRetries int confirmDownSleep time.Duration @@ -120,7 +120,7 @@ func newCluster() *cluster { translationSyncer: NopTranslationSyncer, - InternalClient: newNopInternalClient(), + InternalClient: &InternalClient{}, // TODO might have to fill this out a bit logger: logger.NopLogger, diff --git a/cmd/badloader/badloader.go b/cmd/badloader/badloader.go index 642137575..035b70dea 100644 --- a/cmd/badloader/badloader.go +++ b/cmd/badloader/badloader.go @@ -13,7 +13,7 @@ import ( gohttp "net/http" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/vprint" @@ -22,7 +22,7 @@ import ( "strings" ) -func UploadTar(srcFile string, client *http.InternalClient) error { +func UploadTar(srcFile string, client *pilosa.InternalClient) error { t0 := time.Now() f, err := os.Open(srcFile) if err != nil { @@ -114,7 +114,7 @@ func main() { host := "127.0.0.1:10101" h := &gohttp.Client{} - c, err := http.NewInternalClient(host, h) + c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) vprint.PanicOn(err) tarSrcPath := "q2.tar.gz" diff --git a/cmd/pilosa-bench/main.go b/cmd/pilosa-bench/main.go index 15c7f4e6b..c68e97e14 100644 --- a/cmd/pilosa-bench/main.go +++ b/cmd/pilosa-bench/main.go @@ -16,8 +16,8 @@ import ( "strings" "time" - "github.com/molecula/featurebase/v3" - phttp "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/encoding/proto" "golang.org/x/sync/errgroup" ) @@ -78,7 +78,7 @@ func run(ctx context.Context, args []string) (err error) { rand.Seed(0) // Setup connection to pilosa. - client, err := phttp.NewInternalClient(*hostport, http.DefaultClient) + client, err := pilosa.NewInternalClient(*hostport, http.DefaultClient, pilosa.WithSerializer(proto.Serializer{})) if err != nil { return err } @@ -270,7 +270,7 @@ func generateTopKQuery(index, field string, from, to time.Time) string { } // loadFields returns a mapping of index/field names to field info & identifiers. -func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) { +func loadFields(ctx context.Context, client *pilosa.InternalClient) (map[fieldKey]*fieldInfo, error) { indexes, err := client.Schema(ctx) if err != nil { return nil, err @@ -299,7 +299,7 @@ func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey } // fetchFieldIDs returns a list of field IDs or keys. -func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) { +func fetchFieldIDs(ctx context.Context, client *pilosa.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) { resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`}) if err != nil { return nil, err diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 282e9034b..27529e782 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -16,9 +16,10 @@ import ( "time" "github.com/gogo/protobuf/proto" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/client" - "github.com/molecula/featurebase/v3/http" + fb_proto "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/pb" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/vprint" @@ -162,7 +163,7 @@ func main() { func (cfg *RandomQueryConfig) Run() (err error) { remoteClient := nethttp.DefaultClient - cli, err := http.NewInternalClient(cfg.HostPort, remoteClient) + cli, err := pilosa.NewInternalClient(cfg.HostPort, remoteClient, pilosa.WithSerializer(fb_proto.Serializer{})) if err != nil { return err } diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go index 2c847217f..b29582f1a 100644 --- a/cmd/random-query/main_test.go +++ b/cmd/random-query/main_test.go @@ -9,7 +9,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck @@ -35,7 +34,7 @@ func Test_RandomQuery(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID(nodeid[0]), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), pilosa.OptServerReplicaN(nReplicas), )}, ) diff --git a/cmd/slurp/slurp.go b/cmd/slurp/slurp.go index 8b4eaef85..dfd01f063 100644 --- a/cmd/slurp/slurp.go +++ b/cmd/slurp/slurp.go @@ -18,7 +18,7 @@ import ( "time" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/vprint" ) @@ -32,7 +32,7 @@ type stateMachine struct { lastField string lastShard uint64 state string - client *http.InternalClient + client *pilosa.InternalClient start time.Time profile string @@ -136,7 +136,7 @@ func (r *stateMachine) Upload() error { return nil } -func UploadTar(srcFile string, client *http.InternalClient, profile, host string) error { +func UploadTar(srcFile string, client *pilosa.InternalClient, profile, host string) error { f, err := os.Open(srcFile) if err != nil { @@ -192,7 +192,7 @@ func main() { if profile != "" { startProfile(host) } - c, err := http.NewInternalClient(host, h) + c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) vprint.PanicOn(err) t0 := time.Now() diff --git a/ctl/backup.go b/ctl/backup.go index 4a0e1bade..aed077a37 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -13,7 +13,7 @@ import ( "time" pilosa "github.com/molecula/featurebase/v3" - fb_http "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/topology" "github.com/pkg/errors" @@ -49,7 +49,7 @@ type BackupCommand struct { // nolint: maligned Pprof string `json:"pprof"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO @@ -93,7 +93,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) { } // Create a client to the server. - client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod), fb_http.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)) + client, err := commandClient(cmd, pilosa.WithClientRetryPeriod(cmd.RetryPeriod), pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)) if err != nil { return fmt.Errorf("creating client: %w", err) } @@ -289,9 +289,10 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string, logger := cmd.Logger() logger.Printf("backing up shard: index=%q id=%d", indexName, shard) - client := fb_http.NewInternalClientFromURI(&node.URI, - fb_http.GetHTTPClient(cmd.tlsConfig, fb_http.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)), - fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) + client := pilosa.NewInternalClientFromURI(&node.URI, + pilosa.GetHTTPClient(cmd.tlsConfig, pilosa.ClientResponseHeaderTimeoutOption(cmd.HeaderTimeout)), + pilosa.WithClientRetryPeriod(cmd.RetryPeriod), + pilosa.WithSerializer(proto.Serializer{})) rc, err := client.ShardReader(ctx, indexName, shard) if err != nil { return fmt.Errorf("fetching shard reader: %w", err) diff --git a/ctl/chksum.go b/ctl/chksum.go index 5b10d56c9..037970644 100644 --- a/ctl/chksum.go +++ b/ctl/chksum.go @@ -20,7 +20,7 @@ type ChkSumCommand struct { // nolint: maligned Host string `json:"host"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO diff --git a/ctl/common.go b/ctl/common.go index ecb20d8e9..ae6551e53 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -4,7 +4,8 @@ package ctl import ( "time" - "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/server" "github.com/pkg/errors" @@ -27,14 +28,14 @@ func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string, flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections") } -// AnyClientOption can be either http.InternalClientOption or -// http.ClientOption. The internal options are specific to the +// AnyClientOption can be either pilosa.InternalClientOption or +// pilosa.ClientOption. The internal options are specific to the // featurebase client, whereas the client options are applied to the // Go HTTP client that gets used under the hood. type AnyClientOption interface{} // commandClient returns a pilosa.InternalHTTPClient for the command -func commandClient(cmd CommandWithTLSSupport, opts ...AnyClientOption) (*http.InternalClient, error) { +func commandClient(cmd CommandWithTLSSupport, opts ...AnyClientOption) (*pilosa.InternalClient, error) { internalopts, clientopts, err := separateOptions(opts...) if err != nil { return nil, errors.Wrap(err, "separating client options") @@ -42,13 +43,14 @@ func commandClient(cmd CommandWithTLSSupport, opts ...AnyClientOption) (*http.In // we default dial timeout to 3s in commandClient, but prepend it // to the option list so other options can override it. - clientopts = append([]http.ClientOption{http.ClientDialTimeoutOption(time.Second * 3)}, clientopts...) + clientopts = append([]pilosa.ClientOption{pilosa.ClientDialTimeoutOption(time.Second * 3)}, clientopts...) + internalopts = append([]pilosa.InternalClientOption{pilosa.WithSerializer(proto.Serializer{})}, internalopts...) tls := cmd.TLSConfiguration() tlsConfig, err := server.GetTLSConfig(&tls, cmd.Logger()) if err != nil { return nil, errors.Wrap(err, "getting tls config") } - client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(tlsConfig, clientopts...), internalopts...) + client, err := pilosa.NewInternalClient(cmd.TLSHost(), pilosa.GetHTTPClient(tlsConfig, clientopts...), internalopts...) if err != nil { return nil, errors.Wrap(err, "getting internal client") } @@ -57,15 +59,15 @@ func commandClient(cmd CommandWithTLSSupport, opts ...AnyClientOption) (*http.In // separateOptions splits the list of AnyClientOption into the two // possible types. -func separateOptions(opts ...AnyClientOption) ([]http.InternalClientOption, []http.ClientOption, error) { - internalopts := []http.InternalClientOption{} - clientopts := []http.ClientOption{} +func separateOptions(opts ...AnyClientOption) ([]pilosa.InternalClientOption, []pilosa.ClientOption, error) { + internalopts := []pilosa.InternalClientOption{} + clientopts := []pilosa.ClientOption{} for _, opt := range opts { - if iopt, ok := opt.(http.InternalClientOption); ok { + if iopt, ok := opt.(pilosa.InternalClientOption); ok { internalopts = append(internalopts, iopt) continue } - if copt, ok := opt.(http.ClientOption); ok { + if copt, ok := opt.(pilosa.ClientOption); ok { clientopts = append(clientopts, copt) continue } diff --git a/ctl/import.go b/ctl/import.go index 3b18b8499..09bc4e0c4 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -11,7 +11,7 @@ import ( "strconv" "time" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/server" "github.com/pkg/errors" @@ -48,7 +48,7 @@ type ImportCommand struct { // nolint: maligned Sort bool `json:"sort"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO diff --git a/ctl/restore.go b/ctl/restore.go index 8f370ba0e..0c8cb51b0 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -18,7 +18,6 @@ import ( "github.com/hashicorp/go-retryablehttp" pilosa "github.com/molecula/featurebase/v3" - fb_http "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/topology" @@ -44,7 +43,7 @@ type RestoreCommand struct { Pprof string `json:"pprof"` // Reusable client. - client pilosa.InternalClient + client *pilosa.InternalClient // Standard input/output *pilosa.CmdIO @@ -86,7 +85,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) { return fmt.Errorf("parsing tls config: %w", err) } // Create a client to the server. - client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) + client, err := commandClient(cmd, pilosa.WithClientRetryPeriod(cmd.RetryPeriod)) if err != nil { return fmt.Errorf("creating client: %w", err) } diff --git a/dbshard_test.go b/dbshard_test.go index 6f7281301..95ff92eea 100644 --- a/dbshard_test.go +++ b/dbshard_test.go @@ -7,9 +7,8 @@ import ( "reflect" "testing" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck @@ -23,7 +22,7 @@ func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) { pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() diff --git a/executor_test.go b/executor_test.go index d22db1c9b..5233d07a7 100644 --- a/executor_test.go +++ b/executor_test.go @@ -29,7 +29,6 @@ import ( "github.com/molecula/featurebase/v3/boltdb" "github.com/molecula/featurebase/v3/ctl" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/proto" "github.com/molecula/featurebase/v3/server" @@ -3826,7 +3825,7 @@ func TestExecutor_Execute_Existence(t *testing.T) { c := test.MustRunCluster(t, 1, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), ), }) defer c.Close() @@ -4217,7 +4216,7 @@ func TestExecutor_Execute_All(t *testing.T) { c := test.MustRunCluster(t, 1, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), ), }) defer c.Close() diff --git a/handler.go b/handler.go index c74296b45..c39e03780 100644 --- a/handler.go +++ b/handler.go @@ -76,9 +76,9 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { }) } -// Handler is the interface for the data handler, a wrapper around +// HandlerI is the interface for the data handler, a wrapper around // Pilosa's data store. -type Handler interface { +type HandlerI interface { Serve() error Close() error } @@ -94,7 +94,7 @@ func (n nopHandler) Close() error { } // NopHandler is a no-op implementation of the Handler interface. -var NopHandler Handler = nopHandler{} +var NopHandler HandlerI = nopHandler{} // ImportValueRequest describes the import request structure // for a value (BSI) import. diff --git a/http/error.go b/http/error.go deleted file mode 100644 index 733f3f655..000000000 --- a/http/error.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package http - -// Error defines a standard application error. -type Error struct { - // Human-readable message. - Message string `json:"message"` -} - -// Error returns the string representation of the error message. -func (e *Error) Error() string { - return e.Message -} diff --git a/http/handler.go b/http_handler.go similarity index 90% rename from http/handler.go rename to http_handler.go index 785e0e7f3..e352711e9 100644 --- a/http/handler.go +++ b/http_handler.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http +package pilosa import ( "bytes" @@ -29,10 +29,8 @@ import ( "github.com/felixge/fgprof" "github.com/gorilla/handlers" "github.com/gorilla/mux" - pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/authz" - "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/pql" @@ -51,7 +49,7 @@ import ( type Handler struct { Handler http.Handler - fileSystem pilosa.FileSystem + fileSystem FileSystem logger logger.Logger @@ -60,7 +58,7 @@ type Handler struct { // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec - api *pilosa.API + api *API ln net.Listener // url is used to hold the advertise bind address for printing a log during startup. @@ -68,6 +66,9 @@ type Handler struct { closeTimeout time.Duration + serializer Serializer + roaringSerializer Serializer + server *http.Server middleware []func(http.Handler) http.Handler @@ -96,7 +97,7 @@ type errorResponse struct { Error string `json:"error"` } -// handlerOption is a functional option type for pilosa.Handler +// handlerOption is a functional option type for Handler type handlerOption func(s *Handler) error func OptHandlerMiddleware(middleware func(http.Handler) http.Handler) handlerOption { @@ -116,7 +117,7 @@ func OptHandlerAllowedOrigins(origins []string) handlerOption { } } -func OptHandlerAPI(api *pilosa.API) handlerOption { +func OptHandlerAPI(api *API) handlerOption { return func(h *Handler) error { h.api = api return nil @@ -137,7 +138,7 @@ func OptHandlerAuthZ(gp *authz.GroupPermissions) handlerOption { } } -func OptHandlerFileSystem(fs pilosa.FileSystem) handlerOption { +func OptHandlerFileSystem(fs FileSystem) handlerOption { return func(h *Handler) error { h.fileSystem = fs return nil @@ -158,6 +159,20 @@ func OptHandlerQueryLogger(logger logger.Logger) handlerOption { } } +func OptHandlerSerializer(s Serializer) handlerOption { + return func(h *Handler) error { + h.serializer = s + return nil + } +} + +func OptHandlerRoaringSerializer(s Serializer) handlerOption { + return func(h *Handler) error { + h.roaringSerializer = s + return nil + } +} + // OptHandlerListener set the listener that will be used by the HTTP server. // Url must be the advertised URL. It will be used to show a log to the user // about where the Web UI is. This option is mandatory. @@ -183,15 +198,8 @@ var importOk []byte // NewHandler returns a new instance of Handler with a default logger. func NewHandler(opts ...handlerOption) (*Handler, error) { - makeImportOk.Do(func() { - var err error - importOk, err = proto.DefaultSerializer.Marshal(&pilosa.ImportResponse{Err: ""}) - if err != nil { - panic(fmt.Sprintf("trying to cache import-OK response: %v", err)) - } - }) handler := &Handler{ - fileSystem: pilosa.NopFileSystem, + fileSystem: NopFileSystem, logger: logger.NopLogger, closeTimeout: time.Second * 30, } @@ -202,6 +210,16 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { return nil, errors.Wrap(err, "applying option") } } + if handler.serializer == nil || handler.roaringSerializer == nil { + return nil, errors.New("must use serializer options when creating handler") + } + makeImportOk.Do(func() { + var err error + importOk, err = handler.serializer.Marshal(&ImportResponse{Err: ""}) + if err != nil { + panic(fmt.Sprintf("trying to cache import-OK response: %v", err)) + } + }) // if OptHandlerFileSystem is used, it must be before newRouter is called handler.Handler = newRouter(handler) @@ -350,7 +368,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { queryRequest := r.Context().Value(contextKeyQueryRequest) var queryString string - if req, ok := queryRequest.(*pilosa.QueryRequest); ok { + if req, ok := queryRequest.(*QueryRequest); ok { queryString = req.Query } @@ -378,7 +396,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { stats := h.api.StatsWithTags(statsTags) if stats != nil { - stats.Timing(pilosa.MetricHTTPRequest, dur, 0.1) + stats.Timing(MetricHTTPRequest, dur, 0.1) } }) } @@ -604,7 +622,7 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http // figure out what the user is querying for queryString := "" queryRequest := r.Context().Value(contextKeyQueryRequest) - if req, ok := queryRequest.(*pilosa.QueryRequest); ok { + if req, ok := queryRequest.(*QueryRequest); ok { queryString = req.Query q, err := pql.ParseString(queryString) @@ -734,10 +752,21 @@ func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { h *Handler - Success bool `json:"success"` - Name string `json:"name,omitempty"` - CreatedAt int64 `json:"createdAt,omitempty"` - Error *Error `json:"error,omitempty"` + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + Error *HTTPError `json:"error,omitempty"` +} + +// Error defines a standard application error. +type HTTPError struct { + // Human-readable message. + Message string `json:"message"` +} + +// Error returns the string representation of the error message. +func (e *HTTPError) Error() string { + return e.Message } // check determines success or failure based on the error. @@ -752,18 +781,18 @@ func (r *successResponse) check(err error) (statusCode int) { // Determine HTTP status code based on the error type. switch cause.(type) { - case pilosa.BadRequestError: + case BadRequestError: statusCode = http.StatusBadRequest - case pilosa.ConflictError: + case ConflictError: statusCode = http.StatusConflict - case pilosa.NotFoundError: + case NotFoundError: statusCode = http.StatusNotFound default: statusCode = http.StatusInternalServerError } r.Success = false - r.Error = &Error{Message: err.Error()} + r.Error = &HTTPError{Message: err.Error()} return statusCode } @@ -873,7 +902,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { return } if !h.permissions.IsAdmin(g.([]authn.Group)) { - var filtered []*pilosa.IndexInfo + var filtered []*IndexInfo allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read) for _, s := range schema { for _, index := range allowed { @@ -887,7 +916,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } } - if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { + if err := json.NewEncoder(w).Encode(Schema{Indexes: schema}); err != nil { h.logger.Errorf("write schema response error: %s", err) } } @@ -914,7 +943,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) return } if !h.permissions.IsAdmin(g.([]authn.Group)) { - var filtered []*pilosa.IndexInfo + var filtered []*IndexInfo allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read) for _, s := range schema { for _, index := range allowed { @@ -927,7 +956,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) schema = filtered } } - if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { + if err := json.NewEncoder(w).Encode(Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } @@ -940,7 +969,7 @@ func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { remote = true } - schema := &pilosa.Schema{} + schema := &Schema{} if err := json.NewDecoder(r.Body).Decode(schema); err != nil { http.Error(w, fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err), http.StatusBadRequest) return @@ -981,12 +1010,12 @@ func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) { } if !h.permissions.IsAdmin(g.([]authn.Group)) { allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read) - filteredNodeUsages := map[string]pilosa.NodeUsage{} + filteredNodeUsages := map[string]NodeUsage{} for nodeId, nodeUsage := range nodeUsages { - filteredIndexUsage := pilosa.NodeUsage{ - Disk: pilosa.DiskUsage{ - IndexUsage: map[string]pilosa.IndexUsage{}, + filteredIndexUsage := NodeUsage{ + Disk: DiskUsage{ + IndexUsage: map[string]IndexUsage{}, }, } for index, idxUsage := range nodeUsage.Disk.IndexUsage { @@ -1058,7 +1087,7 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { } type getSchemaResponse struct { - Indexes []*pilosa.IndexInfo `json:"indexes"` + Indexes []*IndexInfo `json:"indexes"` } type getStatusResponse struct { @@ -1068,8 +1097,7 @@ type getStatusResponse struct { ClusterName string `json:"clusterName"` } -func hash(s string) string { - +func httpHash(s string) string { hasher := blake3.New() _, _ = hasher.Write([]byte(s)) var buf [16]byte @@ -1085,11 +1113,11 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Read previouly parsed request from context qreq := r.Context().Value(contextKeyQueryRequest) qerr := r.Context().Value(contextKeyQueryError) - req, ok := qreq.(*pilosa.QueryRequest) + req, ok := qreq.(*QueryRequest) if DoPerQueryProfiling { backend := storage.DefaultBackend - reqHash := hash(req.Query) + reqHash := httpHash(req.Query) qlen := len(req.Query) if qlen > 100 { @@ -1112,7 +1140,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { if err != nil || !ok { w.WriteHeader(http.StatusBadRequest) - e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &QueryResponse{Err: err}) if e != nil { h.logger.Errorf("write query response error: %v (while trying to write another error: %v)", e, err) } @@ -1124,9 +1152,9 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp, err := h.api.Query(r.Context(), req) if err != nil { switch errors.Cause(err) { - case pilosa.ErrTooManyWrites: + case ErrTooManyWrites: w.WriteHeader(http.StatusRequestEntityTooLarge) - case pilosa.ErrTranslateStoreReadOnly: + case ErrTranslateStoreReadOnly: u := h.api.PrimaryReplicaNodeURL() u.Path, u.RawQuery = r.URL.Path, r.URL.RawQuery http.Redirect(w, r, u.String(), http.StatusFound) @@ -1134,7 +1162,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { default: w.WriteHeader(http.StatusBadRequest) } - e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &QueryResponse{Err: err}) if e != nil { h.logger.Errorf("write query response error: %v (while trying to write another error: %v)", e, err) } @@ -1146,7 +1174,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // doing nothing right now. if resp.Err != nil { switch errors.Cause(resp.Err) { - case pilosa.ErrTooManyWrites: + case ErrTooManyWrites: w.WriteHeader(http.StatusRequestEntityTooLarge) default: w.WriteHeader(http.StatusBadRequest) @@ -1282,7 +1310,7 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { } type postIndexRequest struct { - Options pilosa.IndexOptions `json:"options"` + Options IndexOptions `json:"options"` } //_postIndexRequest is necessary to avoid recursion while decoding. @@ -1297,14 +1325,14 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error { return errors.Wrap(err, "unmarshalling unexpected values") } - validIndexOptions := getValidOptions(pilosa.IndexOptions{}) + validIndexOptions := getValidOptions(IndexOptions{}) err := validateOptions(m, validIndexOptions) if err != nil { return err } // Unmarshal expected values. _p := _postIndexRequest{ - Options: pilosa.IndexOptions{ + Options: IndexOptions{ Keys: false, TrackExistence: true, }, @@ -1389,7 +1417,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { // Decode request. req := postIndexRequest{ - Options: pilosa.IndexOptions{ + Options: IndexOptions{ Keys: false, TrackExistence: true, }, @@ -1403,7 +1431,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { if index != nil { resp.CreatedAt = index.CreatedAt() - } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + } else if _, ok = errors.Cause(err).(ConflictError); ok { if index, _ = h.api.Index(r.Context(), indexName); index != nil { resp.CreatedAt = index.CreatedAt() } @@ -1478,13 +1506,13 @@ func (h *Handler) handleGetPastQueries(w http.ResponseWriter, r *http.Request) { } -func fieldOptionsToFunctionalOpts(opt fieldOptions) []pilosa.FieldOption { +func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption { // Convert json options into functional options. - var fos []pilosa.FieldOption + var fos []FieldOption switch opt.Type { - case pilosa.FieldTypeSet: - fos = append(fos, pilosa.OptFieldTypeSet(*opt.CacheType, *opt.CacheSize)) - case pilosa.FieldTypeInt: + case FieldTypeSet: + fos = append(fos, OptFieldTypeSet(*opt.CacheType, *opt.CacheSize)) + case FieldTypeInt: if opt.Min == nil { min := pql.NewDecimal(int64(math.MinInt64), 0) opt.Min = &min @@ -1493,8 +1521,8 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []pilosa.FieldOption { max := pql.NewDecimal(int64(math.MaxInt64), 0) opt.Max = &max } - fos = append(fos, pilosa.OptFieldTypeInt(opt.Min.ToInt64(0), opt.Max.ToInt64(0))) - case pilosa.FieldTypeDecimal: + fos = append(fos, OptFieldTypeInt(opt.Min.ToInt64(0), opt.Max.ToInt64(0))) + case FieldTypeDecimal: scale := int64(0) if opt.Scale != nil { scale = *opt.Scale @@ -1516,27 +1544,27 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []pilosa.FieldOption { minmax = append(minmax, *opt.Max) } } - fos = append(fos, pilosa.OptFieldTypeDecimal(scale, minmax...)) - case pilosa.FieldTypeTimestamp: + fos = append(fos, OptFieldTypeDecimal(scale, minmax...)) + case FieldTypeTimestamp: if opt.Epoch == nil { - epoch := pilosa.DefaultEpoch + epoch := DefaultEpoch opt.Epoch = &epoch } - fos = append(fos, pilosa.OptFieldTypeTimestamp(opt.Epoch.UTC(), *opt.TimeUnit)) - case pilosa.FieldTypeTime: - fos = append(fos, pilosa.OptFieldTypeTime(*opt.TimeQuantum, opt.NoStandardView)) - case pilosa.FieldTypeMutex: - fos = append(fos, pilosa.OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize)) - case pilosa.FieldTypeBool: - fos = append(fos, pilosa.OptFieldTypeBool()) + fos = append(fos, OptFieldTypeTimestamp(opt.Epoch.UTC(), *opt.TimeUnit)) + case FieldTypeTime: + fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, opt.NoStandardView)) + case FieldTypeMutex: + fos = append(fos, OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize)) + case FieldTypeBool: + fos = append(fos, OptFieldTypeBool()) } if opt.Keys != nil { if *opt.Keys { - fos = append(fos, pilosa.OptFieldKeys()) + fos = append(fos, OptFieldKeys()) } } if opt.ForeignIndex != nil { - fos = append(fos, pilosa.OptFieldForeignIndex(*opt.ForeignIndex)) + fos = append(fos, OptFieldForeignIndex(*opt.ForeignIndex)) } return fos } @@ -1580,13 +1608,13 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos := fieldOptionsToFunctionalOpts(req.Options) field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...) - if _, ok = err.(pilosa.BadRequestError); ok { + if _, ok = err.(BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) return } if field != nil { resp.CreatedAt = field.CreatedAt() - } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + } else if _, ok = errors.Cause(err).(ConflictError); ok { if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { resp.CreatedAt = field.CreatedAt() } @@ -1676,7 +1704,7 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { opt.Epoch = fSpec.FieldOptions.Epoch opt.TimeUnit = fSpec.FieldOptions.Unit if fSpec.FieldOptions.TimeQuantum != nil { - timeQuantumVal := pilosa.TimeQuantum(*fSpec.FieldOptions.TimeQuantum) + timeQuantumVal := TimeQuantum(*fSpec.FieldOptions.TimeQuantum) opt.TimeQuantum = &timeQuantumVal } @@ -1694,7 +1722,7 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { // a later error, but if the list of fields is empty, the entire index was new, // and should be cleaned up, in which case there's no need to track or delete // the specific fields separately. -func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *pilosa.Index, returnedFields []string, err error) { +func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) { // create index indexName := schema.IndexName var createdFields []string @@ -1707,7 +1735,7 @@ func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) default: return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType) } - opts := pilosa.IndexOptions{ + opts := IndexOptions{ Keys: useKeys, TrackExistence: true, } @@ -1731,7 +1759,7 @@ func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) case "ensure", "require": index, err = h.api.Index(ctx, indexName) if err != nil { - if _, ok := err.(pilosa.NotFoundError); !ok { + if _, ok := err.(NotFoundError); !ok { return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err) } else { err = nil @@ -1791,7 +1819,7 @@ func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) field, schemaErr := h.api.Field(ctx, indexName, fieldName) if schemaErr != nil { // NotFoundError is fine - if _, ok := schemaErr.(pilosa.NotFoundError); !ok { + if _, ok := schemaErr.(NotFoundError); !ok { return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err) } } @@ -1903,35 +1931,35 @@ type postFieldRequest struct { Options fieldOptions `json:"options"` } -// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, +// fieldOptions tracks FieldOptions. It is made up of pointers to values, // and used for input validation. type fieldOptions struct { - Type string `json:"type,omitempty"` - CacheType *string `json:"cacheType,omitempty"` - CacheSize *uint32 `json:"cacheSize,omitempty"` - Min *pql.Decimal `json:"min,omitempty"` - Max *pql.Decimal `json:"max,omitempty"` - Scale *int64 `json:"scale,omitempty"` - Epoch *time.Time `json:"epoch,omitempty"` - TimeUnit *string `json:"timeUnit,omitempty"` - TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` - Keys *bool `json:"keys,omitempty"` - NoStandardView bool `json:"noStandardView,omitempty"` - ForeignIndex *string `json:"foreignIndex,omitempty"` + Type string `json:"type,omitempty"` + CacheType *string `json:"cacheType,omitempty"` + CacheSize *uint32 `json:"cacheSize,omitempty"` + Min *pql.Decimal `json:"min,omitempty"` + Max *pql.Decimal `json:"max,omitempty"` + Scale *int64 `json:"scale,omitempty"` + Epoch *time.Time `json:"epoch,omitempty"` + TimeUnit *string `json:"timeUnit,omitempty"` + TimeQuantum *TimeQuantum `json:"timeQuantum,omitempty"` + Keys *bool `json:"keys,omitempty"` + NoStandardView bool `json:"noStandardView,omitempty"` + ForeignIndex *string `json:"foreignIndex,omitempty"` } func (o *fieldOptions) validate() error { // Pointers to default values. - defaultCacheType := pilosa.DefaultCacheType - defaultCacheSize := uint32(pilosa.DefaultCacheSize) + defaultCacheType := DefaultCacheType + defaultCacheSize := uint32(DefaultCacheSize) switch o.Type { - case pilosa.FieldTypeSet, "": + case FieldTypeSet, "": // Because FieldTypeSet is the default, its arguments are // not required. Instead, the defaults are applied whenever // a value does not exist. if o.Type == "" { - o.Type = pilosa.FieldTypeSet + o.Type = FieldTypeSet } if o.CacheType == nil { o.CacheType = &defaultCacheType @@ -1940,59 +1968,59 @@ func (o *fieldOptions) validate() error { o.CacheSize = &defaultCacheSize } if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) + return NewBadRequestError(errors.New("min does not apply to field type set")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) + return NewBadRequestError(errors.New("max does not apply to field type set")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) } - case pilosa.FieldTypeInt: + case FieldTypeInt: if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + return NewBadRequestError(errors.New("cacheType does not apply to field type int")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) } - case pilosa.FieldTypeDecimal: + case FieldTypeDecimal: if o.Scale == nil { - return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument")) + return NewBadRequestError(errors.New("decimal field requires a scale argument")) } else if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + return NewBadRequestError(errors.New("cacheType does not apply to field type int")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) - } else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal { - return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.ForeignIndex != nil && o.Type == FieldTypeDecimal { + return NewBadRequestError(errors.New("decimal field cannot be a foreign key")) } - case pilosa.FieldTypeTimestamp: + case FieldTypeTimestamp: if o.TimeUnit == nil { - return pilosa.NewBadRequestError(errors.New("timestamp field requires a timeUnit argument")) - } else if !pilosa.IsValidTimeUnit(*o.TimeUnit) { - return pilosa.NewBadRequestError(errors.New("invalid timeUnit argument")) + return NewBadRequestError(errors.New("timestamp field requires a timeUnit argument")) + } else if !IsValidTimeUnit(*o.TimeUnit) { + return NewBadRequestError(errors.New("invalid timeUnit argument")) } else if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type timestamp")) + return NewBadRequestError(errors.New("cacheType does not apply to field type timestamp")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type timestamp")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type timestamp")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type timestamp")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type timestamp")) } else if o.ForeignIndex != nil { - return pilosa.NewBadRequestError(errors.New("timestamp field cannot be a foreign key")) + return NewBadRequestError(errors.New("timestamp field cannot be a foreign key")) } - case pilosa.FieldTypeTime: + case FieldTypeTime: if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) + return NewBadRequestError(errors.New("cacheType does not apply to field type time")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type time")) } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) + return NewBadRequestError(errors.New("min does not apply to field type time")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) + return NewBadRequestError(errors.New("max does not apply to field type time")) } else if o.TimeQuantum == nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) + return NewBadRequestError(errors.New("timeQuantum is required for field type time")) } - case pilosa.FieldTypeMutex: + case FieldTypeMutex: if o.CacheType == nil { o.CacheType = &defaultCacheType } @@ -2000,27 +2028,27 @@ func (o *fieldOptions) validate() error { o.CacheSize = &defaultCacheSize } if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex")) + return NewBadRequestError(errors.New("min does not apply to field type mutex")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex")) + return NewBadRequestError(errors.New("max does not apply to field type mutex")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) } - case pilosa.FieldTypeBool: + case FieldTypeBool: if o.CacheType != nil { - return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + return NewBadRequestError(errors.New("cacheType does not apply to field type bool")) } else if o.CacheSize != nil { - return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + return NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) } else if o.Min != nil { - return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + return NewBadRequestError(errors.New("min does not apply to field type bool")) } else if o.Max != nil { - return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + return NewBadRequestError(errors.New("max does not apply to field type bool")) } else if o.TimeQuantum != nil { - return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + return NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) } else if o.Keys != nil { - return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + return NewBadRequestError(errors.New("keys does not apply to field type bool")) } else if o.ForeignIndex != nil { - return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key")) + return NewBadRequestError(errors.New("bool field cannot be a foreign key")) } default: return errors.Errorf("invalid field type: %s", o.Type) @@ -2051,7 +2079,7 @@ func (h *Handler) handleGetTransactionList(w http.ResponseWriter, r *http.Reques trnsMap, err := h.api.Transactions(r.Context()) if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary: + case ErrNodeNotPrimary: http.Error(w, err.Error(), http.StatusBadRequest) default: http.Error(w, "problem getting transactions: "+err.Error(), http.StatusInternalServerError) @@ -2060,7 +2088,7 @@ func (h *Handler) handleGetTransactionList(w http.ResponseWriter, r *http.Reques } // Convert the map of transactions to a slice. - trnsList := make([]*pilosa.Transaction, len(trnsMap)) + trnsList := make([]*Transaction, len(trnsMap)) var i int for _, v := range trnsMap { trnsList[i] = v @@ -2086,7 +2114,7 @@ func (h *Handler) handleGetTransactions(w http.ResponseWriter, r *http.Request) trnsMap, err := h.api.Transactions(r.Context()) if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary: + case ErrNodeNotPrimary: http.Error(w, err.Error(), http.StatusBadRequest) default: http.Error(w, "problem getting transactions: "+err.Error(), http.StatusInternalServerError) @@ -2101,18 +2129,18 @@ func (h *Handler) handleGetTransactions(w http.ResponseWriter, r *http.Request) } type TransactionResponse struct { - Transaction *pilosa.Transaction `json:"transaction,omitempty"` - Error string `json:"error,omitempty"` + Transaction *Transaction `json:"transaction,omitempty"` + Error string `json:"error,omitempty"` } -func (h *Handler) doTransactionResponse(w http.ResponseWriter, err error, trns *pilosa.Transaction) { +func (h *Handler) doTransactionResponse(w http.ResponseWriter, err error, trns *Transaction) { if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary, pilosa.ErrTransactionExists: + case ErrNodeNotPrimary, ErrTransactionExists: w.WriteHeader(http.StatusBadRequest) - case pilosa.ErrTransactionExclusive: + case ErrTransactionExclusive: w.WriteHeader(http.StatusConflict) - case pilosa.ErrTransactionNotFound: + case ErrTransactionNotFound: w.WriteHeader(http.StatusNotFound) default: w.WriteHeader(http.StatusInternalServerError) @@ -2147,7 +2175,7 @@ func (h *Handler) handlePostTransaction(w http.ResponseWriter, r *http.Request) http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - reqTrns := &pilosa.Transaction{} + reqTrns := &Transaction{} if err := json.NewDecoder(r.Body).Decode(reqTrns); err != nil || reqTrns.Timeout == 0 { if err == nil { http.Error(w, "timeout is required and cannot be 0", http.StatusBadRequest) @@ -2210,7 +2238,7 @@ func (h *Handler) handleGetIndexShardSnapshot(w http.ResponseWriter, r *http.Req rc, err := h.api.IndexShardSnapshot(r.Context(), indexName, shard) if err != nil { switch errors.Cause(err) { - case pilosa.ErrIndexNotFound: + case ErrIndexNotFound: http.Error(w, err.Error(), http.StatusNotFound) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2227,7 +2255,7 @@ func (h *Handler) handleGetIndexShardSnapshot(w http.ResponseWriter, r *http.Req } // readQueryRequest parses an query parameters from r. -func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { +func (h *Handler) readQueryRequest(r *http.Request) (*QueryRequest, error) { switch r.Header.Get("Content-Type") { case "application/x-protobuf": return h.readProtobufQueryRequest(r) @@ -2247,15 +2275,15 @@ func (w *passthroughWriter) Write(p []byte) (int, error) { } // readProtobufQueryRequest parses query parameters in protobuf from r. -func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { +func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, error) { // Slurp the body. body, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } - qreq := &pilosa.QueryRequest{} - err = proto.DefaultSerializer.Unmarshal(body, qreq) + qreq := &QueryRequest{} + err = h.serializer.Unmarshal(body, qreq) if err != nil { return nil, errors.Wrap(err, "unmarshalling query request") } @@ -2263,7 +2291,7 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques } // readURLQueryRequest parses query parameters from URL parameters from r. -func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { +func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { q := r.URL.Query() // Parse query string. @@ -2289,7 +2317,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er } } - return &pilosa.QueryRequest{ + return &QueryRequest{ Query: query, Shards: shards, Profile: profile, @@ -2297,7 +2325,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er } // writeQueryResponse writes the response from the executor to w. -func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error { +func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error { if !validHeaderAcceptJSON(r.Header) { w.Header().Set("Content-Type", "application/protobuf") return h.writeProtobufQueryResponse(w, resp, headerAcceptRoaringRow(r.Header)) @@ -2307,10 +2335,10 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res } // writeProtobufQueryResponse writes the response from the executor to w as protobuf. -func (h *Handler) writeProtobufQueryResponse(w io.Writer, resp *pilosa.QueryResponse, writeRoaring bool) error { - serializer := proto.DefaultSerializer +func (h *Handler) writeProtobufQueryResponse(w io.Writer, resp *QueryResponse, writeRoaring bool) error { + serializer := h.serializer if writeRoaring { - serializer = proto.RoaringSerializer + serializer = h.roaringSerializer } if buf, err := serializer.Marshal(resp); err != nil { return errors.Wrap(err, "marshalling") @@ -2321,7 +2349,7 @@ func (h *Handler) writeProtobufQueryResponse(w io.Writer, resp *pilosa.QueryResp } // writeJSONQueryResponse writes the response from the executor to w as JSON. -func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse) error { +func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *QueryResponse) error { return json.NewEncoder(w).Encode(resp) } @@ -2413,9 +2441,9 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { if err = h.api.ExportCSV(r.Context(), index, field, shard, w); err != nil { switch errors.Cause(err) { - case pilosa.ErrFragmentNotFound: + case ErrFragmentNotFound: break - case pilosa.ErrClusterDoesNotOwnShard: + case ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2504,9 +2532,9 @@ func (h *Handler) handleGetNodes(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { buf, err := h.api.FragmentBlockData(r.Context(), r.Body) if err != nil { - if _, ok := err.(pilosa.BadRequestError); ok { + if _, ok := err.(BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) - } else if errors.Cause(err) == pilosa.ErrFragmentNotFound { + } else if errors.Cause(err) == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2539,7 +2567,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request blocks, err := h.api.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), q.Get("view"), shard) if err != nil { - if errors.Cause(err) == pilosa.ErrFragmentNotFound { + if errors.Cause(err) == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2557,7 +2585,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request } type getFragmentBlocksResponse struct { - Blocks []pilosa.FragmentBlock `json:"blocks"` + Blocks []FragmentBlock `json:"blocks"` } // handleGetFragmentData handles GET /internal/fragment/data requests. @@ -2609,7 +2637,7 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) // Retrieve partition data from holder. p, err := h.api.TranslateData(r.Context(), q.Get("index"), int(partition)) - if redir, ok := err.(pilosa.RedirectError); ok { + if redir, ok := err.(RedirectError); ok { newURL := *r.URL newURL.Host = redir.HostPort http.Redirect(w, r, newURL.String(), http.StatusSeeOther) @@ -2685,7 +2713,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht removeNode, err := h.api.RemoveNode(req.ID) if err != nil { - if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + if errors.Cause(err) == ErrNodeIDNotExists { http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) } else { http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) @@ -2721,10 +2749,10 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re var msg string if err != nil { switch errors.Cause(err) { - case pilosa.ErrNodeNotPrimary: + case ErrNodeNotPrimary: http.Error(w, err.Error(), http.StatusBadRequest) return - case pilosa.ErrResizeNotRunning: + case ErrResizeNotRunning: msg = err.Error() default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2767,7 +2795,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques err := h.api.ClusterMessage(r.Context(), r.Body) if err != nil { switch err := err.(type) { - case pilosa.MessageProcessingError: + case MessageProcessingError: http.Error(w, err.Error(), http.StatusInternalServerError) default: http.Error(w, err.Error(), http.StatusBadRequest) @@ -2785,14 +2813,14 @@ type defaultClusterMessageResponse struct{} func (h *Handler) handlePostTranslateData(w http.ResponseWriter, r *http.Request) { // Parse offsets for all indexes and fields from POST body. - offsets := make(pilosa.TranslateOffsetMap) + offsets := make(TranslateOffsetMap) if err := json.NewDecoder(r.Body).Decode(&offsets); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Stream all translation data. rd, err := h.api.GetTranslateEntryReader(r.Context(), offsets) - if errors.Cause(err) == pilosa.ErrNotImplemented { + if errors.Cause(err) == ErrNotImplemented { http.Error(w, err.Error(), http.StatusNotImplemented) return } else if err != nil { @@ -2809,7 +2837,7 @@ func (h *Handler) handlePostTranslateData(w http.ResponseWriter, r *http.Request enc := json.NewEncoder(w) for { // Read from store. - var entry pilosa.TranslateEntry + var entry TranslateEntry if err := rd.ReadEntry(&entry); err == io.EOF { return } else if err != nil { @@ -2932,13 +2960,13 @@ func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Re http.Error(w, err.Error(), http.StatusBadRequest) } } - opt := func(o *pilosa.ImportOptions) error { + opt := func(o *ImportOptions) error { o.SimPowerLossAfter = loss return nil } - req := &pilosa.AtomicRecord{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + req := &AtomicRecord{} + if err := h.serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -2952,7 +2980,7 @@ func (h *Handler) handlePostImportAtomicRecord(w http.ResponseWriter, r *http.Re } if err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + case ErrClusterDoesNotOwnShard, ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2980,7 +3008,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] index, err := h.api.Index(r.Context(), indexName) if err != nil { - if errors.Cause(err) == pilosa.ErrIndexNotFound { + if errors.Cause(err) == ErrIndexNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -2990,7 +3018,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { fieldName := mux.Vars(r)["field"] field := index.Field(fieldName) if field == nil { - http.Error(w, pilosa.ErrFieldNotFound.Error(), http.StatusNotFound) + http.Error(w, ErrFieldNotFound.Error(), http.StatusNotFound) return } @@ -2999,9 +3027,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { doClear := q.Get("clear") == "true" doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - opts := []pilosa.ImportOption{ - pilosa.OptImportOptionsClear(doClear), - pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + opts := []ImportOption{ + OptImportOptionsClear(doClear), + OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), } // Read entire body. @@ -3011,11 +3039,11 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } // Unmarshal request based on field type. - if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal || field.Type() == pilosa.FieldTypeTimestamp { + if field.Type() == FieldTypeInt || field.Type() == FieldTypeDecimal || field.Type() == FieldTypeTimestamp { // Field type: Int // Marshal into request object. - req := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + req := &ImportValueRequest{} + if err := h.serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -3025,7 +3053,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.ImportValue(r.Context(), qcx, req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + case ErrClusterDoesNotOwnShard, ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -3040,8 +3068,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } else { // Field type: set, time, mutex // Marshal into request object. - req := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + req := &ImportRequest{} + if err := h.serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -3051,7 +3079,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.Import(r.Context(), qcx, req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + case ErrClusterDoesNotOwnShard, ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -3178,9 +3206,9 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request return } - req := &pilosa.ImportRoaringRequest{} + req := &ImportRoaringRequest{} span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") - err = proto.DefaultSerializer.Unmarshal(body, req) + err = h.serializer.Unmarshal(body, req) span.Finish() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -3193,16 +3221,16 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest) return } - resp := &pilosa.ImportResponse{} + resp := &ImportResponse{} // TODO give meaningful stats for import err = h.api.ImportRoaring(ctx, indexName, fieldName, shard, remote, req) if err != nil { resp.Err = err.Error() - if _, ok := err.(pilosa.BadRequestError); ok { + if _, ok := err.(BadRequestError); ok { w.WriteHeader(http.StatusBadRequest) - } else if _, ok := err.(pilosa.NotFoundError); ok { + } else if _, ok := err.(NotFoundError); ok { w.WriteHeader(http.StatusNotFound) - } else if _, ok := err.(pilosa.PreconditionFailedError); ok { + } else if _, ok := err.(PreconditionFailedError); ok { w.WriteHeader(http.StatusPreconditionFailed) } else { w.WriteHeader(http.StatusInternalServerError) @@ -3210,7 +3238,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request } // Marshal response object. - buf, err := proto.DefaultSerializer.Marshal(resp) + buf, err := h.serializer.Marshal(resp) if err != nil { http.Error(w, fmt.Sprintf("marshal import-roaring response: %v", err), http.StatusInternalServerError) return @@ -3247,7 +3275,7 @@ func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { req := &ingest.ShardedRequest{} span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") - err = proto.DefaultSerializer.Unmarshal(body, req) + err = h.serializer.Unmarshal(body, req) span.Finish() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -3288,10 +3316,10 @@ func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request h.logger.Errorf("writing translate keys response: %v", err) } - case pilosa.ErrTranslatingKeyNotFound: + case ErrTranslatingKeyNotFound: http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusNotFound) - case pilosa.ErrTranslateStoreReadOnly: + case ErrTranslateStoreReadOnly: http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusPreconditionFailed) default: @@ -3528,7 +3556,7 @@ func (h *Handler) handleReserveIDs(w http.ResponseWriter, r *http.Request) { return } - var req pilosa.IDAllocReserveRequest + var req IDAllocReserveRequest req.Offset = ^uint64(0) err = json.Unmarshal(bd, &req) if err != nil { @@ -3538,12 +3566,12 @@ func (h *Handler) handleReserveIDs(w http.ResponseWriter, r *http.Request) { ids, err := h.api.ReserveIDs(req.Key, req.Session, req.Offset, req.Count) if err != nil { - var esync pilosa.ErrIDOffsetDesync + var esync ErrIDOffsetDesync if errors.As(err, &esync) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusConflict) err = json.NewEncoder(w).Encode(struct { - pilosa.ErrIDOffsetDesync + ErrIDOffsetDesync Err string `json:"error"` }{ ErrIDOffsetDesync: esync, @@ -3582,7 +3610,7 @@ func (h *Handler) handleCommitIDs(w http.ResponseWriter, r *http.Request) { return } - var req pilosa.IDAllocCommitRequest + var req IDAllocCommitRequest err = json.Unmarshal(bd, &req) if err != nil { http.Error(w, "failed to decode request", http.StatusBadRequest) diff --git a/http/handler_internal_test.go b/http_handler_internal_test.go similarity index 97% rename from http/handler_internal_test.go rename to http_handler_internal_test.go index 25173ede6..9111bcbf6 100644 --- a/http/handler_internal_test.go +++ b/http_handler_internal_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http +package pilosa import ( "bytes" @@ -17,7 +17,6 @@ import ( "time" "github.com/golang-jwt/jwt" - pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "golang.org/x/oauth2" @@ -33,9 +32,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}}, - {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}}, - {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{TrackExistence: true}}}, + {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: IndexOptions{TrackExistence: false}}}, + {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: IndexOptions{Keys: true, TrackExistence: true}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "unknown key: badKey:test"}, @@ -107,8 +106,8 @@ func decimalPtr(d pql.Decimal) *pql.Decimal { // Test fieldOption validation. func TestFieldOptionValidation(t *testing.T) { - timeQuantum := pilosa.TimeQuantum("YMD") - defaultCacheSize := uint32(pilosa.DefaultCacheSize) + timeQuantum := TimeQuantum("YMD") + defaultCacheSize := uint32(DefaultCacheSize) tests := []struct { json string expected postFieldRequest @@ -116,17 +115,17 @@ func TestFieldOptionValidation(t *testing.T) { }{ // FieldType: Set {json: `{"options": {}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeSet, - CacheType: stringPtr(pilosa.DefaultCacheType), + Type: FieldTypeSet, + CacheType: stringPtr(DefaultCacheType), CacheSize: &defaultCacheSize, }}}, {json: `{"options": {"type": "set"}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeSet, - CacheType: stringPtr(pilosa.DefaultCacheType), + Type: FieldTypeSet, + CacheType: stringPtr(DefaultCacheType), CacheSize: &defaultCacheSize, }}}, {json: `{"options": {"type": "set", "cacheType": "lru"}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeSet, + Type: FieldTypeSet, CacheType: stringPtr("lru"), CacheSize: &defaultCacheSize, }}}, @@ -138,7 +137,7 @@ func TestFieldOptionValidation(t *testing.T) { {json: `{"options": {"type": "int"}}`, err: "min is required for field type int"}, {json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"}, {json: `{"options": {"type": "int", "min": 0, "max": 1001}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeInt, + Type: FieldTypeInt, Min: decimalPtr(pql.NewDecimal(0, 0)), Max: decimalPtr(pql.NewDecimal(1001, 0)), }}}, @@ -149,7 +148,7 @@ func TestFieldOptionValidation(t *testing.T) { // FieldType: Time {json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"}, {json: `{"options": {"type": "time", "timeQuantum": "YMD"}}`, expected: postFieldRequest{Options: fieldOptions{ - Type: pilosa.FieldTypeTime, + Type: FieldTypeTime, TimeQuantum: &timeQuantum, }}}, {json: `{"options": {"type": "time", "timeQuantum": "YMD", "min": 0}}`, err: "min does not apply to field type time"}, diff --git a/http/handler_test.go b/http_handler_test.go similarity index 90% rename from http/handler_test.go rename to http_handler_test.go index c58a4bd3e..a692a8435 100644 --- a/http/handler_test.go +++ b/http_handler_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http_test +package pilosa_test import ( "encoding/json" @@ -10,17 +10,17 @@ import ( "testing" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" ) func TestHandlerOptions(t *testing.T) { - _, err := http.NewHandler() + _, err := pilosa.NewHandler() if err == nil { t.Fatalf("expected error making handler without options, got nil") } - _, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{})) + _, err = pilosa.NewHandler(pilosa.OptHandlerAPI(&pilosa.API{})) if err == nil { t.Fatalf("expected error making handler without options, got nil") } @@ -30,24 +30,30 @@ func TestHandlerOptions(t *testing.T) { t.Fatalf("creating listener: %v", err) } - _, err = http.NewHandler(http.OptHandlerListener(ln, ln.Addr().String())) + _, err = pilosa.NewHandler(pilosa.OptHandlerListener(ln, ln.Addr().String())) if err == nil { t.Fatalf("expected error making handler without options, got nil") } + + _, err = pilosa.NewHandler(pilosa.OptHandlerListener(ln, ln.Addr().String()), pilosa.OptHandlerSerializer(proto.Serializer{}), pilosa.OptHandlerSerializer(proto.RoaringSerializer)) + if err == nil { + t.Fatalf("expected error making handler without enough options, got nil") + } + } func TestMarshalUnmarshalTransactionResponse(t *testing.T) { tests := []struct { name string - tr *http.TransactionResponse + tr *pilosa.TransactionResponse }{ { name: "nil transaction", - tr: &http.TransactionResponse{}, + tr: &pilosa.TransactionResponse{}, }, { name: "empty transaction", - tr: &http.TransactionResponse{Transaction: &pilosa.Transaction{}}, + tr: &pilosa.TransactionResponse{Transaction: &pilosa.Transaction{}}, }, } @@ -58,7 +64,7 @@ func TestMarshalUnmarshalTransactionResponse(t *testing.T) { t.Fatalf("marshaling: %v", err) } - mytr := &http.TransactionResponse{} + mytr := &pilosa.TransactionResponse{} err = json.Unmarshal(data, mytr) if err != nil { t.Fatalf("unmarshalling: %v", err) diff --git a/http/translator.go b/http_translator.go similarity index 75% rename from http/translator.go rename to http_translator.go index 4c161d590..0a28315b7 100644 --- a/http/translator.go +++ b/http_translator.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http +package pilosa import ( "bytes" @@ -12,25 +12,24 @@ import ( "reflect" "sync" - "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/logger" ) -func GetOpenTranslateReaderFunc(client *http.Client) pilosa.OpenTranslateReaderFunc { +func GetOpenTranslateReaderFunc(client *http.Client) OpenTranslateReaderFunc { return GetOpenTranslateReaderWithLockerFunc(client, nopLocker{}) } -func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) pilosa.OpenTranslateReaderFunc { +func GetOpenTranslateReaderWithLockerFunc(client *http.Client, locker sync.Locker) OpenTranslateReaderFunc { lockType := reflect.TypeOf(locker) if lockType.Kind() == reflect.Ptr { lockType = lockType.Elem() } - return func(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap) (pilosa.TranslateEntryReader, error) { + return func(ctx context.Context, nodeURL string, offsets TranslateOffsetMap) (TranslateEntryReader, error) { return openTranslateReader(ctx, nodeURL, offsets, client, reflect.New(lockType).Interface().(sync.Locker)) } } -func openTranslateReader(ctx context.Context, nodeURL string, offsets pilosa.TranslateOffsetMap, client *http.Client, locker sync.Locker) (pilosa.TranslateEntryReader, error) { +func openTranslateReader(ctx context.Context, nodeURL string, offsets TranslateOffsetMap, client *http.Client, locker sync.Locker) (TranslateEntryReader, error) { r := NewTranslateEntryReader(ctx, client) r.locker = locker @@ -47,9 +46,9 @@ type nopLocker struct{} func (nopLocker) Lock() {} func (nopLocker) Unlock() {} -// TranslateEntryReader represents an implementation of pilosa.TranslateEntryReader. +// TranslateEntryReader represents an implementation of TranslateEntryReader. // It consolidates all index & field translate entries into a single reader. -type TranslateEntryReader struct { +type HTTPTranslateEntryReader struct { locker sync.Locker ctx context.Context @@ -60,7 +59,7 @@ type TranslateEntryReader struct { // Lookup of offsets for each index & field. // Must be set before calling Open(). - Offsets pilosa.TranslateOffsetMap + Offsets TranslateOffsetMap // URL to stream entries from. // Must be set before calling Open(). @@ -72,17 +71,17 @@ type TranslateEntryReader struct { } // NewTranslateEntryReader returns a new instance of TranslateEntryReader. -func NewTranslateEntryReader(ctx context.Context, client *http.Client) *TranslateEntryReader { +func NewTranslateEntryReader(ctx context.Context, client *http.Client) *HTTPTranslateEntryReader { if client == nil { client = http.DefaultClient } - r := &TranslateEntryReader{locker: nopLocker{}, HTTPClient: client, Logger: logger.NopLogger} + r := &HTTPTranslateEntryReader{locker: nopLocker{}, HTTPClient: client, Logger: logger.NopLogger} r.ctx, r.cancel = context.WithCancel(ctx) return r } // Open initiates the reader. -func (r *TranslateEntryReader) Open() error { +func (r *HTTPTranslateEntryReader) Open() error { // Serialize map of offsets to request body. requestBody, err := json.Marshal(r.Offsets) if err != nil { @@ -107,7 +106,7 @@ func (r *TranslateEntryReader) Open() error { // Handle error codes. if resp.StatusCode == http.StatusNotImplemented { r.body.Close() - return pilosa.ErrNotImplemented + return ErrNotImplemented } else if resp.StatusCode != http.StatusOK { body, _ := ioutil.ReadAll(resp.Body) r.body.Close() @@ -117,7 +116,7 @@ func (r *TranslateEntryReader) Open() error { } // Close stops the reader. -func (r *TranslateEntryReader) Close() error { +func (r *HTTPTranslateEntryReader) Close() error { if r.cancel != nil { r.cancel() } @@ -132,7 +131,7 @@ func (r *TranslateEntryReader) Close() error { // ReadEntry reads the next entry from the stream into entry. // Returns io.EOF at the end of the stream. -func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { +func (r *HTTPTranslateEntryReader) ReadEntry(entry *TranslateEntry) error { r.locker.Lock() defer r.locker.Unlock() diff --git a/http/translator_test.go b/http_translator_test.go similarity index 92% rename from http/translator_test.go rename to http_translator_test.go index 74a579f9c..8df8d86d0 100644 --- a/http/translator_test.go +++ b/http_translator_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http_test +package pilosa_test import ( "context" @@ -8,8 +8,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/test" ) @@ -41,7 +40,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { } // Connect to server and stream all available data. - r := http.NewTranslateEntryReader(context.Background(), nil) + r := pilosa.NewTranslateEntryReader(context.Background(), nil) r.URL = primary.URL() // Wait to ensure writes make it to translate store @@ -123,7 +122,7 @@ func BenchmarkReadEntryNoMutex(b *testing.B) { defer teardown() for n := 0; n < b.N; n++ { - r, err := http.GetOpenTranslateReaderFunc(nil)(ctx, url, offset) + r, err := pilosa.GetOpenTranslateReaderFunc(nil)(ctx, url, offset) if err != nil { b.Fatalf("opening translate reader: %+v", err) } @@ -138,7 +137,7 @@ func BenchmarkReadEntryWithMutex(b *testing.B) { defer teardown() for n := 0; n < b.N; n++ { - r, err := http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})(ctx, url, offset) + r, err := pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})(ctx, url, offset) if err != nil { b.Fatalf("opening translate reader: %+v", err) } diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index 99a653ca9..0a4325167 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -17,7 +17,7 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/disco" - picli "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/logger" "github.com/pkg/errors" ) @@ -87,15 +87,15 @@ func TestClusterStuff(t *testing.T) { auth = true } - cli1, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil)) + cli1, err := pilosa.NewInternalClient("pilosa1:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { t.Fatalf("getting client: %v", err) } - cli2, err := picli.NewInternalClient("pilosa2:10101", picli.GetHTTPClient(nil)) + cli2, err := pilosa.NewInternalClient("pilosa2:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { t.Fatalf("getting client: %v", err) } - cli3, err := picli.NewInternalClient("pilosa3:10101", picli.GetHTTPClient(nil)) + cli3, err := pilosa.NewInternalClient("pilosa3:10101", pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { t.Fatalf("getting client: %v", err) } @@ -134,7 +134,7 @@ func TestClusterStuff(t *testing.T) { } // Check query results from each node. - for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { + for i, cli := range []*pilosa.InternalClient{cli1, cli2, cli3} { r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) if err != nil { t.Fatalf("count querying pilosa%d: %v", i, err) @@ -157,7 +157,7 @@ func TestClusterStuff(t *testing.T) { t.Log("done waiting for stability") // Check query results from each node. - for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { + for i, cli := range []*pilosa.InternalClient{cli1, cli2, cli3} { r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) if err != nil { t.Fatalf("count querying pilosa%d: %v", i, err) diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index b59dc9be1..164765f26 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -17,7 +17,7 @@ import ( pilosa "github.com/molecula/featurebase/v3" boltdb "github.com/molecula/featurebase/v3/boltdb" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/topology" "github.com/pkg/errors" @@ -54,7 +54,7 @@ func pauseNode(t *testing.T, node string) error { } type keyInserter struct { - client *http.InternalClient + client *pilosa.InternalClient uri *net.URI index string keys []string @@ -69,10 +69,10 @@ func getAddress(node string) string { return node + ":10101" } -func getClients(addrs []string) ([]*http.InternalClient, error) { - clients := make([]*http.InternalClient, 0, len(addrs)) +func getClients(addrs []string) ([]*pilosa.InternalClient, error) { + clients := make([]*pilosa.InternalClient, 0, len(addrs)) for _, addr := range addrs { - c, err := http.NewInternalClient(addr, http.GetHTTPClient(nil)) + c, err := pilosa.NewInternalClient(addr, pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { return nil, err } @@ -93,7 +93,7 @@ func getURIsFromAddresses(addrs []string) ([]*net.URI, error) { return uris, nil } -func readIndexTranslateData(ctx context.Context, client *http.InternalClient, dirPath, index string, partition int) error { +func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient, dirPath, index string, partition int) error { // read translateStore contents from endpoint r, err := client.IndexTranslateDataReader(ctx, index, partition) if err != nil { @@ -177,7 +177,7 @@ var errOpRetriable = errors.New("If operation failed on this error, it can be re func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, keys []string) error { // get client that's connected to node address := getAddress(node) - client, err := http.NewInternalClient(address, http.GetHTTPClient(nil)) + client, err := pilosa.NewInternalClient(address, pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) if err != nil { return err } diff --git a/http/client.go b/internal_client.go similarity index 90% rename from http/client.go rename to internal_client.go index 684c9f726..735d84386 100644 --- a/http/client.go +++ b/internal_client.go @@ -1,5 +1,5 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package http +// Copyright 2022 Molecula Corp. All rights reserved. +package pilosa import ( "bytes" @@ -20,9 +20,7 @@ import ( "time" "github.com/hashicorp/go-retryablehttp" - pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/authn" - "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" @@ -34,7 +32,7 @@ import ( // InternalClient represents a client to the Pilosa cluster. type InternalClient struct { defaultURI *pnet.URI - serializer pilosa.Serializer + serializer Serializer log logger.Logger @@ -42,7 +40,7 @@ type InternalClient struct { httpClient *http.Client retryableClient *retryablehttp.Client // the local node's API, used for operations that we can short-circuit that way - api *pilosa.API + api *API // secret Key for auth across nodes secretKey string @@ -53,7 +51,7 @@ type InternalClient struct { // of going through http. func NewInternalClient(host string, remoteClient *http.Client, opts ...InternalClientOption) (*InternalClient, error) { if host == "" { - return nil, pilosa.ErrHostRequired + return nil, ErrHostRequired } uri, err := pnet.NewURIFromAddress(host) @@ -67,6 +65,12 @@ func NewInternalClient(host string, remoteClient *http.Client, opts ...InternalC type InternalClientOption func(c *InternalClient) +func WithSerializer(s Serializer) InternalClientOption { + return func(c *InternalClient) { + c.serializer = s + } +} + // WithSecretKey adds the secretKey used for inter-node communication when auth // is enabled func WithSecretKey(secretKey string) InternalClientOption { @@ -122,7 +126,6 @@ func retryWith400Policy(ctx context.Context, resp *http.Response, err error) (bo func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client, opts ...InternalClientOption) *InternalClient { ic := &InternalClient{ defaultURI: defaultURI, - serializer: proto.Serializer{}, httpClient: remoteClient, log: logger.NewStandardLogger(os.Stderr), } @@ -167,7 +170,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64 return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -200,7 +203,7 @@ func (c *InternalClient) AvailableShards(ctx context.Context, indexName string) return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -220,7 +223,7 @@ func (c *InternalClient) AvailableShards(ctx context.Context, indexName string) // SchemaNode returns all index and field schema information from the specified // node. -func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*pilosa.IndexInfo, error) { +func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema") defer span.Finish() @@ -234,7 +237,7 @@ func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bo return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -253,7 +256,7 @@ func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bo } // Schema returns all index and field schema information. -func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { +func (c *InternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema") defer span.Finish() @@ -266,7 +269,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -304,7 +307,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf [] req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -320,7 +323,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf [] var msg string // try to decode a JSON response var sr successResponse - qr := &pilosa.QueryResponse{} + qr := &QueryResponse{} if err = json.Unmarshal(buf, &sr); err == nil { msg = sr.Error.Error() } else if err := c.serializer.Unmarshal(buf, qr); err == nil { @@ -355,7 +358,7 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -388,7 +391,7 @@ func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -417,7 +420,7 @@ func (c *InternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, indexNam return nil, errors.Wrap(err, "creating request") } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -434,7 +437,7 @@ func (c *InternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, indexNam return out, err } -func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilosa.Schema, remote bool) error { +func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error { u := uri.Path(fmt.Sprintf("/schema?remote=%v", remote)) buf, err := json.Marshal(s) if err != nil { @@ -448,7 +451,7 @@ func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilos req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -463,7 +466,7 @@ func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilos } // CreateIndex creates a new index on the server. -func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { +func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndex") defer span.Finish() @@ -495,14 +498,14 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusConflict { - return pilosa.ErrIndexExists + return ErrIndexExists } return err } @@ -524,7 +527,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -556,7 +559,7 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -575,21 +578,21 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { } // Query executes query against the index. -func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Query") defer span.Finish() return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } // QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { +func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "QueryNode") defer span.Finish() if index == "" { - return nil, pilosa.ErrIndexRequired + return nil, ErrIndexRequired } else if queryRequest.Query == "" { - return nil, pilosa.ErrQueryRequired + return nil, ErrQueryRequired } buf, err := c.serializer.Marshal(queryRequest) if err != nil { @@ -615,7 +618,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -630,7 +633,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str return nil, errors.Wrap(err, "reading") } - qresp := &pilosa.QueryResponse{} + qresp := &QueryResponse{} if err := c.serializer.Unmarshal(body, qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } else if qresp.Err != nil { @@ -649,12 +652,12 @@ func getPrimaryNode(nodes []*topology.Node) *topology.Node { return nil } -func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { +func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureIndex") defer span.Finish() err := c.CreateIndex(ctx, name, options) - if err == nil || errors.Cause(err) == pilosa.ErrIndexExists { + if err == nil || errors.Cause(err) == ErrIndexExists { return nil } return err @@ -663,21 +666,21 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureField") defer span.Finish() - return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{}) + return c.EnsureFieldWithOptions(ctx, indexName, fieldName, FieldOptions{}) } -func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error { +func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt FieldOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureFieldWithOptions") defer span.Finish() err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt) - if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { + if err == nil || errors.Cause(err) == ErrFieldExists { return nil } return err } // importNode sends a pre-marshaled import request to a node. -func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error { +func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, index, field string, buf []byte, opts *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.importNode") defer span.Finish() @@ -702,7 +705,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -718,7 +721,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in return errors.Wrap(err, "reading") } - var isresp pilosa.ImportResponse + var isresp ImportResponse if err := c.serializer.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) } else if s := isresp.Err; s != "" { @@ -736,7 +739,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in // that in here with a type switch seems messy. Similarly, index/field/shard // exist because we can't access those members of the two slightly different // structs. -func (c *InternalClient) importHelper(ctx context.Context, req pilosa.Message, process func() error, index string, field string, shard uint64, options *pilosa.ImportOptions) error { +func (c *InternalClient) importHelper(ctx context.Context, req Message, process func() error, index string, field string, shard uint64, options *ImportOptions) error { // If we don't actually know what shards we're sending to, and we have // a local API and a qcx, we'll have a process function that uses the local // API. Otherwise, even if we have an API @@ -846,7 +849,7 @@ func (c *InternalClient) importHelper(ctx context.Context, req pilosa.Message, p // // If we get a non-nil qcx, and have an associated API, we'll use that API // directly for the local shard. -func (c *InternalClient) Import(ctx context.Context, qcx *pilosa.Qcx, req *pilosa.ImportRequest, options *pilosa.ImportOptions) error { +func (c *InternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import") defer span.Finish() @@ -874,7 +877,7 @@ func (c *InternalClient) Import(ctx context.Context, qcx *pilosa.Qcx, req *pilos // // If we get a non-nil qcx, and have an associated API, we'll use that API // directly for the local shard. -func (c *InternalClient) ImportValue(ctx context.Context, qcx *pilosa.Qcx, req *pilosa.ImportValueRequest, options *pilosa.ImportOptions) error { +func (c *InternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import") defer span.Finish() @@ -892,14 +895,14 @@ func (c *InternalClient) ImportValue(ctx context.Context, qcx *pilosa.Qcx, req * // ImportRoaring does fast import of raw bits in roaring format (pilosa or // official format, see API.ImportRoaring). -func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error { +func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring") defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } else if field == "" { - return pilosa.ErrFieldRequired + return ErrFieldRequired } if uri == nil { uri = c.defaultURI @@ -923,7 +926,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") httpReq.Header.Set("X-Pilosa-Row", "roaring") - httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("User-Agent", "pilosa/"+Version) httpReq = AddAuthToken(ctx, httpReq) // Execute request against the host. @@ -934,7 +937,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index defer resp.Body.Close() dec := json.NewDecoder(resp.Body) - rbody := &pilosa.ImportResponse{} + rbody := &ImportResponse{} err = dec.Decode(rbody) // Decode can return EOF when no error occurred. helpful! if err != nil && err != io.EOF { @@ -952,9 +955,9 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sha defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } else if field == "" { - return pilosa.ErrFieldRequired + return ErrFieldRequired } // Retrieve a list of nodes that own the shard. @@ -998,7 +1001,7 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *topology.Node, return errors.Wrap(err, "creating request") } req.Header.Set("Accept", "text/csv") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1041,14 +1044,14 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, pilosa.ErrFragmentNotFound + return nil, ErrFragmentNotFound } return nil, err } @@ -1059,19 +1062,19 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateField") defer span.Finish() - return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{}) + return c.CreateFieldWithOptions(ctx, index, field, FieldOptions{}) } // CreateFieldWithOptions creates a new field on the server. -func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { +func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldWithOptions") defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } - // convert pilosa.FieldOptions to fieldOptions + // convert FieldOptions to fieldOptions // // TODO this kind of sucks because it's one more place that needs // changes when we change anything with field options (and there @@ -1082,23 +1085,23 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel Type: opt.Type, } switch fieldOpt.Type { - case pilosa.FieldTypeSet, pilosa.FieldTypeMutex: + case FieldTypeSet, FieldTypeMutex: fieldOpt.CacheType = &opt.CacheType fieldOpt.CacheSize = &opt.CacheSize fieldOpt.Keys = &opt.Keys - case pilosa.FieldTypeInt: + case FieldTypeInt: fieldOpt.Min = &opt.Min fieldOpt.Max = &opt.Max - case pilosa.FieldTypeTime: + case FieldTypeTime: fieldOpt.TimeQuantum = &opt.TimeQuantum - case pilosa.FieldTypeBool: + case FieldTypeBool: // pass - case pilosa.FieldTypeDecimal: + case FieldTypeDecimal: fieldOpt.Min = &opt.Min fieldOpt.Max = &opt.Max fieldOpt.Scale = &opt.Scale default: - fieldOpt.Type = pilosa.DefaultFieldType + fieldOpt.Type = DefaultFieldType fieldOpt.Keys = &opt.Keys } @@ -1131,14 +1134,14 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusConflict { - return pilosa.ErrFieldExists + return ErrFieldExists } return err } @@ -1148,7 +1151,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) { +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentBlocks") defer span.Finish() @@ -1169,7 +1172,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, inde return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -1178,7 +1181,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, inde if err != nil { // Return the appropriate error. if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, pilosa.ErrFragmentNotFound + return nil, ErrFragmentNotFound } return nil, err } @@ -1200,7 +1203,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi if uri == nil { panic("need to pass a URI to BlockData") } - buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{ + buf, err := c.serializer.Marshal(&BlockDataRequest{ Index: index, Field: field, View: view, @@ -1220,7 +1223,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Accept", "application/protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1233,7 +1236,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi defer resp.Body.Close() // Decode response object. - var rsp pilosa.BlockDataResponse + var rsp BlockDataResponse if body, err := ioutil.ReadAll(resp.Body); err != nil { return nil, nil, errors.Wrap(err, "reading") } else if err := c.serializer.Unmarshal(body, &rsp); err != nil { @@ -1254,7 +1257,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []b } req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req.Header.Set("Connection", "keep-alive") if c.secretKey != "" { @@ -1272,16 +1275,16 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []b } // TranslateKeysNode function is mainly called to translate keys from primary node. -// If primary node returns 404 error the function wraps it with pilosa.ErrTranslatingKeyNotFound. +// If primary node returns 404 error the function wraps it with ErrTranslatingKeyNotFound. func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode") defer span.Finish() if index == "" { - return nil, pilosa.ErrIndexRequired + return nil, ErrIndexRequired } - buf, err := c.serializer.Marshal(&pilosa.TranslateKeysRequest{ + buf, err := c.serializer.Marshal(&TranslateKeysRequest{ Index: index, Field: field, Keys: keys, @@ -1302,14 +1305,14 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, errors.Wrap(pilosa.ErrTranslatingKeyNotFound, err.Error()) + return nil, errors.Wrap(ErrTranslatingKeyNotFound, err.Error()) } return nil, err } @@ -1321,7 +1324,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i return nil, errors.Wrap(err, "reading") } - tkresp := &pilosa.TranslateKeysResponse{} + tkresp := &TranslateKeysResponse{} if err := c.serializer.Unmarshal(body, tkresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1334,10 +1337,10 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in defer span.Finish() if index == "" { - return nil, pilosa.ErrIndexRequired + return nil, ErrIndexRequired } - buf, err := c.serializer.Marshal(&pilosa.TranslateIDsRequest{ + buf, err := c.serializer.Marshal(&TranslateIDsRequest{ Index: index, Field: field, IDs: ids, @@ -1357,7 +1360,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1373,7 +1376,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in return nil, errors.Wrap(err, "reading") } - tkresp := &pilosa.TranslateIDsResponse{} + tkresp := &TranslateIDsResponse{} if err := c.serializer.Unmarshal(body, tkresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1381,7 +1384,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in } // GetNodeUsage retrieves the size-on-disk information for the specified node. -func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]pilosa.NodeUsage, error) { +func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) { u := uri.Path("/ui/usage?remote=true") req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -1389,7 +1392,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1405,7 +1408,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s return nil, errors.Wrap(err, "reading") } - nodeUsages := make(map[string]pilosa.NodeUsage) // map of size 1 + nodeUsages := make(map[string]NodeUsage) // map of size 1 if err := json.Unmarshal(body, &nodeUsages); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1413,7 +1416,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s } // GetPastQueries retrieves the query history log for the specified node. -func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]pilosa.PastQueryStatus, error) { +func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) { u := uri.Path("/query-history?remote=true") req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -1421,7 +1424,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]p } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request against the host. @@ -1437,7 +1440,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]p return nil, errors.Wrap(err, "reading") } - queries := make([]pilosa.PastQueryStatus, 100) + queries := make([]PastQueryStatus, 100) if err := json.Unmarshal(body, &queries); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) } @@ -1463,7 +1466,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1512,7 +1515,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1562,7 +1565,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1615,7 +1618,7 @@ func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, req.Header.Set("Content-Length", strconv.Itoa(len(reqData))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1660,7 +1663,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, // Apply headers. req.Header.Set("Content-Length", strconv.Itoa(len(like))) req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Send the request. @@ -1690,7 +1693,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, return matches, nil } -func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.Transaction, error) { +func (c *InternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Transactions") defer span.Finish() @@ -1700,7 +1703,7 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.T return nil, errors.Wrap(err, "creating transactions request") } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1711,15 +1714,15 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.T _, _ = io.Copy(ioutil.Discard, resp.Body) _ = resp.Body.Close() }() - trnsMap := make(map[string]*pilosa.Transaction) + trnsMap := make(map[string]*Transaction) err = json.NewDecoder(resp.Body).Decode(&trnsMap) return trnsMap, errors.Wrap(err, "json decoding") } -func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*pilosa.Transaction, error) { +func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.StartTransaction") defer span.Finish() - buf, err := json.Marshal(&pilosa.Transaction{ + buf, err := json.Marshal(&Transaction{ ID: id, Timeout: timeout, Exclusive: exclusive, @@ -1739,7 +1742,7 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -1756,14 +1759,14 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou return nil, errors.Wrap(err, "decoding response") } if resp.StatusCode == 409 { - err = pilosa.ErrTransactionExclusive + err = ErrTransactionExclusive } else if tr.Error != "" { err = errors.New(tr.Error) } return tr.Transaction, err } -func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) { +func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FinishTransaction") defer span.Finish() @@ -1774,7 +1777,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pil } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -1797,7 +1800,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pil return tr.Transaction, err } -func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa.Transaction, error) { +func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.GetTransaction") defer span.Finish() @@ -1811,7 +1814,7 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa return nil, errors.Wrap(err, "creating get transaction request") } req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) @@ -1922,7 +1925,7 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp var msg string // try to decode a JSON response var sr successResponse - qr := &pilosa.QueryResponse{} + qr := &QueryResponse{} if err = json.Unmarshal(buf, &sr); err == nil { msg = sr.Error.Error() } else if err := c.serializer.Unmarshal(buf, qr); err == nil { @@ -1936,7 +1939,7 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp } // Bits is a slice of Bit. -type Bits []pilosa.Bit +type Bits []Bit func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Bits) Len() int { return len(p) } @@ -2029,10 +2032,10 @@ func (p Bits) Timestamps() []int64 { } // GroupByShard returns a map of bits by shard. -func (p Bits) GroupByShard() map[uint64][]pilosa.Bit { - m := make(map[uint64][]pilosa.Bit) +func (p Bits) GroupByShard() map[uint64][]Bit { + m := make(map[uint64][]Bit) for _, bit := range p { - shard := bit.ColumnID / pilosa.ShardWidth + shard := bit.ColumnID / ShardWidth m[shard] = append(m[shard], bit) } @@ -2045,7 +2048,7 @@ func (p Bits) GroupByShard() map[uint64][]pilosa.Bit { } // FieldValues represents a slice of field values. -type FieldValues []pilosa.FieldValue +type FieldValues []FieldValue func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p FieldValues) Len() int { return len(p) } @@ -2098,10 +2101,10 @@ func (p FieldValues) Values() []int64 { } // GroupByShard returns a map of field values by shard. -func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue { - m := make(map[uint64][]pilosa.FieldValue) +func (p FieldValues) GroupByShard() map[uint64][]FieldValue { + m := make(map[uint64][]FieldValue) for _, val := range p { - shard := val.ColumnID / pilosa.ShardWidth + shard := val.ColumnID / ShardWidth m[shard] = append(m[shard], val) } @@ -2114,7 +2117,7 @@ func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue { } // BitsByPos is a slice of bits sorted row then column. -type BitsByPos []pilosa.Bit +type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } @@ -2126,11 +2129,6 @@ func (p BitsByPos) Less(i, j int) bool { return p0 < p1 } -// pos returns the row position of a row/column pair. -func pos(rowID, columnID uint64) uint64 { - return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth) -} - func uriPathToURL(uri *pnet.URI, path string) url.URL { return url.URL{ Scheme: uri.Scheme, @@ -2170,14 +2168,14 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { - return nil, pilosa.ErrFragmentNotFound + return nil, ErrFragmentNotFound } return nil, err } @@ -2189,7 +2187,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } if uri == nil { @@ -2205,7 +2203,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind if err != nil { return errors.Wrap(err, "creating request") } - httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("User-Agent", "pilosa/"+Version) token, ok := ctx.Value("token").(string) if ok && token != "" { httpReq.Header.Set("Authorization", token) @@ -2225,7 +2223,7 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind defer span.Finish() if index == "" { - return pilosa.ErrIndexRequired + return ErrIndexRequired } if uri == nil { @@ -2241,7 +2239,7 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind if err != nil { return errors.Wrap(err, "creating request") } - httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("User-Agent", "pilosa/"+Version) token, ok := ctx.Value("token").(string) if ok && token != "" { @@ -2271,7 +2269,7 @@ func (c *InternalClient) ShardReader(ctx context.Context, index string, shard ui return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2294,7 +2292,7 @@ func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2318,7 +2316,7 @@ func (c *InternalClient) IDAllocDataWriter(ctx context.Context, f io.Reader, pri return errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2345,7 +2343,7 @@ func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index str return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2353,7 +2351,7 @@ func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index str resp, err := c.executeRequest(req.WithContext(ctx), forwardAuthHeader(true)) if resp != nil && resp.StatusCode == http.StatusNotFound { resp.Body.Close() - return nil, pilosa.ErrTranslateStoreNotFound + return nil, ErrTranslateStoreNotFound } else if err != nil { return nil, err } @@ -2375,7 +2373,7 @@ func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, fi return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/octet-stream") req = AddAuthToken(ctx, req) @@ -2383,16 +2381,14 @@ func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, fi resp, err := c.executeRequest(req.WithContext(ctx)) if resp != nil && resp.StatusCode == http.StatusNotFound { resp.Body.Close() - return nil, pilosa.ErrTranslateStoreNotFound + return nil, ErrTranslateStoreNotFound } else if err != nil { return nil, err } return resp.Body, nil } -// Status function is just a public function for this particular implementation of InternalClient. -// It's not require by pilosa.InternalClient interface. -// The function returns pilosa cluster state as a string ("NORMAL", "DEGRADED", "DOWN", "RESIZING", ...) +// Status returns pilosa cluster state as a string ("NORMAL", "DEGRADED", "DOWN", "RESIZING", ...) func (c *InternalClient) Status(ctx context.Context) (string, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Status") defer span.Finish() @@ -2406,7 +2402,7 @@ func (c *InternalClient) Status(ctx context.Context) (string, error) { return "", errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -2438,7 +2434,7 @@ func (c *InternalClient) PartitionNodes(ctx context.Context, partitionID int) ([ return nil, errors.Wrap(err, "creating request") } - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("User-Agent", "pilosa/"+Version) req.Header.Set("Accept", "application/json") req = AddAuthToken(ctx, req) @@ -2456,6 +2452,6 @@ func (c *InternalClient) PartitionNodes(ctx context.Context, partitionID int) ([ return a, nil } -func (c *InternalClient) SetInternalAPI(api *pilosa.API) { +func (c *InternalClient) SetInternalAPI(api *API) { c.api = api } diff --git a/http/client_test.go b/internal_client_test.go similarity index 97% rename from http/client_test.go rename to internal_client_test.go index 8625c1e2e..8d1531137 100644 --- a/http/client_test.go +++ b/internal_client_test.go @@ -1,5 +1,5 @@ // Copyright 2021 Molecula Corp. All rights reserved. -package http_test +package pilosa_test import ( "bufio" @@ -15,7 +15,7 @@ import ( "github.com/davecgh/go-spew/spew" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" @@ -122,9 +122,9 @@ func TestClient_MultiNode(t *testing.T) { // Connect to each node to compare results. client := make([]*Client, 3) - client[0] = MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil)) - client[1] = MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil)) - client[2] = MustNewClient(c.GetNode(2).URL(), http.GetHTTPClient(nil)) + client[0] = MustNewClient(c.GetNode(0).URL(), pilosa.GetHTTPClient(nil)) + client[1] = MustNewClient(c.GetNode(1).URL(), pilosa.GetHTTPClient(nil)) + client[2] = MustNewClient(c.GetNode(2).URL(), pilosa.GetHTTPClient(nil)) topN := 4 queryRequest := &pilosa.QueryRequest{ @@ -188,7 +188,7 @@ func TestClient_Export(t *testing.T) { cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys()) cmd.MustCreateField(t, "unkeyed", "unkeyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000)) - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) data := []pilosa.Bit{ {RowID: 1, ColumnID: 100, RowKey: "row1", ColumnKey: "col100"}, {RowID: 1, ColumnID: 101, RowKey: "row1", ColumnKey: "col101"}, @@ -376,7 +376,7 @@ func TestClient_Import(t *testing.T) { recIDs := []uint64{0, 3, 7} valueIDs := []uint64{0, 3, 7} - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) // set API to point at the local node c.SetInternalAPI(cmd.API) @@ -532,7 +532,7 @@ func TestClient_ImportRoaring(t *testing.T) { // Send import request. host := cluster.GetNode(0).URL() - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { @@ -656,7 +656,7 @@ func TestClient_ImportRoaring_MultiView(t *testing.T) { // Send import request. host := cluster.GetNode(0).URL() - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportRoaringRequest{Views: map[string][]byte{}} req.Views["a"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100") req.Views["b"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100") @@ -681,7 +681,7 @@ func TestClient_ImportKeys(t *testing.T) { cmd.MustCreateField(t, "unkeyed", "keyedf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys()) // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) baseReq := &pilosa.ImportRequest{ Index: "keyed", Field: "keyedf", @@ -774,8 +774,8 @@ func TestClient_ImportKeys(t *testing.T) { cmd0.MustCreateField(t, "keyed", "keyedf1", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 1000), pilosa.OptFieldKeys()) // Send import request. - c0 := MustNewClient(host0, http.GetHTTPClient(nil)) - c1 := MustNewClient(host1, http.GetHTTPClient(nil)) + c0 := MustNewClient(host0, pilosa.GetHTTPClient(nil)) + c1 := MustNewClient(host1, pilosa.GetHTTPClient(nil)) // Import to node0. t.Run("Import node0", func(t *testing.T) { @@ -852,7 +852,7 @@ func TestClient_ImportKeys(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: "i", Field: "f", @@ -931,7 +931,7 @@ func TestClient_ImportIDs(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: idxName, Field: fldName, @@ -999,7 +999,7 @@ func TestClient_ImportValue(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: "i", Field: "f", @@ -1078,7 +1078,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportRequest{ Index: "iset", Field: "fset", @@ -1114,7 +1114,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Send import request. - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) req := &pilosa.ImportValueRequest{ Index: "iint", Field: "fint", @@ -1155,7 +1155,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Set a bit on a different shard. hldr.SetBit("i", "f", 0, 1) - c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil)) + c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil)) blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", "standard", 0) if err != nil { t.Fatal(err) @@ -1180,7 +1180,7 @@ func TestClient_CreateDecimalField(t *testing.T) { defer cluster.Close() cmd := cluster.GetNode(0) - c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil)) + c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil)) index := "cdf" err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{}) @@ -1290,8 +1290,8 @@ func TestClientTransactions(t *testing.T) { coord := c.GetPrimary() other := c.GetNonPrimary() - client0 := MustNewClient(coord.URL(), http.GetHTTPClient(nil)) - client1 := MustNewClient(other.URL(), http.GetHTTPClient(nil)) + client0 := MustNewClient(coord.URL(), pilosa.GetHTTPClient(nil)) + client1 := MustNewClient(other.URL(), pilosa.GetHTTPClient(nil)) // can create, list, get, and finish a transaction var expDeadline time.Time @@ -1444,12 +1444,12 @@ func TestClientTransactions(t *testing.T) { // Client represents a test wrapper for pilosa.Client. type Client struct { - *http.InternalClient + *pilosa.InternalClient } // MustNewClient returns a new instance of Client. Panic on error. func MustNewClient(host string, h *gohttp.Client) *Client { - c, err := http.NewInternalClient(host, h) + c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) if err != nil { panic(err) } @@ -1497,7 +1497,7 @@ func TestClient_ImportRoaringExists(t *testing.T) { } // Send import request. host := node.URL() - c := MustNewClient(host, http.GetHTTPClient(nil)) + c := MustNewClient(host, pilosa.GetHTTPClient(nil)) // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") diff --git a/server.go b/server.go index 66332c9fb..3475b670a 100644 --- a/server.go +++ b/server.go @@ -86,7 +86,7 @@ type Server struct { // nolint: maligned // HolderConfig stashes server options that are really Holder options. holderConfig *HolderConfig - defaultClient InternalClient + defaultClient *InternalClient dataDir string // Threshold for logging long-running queries @@ -193,7 +193,7 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { // OptServerInternalClient is a functional option on Server // used to set the implementation of InternalClient. -func OptServerInternalClient(c InternalClient) ServerOption { +func OptServerInternalClient(c *InternalClient) ServerOption { return func(s *Server) error { s.defaultClient = c s.cluster.InternalClient = c @@ -405,7 +405,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { cluster: cluster, diagnostics: newDiagnosticsCollector(defaultDiagnosticServer), systemInfo: newNopSystemInfo(), - defaultClient: nopInternalClient{}, + defaultClient: &InternalClient{}, // TODO may need to make this a valid thing gcNotifier: NopGCNotifier, @@ -511,7 +511,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { return s, nil } -func (s *Server) InternalClient() InternalClient { +func (s *Server) InternalClient() *InternalClient { return s.defaultClient } diff --git a/server/handler_test.go b/server/handler_test.go index 48dddde78..6c195900c 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -21,7 +21,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" "github.com/molecula/featurebase/v3/encoding/proto" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" @@ -31,7 +30,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() cmd := cluster.GetNode(0) - h := cmd.Handler.(*http.Handler).Handler + h := cmd.Handler.(*pilosa.Handler).Handler t.Run("PostSchema", func(t *testing.T) { w := httptest.NewRecorder() @@ -70,7 +69,7 @@ func TestHandler_Endpoints(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() cmd := cluster.GetNode(0) - h := cmd.Handler.(*http.Handler).Handler + h := cmd.Handler.(*pilosa.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -1120,7 +1119,7 @@ func TestHandler_Endpoints(t *testing.T) { clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) defer clus.Close() w = httptest.NewRecorder() - h1 := clus.GetNode(0).Handler.(*http.Handler).Handler + h1 := clus.GetNode(0).Handler.(*pilosa.Handler).Handler h1.ServeHTTP(w, req) result = w.Result() @@ -1383,7 +1382,7 @@ func TestHandler_Endpoints(t *testing.T) { clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) defer clus.Close() w = httptest.NewRecorder() - h := clus.GetNode(0).Handler.(*http.Handler).Handler + h := clus.GetNode(0).Handler.(*pilosa.Handler).Handler h.ServeHTTP(w, req) result = w.Result() @@ -1402,7 +1401,7 @@ func TestCluster_TranslateStore(t *testing.T) { cluster.Nodes[0] = test.NewCommandNode(t, server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), ), ) @@ -1423,7 +1422,7 @@ func TestClusterTranslator(t *testing.T) { []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), )}, []server.CommandOption{ server.OptCommandServerOptions( @@ -1487,7 +1486,7 @@ func TestClusterTranslator(t *testing.T) { // defer cluster.Close() // cmd := cluster.GetNode(0) -// h := cmd.Handler.(*http.Handler).Handler +// h := cmd.Handler.(*pilosa.Handler).Handler // w := httptest.NewRecorder() diff --git a/server/server.go b/server/server.go index 8212f6816..559f1c1e7 100644 --- a/server/server.go +++ b/server/server.go @@ -36,7 +36,6 @@ import ( petcd "github.com/molecula/featurebase/v3/etcd" "github.com/molecula/featurebase/v3/gcnotify" "github.com/molecula/featurebase/v3/gopsutil" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/prometheus" @@ -74,7 +73,7 @@ type Command struct { logger loggerLogger queryLogger loggerLogger - Handler pilosa.Handler + Handler pilosa.HandlerI grpcServer *grpcServer grpcLn net.Listener API *pilosa.API @@ -405,7 +404,7 @@ func (m *Command) SetupServer() error { // Save listenURI for later reference. m.listenURI = uri - c := http.GetHTTPClient(m.tlsConfig) + c := pilosa.GetHTTPClient(m.tlsConfig) // Get advertise address as uri. advertiseURI, err := pilosa.AddressWithDefaults(m.Config.Advertise) @@ -473,7 +472,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerDiagnosticsInterval(diagnosticsInterval), pilosa.OptServerExecutorPoolSize(m.Config.WorkerPoolSize), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})), pilosa.OptServerOpenIDAllocator(pilosa.OpenIDAllocator), pilosa.OptServerLogger(m.logger), pilosa.OptServerQueryLogger(m.queryLogger), @@ -498,9 +497,9 @@ func (m *Command) SetupServer() error { serverOptions = append(serverOptions, m.serverOptions...) if m.Config.Auth.Enable { - serverOptions = append(serverOptions, pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c, http.WithSecretKey(m.Config.Auth.SecretKey)))) + serverOptions = append(serverOptions, pilosa.OptServerInternalClient(pilosa.NewInternalClientFromURI(uri, c, pilosa.WithSecretKey(m.Config.Auth.SecretKey), pilosa.WithSerializer(proto.Serializer{})))) } else { - serverOptions = append(serverOptions, pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c))) + serverOptions = append(serverOptions, pilosa.OptServerInternalClient(pilosa.NewInternalClientFromURI(uri, c, pilosa.WithSerializer(proto.Serializer{})))) } m.Server, err = pilosa.NewServer(serverOptions...) @@ -573,17 +572,19 @@ func (m *Command) SetupServer() error { OptGRPCServerQueryLogger(m.queryLogger), ) - m.Handler, err = http.NewHandler( - http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), - http.OptHandlerAPI(m.API), - http.OptHandlerLogger(m.logger), - http.OptHandlerQueryLogger(m.queryLogger), - http.OptHandlerFileSystem(&statik.FileSystem{}), - http.OptHandlerListener(m.ln, m.Config.Advertise), - http.OptHandlerCloseTimeout(m.closeTimeout), - http.OptHandlerMiddleware(m.grpcServer.middleware(m.Config.Handler.AllowedOrigins)), - http.OptHandlerAuthN(m.auth), - http.OptHandlerAuthZ(&p), + m.Handler, err = pilosa.NewHandler( + pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), + pilosa.OptHandlerAPI(m.API), + pilosa.OptHandlerLogger(m.logger), + pilosa.OptHandlerQueryLogger(m.queryLogger), + pilosa.OptHandlerFileSystem(&statik.FileSystem{}), + pilosa.OptHandlerListener(m.ln, m.Config.Advertise), + pilosa.OptHandlerCloseTimeout(m.closeTimeout), + pilosa.OptHandlerMiddleware(m.grpcServer.middleware(m.Config.Handler.AllowedOrigins)), + pilosa.OptHandlerAuthN(m.auth), + pilosa.OptHandlerAuthZ(&p), + pilosa.OptHandlerSerializer(proto.Serializer{}), + pilosa.OptHandlerRoaringSerializer(proto.RoaringSerializer), ) return errors.Wrap(err, "new handler") } diff --git a/server/server_test.go b/server/server_test.go index 3dfd92728..c1efbc8a7 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -19,7 +19,7 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/encoding/proto" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/server" @@ -54,7 +54,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) + client, err := pilosa.NewInternalClient(m.API.Node().URI.HostPort(), pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) client.SetInternalAPI(m.API) if err != nil { t.Fatal(err) @@ -904,7 +904,7 @@ func TestQueryingWithQuotesAndStuff(t *testing.T) { m := test.RunCommand(t) defer m.Close() - client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) + client, err := pilosa.NewInternalClient(m.API.Node().URI.HostPort(), pilosa.GetHTTPClient(nil), pilosa.WithSerializer(proto.Serializer{})) client.SetInternalAPI(m.API) if err != nil { t.Fatal(err) diff --git a/stats/stats_test.go b/stats/stats_test.go index b2867d184..81b62a5f2 100644 --- a/stats/stats_test.go +++ b/stats/stats_test.go @@ -9,8 +9,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/test" @@ -143,7 +142,7 @@ func TestStatsCount_APICalls(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() cmd := cluster.GetNode(0) - h := cmd.Handler.(*http.Handler).Handler + h := cmd.Handler.(*pilosa.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/test/pilosa.go b/test/pilosa.go index 83041998c..d6663afd8 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -16,7 +16,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/disco" "github.com/molecula/featurebase/v3/encoding/proto" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/testhook" ) @@ -165,8 +164,8 @@ func (m *Command) IsPrimary() bool { } // Client returns a client to connect to the program. -func (m *Command) Client() *http.InternalClient { - return m.Server.InternalClient().(*http.InternalClient) +func (m *Command) Client() *pilosa.InternalClient { + return m.Server.InternalClient() } // Query executes a query against the program through the HTTP API. diff --git a/translator_test.go b/translator_test.go index 38cc47c51..5df16b4a9 100644 --- a/translator_test.go +++ b/translator_test.go @@ -13,7 +13,6 @@ import ( "github.com/google/go-cmp/cmp" pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/mock" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" @@ -156,25 +155,25 @@ func TestTranslation_KeyNotFound(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node3"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -312,19 +311,19 @@ func TestTranslation_Primary(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() @@ -388,25 +387,25 @@ func TestTranslation_TranslateIDsOnCluster(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node2"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node3"), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() diff --git a/tx_test.go b/tx_test.go index 95c82f4af..80d72b51c 100644 --- a/tx_test.go +++ b/tx_test.go @@ -7,7 +7,6 @@ import ( "testing" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/http" "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck @@ -51,7 +50,7 @@ func TestAPI_ImportAtomicRecord(t *testing.T) { server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close()