From 738dead3743dbbdfb6dbc62e713082134d9fd26f Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 30 May 2018 16:09:48 +0300 Subject: [PATCH] Remove view from API, handler, docs --- api.go | 20 ++++++------ client.go | 65 +++++++-------------------------------- client_test.go | 2 +- cluster.go | 2 +- ctl/export.go | 2 +- docs/administration.md | 4 +-- docs/examples.md | 32 -------------------- executor.go | 5 +-- fragment.go | 4 +-- handler.go | 69 +++++------------------------------------- 10 files changed, 36 insertions(+), 169 deletions(-) diff --git a/api.go b/api.go index d12204fd2..252480d52 100644 --- a/api.go +++ b/api.go @@ -283,9 +283,9 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str return nil } -// ExportCSV encodes the fragment designated by the index,frame,view,slice as +// ExportCSV encodes the fragment designated by the index,frame,slice as // CSV of the form , -func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { +func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { return errors.Wrap(err, "validating api method") } @@ -297,7 +297,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin } // Find the fragment. - f := api.Holder.Fragment(indexName, frameName, viewName, slice) + f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) if f == nil { return ErrFragmentNotFound } @@ -333,13 +333,13 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { +func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, viewName, slice) + f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) if f == nil { return nil, ErrFragmentNotFound } @@ -349,7 +349,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // UnmarshalFragment creates a new fragment (if necessary) and reads data from a // Reader which was previously written by MarshalFragment to populate the // fragment's data. -func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { return errors.Wrap(err, "validating api method") } @@ -361,7 +361,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa } // Retrieve view. - view, err := f.CreateViewIfNotExists(viewName) + view, err := f.CreateViewIfNotExists(ViewStandard) if err != nil { return errors.Wrap(err, "creating view") } @@ -397,7 +397,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // Retrieve fragment from holder. - f := api.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) + f := api.Holder.Fragment(req.Index, req.Frame, ViewStandard, req.Slice) if f == nil { return nil, ErrFragmentNotFound } @@ -415,13 +415,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. -func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { +func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, viewName, slice) + f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) if f == nil { return nil, ErrFragmentNotFound } diff --git a/client.go b/client.go index 3ff445a22..f3e0e5dce 100644 --- a/client.go +++ b/client.go @@ -508,13 +508,11 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error { +func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } else if view != ViewStandard { - return ErrInvalidView } // Retrieve a list of nodes that own the slice. @@ -528,7 +526,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) continue } else { @@ -540,13 +538,12 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s } // exportNode copies a CSV export from a node to w. -func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error { +func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { // Create URL. u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ "index": {index}, "frame": {frame}, - "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -578,19 +575,18 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde return nil } -func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) { +func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame string, slice uint64, uri URI) (io.ReadCloser, error) { node := &Node{ URI: uri, } - return c.backupSliceNode(ctx, index, frame, view, slice, node) + return c.backupSliceNode(ctx, index, frame, slice, node) } -func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { +func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame string, slice uint64, node *Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, "frame": {frame}, - "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -669,50 +665,13 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin } } -// FrameViews returns a list of view names for a frame. -func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string) ([]string, error) { - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame)) - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - case http.StatusNotFound: - return nil, ErrFrameNotFound - default: - body, _ := ioutil.ReadAll(resp.Body) - return nil, errors.New(string(body)) - } - - // Decode response. - var rsp getFrameViewsResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Views, nil -} - // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) { +func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) { u := uriPathToURL(c.defaultURI, "/fragment/blocks") u.RawQuery = url.Values{ "index": {index}, "frame": {frame}, - "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode() @@ -749,11 +708,10 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v } // BlockData returns row/column id pairs for a block. -func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, Frame: frame, - View: view, Slice: slice, Block: uint64(block), }) @@ -1101,11 +1059,10 @@ type InternalClient interface { EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error - ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error + ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error - FrameViews(ctx context.Context, index, frame string) ([]string, error) - FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) + FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error diff --git a/client_test.go b/client_test.go index e4287726b..6e5f2dc41 100644 --- a/client_test.go +++ b/client_test.go @@ -335,7 +335,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) - blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0) + blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0) if err != nil { t.Fatal(err) } else if len(blocks) != 2 { diff --git a/cluster.go b/cluster.go index d6026274a..63f4f62b8 100644 --- a/cluster.go +++ b/cluster.go @@ -1259,7 +1259,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Stream slice from remote node. c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a slice has been skipped and diff --git a/ctl/export.go b/ctl/export.go index 2c40b3bc4..5ee8d2120 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -89,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, pilosa.ViewStandard, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/docs/administration.md b/docs/administration.md index d25f8a8d1..dc2e923f2 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -70,9 +70,9 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count #### Exporting -Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index, frame, and view (default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column. +Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the frame. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column. ```request -curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \ +curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0" \ --header "Accept: text/csv" ``` ```response diff --git a/docs/examples.md b/docs/examples.md index ab04b2bb5..6c4f70cc1 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -244,29 +244,6 @@ threshold = 90 return the set of molecules that have at least a 90% similarity with the given molecule. -The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint. - -Standard View is used to calculate similarity -``` -Index: mole - View: Standard - Col: chembl_id - Frame: fingerprint - Row: position_id ("on" bit positions of a fingerprint) -``` - -Inverse View is used for finding chembl_id based on given SMILES. -From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint. -``` -Index: mole - View: Inverse - Col: position_id ("on" bit positions of a fingerprint) - Frame: fingerprint - Row: chembl_id -``` - -After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold. - #### Import process To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key. @@ -312,15 +289,6 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits()) ``` -* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id - - ```python - bit_maps = ["Bitmap(col=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] - bitmap_string = ', '.join(bit_maps) - intersection = "Intersect(%s)" % bitmap_string - mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] - ``` - * From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist. ```python diff --git a/executor.go b/executor.go index 59cd385a6..97f42acf1 100644 --- a/executor.go +++ b/executor.go @@ -584,10 +584,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca frame = DefaultFrame } - // Determine view. - view := ViewStandard - - f := e.Holder.Fragment(index, frame, view, slice) + f := e.Holder.Fragment(index, frame, ViewStandard, slice) if f == nil { return nil, nil } diff --git a/fragment.go b/fragment.go index f4e9906a1..8044f579b 100644 --- a/fragment.go +++ b/fragment.go @@ -1783,7 +1783,7 @@ func (s *FragmentSyncer) SyncFragment() error { // Retrieve remote blocks. client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice()) + blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1862,7 +1862,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { clients = append(clients, client) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), ViewStandard, f.Slice(), id) + rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), f.Slice(), id) if err != nil { return errors.Wrap(err, "getting block") } diff --git a/handler.go b/handler.go index bfff395c3..9ddabaeeb 100644 --- a/handler.go +++ b/handler.go @@ -108,10 +108,10 @@ func (h *Handler) populateValidators() { h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") h.validators["GetSliceMax"] = queryValidationSpecRequired() h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "slice") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -172,8 +172,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") - router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") - router.HandleFunc("/index/{index}/frame/{frame}/view/{view}", handler.handleDeleteView).Methods("DELETE") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -703,59 +701,6 @@ type getFrameFieldsResponse struct { type deleteFrameFieldResponse struct{} -// handleGetFrameViews handles GET /frame/views request. -func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - views, err := h.API.Views(r.Context(), indexName, frameName) - if err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - names := make([]string, len(views)) - for i := range views { - names[i] = views[i].Name() - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -// handleDeleteView handles Delete /frame/view request. -func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - viewName := mux.Vars(r)["view"] - - if err := h.API.DeleteView(r.Context(), indexName, frameName, viewName); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusBadRequest) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteViewResponse struct{} - -type getFrameViewsResponse struct { - Views []string `json:"views,omitempty"` -} - // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] @@ -990,7 +935,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // Parse query parameters. q := r.URL.Query() - index, frame, view := q.Get("index"), q.Get("frame"), q.Get("view") + index, frame := q.Get("index"), q.Get("frame") slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) if err != nil { @@ -998,7 +943,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { + if err = h.API.ExportCSV(r.Context(), index, frame, slice, w); err != nil { switch errors.Cause(err) { case ErrFragmentNotFound: break @@ -1066,7 +1011,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request return } - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), slice) if err != nil { if errors.Cause(err) == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound)