Remove view from API, handler, docs

This commit is contained in:
Yuce Tekol 2018-05-30 16:09:48 +03:00
parent 9456a2076d
commit 738dead374
No known key found for this signature in database
GPG key ID: CB59E46D2FB90573
10 changed files with 36 additions and 169 deletions

20
api.go
View file

@ -283,9 +283,9 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str
return nil
}
// ExportCSV encodes the fragment designated by the index,frame,view,slice as
// ExportCSV encodes the fragment designated by the index,frame,slice as
// CSV of the form <row>,<col>
func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error {
func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, slice uint64, w io.Writer) error {
if err := api.validate(apiExportCSV); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -297,7 +297,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
}
// Find the fragment.
f := api.Holder.Fragment(indexName, frameName, viewName, slice)
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
if f == nil {
return ErrFragmentNotFound
}
@ -333,13 +333,13 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64)
// MarshalFragment returns an object which can write the specified fragment's data
// to an io.Writer. The serialized data can be read back into a fragment with
// the UnmarshalFragment API call.
func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) {
func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64) (io.WriterTo, error) {
if err := api.validate(apiMarshalFragment); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, frameName, viewName, slice)
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -349,7 +349,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName
// UnmarshalFragment creates a new fragment (if necessary) and reads data from a
// Reader which was previously written by MarshalFragment to populate the
// fragment's data.
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error {
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64, reader io.ReadCloser) error {
if err := api.validate(apiUnmarshalFragment); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -361,7 +361,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa
}
// Retrieve view.
view, err := f.CreateViewIfNotExists(viewName)
view, err := f.CreateViewIfNotExists(ViewStandard)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -397,7 +397,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice)
f := api.Holder.Fragment(req.Index, req.Frame, ViewStandard, req.Slice)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -415,13 +415,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment.
func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) {
func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, slice uint64) ([]FragmentBlock, error) {
if err := api.validate(apiFragmentBlocks); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, frameName, viewName, slice)
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
if f == nil {
return nil, ErrFragmentNotFound
}

View file

@ -508,13 +508,11 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu
}
// ExportCSV bulk exports data for a single slice from a host to CSV format.
func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error {
func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if view != ViewStandard {
return ErrInvalidView
}
// Retrieve a list of nodes that own the slice.
@ -528,7 +526,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
for _, i := range rand.Perm(len(nodes)) {
node := nodes[i]
if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil {
if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil {
e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err)
continue
} else {
@ -540,13 +538,12 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
}
// exportNode copies a CSV export from a node to w.
func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error {
func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error {
// Create URL.
u := nodePathToURL(node, "/export")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
@ -578,19 +575,18 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
return nil
}
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame string, slice uint64, uri URI) (io.ReadCloser, error) {
node := &Node{
URI: uri,
}
return c.backupSliceNode(ctx, index, frame, view, slice, node)
return c.backupSliceNode(ctx, index, frame, slice, node)
}
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame string, slice uint64, node *Node) (io.ReadCloser, error) {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
@ -669,50 +665,13 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
}
}
// FrameViews returns a list of view names for a frame.
func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string) ([]string, error) {
// Create URL & HTTP request.
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame))
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Handle response based on status code.
switch resp.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return nil, ErrFrameNotFound
default:
body, _ := ioutil.ReadAll(resp.Body)
return nil, errors.New(string(body))
}
// Decode response.
var rsp getFrameViewsResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, errors.Wrap(err, "decoding")
}
return rsp.Views, nil
}
// FragmentBlocks returns a list of block checksums for a fragment on a host.
// Only returns blocks which contain data.
func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) {
func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) {
u := uriPathToURL(c.defaultURI, "/fragment/blocks")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
@ -749,11 +708,10 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v
}
// BlockData returns row/column id pairs for a block.
func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) {
buf, err := proto.Marshal(&internal.BlockDataRequest{
Index: index,
Frame: frame,
View: view,
Slice: slice,
Block: uint64(block),
})
@ -1101,11 +1059,10 @@ type InternalClient interface {
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error
ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error
ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error
ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error
CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error
FrameViews(ctx context.Context, index, frame string) ([]string, error)
FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error)
FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, pb proto.Message) error

View file

@ -335,7 +335,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
// Retrieve blocks.
c := test.MustNewClient(s.Host(), defaultClient)
blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0)
blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0)
if err != nil {
t.Fatal(err)
} else if len(blocks) != 2 {

View file

@ -1259,7 +1259,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
// Stream slice from remote node.
c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.Slice, srcURI)
if err != nil {
// For now it is an acceptable error if the fragment is not found
// on the remote node. This occurs when a slice has been skipped and

View file

@ -89,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, pilosa.ViewStandard, slice, w); err != nil {
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil {
return errors.Wrap(err, "exporting")
}
}

View file

@ -70,9 +70,9 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
#### Exporting
Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index, frame, and view (default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column.
Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the frame. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column.
```request
curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \
curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0" \
--header "Accept: text/csv"
```
```response

View file

@ -244,29 +244,6 @@ threshold = 90
return the set of molecules that have at least a 90% similarity with the given molecule.
The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint.
Standard View is used to calculate similarity
```
Index: mole
View: Standard
Col: chembl_id
Frame: fingerprint
Row: position_id ("on" bit positions of a fingerprint)
```
Inverse View is used for finding chembl_id based on given SMILES.
From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint.
```
Index: mole
View: Inverse
Col: position_id ("on" bit positions of a fingerprint)
Frame: fingerprint
Row: chembl_id
```
After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold.
#### Import process
To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key.
@ -312,15 +289,6 @@ Return chembl_id = 6223. This script uses Pilosas Intersection query to get a
fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits())
```
* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id
```python
bit_maps = ["Bitmap(col=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
bitmap_string = ', '.join(bit_maps)
intersection = "Intersect(%s)" % bitmap_string
mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"]
```
* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist.
```python

View file

@ -584,10 +584,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
frame = DefaultFrame
}
// Determine view.
view := ViewStandard
f := e.Holder.Fragment(index, frame, view, slice)
f := e.Holder.Fragment(index, frame, ViewStandard, slice)
if f == nil {
return nil, nil
}

View file

@ -1783,7 +1783,7 @@ func (s *FragmentSyncer) SyncFragment() error {
// Retrieve remote blocks.
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.Slice())
if err != nil && err != ErrFragmentNotFound {
return errors.Wrap(err, "getting blocks")
}
@ -1862,7 +1862,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
clients = append(clients, client)
// Only sync the standard block.
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), ViewStandard, f.Slice(), id)
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), f.Slice(), id)
if err != nil {
return errors.Wrap(err, "getting block")
}

View file

@ -108,10 +108,10 @@ func (h *Handler) populateValidators() {
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
h.validators["GetSliceMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "slice")
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "slice")
}
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
@ -172,8 +172,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE")
router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET")
router.HandleFunc("/index/{index}/frame/{frame}/view/{view}", handler.handleDeleteView).Methods("DELETE")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
@ -703,59 +701,6 @@ type getFrameFieldsResponse struct {
type deleteFrameFieldResponse struct{}
// handleGetFrameViews handles GET /frame/views request.
func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
views, err := h.API.Views(r.Context(), indexName, frameName)
if err != nil {
if errors.Cause(err) == ErrFrameNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
names := make([]string, len(views))
for i := range views {
names[i] = views[i].Name()
}
// Encode response.
if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
// handleDeleteView handles Delete /frame/view request.
func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
viewName := mux.Vars(r)["view"]
if err := h.API.DeleteView(r.Context(), indexName, frameName, viewName); err != nil {
if errors.Cause(err) == ErrFrameNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusBadRequest)
}
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
type deleteViewResponse struct{}
type getFrameViewsResponse struct {
Views []string `json:"views,omitempty"`
}
// handlePostFrameAttrDiff handles POST /frame/attr/diff requests.
func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
@ -990,7 +935,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
// Parse query parameters.
q := r.URL.Query()
index, frame, view := q.Get("index"), q.Get("frame"), q.Get("view")
index, frame := q.Get("index"), q.Get("frame")
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
@ -998,7 +943,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
return
}
if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil {
if err = h.API.ExportCSV(r.Context(), index, frame, slice, w); err != nil {
switch errors.Cause(err) {
case ErrFragmentNotFound:
break
@ -1066,7 +1011,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
return
}
blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice)
blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), slice)
if err != nil {
if errors.Cause(err) == ErrFragmentNotFound {
http.Error(w, err.Error(), http.StatusNotFound)