From 1ed4ecef25f81578b3759e248ab85d630ec3f65a Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 17:27:08 -0500 Subject: [PATCH 1/8] Don't try to create inverse views on Import() when inverseEnabled is false --- frame.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/frame.go b/frame.go index c70a07b68..414af55c6 100644 --- a/frame.go +++ b/frame.go @@ -500,18 +500,25 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) dataByFragment[key] = data } - // Attach reversed bits to each inverse view. - for _, name := range inverse { - key := importKey{View: name, Slice: bitmapID / SliceWidth} - data := dataByFragment[key] - data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed - data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed - dataByFragment[key] = data + if f.inverseEnabled { + // Attach reversed bits to each inverse view. + for _, name := range inverse { + key := importKey{View: name, Slice: bitmapID / SliceWidth} + data := dataByFragment[key] + data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed + data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed + dataByFragment[key] = data + } } } // Import into each fragment. for key, data := range dataByFragment { + // Skip inverse data if inverse is not enabled. + if !f.inverseEnabled && IsInverseView(key.View) { + continue + } + // Re-sort data for inverse views. if IsInverseView(key.View) { sort.Sort(importBitSet{ From 5c50ab8f0e64e8f820d20a9edb66d5c748ab0c9c Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 18:05:22 -0500 Subject: [PATCH 2/8] add a client test for imports to an inverseEnabled frame --- client_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/client_test.go b/client_test.go index f808514a0..f78ee69c4 100644 --- a/client_test.go +++ b/client_test.go @@ -190,6 +190,61 @@ func TestClient_Import(t *testing.T) { } } +// Ensure client can bulk import data to an inverse frame. +func TestClient_ImportInverseEnabled(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + d := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + frameOpts := pilosa.FrameOptions{ + InverseEnabled: true, + } + frame, err := d.CreateFrameIfNotExists("f", frameOpts) + if err != nil { + panic(err) + } + v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse) + if err != nil { + panic(err) + } + f, err := v.CreateFragmentIfNotExists(0) + if err != nil { + panic(err) + } + + // Load bitmap into cache to ensure cache gets updated. + f.Bitmap(0) + + s := NewServer() + defer s.Close() + s.Handler.Host = s.Host() + s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Index = idx.Index + + // Send import request. + c := MustNewClient(s.Host()) + if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{ + {BitmapID: 0, ProfileID: 1}, + {BitmapID: 0, ProfileID: 5}, + {BitmapID: 200, ProfileID: 5}, + {BitmapID: 200, ProfileID: 6}, + }); err != nil { + t.Fatal(err) + } + + // Verify data. + if a := f.Bitmap(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) { + t.Fatalf("unexpected bits: %+v", a) + } + if a := f.Bitmap(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) { + t.Fatalf("unexpected bits: %+v", a) + } + if a := f.Bitmap(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) { + t.Fatalf("unexpected bits: %+v", a) + } +} + // Ensure client backup and restore a frame. func TestClient_BackupRestore(t *testing.T) { idx := MustOpenIndex() From 8ceaf82f037d245e6f361f9a2be0e91f1246f59c Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Sun, 30 Apr 2017 00:24:28 -0500 Subject: [PATCH 3/8] create/delete/use command, expand output --- webui/assets/main.js | 184 ++++++++++++++++++++++++++++++++++------- webui/assets/style.css | 11 ++- 2 files changed, 162 insertions(+), 33 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index 4e17db19a..9b223218d 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -116,42 +116,90 @@ class REPL { process_query(query) { var xhr = new XMLHttpRequest(); + var url, data, request, command_name; var e = document.getElementById("index-dropdown"); var indexname = e.options[e.selectedIndex].text; - xhr.open('POST', '/index/' + indexname + '/query'); + var repl = this; + if (query.startsWith(":")) { + var parsed_query = parse_query(query); + if (Object.keys(parsed_query).length === 0) { + repl.createSingleOutput({ + "input": query, + "output": "invalid query", + "status": 400, + "indexname": indexname, + }); + return; + } else { + // set selectedIndex from dropdown list + if (parsed_query.command === "use") { + for (var i = 0; i < e.options.length; i++) { + if (e.options[i].text === parsed_query.command_name) { + e.selectedIndex = i; + break; + } + } + return; + } + url = parsed_query.url; + data = parsed_query.data; + request = parsed_query.request; + command_name = parsed_query.command_name; + } + } + else { + url = '/index/' + indexname + '/query' + request = "POST" + data = query + } + xhr.open(request, url); xhr.setRequestHeader('Content-Type', 'application/text'); - var repl = this var start_time = new Date().getTime(); - xhr.send(query) - xhr.onload = function() { - var end_time = new Date().getTime() + xhr.onload = function () { + var end_time = new Date().getTime(); repl.result_number++ repl.createSingleOutput({ - "input": query, - "output": xhr.responseText, - "indexname": indexname, - "querytime_ms": end_time - start_time, - }) - } + "input": query, + "output": xhr.responseText, + "status": xhr.status, + "indexname": indexname, + "querytime_ms": end_time - start_time, + }); + }; + xhr.send(data); + // Remove index from dropdown with delete index command + if (request === 'DELETE' && url === '/index/' + command_name){ + for (var i = 0; i < e.options.length; i++) { + if (e.options[i].text === command_name) { + e.remove(i); + break; + } + } + } } createSingleOutput(res) { - var node = document.createElement("div"); - node.classList.add('output'); - var output_string = res['output'] - var output_json = JSON.parse(output_string) - var result_class = "result-output" - var getting_started_errors = [ - 'index not found', - 'frame not found', - ] - - if("error" in output_json) { - result_class = "result-error" - if(getting_started_errors.indexOf(output_json['error']) >= 0) { - output_string += `
+ var node = document.createElement("div"); + node.classList.add('output'); + var output_string = res['output'] + var result_class = "result-output" + var getting_started_errors = [ + 'index not found', + 'frame not found', + ] + var output_json; + if (isJSON(output_string)) { + output_json = JSON.parse(output_string) + } + // handle output formatting + if (res["status"] != 200) { + result_class = "result-error"; + if (output_json) { + if ("error" in output_json) { + if (getting_started_errors.indexOf(output_json['error']) >= 0) { + output_string += `

Just getting started? Try this:
$ curl -XPOST "http://127.0.0.1:10101/index/test" -d '{"options": {"columnLabel": "col"}}' # create index "test"
@@ -159,8 +207,10 @@ class REPL { # Select "test" in the index dropdown above
SetBit(row=0, col=0, frame=foo) # Use PQL to set a bit ` + } + } + } } - } var markup =` @@ -184,7 +234,9 @@ class REPL {
${output_string} -
+ +
Expand
 +
@@ -195,8 +247,21 @@ class REPL {
` - node.innerHTML = markup; - this.output.insertBefore(node, this.output.firstChild) + node.innerHTML = markup; + this.output.insertBefore(node, this.output.firstChild); + + // Expand when overflow + var element = this.output.firstChild.getElementsByClassName(result_class)[0]; + var expand = this.output.firstChild.getElementsByClassName("expand")[0]; + if (element.clientHeight < element.scrollHeight) { + expand.style.display = 'block'; + } else { + expand.style.display = 'none'; + } + expand.onclick = function () { + element.style.height = element.scrollHeight + "px" + return false; + }; } populate_index_dropdown() { @@ -367,7 +432,7 @@ function check_anchor_uri() { } } -Date.prototype.today = function () { +Date.prototype.today = function () { return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate(); } @@ -388,3 +453,62 @@ repl.bind_events() input.focus() check_anchor_uri() + +function isJSON(str) { + try { + JSON.parse(str) + } catch (e) { + return false + } + return true +} + +function parse_query(query) { + var valid_command = [":create", ":use", ":delete"]; + // probably separate to a different function when option getting bigger + var keys = query.replace(/\s+/g, " ").split(" "); + var command = keys[0]; + var command_type = keys[1]; + var command_name = keys[2]; + if (!command_name){ + return {} + } + + var parsed_query = {}; + parsed_query["command"] = command.substr(1, command.length); + parsed_query["command_name"] = command_name; + switch (command) { + case ":create": + parsed_query["request"] = "POST"; + switch (command_type){ + case "index": + parsed_query["url"] = '/index/' + command_name; + parsed_query["data"] = ""; + break; + case "frame": + parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; + parsed_query["data"] = ""; + break + } + break; + case ":delete": + parsed_query["request"] = "DELETE"; + switch (command_type){ + case "index": + parsed_query["url"] = '/index/' + command_name; + parsed_query["data"] = ""; + break; + case "frame": + parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; + parsed_query["data"] = ""; + break; + } + break; + case ":use": + console.log(parsed_query) + break; + default: + return {} + } + return parsed_query; +} diff --git a/webui/assets/style.css b/webui/assets/style.css index a3a493e16..797559981 100644 --- a/webui/assets/style.css +++ b/webui/assets/style.css @@ -203,8 +203,6 @@ em{ display: block; } - - .result-io-header{ display: flex; align-items: center; @@ -214,6 +212,7 @@ em{ .result-input, .result-output, .result-error{ + height: 60px; border-radius: 2px; background-color: #fafafa; border: solid 1.5px #e4eff4; @@ -224,6 +223,8 @@ em{ color: #102445; padding: 15px; margin-bottom: 15px; + word-wrap: break-word; + overflow:hidden; } @@ -290,4 +291,8 @@ td{ .number { color: darkorange; } .boolean { color: blue; } .null { color: magenta; } -.key { color: red; } \ No newline at end of file +.key { color: red; } + +.expand { + text-align: center; +} \ No newline at end of file From 9ed5d2dd142e1a60b7f9156a9e887d814f5492a0 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 1 May 2017 10:24:00 -0500 Subject: [PATCH 4/8] remove expand link after expanding output --- webui/assets/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index 9b223218d..4e6b3336e 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -259,7 +259,8 @@ class REPL { expand.style.display = 'none'; } expand.onclick = function () { - element.style.height = element.scrollHeight + "px" + element.style.height = element.scrollHeight + "px"; + expand.style.display = 'none'; return false; }; } @@ -505,7 +506,6 @@ function parse_query(query) { } break; case ":use": - console.log(parsed_query) break; default: return {} From 0a6c4d23fda128013a5a175d779e817c86a9474c Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 1 May 2017 10:30:36 -0500 Subject: [PATCH 5/8] add indexname to parse_query func --- webui/assets/main.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index 4e6b3336e..9582d21ae 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -121,7 +121,7 @@ class REPL { var indexname = e.options[e.selectedIndex].text; var repl = this; if (query.startsWith(":")) { - var parsed_query = parse_query(query); + var parsed_query = parse_query(query, indexname); if (Object.keys(parsed_query).length === 0) { repl.createSingleOutput({ "input": query, @@ -464,9 +464,7 @@ function isJSON(str) { return true } -function parse_query(query) { - var valid_command = [":create", ":use", ":delete"]; - // probably separate to a different function when option getting bigger +function parse_query(query, indexname) { var keys = query.replace(/\s+/g, " ").split(" "); var command = keys[0]; var command_type = keys[1]; From f17a49a8c5d03fa81375b64b8b32ae5db819acfe Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 1 May 2017 11:48:32 -0500 Subject: [PATCH 6/8] consistent function's name --- webui/assets/main.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index 9582d21ae..a7a9a5de8 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -123,7 +123,7 @@ class REPL { if (query.startsWith(":")) { var parsed_query = parse_query(query, indexname); if (Object.keys(parsed_query).length === 0) { - repl.createSingleOutput({ + repl.create_single_output({ "input": query, "output": "invalid query", "status": 400, @@ -159,7 +159,7 @@ class REPL { xhr.onload = function () { var end_time = new Date().getTime(); repl.result_number++ - repl.createSingleOutput({ + repl.create_single_output({ "input": query, "output": xhr.responseText, "status": xhr.status, @@ -180,7 +180,7 @@ class REPL { } } - createSingleOutput(res) { + create_single_output(res) { var node = document.createElement("div"); node.classList.add('output'); var output_string = res['output'] From 547c64397c244f5e8895bf56c33520a4f9ad0b14 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 2 May 2017 11:35:29 -0500 Subject: [PATCH 7/8] break word into multiple lines --- webui/assets/style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/assets/style.css b/webui/assets/style.css index 797559981..515335ef9 100644 --- a/webui/assets/style.css +++ b/webui/assets/style.css @@ -223,7 +223,7 @@ em{ color: #102445; padding: 15px; margin-bottom: 15px; - word-wrap: break-word; + word-break: break-all; overflow:hidden; } From 69af3246db072fe8dadf9629d3a835f85cf4cfaf Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 2 May 2017 11:48:34 -0500 Subject: [PATCH 8/8] fix use command --- webui/assets/main.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index a7a9a5de8..78ed7470c 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -469,9 +469,13 @@ function parse_query(query, indexname) { var command = keys[0]; var command_type = keys[1]; var command_name = keys[2]; - if (!command_name){ - return {} + + if (command !== ":use") { + if (!command_name){ + return {} + } } + var parsed_query = {}; parsed_query["command"] = command.substr(1, command.length); @@ -504,6 +508,7 @@ function parse_query(query, indexname) { } break; case ":use": + parsed_query["command_name"] = keys[1]; break; default: return {}