From 9de9b7cc30bc881c02c73ce8c6cc9cc7d2dbad05 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 21 Nov 2017 23:36:20 +0300 Subject: [PATCH 1/8] Deprecate index time quantum in the docs; resolves #978 --- cmd/import.go | 2 +- docs/api-reference.md | 40 +--------------------------------------- docs/webui.md | 2 +- 3 files changed, 3 insertions(+), 41 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index f998bd4b3..250dccfd3 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -59,7 +59,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index") + flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame") flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame") diff --git a/docs/api-reference.md b/docs/api-reference.md index 5129f4061..206c30f55 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -45,15 +45,9 @@ Response: Creates an index with the given name. -The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: - -* `timeQuantum` (string): time quantum of the index. - Request: ``` -curl localhost:10101/index/user \ - -X POST \ - -d '{"options": {"timeQuantum": "YMDH"}}' +curl -XPOST localhost:10101/index/user ``` Response: @@ -117,38 +111,6 @@ Response: By default, all bits and attributes (*for `Bitmap` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`. -### Change index time quantum - -`PATCH /index//time-quantum` - -Changes the time quantum for the given index. This endpoint should be called at most once right after creating a database. - -The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are: - -* (Empty string) -* Y: year -* M: month -* D: day -* H: hour -* YM: year and month -* MD: month and day -* DH: day and hour -* YMD: year, month and day -* MDH: month, day and hour -* YMDH: year, month, day and hour - -Request: -``` -curl localhost:10101/index/user/time-quantum \ - -X POST \ - -d '{"timeQuantum": "YM"}' -``` - -Response: -``` -{} -``` - ### Create frame `POST /index//frame/` diff --git a/docs/webui.md b/docs/webui.md index a348f59a6..cef8e570d 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -30,7 +30,7 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create frame ` - `:delete frame ` -Index and frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating new index or new frame, add options by using the keys documented in [API reference](../api-reference). +Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference). - `:create index timeQuantum=YM` - `:create frame inverseEnabled=true cacheSize=10000` From 489b7a59c4ff72c49d88c269b1ca67271cc3c344 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 7 Dec 2017 14:02:25 -0600 Subject: [PATCH 2/8] protect against accessing pointers to memory which was unmapped --- roaring/roaring.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 97994c37a..ad7b33bc4 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -665,13 +665,19 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { c := b.containers[i] switch c.container_type { case ContainerRun: + c.array = nil + c.bitmap = nil runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount] opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size case ContainerArray: + c.runs = nil + c.bitmap = nil c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32) case ContainerBitmap: + c.array = nil + c.runs = nil c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] opsOffset = int(offset) + len(c.bitmap)*8 // sizeof(uint64) } @@ -1019,17 +1025,16 @@ func (c *container) unmap() { return } - if c.array != nil { + switch c.container_type { + case ContainerArray: tmp := make([]uint16, len(c.array)) copy(tmp, c.array) c.array = tmp - } - if c.bitmap != nil { + case ContainerBitmap: tmp := make([]uint64, len(c.bitmap)) copy(tmp, c.bitmap) c.bitmap = tmp - } - if c.runs != nil { + case ContainerRun: tmp := make([]interval16, len(c.runs)) copy(tmp, c.runs) c.runs = tmp @@ -1614,21 +1619,17 @@ func (c *container) runToArray() { func (c *container) clone() *container { other := &container{n: c.n, container_type: c.container_type} - if c.array != nil { + switch c.container_type { + case ContainerArray: other.array = make([]uint16, len(c.array)) copy(other.array, c.array) - } - - if c.bitmap != nil { + case ContainerBitmap: other.bitmap = make([]uint64, len(c.bitmap)) copy(other.bitmap, c.bitmap) - } - - if c.runs != nil { + case ContainerRun: other.runs = make([]interval16, len(c.runs)) copy(other.runs, c.runs) } - return other } From 907aa3495f0b23bed7a7c737e0d84465875db5f0 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 7 Dec 2017 14:58:11 -0600 Subject: [PATCH 3/8] add container types and set c.n to get tests working --- roaring/roaring_internal_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index b36723b26..aae6f9b22 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1267,8 +1267,8 @@ func TestBitmapZeroRange(t *testing.T) { } func TestUnionBitmapRun(t *testing.T) { - a := &container{bitmap: make([]uint64, bitmapN)} - b := &container{} + a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &container{container_type: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1288,6 +1288,7 @@ func TestUnionBitmapRun(t *testing.T) { } a.n = a.bitmapCountRange(0, 65535) b.runs = test.runs + b.n = b.runCountRange(0, 65535) ret := unionBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() @@ -1556,8 +1557,8 @@ func TestDifferenceRunBitmap(t *testing.T) { } func TestDifferenceBitmapRun(t *testing.T) { - a := &container{bitmap: make([]uint64, bitmapN)} - b := &container{} + a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &container{container_type: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 From 4785b0e7931c38f1a61d73b6ceccd82f9b17569e Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 7 Dec 2017 15:20:53 -0600 Subject: [PATCH 4/8] add container types to other tests (though they were passing already) --- roaring/roaring_internal_test.go | 65 ++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index aae6f9b22..f5c12aa83 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -347,8 +347,8 @@ func TestRunMax(t *testing.T) { } func TestIntersectionCountArrayRun(t *testing.T) { - a := &container{array: []uint16{1, 5, 10, 11, 12}} - b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + a := &container{container_type: ContainerArray, array: []uint16{1, 5, 10, 11, 12}} + b := &container{container_type: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} ret := intersectionCountArrayRun(a, b) if ret != 3 { @@ -357,16 +357,16 @@ func TestIntersectionCountArrayRun(t *testing.T) { } func TestIntersectionCountBitmapRun(t *testing.T) { - a := &container{bitmap: []uint64{0x8000000000000000}} - b := &container{runs: []interval16{{start: 63, last: 64}}} + a := &container{container_type: ContainerBitmap, bitmap: []uint64{0x8000000000000000}} + b := &container{container_type: ContainerRun, runs: []interval16{{start: 63, last: 64}}} ret := intersectionCountBitmapRun(a, b) if ret != 1 { t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret) } - a = &container{bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} - b = &container{runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} + a = &container{container_type: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} + b = &container{container_type: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} ret = intersectionCountBitmapRun(a, b) if ret != 14 { @@ -414,6 +414,8 @@ func TestIntersectionCountRunRun(t *testing.T) { bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, } for i, test := range tests { + a.container_type = ContainerRun + b.container_type = ContainerRun a.runs = test.aruns b.runs = test.bruns ret := intersectionCountRunRun(a, b) @@ -454,6 +456,8 @@ func TestIntersectArrayRun(t *testing.T) { } for i, test := range tests { + a.container_type = ContainerArray + b.container_type = ContainerRun a.array = test.array b.runs = test.runs ret := intersectArrayRun(a, b) @@ -510,6 +514,8 @@ func TestIntersectRunRun(t *testing.T) { }, } for i, test := range tests { + a.container_type = ContainerRun + b.container_type = ContainerRun a.runs = test.aruns b.runs = test.bruns ret := intersectRunRun(a, b) @@ -573,6 +579,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { for i, v := range test.exp { exp[i] = v } + a.container_type = ContainerBitmap + b.container_type = ContainerRun ret := intersectBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() @@ -632,6 +640,8 @@ func TestIntersectBitmapRunArray(t *testing.T) { a.bitmap[i] = v } b.runs = test.runs + a.container_type = ContainerBitmap + b.container_type = ContainerRun ret := intersectBitmapRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -873,6 +883,8 @@ func TestUnionRunRun(t *testing.T) { for i, test := range tests { a.runs = test.aruns b.runs = test.bruns + a.container_type = ContainerRun + b.container_type = ContainerRun ret := unionRunRun(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) @@ -913,6 +925,8 @@ func TestUnionArrayRun(t *testing.T) { for i, test := range tests { a.array = test.array b.runs = test.runs + a.container_type = ContainerArray + b.container_type = ContainerRun ret := unionArrayRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -921,7 +935,7 @@ func TestUnionArrayRun(t *testing.T) { } func TestBitmapSetRange(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN)} + c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -961,7 +975,7 @@ func TestBitmapSetRange(t *testing.T) { } func TestArrayToBitmap(t *testing.T) { - a := &container{} + a := &container{container_type: ContainerArray} tests := []struct { array []uint16 exp []uint64 @@ -992,7 +1006,7 @@ func TestArrayToBitmap(t *testing.T) { } func TestBitmapToArray(t *testing.T) { - a := &container{} + a := &container{container_type: ContainerBitmap} tests := []struct { bitmap []uint64 exp []uint16 @@ -1023,7 +1037,7 @@ func TestBitmapToArray(t *testing.T) { } func TestRunToBitmap(t *testing.T) { - a := &container{} + a := &container{container_type: ContainerRun} tests := []struct { runs []interval16 exp []uint64 @@ -1077,7 +1091,7 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := &container{} + a := &container{container_type: ContainerBitmap} tests := []struct { bitmap []uint64 exp []interval16 @@ -1155,7 +1169,7 @@ func TestBitmapToRun(t *testing.T) { } func TestArrayToRun(t *testing.T) { - a := &container{} + a := &container{container_type: ContainerArray} tests := []struct { array []uint16 exp []interval16 @@ -1189,7 +1203,7 @@ func TestArrayToRun(t *testing.T) { } func TestRunToArray(t *testing.T) { - a := &container{} + a := &container{container_type: ContainerRun} tests := []struct { runs []interval16 exp []uint16 @@ -1223,7 +1237,7 @@ func TestRunToArray(t *testing.T) { } func TestBitmapZeroRange(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN)} + c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -1306,7 +1320,7 @@ func TestUnionBitmapRun(t *testing.T) { } func TestBitmapCountRuns(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN)} + c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 exp int @@ -1356,7 +1370,7 @@ func TestBitmapCountRuns(t *testing.T) { } func TestArrayCountRuns(t *testing.T) { - c := &container{} + c := &container{container_type: ContainerArray} tests := []struct { array []uint16 exp int @@ -1397,8 +1411,8 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := &container{} - b := &container{} + a := &container{container_type: ContainerArray} + b := &container{container_type: ContainerRun} tests := []struct { array []uint16 runs []interval16 @@ -1423,8 +1437,8 @@ func TestDifferenceArrayRun(t *testing.T) { } func TestDifferenceRunArray(t *testing.T) { - a := &container{} - b := &container{} + a := &container{container_type: ContainerRun} + b := &container{container_type: ContainerArray} tests := []struct { runs []interval16 array []uint16 @@ -1494,8 +1508,8 @@ func MakeLastBitSet() []uint64 { } func TestDifferenceRunBitmap(t *testing.T) { - a := &container{} - b := &container{bitmap: make([]uint64, bitmapN)} + a := &container{container_type: ContainerRun} + b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { runs []interval16 bitmap []uint64 @@ -1585,7 +1599,7 @@ func TestDifferenceBitmapRun(t *testing.T) { } func TestDifferenceBitmapArray(t *testing.T) { - b := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} a := &container{container_type: ContainerArray} tests := []struct { bitmap []uint64 @@ -2514,8 +2528,9 @@ func TestSearc64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := &container{}, &container{ - bitmap: make([]uint64, bitmapN), + a, b := &container{container_type: ContainerArray}, &container{ + container_type: ContainerBitmap, + bitmap: make([]uint64, bitmapN), } tests := []struct { array []uint16 From 2bf1e1a4e334ebe103bd1ab52ee5e4f82b8608c2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 12 Dec 2017 16:37:56 -0600 Subject: [PATCH 5/8] Release v0.8.3 --- CHANGELOG.md | 25 +++++++++++++++++++++++++ docs/installation.md | 12 ++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a54a5b73..9676b4f89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.8.3] - 2017-12-12 + +This version contains 1 contribution from 1 contributor. There are 2 files changed, 59 insertions, and 42 deletions. + +### Fixed + +- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000)) + +## [0.8.2] - 2017-12-05 + +This version contains 1 contribution from 1 contributor. There are 15 files changed, 127 insertions, and 98 deletions. + +### Fixed + +- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994)) + +## [0.8.1] - 2017-11-15 + +This version contains 2 contributions from 2 contributors. There are 4 files changed, 27 insertions, and 14 deletions. + +### Fixed + +- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969)) +- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968)) + ## [0.8.0] - 2017-11-15 This version contains 31 contributions from 8 contributors. There are 84 files changed, 3,732 insertions, and 1,428 deletions. diff --git a/docs/installation.md b/docs/installation.md index 33528f969..5a0ea3086 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.0/pilosa-v0.8.0-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.0-darwin-amd64.tar.gz + tar xfz pilosa-v0.8.3-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.0-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.8.3-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -228,19 +228,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.0/pilosa-v0.8.0-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.0-linux-amd64.tar.gz + tar xfz pilosa-v0.8.3-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.0-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.8.3-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: From ab19576dbba6c8d1a8183dcb7efc0cdcba4ddac2 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 13 Dec 2017 17:27:08 +0300 Subject: [PATCH 6/8] remove frame options from getting started --- docs/getting-started.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 3a0306a51..a50dccaa5 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -59,21 +59,12 @@ curl localhost:10101/index/repository -X POST Let's create the `stargazer` frame which has user IDs of stargazers as its rows: ``` -curl localhost:10101/index/repository/frame/stargazer \ - -X POST \ - -d '{"options": {"timeQuantum": "YMD", - "inverseEnabled": true}}' +curl localhost:10101/index/repository/frame/stargazer -X POST ``` -Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. - -We set `inverseEnabled` to `true` in order to allow queries over columns as well as rows. - Next up is the `language` frame, which will contain IDs for programming languages: ``` -curl localhost:10101/index/repository/frame/language \ - -X POST \ - -d '{"options": {"inverseEnabled": true}}' +curl localhost:10101/index/repository/frame/language -X POST ``` #### Import Data From CSV Files From 0f48e4ee17a533fb81bc46e310282b563fe33eb7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 13 Dec 2017 15:35:02 -0600 Subject: [PATCH 7/8] Use `go test` instead of `goveralls` as main CI test command. Call `goveralls` in after_success block. Fixes #1009. --- .travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c368fbd49..2ddff4f4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,11 +7,14 @@ env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" -addons: -before_install: - - go get github.com/mattn/goveralls +install: + - make vendor script: - - make vendor && $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go" + - make test +# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 +after_success: + - go get github.com/mattn/goveralls + - $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go" before_deploy: - pip install awscli --user `whoami` deploy: From 1448171a2c1bfdf17267b2d5fe4673b018c58fa6 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 15 Dec 2017 18:12:28 +0300 Subject: [PATCH 8/8] Adds local and cluster IDs --- Gopkg.lock | 8 +++++++- Gopkg.toml | 4 ++++ client.go | 25 +++++++++++++++++++++++++ handler.go | 8 ++++++++ holder.go | 25 +++++++++++++++++++++++++ server.go | 32 +++++++++++++++++++++++++++++++- 6 files changed, 100 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index ed772ef39..4114168b3 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -157,6 +157,12 @@ revision = "fd36b3595eb2ec8da4b8153b107f7ea08504899d" version = "v0.1.1" +[[projects]] + name = "github.com/satori/go.uuid" + packages = ["."] + revision = "879c5887cd475cd7864858769793b2ceb0d44feb" + version = "v1.1.0" + [[projects]] branch = "master" name = "github.com/sean-/seed" @@ -238,6 +244,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "75badb0bcc3bb356b04af17979e0af61b4b66c5e0a483f09e39cf1f9b5e5de2c" + inputs-digest = "210f654a7a072d5751f0814e4d71ef0758dd53b3dc59ed462619396ef8621d81" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 7ffa67a2f..527744da4 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,3 +1,7 @@ # This file intentionally left blank as all needed dependencies are imported by # the project and thus tracked by `dep`. # See https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md for details. + +[[constraint]] + name = "github.com/satori/go.uuid" + version = "1.1.0" diff --git a/client.go b/client.go index f71cf57d4..8f4f282f8 100644 --- a/client.go +++ b/client.go @@ -1087,6 +1087,30 @@ func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI { return clientURI } +func (c *InternalHTTPClient) NodeID(uri *URI) (string, error) { + u := uriPathToURL(uri, "/id") + req, err := http.NewRequest("GET", u.String(), nil) + resp, err := c.HTTPClient.Do(req) + if err != nil { + return "", fmt.Errorf("executing http request: %v", err) + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("reading response body: %v", err) + } + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return "", fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) + } + return string(body), nil +} + // Bit represents the location of a single bit. type Bit struct { RowID uint64 @@ -1262,4 +1286,5 @@ type InternalClient interface { ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, pb proto.Message) error + NodeID(uri *URI) (string, error) } diff --git a/handler.go b/handler.go index 7a094bdc9..a5b5c4808 100644 --- a/handler.go +++ b/handler.go @@ -138,6 +138,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") + router.HandleFunc("/id", handler.handleGetID).Methods("GET") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: @@ -2026,4 +2027,11 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } } +func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(h.Holder.LocalID)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + type defaultClusterMessageResponse struct{} diff --git a/holder.go b/holder.go index 7cbb8ca42..3f6ccc4fa 100644 --- a/holder.go +++ b/holder.go @@ -19,14 +19,19 @@ import ( "errors" "fmt" "io" + "io/ioutil" "log" "net/http" "os" + "path" "path/filepath" "sort" + "strings" "sync" "syscall" "time" + + uuid "github.com/satori/go.uuid" ) const ( @@ -59,6 +64,8 @@ type Holder struct { CacheFlushInterval time.Duration LogOutput io.Writer + + LocalID string } // NewHolder returns a new instance of Holder. @@ -426,6 +433,24 @@ func (h *Holder) setFileLimit() { func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) } +func (h *Holder) loadLocalID() error { + idPath := path.Join(h.Path, "ID") + localID := "" + localIDBytes, err := ioutil.ReadFile(idPath) + if err == nil { + localID = strings.TrimSpace(string(localIDBytes)) + } else { + u := uuid.NewV4() + localID = u.String() + err = ioutil.WriteFile(idPath, []byte(localID), 0600) + if err != nil { + return err + } + } + h.LocalID = localID + return nil +} + // HolderSyncer is an active anti-entropy tool that compares the local holder // with a remote holder based on block checksums and resolves differences. type HolderSyncer struct { diff --git a/server.go b/server.go index 68e9c6a74..a285d1cca 100644 --- a/server.go +++ b/server.go @@ -73,6 +73,7 @@ type Server struct { URI *URI Cluster *Cluster diagnostics *diagnostics.Diagnostics + ClusterID string // Background monitoring intervals. AntiEntropyInterval time.Duration @@ -206,6 +207,15 @@ func (s *Server) Open() error { } }() + // load local ID + if err := s.Holder.loadLocalID(); err != nil { + s.Logger().Println(err) + } + + if err := s.loadClusterID(); err != nil { + s.Logger().Println(err) + } + // Start background monitoring. s.wg.Add(4) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() @@ -569,7 +579,8 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) - // TODO: unique cluster ID + s.diagnostics.Set("LocalID", s.Holder.LocalID) + s.diagnostics.Set("ClusterID", s.ClusterID) // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { @@ -664,6 +675,25 @@ func (s *Server) createDefaultClient(remoteClient *http.Client) { s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient) } +func (s *Server) loadClusterID() error { + // If this is the first node in the cluster, set the ClusterID to its ID + node0URI, err := s.Cluster.Nodes[0].URI() + if err == nil { + if s.URI.Equals(node0URI) { + s.ClusterID = s.Holder.LocalID + return nil + } + } else { + return err + } + if clusterID, err := s.defaultClient.NodeID(node0URI); err == nil { + s.ClusterID = clusterID + return nil + } else { + return err + } +} + // CountOpenFiles on operating systems that support lsof. func CountOpenFiles() (int, error) { switch runtime.GOOS {