From 9954bdd340735ea5f7576b781e3889523d0ea106 Mon Sep 17 00:00:00 2001 From: Charlie Andrews Date: Mon, 9 Oct 2017 17:07:39 -0500 Subject: [PATCH 1/4] Add /recalculatecaches endpoint for triggering the recalculating of caches on demand --- handler.go | 6 ++++++ handler_test.go | 16 ++++++++++++++++ holder.go | 22 ++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/handler.go b/handler.go index d8463862a..b90d6be83 100644 --- a/handler.go +++ b/handler.go @@ -134,6 +134,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + router.HandleFunc("/recalculatecaches", handler.handleRecalculateCaches).Methods("POST") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: @@ -1962,6 +1963,11 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, return setBits, nil } +func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { + h.Holder.RecalculateCaches() + w.WriteHeader(http.StatusNoContent) +} + // GetTimeStamp retrieves unix timestamp from Input data. func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) { tmstamp, ok := data[timeField] diff --git a/handler_test.go b/handler_test.go index 6e8ebab88..a57c0da50 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1782,3 +1782,19 @@ func MustReadAll(r io.Reader) []byte { } return buf } + +func TestHandler_RecalculateCaches(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.NewHandler() + h.Holder = hldr.Holder + h.Cluster = test.NewCluster(1) + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculatecaches", nil)) + if w.Code != http.StatusNoContent { + t.Fatalf("unexpected status code: %d", w.Code) + } + +} diff --git a/holder.go b/holder.go index 0cf5f6901..31aee1c21 100644 --- a/holder.go +++ b/holder.go @@ -361,6 +361,28 @@ func (h *Holder) flushCaches() { } } +// RecalculateCaches calls RecalculateCache on every fragment of every frame of +// every index. This is probably not practical to call in real-world workloads, +// but makes writing integration tests much eaiser, since one doesn't have to +// wait 10 seconds after setting bits to get expected response. +func (h *Holder) RecalculateCaches() { + for _, index := range h.Indexes() { + for _, frame := range index.Frames() { + for _, view := range frame.Views() { + for _, fragment := range view.Fragments() { + select { + case <-h.closing: + return + default: + } + + fragment.RecalculateCache() + } + } + } + } +} + // setFileLimit attempts to set the open file limit to the FileLimit constant defined above. func (h *Holder) setFileLimit() { oldLimit := &syscall.Rlimit{} From b6509a3ad71eea83dcdb74ef1cb32dd1c7f6dd56 Mon Sep 17 00:00:00 2001 From: Charlie Andrews Date: Wed, 11 Oct 2017 09:36:32 -0500 Subject: [PATCH 2/4] Add RecalculateCaches to every level in hierarchy This to decouple the implementation of recalculating caches from the structure of the hierarchy: holder > index > frame > view > fragment. See [Law of Demeter](https://en.wikipedia.org/wiki/Law_of_Demeter). --- frame.go | 7 +++++++ holder.go | 22 +++++----------------- index.go | 7 +++++++ view.go | 7 +++++++ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/frame.go b/frame.go index 141772a93..21f160a64 100644 --- a/frame.go +++ b/frame.go @@ -543,6 +543,13 @@ func (f *Frame) Views() []*View { return other } +// RecalculateCaches recalculates caches on every view in the frame. +func (f *Frame) RecalculateCaches() { + for _, view := range f.Views() { + view.RecalculateCaches() + } +} + // CreateViewIfNotExists returns the named view, creating it if necessary. func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { // Don't create inverse views if they are not enabled. diff --git a/holder.go b/holder.go index 31aee1c21..0b091045f 100644 --- a/holder.go +++ b/holder.go @@ -361,25 +361,13 @@ func (h *Holder) flushCaches() { } } -// RecalculateCaches calls RecalculateCache on every fragment of every frame of -// every index. This is probably not practical to call in real-world workloads, -// but makes writing integration tests much eaiser, since one doesn't have to -// wait 10 seconds after setting bits to get expected response. +// RecalculateCaches recalculates caches on every index in the holder. This is +// probably not practical to call in real-world workloads, but makes writing +// integration tests much eaiser, since one doesn't have to wait 10 seconds +// after setting bits to get expected response. func (h *Holder) RecalculateCaches() { for _, index := range h.Indexes() { - for _, frame := range index.Frames() { - for _, view := range frame.Views() { - for _, fragment := range view.Fragments() { - select { - case <-h.closing: - return - default: - } - - fragment.RecalculateCache() - } - } - } + index.RecalculateCaches() } } diff --git a/index.go b/index.go index 6e38b19aa..3e6cb2518 100644 --- a/index.go +++ b/index.go @@ -392,6 +392,13 @@ func (i *Index) Frames() []*Frame { return a } +// RecalculateCaches recalculates caches on every frame in the index. +func (i *Index) RecalculateCaches() { + for _, frame := range i.Frames() { + frame.RecalculateCaches() + } +} + // CreateFrame creates a frame. func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { i.mu.Lock() diff --git a/view.go b/view.go index 4ee863c8b..5da97d644 100644 --- a/view.go +++ b/view.go @@ -213,6 +213,13 @@ func (v *View) Fragments() []*Fragment { return other } +// RecalculateCaches recalculates the cache on every fragment in the view. +func (v *View) RecalculateCaches() { + for _, fragment := range v.Fragments() { + fragment.RecalculateCache() + } +} + // CreateFragmentIfNotExists returns a fragment in the view by slice. func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { v.mu.Lock() From 5418dc152eccc53af2af9e13ce3928bf4e9eeb3b Mon Sep 17 00:00:00 2001 From: Charlie Andrews Date: Wed, 11 Oct 2017 09:45:20 -0500 Subject: [PATCH 3/4] Add docs for recalculate caches --- docs/api-reference.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/api-reference.md b/docs/api-reference.md index df1bfd82a..887bd5e5f 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -412,3 +412,14 @@ Response: {"version":"v0.6.0"} ``` +### Recalculate Caches + +`POST /recalculatecaches` + +Recalculates the caches on demand. The cache is recalculated every 10 +secondes by default. This endpoint can be used to recalculate the cache +before the 10 second interval. This should probably only be used in +integration tests and not in a typical production workflow. + +Response: `204 No Content` + From a7ec49c6adcb034c68e83eeff80341c6f6895ef5 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 11 Oct 2017 10:56:30 -0500 Subject: [PATCH 4/4] Adjust the documentation. Rename endpoint to `recalculate-caches` --- docs/api-reference.md | 8 +++++--- handler.go | 2 +- handler_test.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 887bd5e5f..5129f4061 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -414,12 +414,14 @@ Response: ### Recalculate Caches -`POST /recalculatecaches` +`POST /recalculate-caches` Recalculates the caches on demand. The cache is recalculated every 10 -secondes by default. This endpoint can be used to recalculate the cache +seconds by default. This endpoint can be used to recalculate the cache before the 10 second interval. This should probably only be used in -integration tests and not in a typical production workflow. +integration tests and not in a typical production workflow. Note that +in a multi-node cluster, the cache is only recalculated on the node +that receives the request. Response: `204 No Content` diff --git a/handler.go b/handler.go index b90d6be83..bae161515 100644 --- a/handler.go +++ b/handler.go @@ -134,7 +134,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - router.HandleFunc("/recalculatecaches", handler.handleRecalculateCaches).Methods("POST") + router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: diff --git a/handler_test.go b/handler_test.go index a57c0da50..1524a30b8 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1792,7 +1792,7 @@ func TestHandler_RecalculateCaches(t *testing.T) { h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculatecaches", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) if w.Code != http.StatusNoContent { t.Fatalf("unexpected status code: %d", w.Code) }