From 55a9952b9278226698d18579fb28681773b74565 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Mon, 25 Jan 2021 16:19:12 -0600 Subject: [PATCH 1/5] mmap limit comparison error message --- server/server.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/server.go b/server/server.go index 291c986e1..92de59e3e 100644 --- a/server/server.go +++ b/server/server.go @@ -23,11 +23,13 @@ import ( "bytes" "context" "crypto/tls" + "encoding/binary" "io" "log" "math/rand" "net" "os" + "os/exec" "os/signal" "runtime" "strconv" @@ -152,6 +154,12 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "setting up server") } + cmd, err := exec.Command("sysctl vm.max_map_count").Output() + data := binary.BigEndian.Uint64(cmd) + if m.Config.MaxMapCount >= data { + m.logger.Printf("grpc server error: Config max map limit (%v) is greater than current system limits (%v)", m.Config.MaxMapCount, data) + } + // Set up networking (i.e. gossip) err = m.setupNetworking() if err != nil { From 3809fe673440df5c5bbb590b26f19fcd59d2130c Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Tue, 26 Jan 2021 09:35:51 -0600 Subject: [PATCH 2/5] add error check --- server/server.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index 92de59e3e..8ab8402b9 100644 --- a/server/server.go +++ b/server/server.go @@ -24,6 +24,7 @@ import ( "context" "crypto/tls" "encoding/binary" + "fmt" "io" "log" "math/rand" @@ -154,10 +155,14 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "setting up server") } - cmd, err := exec.Command("sysctl vm.max_map_count").Output() + cmd, err := exec.Command("sysctl", "vm.max_map_count").Output() data := binary.BigEndian.Uint64(cmd) - if m.Config.MaxMapCount >= data { - m.logger.Printf("grpc server error: Config max map limit (%v) is greater than current system limits (%v)", m.Config.MaxMapCount, data) + if err != nil { + fmt.Println("Error: ", err) + } else { + if m.Config.MaxMapCount >= data { + m.logger.Printf("grpc server error: Config max map limit (%v) is greater than current system limits (%v)", m.Config.MaxMapCount, data) + } } // Set up networking (i.e. gossip) From 3982a8e970fd07edd60dc756b57fa74a50382473 Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Tue, 26 Jan 2021 11:26:36 -0600 Subject: [PATCH 3/5] format messages and change mmap comparison logic --- server/server.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/server/server.go b/server/server.go index 8ab8402b9..ac06bc0fa 100644 --- a/server/server.go +++ b/server/server.go @@ -24,7 +24,6 @@ import ( "context" "crypto/tls" "encoding/binary" - "fmt" "io" "log" "math/rand" @@ -155,13 +154,13 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "setting up server") } - cmd, err := exec.Command("sysctl", "vm.max_map_count").Output() - data := binary.BigEndian.Uint64(cmd) + result, err := exec.Command("sysctl", "vm.max_map_count").Output() if err != nil { - fmt.Println("Error: ", err) + m.logger.Printf("Tried unsuccessfully to check system mmap limit: %v", err) } else { - if m.Config.MaxMapCount >= data { - m.logger.Printf("grpc server error: Config max map limit (%v) is greater than current system limits (%v)", m.Config.MaxMapCount, data) + sysMmapLimit := binary.BigEndian.Uint64(result) + if m.Config.MaxMapCount > sysMmapLimit { + m.logger.Printf("WARNING: Config max map limit (%v) is greater than current system limits (%v)", m.Config.MaxMapCount, sysMmapLimit) } } From 48ac989e6a5084e262f836649c7fd2e1de82091f Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 26 Jan 2021 15:41:47 -0600 Subject: [PATCH 4/5] Return zero-bit row (with Index/Field) instead of nil in executeDistinctShardSet --- executor.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index b1a0ae1eb..9fa829aac 100644 --- a/executor.go +++ b/executor.go @@ -143,7 +143,6 @@ func (e *executor) Close() error { // Execute executes a PQL query. func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute") span.LogKV("pql", q.String()) defer span.Finish() @@ -1505,7 +1504,18 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0) switch errors.Cause(err) { case ViewNotFound, FragmentNotFound: - return nil, nil + // It may seem reasonable to return `nil` here in the case where the + // fragment for this shard does not exist. The problem with doing that + // is that if this operation is being performed on a remote node, then + // this result is going to get serialized as a QueryResponse and sent + // back to the original, non-remote node. When this happens, the + // encodeRow/decodeRow logic replaces `nil` with an empty Row. An empty + // Row will cause problems during the union step of the reduce phase if + // it is the "left" side of the union, because then the resulting Row + // after the union will have blank Index and Field values. Here, we + // ensure that we send a non-nil Row with valid Index and Field values + // so that the union step doesn't cause problems. + return &Row{Index: index, Field: fieldName}, nil case nil: default: return nil, errors.Wrap(err, "getting fragment data") From 927db378b403923a138df1b0fdd4b0ac2b66317e Mon Sep 17 00:00:00 2001 From: Maxton Huff Date: Wed, 27 Jan 2021 09:52:13 -0600 Subject: [PATCH 5/5] add linux OS check and the way mmap limit is read --- lattice | 2 +- server/server.go | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lattice b/lattice index 36f453c1e..28c2313ec 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit 36f453c1ea3bf86c546a8ad4a88f2a926724d683 +Subproject commit 28c2313ecfcd7e083d42d4e409483e968b4c421b diff --git a/server/server.go b/server/server.go index ac06bc0fa..530e4102a 100644 --- a/server/server.go +++ b/server/server.go @@ -23,16 +23,16 @@ import ( "bytes" "context" "crypto/tls" - "encoding/binary" "io" + "io/ioutil" "log" "math/rand" "net" "os" - "os/exec" "os/signal" "runtime" "strconv" + "strings" "sync" "syscall" "time" @@ -154,13 +154,17 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "setting up server") } - result, err := exec.Command("sysctl", "vm.max_map_count").Output() - if err != nil { - m.logger.Printf("Tried unsuccessfully to check system mmap limit: %v", err) - } else { - sysMmapLimit := binary.BigEndian.Uint64(result) - if m.Config.MaxMapCount > sysMmapLimit { - m.logger.Printf("WARNING: Config max map limit (%v) is greater than current system limits (%v)", m.Config.MaxMapCount, sysMmapLimit) + if runtime.GOOS == "linux" { + result, err := ioutil.ReadFile("/proc/sys/vm/max_map_count") + if err != nil { + m.logger.Printf("Tried unsuccessfully to check system mmap limit: %v", err) + } else { + sysMmapLimit, err := strconv.ParseUint(strings.TrimSuffix(string(result), "\n"), 10, 64) + if err != nil { + m.logger.Printf("Tried unsuccessfully to check system mmap limit: %v", err) + } else if m.Config.MaxMapCount > sysMmapLimit { + m.logger.Printf("WARNING: Config max map limit (%v) is greater than current system limits (%v)", m.Config.MaxMapCount, sysMmapLimit) + } } }