[FB-1388] Add optional support for continuous profiling in datadog (#2037)

Enable ability to use DataDog continuous profiling

cmd/server.go

@@ -11,6 +11,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	jaegercfg "github.com/uber/jaeger-client-go/config"
+	"gopkg.in/DataDog/dd-trace-go.v1/profiler"
 )
 
 // Server is global so that tests can control and verify it.
@@ -59,6 +60,37 @@ on the configured port.`,
 			if err := Server.Start(); err != nil {
 				return errors.Wrap(err, "running server")
 			}
+			if Server.Config.DataDog.Enable {
+				opts := make([]profiler.ProfileType, 0)
+				if Server.Config.DataDog.CPUProfile {
+					opts = append(opts, profiler.CPUProfile)
+				}
+				if Server.Config.DataDog.HeapProfile {
+					opts = append(opts, profiler.HeapProfile)
+				}
+				if Server.Config.DataDog.BlockProfile {
+					opts = append(opts, profiler.BlockProfile)
+				}
+				if Server.Config.DataDog.GoroutineProfile {
+					opts = append(opts, profiler.GoroutineProfile)
+				}
+				if Server.Config.DataDog.MutexProfile {
+					opts = append(opts, profiler.MutexProfile)
+				}
+				err := profiler.Start(
+					profiler.WithService(Server.Config.DataDog.Service),
+					profiler.WithEnv(Server.Config.DataDog.Env),
+					profiler.WithVersion(Server.Config.DataDog.Version),
+					profiler.WithTags(Server.Config.DataDog.Tags),
+					profiler.WithProfileTypes(
+						opts...,
+					),
+				)
+				if err != nil {
+					return errors.Wrap(err, "starting datadog")
+				}
+				defer profiler.Stop()
+			}
 
 			if Server.Config.Tracing.SamplerType != "off" {
 				// Initialize tracing in the command since it is global.

ctl/server.go

@@ -109,4 +109,14 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.StringVar(&srv.Config.Auth.PermissionsFile, "auth.permissions", srv.Config.Auth.PermissionsFile, "Permissions' file with group authorization.")
 	flags.StringVar(&srv.Config.Auth.QueryLogPath, "auth.query-log-path", srv.Config.Auth.QueryLogPath, "Path to log user queries")
 
+	flags.BoolVar(&srv.Config.DataDog.Enable, "datadog.enable", false, "enable continuous profiling with DataDog cloud service, Note you must have DataDog agent installed")
+	flags.StringVar(&srv.Config.DataDog.Service, "datadog.service", "default-service", "The Datadog service name, for example my-web-app")
+	flags.StringVar(&srv.Config.DataDog.Env, "datadog.env", "default-env", "The Datadog environment name, for example, production")
+	flags.StringVar(&srv.Config.DataDog.Version, "datadog.version", "default-version", "The version of your application")
+	flags.StringVar(&srv.Config.DataDog.Tags, "datadog.tags", "molecula", "The tags to apply to an uploaded profile. Must be a list of in the format <KEY1>:<VALUE1>,<KEY2>:<VALUE2>")
+	flags.BoolVar(&srv.Config.DataDog.CPUProfile, "datadog.cpu-profile", true, "golang pprof cpu profile ")
+	flags.BoolVar(&srv.Config.DataDog.HeapProfile, "datadog.heap-profile", true, "golang pprof heap profile")
+	flags.BoolVar(&srv.Config.DataDog.MutexProfile, "datadog.mutex-profile", false, "golang pprof mutex profile")
+	flags.BoolVar(&srv.Config.DataDog.GoroutineProfile, "datadog.goroutine-profile", false, "golang pprof goroutine profile")
+	flags.BoolVar(&srv.Config.DataDog.BlockProfile, "datadog.block-profile", false, "golang pprof goroutine ")
 }

fragment.go

@@ -696,7 +696,6 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint64, value int
 func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint64, value int64, clear bool) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-
 	tx := txOrig
 	if NilInside(tx) {
 		tx = f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard})

go.mod

@@ -1,39 +1,44 @@
 module github.com/molecula/featurebase/v3
 
 require (
-	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
-	github.com/DataDog/datadog-go v2.2.0+incompatible
+	github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a
+	github.com/DataDog/datadog-go v4.8.3+incompatible
+	github.com/DataDog/datadog-go/v5 v5.1.0 // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/beevik/ntp v0.3.0
 	github.com/benbjohnson/immutable v0.3.0
 	github.com/buger/jsonparser v1.1.1
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash v1.1.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
 	github.com/felixge/fgprof v0.9.1
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-test/deep v1.0.7
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.2
-	github.com/google/go-cmp v0.5.5
+	github.com/google/go-cmp v0.5.7
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
 	github.com/gorilla/handlers v1.3.0
-	github.com/gorilla/mux v1.7.0
+	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/securecookie v1.1.1
-	github.com/hashicorp/go-retryablehttp v0.7.0
-	github.com/improbable-eng/grpc-web v0.13.0
-	github.com/lib/pq v1.8.0
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.1
+	github.com/improbable-eng/grpc-web v0.15.0
+	github.com/jonboulle/clockwork v0.3.0 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/lib/pq v1.10.5
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b
-	github.com/opentracing/opentracing-go v1.1.0
-	github.com/pelletier/go-toml v1.4.0
+	github.com/opentracing/opentracing-go v1.2.0
+	github.com/pelletier/go-toml v1.9.5
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/prom2json v1.3.0
 	github.com/rakyll/statik v0.1.7
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
-	github.com/rs/cors v1.7.0 // indirect
+	github.com/rs/cors v1.8.2 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil/v3 v3.21.9
 	github.com/spf13/cobra v1.1.3
@@ -54,8 +59,8 @@ require (
 	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
 	google.golang.org/grpc v1.38.0
+	gopkg.in/DataDog/dd-trace-go.v1 v1.38.1
 	gopkg.in/yaml.v2 v2.4.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0

server/config.go

@@ -223,6 +223,19 @@ type Config struct {
 		Rename bool `toml:"rename"`
 	} `toml:"future"`
 
+	DataDog struct {
+		Enable           bool   `toml:"enable"`
+		Service          string `toml:"service"`
+		Env              string `toml:"env"`
+		Version          string `toml:"version"`
+		Tags             string `toml:"tags"`
+		CPUProfile       bool   `toml:"cpu-profile"`
+		MutexProfile     bool   `toml:"mutex-profile"`
+		GoroutineProfile bool   `toml:"goroutine-profile"`
+		BlockProfile     bool   `toml:"block-profile"`
+		HeapProfile      bool   `toml:"heap-profile"`
+	} `toml:"datadog"`
+
 	Auth Auth
 }
 

util.go

@@ -117,7 +117,8 @@ func EtcdUnixSocket(tb testing.TB) string {
 	addr := fmt.Sprintf("fake:%d", clientPort)
 	tb.Cleanup(func() {
 		err := os.Remove(addr)
-		if err != nil {
+
+		if err != nil && !os.IsNotExist(err) { //not an error if the socket is not present
 			tb.Logf("could not remove '%s', %v", addr, err)
 		}
 	})