scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1200 from jaffee/1170-global-defaults

Browse source 
remove global defaults from config.go
This commit is contained in:
Matthew Jaffee 2018-04-15 17:59:21 -05:00 • committed by GitHub
commit d34f3354bb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 121 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
cluster.go
View file

@ -275,11 +275,10 @@ type Cluster struct {
// NewCluster returns a new instance of Cluster with defaults.
func NewCluster() *Cluster {
return &Cluster{
Hasher: &jmphasher{},
PartitionN: DefaultPartitionN,
ReplicaN: DefaultReplicaN,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
EventReceiver: NopEventReceiver,
Hasher: &jmphasher{},
PartitionN: DefaultPartitionN,
ReplicaN: DefaultReplicaN,
EventReceiver: NopEventReceiver,
joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
jobs: make(map[int64]*ResizeJob),

215
config.go
View file

@ -25,94 +25,6 @@ const (
ClusterGossip = "gossip"
)
const (
// DefaultDataDir is the default data directory.
DefaultDataDir = "~/.pilosa"
// DefaultHost is the default hostname to use.
DefaultHost = "localhost"
// DefaultPort is the default port to use with the hostname.
DefaultPort = "10101"
// DefaultClusterDisabled sets the node intercommunication method.
DefaultClusterDisabled = false
// DefaultMetrics sets the internal metrics to no-op.
DefaultMetrics = "nop"
// DefaultMaxWritesPerRequest is the default number of writes per request.
DefaultMaxWritesPerRequest = 5000
// Gossip config based on memberlist.Config.
// Port indicates the port to which pilosa should bind for internal state sharing.
DefaultGossipPort = "14000"
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
DefaultGossipStreamTimeout = 10 * time.Second
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
DefaultGossipSuspicionMult = 4
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
DefaultGossipPushPullInterval = 30 * time.Second
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
DefaultGossipProbeInterval = 1 * time.Second
DefaultGossipProbeTimeout = 500 * time.Millisecond
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
DefaultGossipInterval = 200 * time.Millisecond
DefaultGossipNodes = 3
DefaultGossipToTheDeadTime = 30 * time.Second
DefaultMetricPollInterval = 0 * time.Minute
)
// TLSConfig contains TLS configuration
type TLSConfig struct {
// CertificatePath contains the path to the certificate (.crt or .pem file)
@ -125,20 +37,28 @@ type TLSConfig struct {
// Config represents the configuration for the command.
type Config struct {
// DataDir is the directory where Pilosa stores both indexed data and
// running state such as cluster topology information.
DataDir string `toml:"data-dir"`
Bind string `toml:"bind"`
// Bind is the host:port on which Pilosa will listen.
Bind string `toml:"bind"`
// Limits the number of mutating commands that can be in a single request to
// the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs.
// MaxWritesPerRequest limits the number of mutating commands that can be in
// a single request to the server. This includes SetBit, ClearBit,
// SetRowAttrs & SetColumnAttrs.
MaxWritesPerRequest int `toml:"max-writes-per-request"`
// LogPath configures where Pilosa will write logs.
LogPath string `toml:"log-path"`
Verbose bool `toml:"verbose"`
// Verbose toggles verbose logging which can be useful for debugging.
Verbose bool `toml:"verbose"`
// TLS
TLS TLSConfig
Cluster struct {
// Disabled controls whether clustering functionality is enabled.
Disabled bool `toml:"disabled"`
Coordinator bool `toml:"coordinator"`
ReplicaN int `toml:"replicas"`
@ -146,18 +66,69 @@ type Config struct {
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
// Gossip config is based around memberlist.Config.
Gossip struct {
Port string `toml:"port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
StreamTimeout Duration `toml:"stream-timeout"`
SuspicionMult int `toml:"suspicion-mult"`
// Port indicates the port to which pilosa should bind for internal state sharing.
Port string `toml:"port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
StreamTimeout Duration `toml:"stream-timeout"`
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
SuspicionMult int `toml:"suspicion-mult"`
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
PushPullInterval Duration `toml:"push-pull-interval"`
ProbeTimeout Duration `toml:"probe-timeout"`
ProbeInterval Duration `toml:"probe-interval"`
Nodes int `toml:"nodes"`
Interval Duration `toml:"interval"`
ToTheDeadTime Duration `toml:"to-the-dead-time"`
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
ProbeInterval Duration `toml:"probe-interval"`
ProbeTimeout Duration `toml:"probe-timeout"`
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
Interval Duration `toml:"interval"`
Nodes int `toml:"nodes"`
ToTheDeadTime Duration `toml:"to-the-dead-time"`
} `toml:"gossip"`
AntiEntropy struct {
@ -165,51 +136,55 @@ type Config struct {
} `toml:"anti-entropy"`
Metric struct {
Service string `toml:"service"`
// Service can be statsd, expvar, or none.
Service string `toml:"service"`
// Host tells the statsd client where to write.
Host string `toml:"host"`
PollInterval Duration `toml:"poll-interval"`
Diagnostics bool `toml:"diagnostics"`
// Diagnostics toggles sending some limited diagnostic information to
// Pilosa's developers.
Diagnostics bool `toml:"diagnostics"`
} `toml:"metric"`
}
// NewConfig returns an instance of Config with default options.
func NewConfig() *Config {
c := &Config{
DataDir: DefaultDataDir,
Bind: ":" + DefaultPort,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
DataDir: "~/.pilosa",
Bind: ":10101",
MaxWritesPerRequest: 5000,
// LogPath: "",
// Verbose: false,
TLS: TLSConfig{},
}
// Cluster config.
c.Cluster.Disabled = DefaultClusterDisabled
c.Cluster.Disabled = false
// c.Cluster.Coordinator = false
c.Cluster.ReplicaN = DefaultReplicaN
c.Cluster.Hosts = []string{}
c.Cluster.LongQueryTime = Duration(time.Minute)
// Gossip config.
// c.Gossip.Port = ""
c.Gossip.Port = "14000"
// c.Gossip.Seeds = []string{}
// c.Gossip.Key = ""
c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout)
c.Gossip.SuspicionMult = DefaultGossipSuspicionMult
c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval)
c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout)
c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval)
c.Gossip.Nodes = DefaultGossipNodes
c.Gossip.Interval = Duration(DefaultGossipInterval)
c.Gossip.ToTheDeadTime = Duration(DefaultGossipToTheDeadTime)
c.Gossip.StreamTimeout = Duration(10 * time.Second)
c.Gossip.SuspicionMult = 4
c.Gossip.PushPullInterval = Duration(30 * time.Second)
c.Gossip.ProbeInterval = Duration(1 * time.Second)
c.Gossip.ProbeTimeout = Duration(500 * time.Millisecond)
c.Gossip.Interval = Duration(200 * time.Millisecond)
c.Gossip.Nodes = 3
c.Gossip.ToTheDeadTime = Duration(30 * time.Second)
// AntiEntropy config.
c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval)
c.AntiEntropy.Interval = Duration(10 * time.Minute)
// Metric config.
c.Metric.Service = DefaultMetrics
c.Metric.Service = "none"
// c.Metric.Host = ""
c.Metric.PollInterval = Duration(DefaultMetricPollInterval)
c.Metric.PollInterval = Duration(0 * time.Minute)
c.Metric.Diagnostics = true
return c

2
config_test.go
View file

@ -25,7 +25,7 @@ import (
func Test_NewConfig(t *testing.T) {
c := pilosa.NewConfig()
if c.Cluster.Disabled != pilosa.DefaultClusterDisabled {
if c.Cluster.Disabled {
t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled)
}

3
ctl/generate_config_test.go
View file

@ -17,7 +17,6 @@ package ctl
import (
"bytes"
"context"
"github.com/pilosa/pilosa"
"io"
"os"
"strings"
@ -35,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) {
io.Copy(&buf, r)
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), pilosa.DefaultHost) {
} else if !strings.Contains(buf.String(), "localhost:10101") {
t.Fatalf("Unexpected config: %s", buf.String())
}
}

2
ctl/import_test.go
View file

@ -134,7 +134,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = pilosa.DefaultHost
cm.Host = "anyhost"
cm.Index = "i"
cm.Frame = "f"
file, err := ioutil.TempFile("", "import.csv")

10
docs/configuration.md
View file

@ -197,12 +197,12 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
```toml
[profile]
cpu = "/path/to/somewhere"
cpu = "/path/to/somewhere"
```
#### Profile CPU Time
* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set.
* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set.
* Flag: `--profile.cpu-time="30s"`
* Env: `PILOSA_PROFILE_CPU_TIME="30s"
* Config:
@ -213,7 +213,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
```
#### Metric Service
* Description: Which stats service to use. Choose from [statsd, expvar].
* Description: Which stats service to use. Choose from [statsd, expvar, none].
* Flag: `--metric.service=statsd`
* Env: `PILOSA_METRIC_SERVICE=statsd'
* Config:
@ -236,7 +236,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
#### Metric Poll Interval
* Description: Polling interval for runtime metrics.
* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected.
* Flag: `metric.poll-interval=”0m15s”`
* Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s`
* Config:
@ -248,7 +248,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
#### Metric Diagnostics
* Description: Enable diagnostic reporting. To disable diagnostics set to false.
* Description: Enable reporting of limited usage statistics to Pilosa developers. To disable, set to false.
* Flag: `metric.diagnostics`
* Env: `PILOSA_METRIC_DIAGNOSTICS`
* Config:

10
fragment.go
View file

@ -1832,15 +1832,19 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest.
total := len(set.ColumnIDs) + len(clear.ColumnIDs)
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(s.Cluster.MaxWritesPerRequest))))
maxWrites := s.Cluster.MaxWritesPerRequest
if maxWrites <= 0 {
maxWrites = 5000
}
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites))))
// Only sync the standard block.
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
count++
}
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
count++
}

5
server.go
View file

@ -37,8 +37,7 @@ import (
// Default server settings.
const (
DefaultAntiEntropyInterval = 10 * time.Minute
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
)
// Ensure Server implements interfaces.
@ -108,7 +107,7 @@ func NewServer() *Server {
NewAttrStore: NewNopAttrStore,
AntiEntropyInterval: DefaultAntiEntropyInterval,
AntiEntropyInterval: time.Duration(NewConfig().AntiEntropy.Interval),
MetricInterval: 0,
DiagnosticInterval: 0,

14
server/server.go
View file

@ -20,7 +20,6 @@
package server
import (
"errors"
"fmt"
"io"
"log"
@ -40,6 +39,7 @@ import (
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/statsd"
"github.com/pkg/errors"
)
func init() {
@ -255,13 +255,7 @@ func (m *Command) SetupNetworking() error {
return nil
}
// Set internal port (string).
gossipPortStr := pilosa.DefaultGossipPort
if m.Config.Gossip.Port != "" {
gossipPortStr = m.Config.Gossip.Port
}
gossipPort, err := strconv.Atoi(gossipPortStr)
gossipPort, err := strconv.Atoi(m.Config.Gossip.Port)
if err != nil {
return err
}
@ -318,7 +312,9 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
return pilosa.NewExpvarStatsClient(), nil
case "statsd":
return statsd.NewStatsClient(host)
default:
case "nop", "none":
return pilosa.NopStatsClient, nil
default:
return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].")
}
}