From 02c3c6d3e69ccc8ccdc12e0101c5d267ecba4f3d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 3 Mar 2017 13:03:57 -0600 Subject: [PATCH] use cobra/viper and move cmd/pilosa to server subcommand --- cmd/pilosa/main.go | 190 +---------------- cmd/root.go | 14 ++ cmd/server.go | 90 ++++++++ glide.lock | 50 ++++- glide.yaml | 2 + server/server.go | 197 ++++++++++++++++++ .../main_test.go => server/server_test.go | 10 +- 7 files changed, 360 insertions(+), 193 deletions(-) create mode 100644 cmd/root.go create mode 100644 cmd/server.go create mode 100644 server/server.go rename cmd/pilosa/main_test.go => server/server_test.go (99%) diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 4f7474996..255aef606 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -1,197 +1,15 @@ package main import ( - "errors" - "flag" "fmt" - "io" - "math/rand" "os" - "os/signal" - "path/filepath" - "runtime/pprof" - "strings" - "time" - "github.com/BurntSushi/toml" - "github.com/pilosa/pilosa" -) - -// Version and BuildTime hold the version/build time information passed in at compile time. -var ( - Version string - BuildTime string -) - -func init() { - if Version == "" { - Version = "v0.0.0" - } - if BuildTime == "" { - BuildTime = "not recorded" - } - - rand.Seed(time.Now().UTC().UnixNano()) -} - -const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" + "github.com/pilosa/pilosa/cmd" ) func main() { - m := NewMain() - m.Server.Handler.Version = Version - fmt.Fprintf(m.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime) - - // Parse command line arguments. - if err := m.ParseFlags(os.Args[1:]); err != nil { - fmt.Fprintln(m.Stderr, err) - os.Exit(2) - } - - // Start CPU profiling. - if m.CPUProfile != "" { - f, err := os.Create(m.CPUProfile) - if err != nil { - fmt.Fprintf(m.Stderr, "create cpu profile: %v", err) - os.Exit(1) - } - defer f.Close() - - fmt.Fprintln(m.Stderr, "Starting cpu profile") - pprof.StartCPUProfile(f) - time.AfterFunc(m.CPUTime, func() { - fmt.Fprintln(m.Stderr, "Stopping cpu profile") - pprof.StopCPUProfile() - f.Close() - }) - } - - // Execute the program. - if err := m.Run(); err != nil { - fmt.Fprintln(m.Stderr, err) - fmt.Fprintln(m.Stderr, "stopping profile") - os.Exit(1) - } - - // First SIGKILL causes server to shut down gracefully. - c := make(chan os.Signal, 2) - signal.Notify(c, os.Interrupt) - sig := <-c - fmt.Fprintf(m.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) - - // Second signal causes a hard shutdown. - go func() { <-c; os.Exit(1) }() - - if err := m.Close(); err != nil { - fmt.Fprintln(m.Stderr, err) - os.Exit(1) + if err := cmd.RootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(-1) } } - -// Main represents the main program execution. -type Main struct { - Server *pilosa.Server - - // Configuration options. - ConfigPath string - Config *pilosa.Config - - // Profiling options. - CPUProfile string - CPUTime time.Duration - - // Standard input/output - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewMain returns a new instance of Main. -func NewMain() *Main { - return &Main{ - Server: pilosa.NewServer(), - Config: pilosa.NewConfig(), - - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - } -} - -// Run executes the main program execution. -func (m *Main) Run(args ...string) error { - // Notify user of config file. - if m.ConfigPath != "" { - fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath) - } - - // Setup logging output. - m.Server.LogOutput = m.Stderr - - // Configure index. - fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir) - m.Server.Index.Path = m.Config.DataDir - m.Server.Index.Stats = pilosa.NewExpvarStatsClient() - - // Build cluster from config file. - m.Server.Host = m.Config.Host - m.Server.Cluster = m.Config.PilosaCluster() - - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - - // Initialize server. - if err := m.Server.Open(); err != nil { - return err - } - - fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) - - return nil -} - -// Close shuts down the server. -func (m *Main) Close() error { - return m.Server.Close() -} - -// ParseFlags parses command line flags from args. -func (m *Main) ParseFlags(args []string) error { - fs := flag.NewFlagSet("pilosa", flag.ContinueOnError) - fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile") - fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration") - fs.StringVar(&m.ConfigPath, "config", "", "config path") - fs.SetOutput(m.Stderr) - if err := fs.Parse(args); err != nil { - return err - } - - // Load config, if specified. - if m.ConfigPath != "" { - if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil { - return err - } - } - - // Use default data directory if one is not specified. - if m.Config.DataDir == "" { - m.Config.DataDir = DefaultDataDir - } - - // Expand home directory. - prefix := "~" + string(filepath.Separator) - if strings.HasPrefix(m.Config.DataDir, prefix) { - // u, err := user.Current() - HomeDir := os.Getenv("HOME") - /*if err != nil { - return err - } else*/if HomeDir == "" { - return errors.New("data directory not specified and no home dir available") - } - m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) - } - - return nil -} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 000000000..91d866a18 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,14 @@ +package cmd + +import "github.com/spf13/cobra" + +var RootCmd = &cobra.Command{ + Use: "pilosa", + Short: "pilosa - A Distributed In-memory Binary Bitmap Index", + Long: `Pilosa is a fast index to turbocharge your database. + +This binary contains Pilosa itself, as well as common +tools for administering pilosa, importing/exporting data, +backing up, and more. Complete documentation is available +at http://pilosa.com/docs`, // TODO - is documentation actually there? +} diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 000000000..cc8bbec49 --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "os/signal" + "runtime/pprof" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pilosa/pilosa/server" +) + +var serve = server.NewMain() + +var serveCmd = &cobra.Command{ + Use: "server", + Short: "server - run the pilosa server", + Long: `pilosa server runs Pilosa. + +It will load existing data from the configured +directory, and start listening client connections +on the configured port.`, + Run: func(cmd *cobra.Command, args []string) { + serve.Server.Handler.Version = server.Version + fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", server.Version, server.BuildTime) + + // Parse command line arguments. + if err := serve.ParseFlags(os.Args[1:]); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(2) + } + + // Start CPU profiling. + if serve.CPUProfile != "" { + f, err := os.Create(serve.CPUProfile) + if err != nil { + fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err) + os.Exit(1) + } + defer f.Close() + + fmt.Fprintln(serve.Stderr, "Starting cpu profile") + pprof.StartCPUProfile(f) + time.AfterFunc(serve.CPUTime, func() { + fmt.Fprintln(serve.Stderr, "Stopping cpu profile") + pprof.StopCPUProfile() + f.Close() + }) + } + + // Execute the program. + if err := serve.Run(); err != nil { + fmt.Fprintln(serve.Stderr, err) + fmt.Fprintln(serve.Stderr, "stopping profile") + os.Exit(1) + } + + // First SIGKILL causes server to shut down gracefully. + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt) + sig := <-c + fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) + + // Second signal causes a hard shutdown. + go func() { <-c; os.Exit(1) }() + + if err := serve.Close(); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(1) + } + + }, +} + +func init() { + serveCmd.Flags().StringVarP(&serve.ConfigPath, "config", "c", "", "Configuration file to read from") + serveCmd.Flags().StringVarP(&serve.CPUProfile, "cpuprofile", "", "", "Where to store CPU profile") + serveCmd.Flags().DurationVarP(&serve.CPUTime, "cputime", "", 30*time.Second, "CPU profile duration") + + err := viper.BindPFlags(serveCmd.Flags()) + if err != nil { + log.Fatalf("Error binding server flags: %v", err) + } + + RootCmd.AddCommand(serveCmd) +} diff --git a/glide.lock b/glide.lock index af94d5502..001257bf1 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 469de49a1736f34a11e9b0e490f7c1da1d8cb0219fed4bf3ad9e71344ca7f58a -updated: 2017-02-09T17:03:01.816613507-06:00 +hash: 7de62dbaf3cc1dc4959f4f6d8213102cb182b4dd7a87b3ac29260ad6bc1b0cef +updated: 2017-03-03T12:25:48.088390296-06:00 imports: - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda @@ -13,6 +13,8 @@ imports: version: 346938d642f2ec3594ed81d874461961cd0faa76 subpackages: - spew +- name: github.com/fsnotify/fsnotify + version: 7d7316ed6e1ed2de075aab8dfc76de5d158d66e1 - name: github.com/gogo/protobuf version: a9cd0c35b97daf74d0ebf3514c5254814b2703b4 subpackages: @@ -23,10 +25,54 @@ imports: - lru - name: github.com/golang/protobuf version: 888eb0692c857ec880338addf316bd662d5e630e + subpackages: + - proto +- name: github.com/hashicorp/hcl + version: 630949a3c5fa3c613328e1b8256052cbc2327c9b + subpackages: + - hcl/ast + - hcl/parser + - hcl/scanner + - hcl/strconv + - hcl/token + - json/parser + - json/scanner + - json/token +- name: github.com/inconshreveable/mousetrap + version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 +- name: github.com/magiconair/properties + version: b3b15ef068fd0b17ddf408a23669f20811d194d2 +- name: github.com/mitchellh/mapstructure + version: db1efb556f84b25a0a13a04aad883943538ad2e0 +- name: github.com/pelletier/go-buffruneio + version: c37440a7cf42ac63b919c752ca73a85067e05992 +- name: github.com/pelletier/go-toml + version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a - name: github.com/satori/go.uuid version: 879c5887cd475cd7864858769793b2ceb0d44feb +- name: github.com/spf13/afero + version: 9be650865eab0c12963d8753212f4f9c66cdcf12 + subpackages: + - mem +- name: github.com/spf13/cast + version: 4f1683a2242a92e62d6ff705a30e435cbf2b50a3 +- name: github.com/spf13/cobra + version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c +- name: github.com/spf13/jwalterweatherman + version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 +- name: github.com/spf13/pflag + version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 +- name: github.com/spf13/viper + version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4 - name: golang.org/x/sys version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: - unix +- name: golang.org/x/text + version: 5a42fa2464759cbb7ee0af9de00b54d69f09a29c + subpackages: + - transform + - unicode/norm +- name: gopkg.in/yaml.v2 + version: a3f3340b5840cee44f372bddb5880fcbc419b46a testImports: [] diff --git a/glide.yaml b/glide.yaml index 10c8509d0..1c4f35dae 100644 --- a/glide.yaml +++ b/glide.yaml @@ -27,3 +27,5 @@ import: - package: github.com/golang/protobuf - package: github.com/satori/go.uuid version: ^1.1.0 +- package: github.com/spf13/cobra +- package: github.com/spf13/viper diff --git a/server/server.go b/server/server.go new file mode 100644 index 000000000..bb151a37d --- /dev/null +++ b/server/server.go @@ -0,0 +1,197 @@ +package server + +import ( + "errors" + "flag" + "fmt" + "io" + "math/rand" + "os" + "os/signal" + "path/filepath" + "runtime/pprof" + "strings" + "time" + + "github.com/BurntSushi/toml" + "github.com/pilosa/pilosa" +) + +// Version and BuildTime hold the version/build time information passed in at compile time. +var ( + Version string + BuildTime string +) + +func init() { + if Version == "" { + Version = "v0.0.0" + } + if BuildTime == "" { + BuildTime = "not recorded" + } + + rand.Seed(time.Now().UTC().UnixNano()) +} + +const ( + // DefaultDataDir is the default data directory. + DefaultDataDir = "~/.pilosa" +) + +func mainz() { + serve := NewMain() + serve.Server.Handler.Version = Version + fmt.Fprintf(serve.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime) + + // Parse command line arguments. + if err := serve.ParseFlags(os.Args[1:]); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(2) + } + + // Start CPU profiling. + if serve.CPUProfile != "" { + f, err := os.Create(serve.CPUProfile) + if err != nil { + fmt.Fprintf(serve.Stderr, "create cpu profile: %v", err) + os.Exit(1) + } + defer f.Close() + + fmt.Fprintln(serve.Stderr, "Starting cpu profile") + pprof.StartCPUProfile(f) + time.AfterFunc(serve.CPUTime, func() { + fmt.Fprintln(serve.Stderr, "Stopping cpu profile") + pprof.StopCPUProfile() + f.Close() + }) + } + + // Execute the program. + if err := serve.Run(); err != nil { + fmt.Fprintln(serve.Stderr, err) + fmt.Fprintln(serve.Stderr, "stopping profile") + os.Exit(1) + } + + // First SIGKILL causes server to shut down gracefully. + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt) + sig := <-c + fmt.Fprintf(serve.Stderr, "Received %s; gracefully shutting down...\n", sig.String()) + + // Second signal causes a hard shutdown. + go func() { <-c; os.Exit(1) }() + + if err := serve.Close(); err != nil { + fmt.Fprintln(serve.Stderr, err) + os.Exit(1) + } +} + +// Main represents the main program execution. +type Main struct { + Server *pilosa.Server + + // Configuration options. + ConfigPath string + Config *pilosa.Config + + // Profiling options. + CPUProfile string + CPUTime time.Duration + + // Standard input/output + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewMain returns a new instance of Main. +func NewMain() *Main { + return &Main{ + Server: pilosa.NewServer(), + Config: pilosa.NewConfig(), + + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } +} + +// Run executes the main program execution. +func (m *Main) Run(args ...string) error { + // Notify user of config file. + if m.ConfigPath != "" { + fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath) + } + + // Setup logging output. + m.Server.LogOutput = m.Stderr + + // Configure index. + fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir) + m.Server.Index.Path = m.Config.DataDir + m.Server.Index.Stats = pilosa.NewExpvarStatsClient() + + // Build cluster from config file. + m.Server.Host = m.Config.Host + m.Server.Cluster = m.Config.PilosaCluster() + + // Set configuration options. + m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) + + // Initialize server. + if err := m.Server.Open(); err != nil { + return err + } + + fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) + + return nil +} + +// Close shuts down the server. +func (m *Main) Close() error { + return m.Server.Close() +} + +// ParseFlags parses command line flags from args. +func (m *Main) ParseFlags(args []string) error { + fs := flag.NewFlagSet("pilosa", flag.ContinueOnError) + fs.StringVar(&m.CPUProfile, "cpuprofile", "", "cpu profile") + fs.DurationVar(&m.CPUTime, "cputime", 30*time.Second, "cpu profile duration") + fs.StringVar(&m.ConfigPath, "config", "", "config path") + fs.SetOutput(m.Stderr) + if err := fs.Parse(args); err != nil { + return err + } + + // Load config, if specified. + if m.ConfigPath != "" { + if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil { + return err + } + } + + // Use default data directory if one is not specified. + if m.Config.DataDir == "" { + m.Config.DataDir = DefaultDataDir + } + + // Expand home directory. + prefix := "~" + string(filepath.Separator) + if strings.HasPrefix(m.Config.DataDir, prefix) { + // u, err := user.Current() + HomeDir := os.Getenv("HOME") + /*if err != nil { + return err + } else*/if HomeDir == "" { + return errors.New("data directory not specified and no home dir available") + } + m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) + } + + return nil +} diff --git a/cmd/pilosa/main_test.go b/server/server_test.go similarity index 99% rename from cmd/pilosa/main_test.go rename to server/server_test.go index 184f5c334..bb772ab45 100644 --- a/cmd/pilosa/main_test.go +++ b/server/server_test.go @@ -1,4 +1,4 @@ -package main_test +package server_test import ( "bytes" @@ -18,7 +18,7 @@ import ( "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" - main "github.com/pilosa/pilosa/cmd/pilosa" + "github.com/pilosa/pilosa/server" ) // Ensure program can process queries and maintain consistency. @@ -304,7 +304,7 @@ path = "/path/to/plugins" // Main represents a test wrapper for main.Main. type Main struct { - *main.Main + *server.Main Stdin bytes.Buffer Stdout bytes.Buffer @@ -318,7 +318,7 @@ func NewMain() *Main { panic(err) } - m := &Main{Main: main.NewMain()} + m := &Main{Main: server.NewMain()} m.Config.DataDir = path m.Config.Host = "localhost:0" m.Main.Stdin = &m.Stdin @@ -356,7 +356,7 @@ func (m *Main) Reopen() error { // Create new main with the same config. config := m.Config - m.Main = main.NewMain() + m.Main = server.NewMain() m.Config = config // Run new program.