diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index a575c45d2..5addecc66 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -68,7 +68,7 @@ func (m *Main) Run(args ...string) error { // Generate an ID if one is not specified in the config. id := config.ID if id == nil { - *id = pilosa.RandomUUID() + *id = pilosa.NewGUID() } // Set up profiling. diff --git a/core/batch.go b/core/batch.go index a140607a8..647d0e59a 100644 --- a/core/batch.go +++ b/core/batch.go @@ -58,7 +58,7 @@ func (b *Batcher) Batch(database_name, frame, compressed_bitmap string, bitmap_i fragment, err := database.GetFragmentForBitmap(oslice, &db.Bitmap{Id: bitmap_id, FrameType: frame, Filter: filter}) if err == nil { - id := pilosa.RandomUUID() + id := pilosa.NewGUID() batch := db.Message{Data: BatchRequest{Id: &id, Source: &b.ID, Fragment_id: fragment.GetId(), Bitmap_id: bitmap_id, Compressed_bitmap: compressed_bitmap}} dest_id := fragment.GetProcess().Id() b.Transport.Send(&batch, &dest_id) diff --git a/core/etcd.go b/core/etcd.go index 436bd95ae..d2be6c597 100644 --- a/core/etcd.go +++ b/core/etcd.go @@ -193,7 +193,7 @@ func (self *TopologyMapper) AllocateFragment(process_guid, db, frame string, sli //to create the node, just write off the items to etcd and the watch should spawn //be nice if something would notify perhaps queue //so i need db, frame, slice , fragment_id - fuid := pilosa.SUUID_to_Hex(pilosa.Id()) + fuid := pilosa.NewSUUID().String() fragment_key := fmt.Sprintf("%s/db/%s/frame/%s/slice/%d/fragment/%s/process", self.namespace, db, frame, slice_int, fuid) // need to check value to see how many we have left log.Warn("ALLOC:", process_guid, len(process_guid)) @@ -268,7 +268,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error { } } if len(bits) > 7 { - fragment_id = pilosa.Hex_to_SUUID(bits[7]) + fragment_id = pilosa.ParseSUUID(bits[7]) fragment = database.GetOrCreateFragment(frame, slice, fragment_id) } @@ -284,7 +284,7 @@ func (self *TopologyMapper) handlenode(node *etcd.Node) error { process = db.NewProcess(&process_uuid) fragment.SetProcess(process) - if pilosa.Equal(&self.ID, &process_uuid) { + if self.ID.Equals(&process_uuid) { self.Index.AddFragment(bits[1], bits[3], slice_int, fragment_id) } diff --git a/core/ping.go b/core/ping.go index 9c35d015c..0d281580f 100644 --- a/core/ping.go +++ b/core/ping.go @@ -48,7 +48,7 @@ func NewPinger(id pilosa.GUID) *Pinger { } func (self *Pinger) Ping(process_id *pilosa.GUID) (*time.Duration, error) { - id := pilosa.RandomUUID() + id := pilosa.NewGUID() ping := db.Message{Data: PingRequest{Id: &id, Source: &self.ID}} start := time.Now() self.Transport.Send(&ping, process_id) diff --git a/core/remotebits.go b/core/remotebits.go index 4283befc7..47649b7e6 100644 --- a/core/remotebits.go +++ b/core/remotebits.go @@ -55,7 +55,7 @@ func (self *RemoteSetBit) Request() { self.requests = make([]remote_task, 0) source_process, _ := self.ProcessMap.GetProcess(&self.ID) for process, request := range self.cluster { - random_id := pilosa.RandomUUID() + random_id := pilosa.NewGUID() msg := new(db.Message) msg.Data = BitsRequest{ Bits: request, diff --git a/db/topology.go b/db/topology.go index a15b2a6eb..29e8667fd 100644 --- a/db/topology.go +++ b/db/topology.go @@ -323,7 +323,7 @@ func (self *FrameSliceIntersect) GetFragment(fragment_id pilosa.SUUID) (*Fragmen func (self *FrameSliceIntersect) AddFragment(fragment *Fragment) { self.fragments = append(self.fragments, fragment) - self.hashring.Add(pilosa.SUUID_to_Hex(fragment.id)) + self.hashring.Add(fragment.id.String()) } ///////// FRAGMENTS @@ -373,7 +373,7 @@ func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment log.Warn(err) return nil, err } - frag_id := pilosa.Hex_to_SUUID(frag_id_s) + frag_id := pilosa.ParseSUUID(frag_id_s) return fsi.GetFragment(frag_id) } @@ -391,7 +391,7 @@ func (d *Database) GetFragmentForFrameSlice(frame *Frame, slice *Slice) (*Fragme log.Warn(err) return nil, err } - frag_id := pilosa.Hex_to_SUUID(frag_id_s) + frag_id := pilosa.ParseSUUID(frag_id_s) return fsi.GetFragment(frag_id) } diff --git a/executor/executor.go b/executor/executor.go index ec7a48f90..4f16d6d10 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -444,7 +444,7 @@ func (self *Executor) GetQueryStepHandler(msg *db.Message) { bh, err := self.Index.Get(qs.Location.FragmentId, qs.Bitmap.Id) if err != nil { spew.Dump(err) - log.Error("GetQueryStepHandler1", pilosa.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id) + log.Error("GetQueryStepHandler1", qs.Location.FragmentId.String(), qs.Bitmap.Id) log.Error("GetQueryStepHandler2", err) } @@ -455,7 +455,7 @@ func (self *Executor) GetQueryStepHandler(msg *db.Message) { bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh) if err != nil { spew.Dump(err) - log.Error("GetQueryStepHandlerr3", pilosa.SUUID_to_Hex(qs.Location.FragmentId), qs.Bitmap.Id) + log.Error("GetQueryStepHandlerr3", qs.Location.FragmentId.String(), qs.Bitmap.Id) log.Error("GetQueryStepHandler4", err) } result = bm @@ -709,7 +709,7 @@ func newtask(p pilosa.GUID) *Task { result := new(Task) result.processid = p result.f = make(map[pilosa.SUUID]pilosa.FillArgs) - result.hold_id = pilosa.RandomUUID() + result.hold_id = pilosa.NewGUID() return result } diff --git a/fragment_container.go b/fragment_container.go index e51310d45..c09645593 100644 --- a/fragment_container.go +++ b/fragment_container.go @@ -173,7 +173,7 @@ func (self *FragmentContainer) TopN(frag_id SUUID, bh BitmapHandle, n int, categ statsd.SendTimer("fragmant_container_TopN", result.exec_time.Nanoseconds()) return result.answer.([]Pair), nil } - return nil, errors.New(fmt.Sprintf("Fragment not found:%s", SUUID_to_Hex(frag_id))) + return nil, errors.New(fmt.Sprintf("Fragment not found:%s", frag_id.String())) } func (self *FragmentContainer) TopNAll(frag_id SUUID, n int, categories []uint64) ([]Pair, error) { @@ -185,7 +185,7 @@ func (self *FragmentContainer) TopNAll(frag_id SUUID, n int, categories []uint64 statsd.SendTimer("fragmant_container_TopNAll", result.exec_time.Nanoseconds()) return result.answer.([]Pair), nil } - return nil, errors.New(fmt.Sprintf("Fragment not found:%s", SUUID_to_Hex(frag_id))) + return nil, errors.New(fmt.Sprintf("Fragment not found:%s", frag_id.String())) } func (self *FragmentContainer) TopFillBatch(args []FillArgs) ([]Pair, error) { @@ -319,7 +319,7 @@ func (self *FragmentContainer) AddFragment(db string, frame string, slice int, i _, ok := self.fragments[id] if !ok { // dumpHandlesToLog() - log.Warn("ADD FRAGMENT", frame, db, slice, SUUID_to_Hex(id)) + log.Warn("ADD FRAGMENT", frame, db, slice, id.String()) f := NewFragment(id, db, slice, frame) loader := make(chan Command) self.fragments[id] = f diff --git a/id.go b/pilosa.go similarity index 57% rename from id.go rename to pilosa.go index e747d2186..a480b3946 100644 --- a/id.go +++ b/pilosa.go @@ -1,36 +1,27 @@ package pilosa import ( - "bytes" "encoding/binary" "encoding/hex" "fmt" "math/rand" - "os" "strings" "time" - log "github.com/cihub/seelog" "github.com/gocql/gocql" ) -var ( - counter = uint64(0) - Random *os.File -) +var counter = uint64(0) func init() { rand.Seed(time.Now().UTC().UnixNano()) - f, err := os.Open("/dev/urandom") - if err != nil { - log.Warn(err) - } - Random = f } +// SUUID represents a sequential UUID. type SUUID uint64 -func Id() SUUID { +// SUUID returns a new SUUID. +func NewSUUID() SUUID { millis := uint64(time.Now().UTC().UnixNano()) id := millis << (64 - 41) id |= uint64(rand.Intn(128)) << (64 - 41 - 13) @@ -39,26 +30,24 @@ func Id() SUUID { return SUUID(id) } -func SUUID_to_Hex(a SUUID) string { - buf := new(bytes.Buffer) - binary.Write(buf, binary.BigEndian, a) - return hex.EncodeToString(buf.Bytes()) +// String returns a string representation of id. +func (id SUUID) String() string { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], uint64(id)) + return hex.EncodeToString(buf[:]) } -func Hex_to_SUUID(str string) SUUID { - l := len(str) - var m string - if l < 16 { - m = strings.Repeat("0", 16-l) + str - } else { - m = str +// ParseSUUID parses s into an SUUID. +func ParseSUUID(s string) SUUID { + if n := len(s); n < 16 { + s = strings.Repeat("0", 16-n) + s } - b, _ := hex.DecodeString(m) - num := binary.BigEndian.Uint64(b) - return SUUID(num) + b, _ := hex.DecodeString(s) + return SUUID(binary.BigEndian.Uint64(b)) } +// GUID represents a globally unique identifier. type GUID [16]byte // UnmarshalText parses a text value into a GUID. @@ -73,11 +62,11 @@ func (id *GUID) UnmarshalText(text []byte) error { return nil } -func (self GUID) String() string { +func (id GUID) String() string { var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34} const hexString = "0123456789abcdef" r := make([]byte, 36) - for i, b := range self { + for i, b := range id { r[offsets[i]] = hexString[b>>4] r[offsets[i]+1] = hexString[b&0xF] } @@ -89,26 +78,29 @@ func (self GUID) String() string { } -func Equal(a, b *GUID) bool { - for i, v := range a { - if v != b[i] { +// Equals returns true if id equals other. +func (id *GUID) Equals(other *GUID) bool { + for i, v := range id { + if v != other[i] { return false } } return true } -func RandomUUID() GUID { +// NewGUID returns a random GUID. +func NewGUID() GUID { uid, _ := gocql.RandomUUID() - var r GUID - copy(r[:], uid[:]) - return r + var id GUID + copy(id[:], uid[:]) + return id } -func ParseGUID(input string) (GUID, error) { +// ParseGUID parses s into a GUID. +func ParseGUID(s string) (GUID, error) { var u GUID j := 0 - for _, r := range input { + for _, r := range s { switch { case r == '-' && j&1 == 0: continue @@ -119,12 +111,12 @@ func ParseGUID(input string) (GUID, error) { case r >= 'A' && r <= 'F' && j < 32: u[j/2] |= byte(r-'A'+10) << uint(4-j&1*4) default: - return GUID{}, fmt.Errorf("invalid GUID %q", input) + return GUID{}, fmt.Errorf("invalid GUID %q", s) } j += 1 } if j != 32 { - return GUID{}, fmt.Errorf("invalid GUID %q", input) + return GUID{}, fmt.Errorf("invalid GUID %q", s) } return u, nil } diff --git a/id_test.go b/pilosa_test.go similarity index 50% rename from id_test.go rename to pilosa_test.go index f4c047ca3..f1cb30bde 100644 --- a/id_test.go +++ b/pilosa_test.go @@ -8,44 +8,44 @@ import ( ) // Ensure id can be parsed from string. -func TestId_Small(t *testing.T) { - if v := pilosa.Hex_to_SUUID("1"); v != 1 { +func TestSUUID_Small(t *testing.T) { + if v := pilosa.ParseSUUID("1"); v != 1 { t.Fatalf("unexpected SUUID: %v", v) } } // Ensure generated IDs are unique. -func TestId_Unique(t *testing.T) { - a, b := pilosa.Id(), pilosa.Id() +func TestSUUID_Unique(t *testing.T) { + a, b := pilosa.NewSUUID(), pilosa.NewSUUID() if a == b { t.Fatalf("ids should be unique: %v != %v", a, b) } } // Ensure ids can be converted to and from hex. -func TestId_Hex(t *testing.T) { - a := pilosa.Id() - b := pilosa.Hex_to_SUUID(pilosa.SUUID_to_Hex(a)) +func TestSUUID_Hex(t *testing.T) { + a := pilosa.NewSUUID() + b := pilosa.ParseSUUID(a.String()) if a != b { t.Fatalf("ids not equal: %v != %v", a, b) } } // Ensure ids can be generated in sequence. -func TestId_Multiple(t *testing.T) { +func TestSUUID_Multiple(t *testing.T) { for i := 0; i < 10; i++ { - println(pilosa.SUUID_to_Hex(pilosa.Id())) + println(pilosa.NewSUUID().String()) } } -// Ensure a random UUID can be converted to a string. -func TestRandomUUID_String(t *testing.T) { - fmt.Println(pilosa.RandomUUID().String()) +// Ensure a random GUID can be converted to a string. +func TestGUID_String(t *testing.T) { + fmt.Println(pilosa.NewGUID().String()) } -func BenchmarkId(b *testing.B) { +func BenchmarkSUUID(b *testing.B) { // run the Fib function b.N times for n := 0; n < b.N; n++ { - pilosa.Id() + pilosa.NewSUUID() } } diff --git a/query/parser.go b/query/parser.go index 63a68487f..d15fec54a 100644 --- a/query/parser.go +++ b/query/parser.go @@ -52,7 +52,7 @@ func (self *QueryParser) Parse() (query *Query, err error) { }() var token *Token - id := pilosa.RandomUUID() + id := pilosa.NewGUID() query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})} token = self.next() diff --git a/query/planner.go b/query/planner.go index 0f7d962de..bc7de4391 100644 --- a/query/planner.go +++ b/query/planner.go @@ -75,7 +75,7 @@ func (self *BaseQueryStep) GetLocation() *db.Location { func (self *BaseQueryStep) LocIsDest() bool { log.Trace("BaseQueryStep.LocIsDest") - if pilosa.Equal(self.Location.ProcessId, self.Destination.ProcessId) && + if self.Location.ProcessId.Equals(self.Destination.ProcessId) && self.Location.FragmentId == self.Destination.FragmentId { log.Trace("BaseQueryStep.LocIsDest Return true") return true @@ -616,7 +616,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Lo } step := CatQueryStep{&BaseQueryStep{id, "cat", loc, location}, inputs, cat.N} for index, subq := range cat.subqueries { - sub_id := pilosa.RandomUUID() + sub_id := pilosa.NewGUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -634,7 +634,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Lo } step := StashQueryStep{&BaseQueryStep{id, "stash", loc, location}, inputs, stash.N} for index, subq := range stash.subqueries { - sub_id := pilosa.RandomUUID() + sub_id := pilosa.NewGUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -651,7 +651,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Lo } step := UnionQueryStep{&BaseQueryStep{id, "union", loc, location}, inputs} for index, subq := range union.subqueries { - sub_id := pilosa.RandomUUID() + sub_id := pilosa.NewGUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -668,7 +668,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Lo } step := IntersectQueryStep{&BaseQueryStep{id, "intersect", loc, location}, inputs} for index, subq := range intersect.subqueries { - sub_id := pilosa.RandomUUID() + sub_id := pilosa.NewGUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -685,7 +685,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Lo } step := DifferenceQueryStep{&BaseQueryStep{id, "difference", loc, location}, inputs} for index, subq := range difference.subqueries { - sub_id := pilosa.RandomUUID() + sub_id := pilosa.NewGUID() step.Inputs[index] = &sub_id subq_steps, err := self.flatten(subq, &sub_id, loc) if err != nil { @@ -727,7 +727,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Lo plan := QueryPlan{step} return &plan, nil } else if cnt, ok := qt.(*CountQueryTree); ok { - sub_id := pilosa.RandomUUID() + sub_id := pilosa.NewGUID() loc, err := cnt.getLocation(self.Database) if err != nil { return nil, err @@ -740,7 +740,7 @@ func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Lo plan = append(plan, *subq_steps...) plan = append(plan, step) } else if topn, ok := qt.(*TopNQueryTree); ok { - sub_id := pilosa.RandomUUID() + sub_id := pilosa.NewGUID() loc, err := topn.getLocation(self.Database) if err != nil { return nil, err diff --git a/query/query.go b/query/query.go index f012d4d27..b50c904e8 100644 --- a/query/query.go +++ b/query/query.go @@ -68,7 +68,7 @@ func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.L func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) (*QueryPlan, error) { log.Trace("QueryPlanForQuery", database, query, destination) query_planner := QueryPlanner{Database: database, Query: query} - id := pilosa.RandomUUID() + id := pilosa.NewGUID() query_plan, err := query_planner.Plan(query, &id, destination) if err != nil { return nil, err diff --git a/storage/leveldb/leveldb.go b/storage/leveldb/leveldb.go index 4af2920cd..5a78afca6 100644 --- a/storage/leveldb/leveldb.go +++ b/storage/leveldb/leveldb.go @@ -40,7 +40,7 @@ func NewStorage(opt pilosa.StorageOptions) *Storage { opt.DB, strconv.Itoa(opt.Slice), opt.Frame, - pilosa.SUUID_to_Hex(opt.FragmentID), + opt.FragmentID.String(), ) return &Storage{path: path}