scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-11 07:11:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

adjusted loading routines to match intel littleEndian

Browse source
This commit is contained in:
Todd Gruben 2015-06-30 16:24:20 +00:00
parent 58247dabc5
commit 98f13fecd8
9 changed files with 53 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
commands/pilosa-cruncher/cruncher.go
View file

@ -5,8 +5,11 @@ import (
log "github.com/cihub/seelog"
"github.com/mitchellh/panicwrap"
"os"
"pilosa/config"
"pilosa/core"
"pilosa/cruncher"
"pilosa/index"
"pilosa/util"
"runtime/pprof"
)
@ -39,6 +42,10 @@ func main() {
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
config.SetupConfig()
util.SetupUtil()
index.SetupCassandra()
cruncher := cruncher.NewCruncher()
cruncher.Run()
log.Warn("STOP")

2
config/config.go
View file

@ -19,7 +19,7 @@ type Config struct {
var config *Config
func init() {
func SetupConfig() {
config = NewConfig("")
}

2
core/batch.go
View file

@ -2,6 +2,7 @@ package core
import (
"encoding/gob"
log "github.com/cihub/seelog"
"pilosa/db"
"pilosa/util"
)
@ -32,6 +33,7 @@ func init() {
}
func (self *Service) Batch(database_name, frame, compressed_bitmap string, bitmap_id uint64, slice int, filter uint64) error {
log.Trace("Batch:", "db:", database_name, " frame:", frame, " slice:", slice, " cb:", compressed_bitmap, " bid:", bitmap_id, "f:", filter)
//determine the fragment_id from database/frame/slice
database := self.Cluster.GetOrCreateDatabase(database_name)
oslice := database.GetOrCreateSlice(slice)

10
core/load.go
View file

@ -7,6 +7,7 @@ import (
"compress/gzip"
"encoding/base64"
"encoding/binary"
"errors"
log "github.com/cihub/seelog"
"pilosa/index"
)
@ -19,6 +20,11 @@ func copy_raw(src [32]uint64) index.BlockArray {
return index.BlockArray{o}
}
func sendBitmap(service *Service, bitmap index.IBitmap, db string, frame string, bitmap_id, filter uint64, slice int, finish chan error) {
if slice < 0 {
log.Warn("Bad split", db, frame, slice, bitmap_id)
finish <- errors.New("BadSplit")
return
}
compressed_bitmap := bitmap.ToCompressString()
results := service.Batch(db, frame, compressed_bitmap, bitmap_id, slice, filter)
finish <- results
@ -36,7 +42,7 @@ func FromApiString(service *Service, db string, frame string, api_string string,
return "Bad"
}
var numChunks uint64
err = binary.Read(reader, binary.BigEndian, &numChunks)
err = binary.Read(reader, binary.LittleEndian, &numChunks)
if err != nil {
log.Warn(err)
return "Bad"
@ -52,7 +58,7 @@ func FromApiString(service *Service, db string, frame string, api_string string,
Key uint64
Block [32]uint64
}
binary.Read(reader, binary.BigEndian, &raw)
binary.Read(reader, binary.LittleEndian, &raw)
slice := raw.Key >> 5
if slice != last_slice {
if first {

20
index/bitmap.go
View file

@ -6,6 +6,7 @@ import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/binary"
"encoding/gob"
log "github.com/cihub/seelog"
"io/ioutil"
@ -413,6 +414,7 @@ type IBitmap interface {
ToBytes() []byte
FromBytes([]byte)
ToCompressString() string
ToRawCompressString() (string, int)
FromCompressString(string)
}
@ -457,6 +459,24 @@ func (b *Bitmap) Get(a *Chunk) *Chunk {
}
return nil
}
func (b *Bitmap) ToRawCompressString() (string, int) {
var bt bytes.Buffer
buf := gzip.NewWriter(&bt)
binary.Write(buf, binary.LittleEndian, uint64(b.nodes.Len()))
max_slice := 0
for i := b.nodes.Min(); !i.Limit(); i = i.Next() {
obj := i.Item().(*Chunk)
max_slice = int(obj.Key)
binary.Write(buf, binary.LittleEndian, obj.Key)
for _, v := range obj.Value.Block {
binary.Write(buf, binary.LittleEndian, v)
}
}
buf.Flush()
//buf.Close()
max_slice = max_slice / 32
return base64.StdEncoding.EncodeToString(bt.Bytes()), max_slice
}
func (b *Bitmap) ToBytes() []byte {
var (

12
index/storage_cass.go
View file

@ -27,7 +27,7 @@ type CassandraStorage struct {
var cluster *gocql.ClusterConfig
var session *gocql.Session
func init() {
func SetupCassandra() {
var err error
hosts := config.GetStringArrayDefault("cassandra_hosts", []string{"localhost"})
keyspace := config.GetStringDefault("cassandra_keyspace", "pilosa")
@ -44,12 +44,12 @@ func init() {
func BuildSchema() {
/*
"CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1"
create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true;
CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true;
"CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1"
create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true;
CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true;
CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) )
"
CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) )
"
*/
}

10
util/constants.go
View file

@ -30,10 +30,12 @@ func ByteToInt64(data []byte) int64 {
return value
}
func Uint64ToInt64(before uint64) int64 {
buf := Uint64ToByte(before)
return ByteToInt64(buf)
return int64(before)
//buf := Uint64ToByte(before)
//return ByteToInt64(buf)
}
func Int64ToUint64(before int64) uint64 {
buf := Int64ToByte(before)
return ByteToUint64(buf)
return uint64(before)
//buf := Int64ToByte(before)
//return ByteToUint64(buf)
}

3
util/statd.go
View file

@ -20,7 +20,8 @@ var (
end chan bool
)
func init() {
func SetupUtil() {
setup_storage()
timer = make(chan args, 32768)
count = make(chan string, 32768)
end = make(chan bool)

2
util/storage.go
View file

@ -9,7 +9,7 @@ import (
"github.com/kr/s3/s3util"
)
func init() {
func setup_storage() {
access_key := config.GetString("AWS_ACCESS_KEY_ID")
secret := config.GetString("AWS_SECRET_ACCESS_KEY")