mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
133 lines
4 KiB
Go
133 lines
4 KiB
Go
// Copyright 2020 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pilosa
|
|
|
|
// util.go: a place for generic, reusable utilities.
|
|
|
|
import (
|
|
"os"
|
|
"reflect"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/molecula/featurebase/v2/roaring"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
// LeftShifted16MaxContainerKey is 0xffffffffffff0000. It is similar
|
|
// to the roaring.maxContainerKey 0x0000ffffffffffff, but
|
|
// shifted 16 bits to the left so its domain is the full [0, 2^64) bit space.
|
|
// It is used to match the semantics of the roaring.OffsetRange() API.
|
|
// This is the maximum endx value for Tx.OffsetRange(), because the lowbits,
|
|
// as in the roaring.OffsetRange(), are not allowed to be set.
|
|
// It is used in Tx.RoaringBitamp() to obtain the full contents of a fragment
|
|
// from a call from tx.OffsetRange() by requesting [0, LeftShifted16MaxContainerKey)
|
|
// with an offset of 0.
|
|
const LeftShifted16MaxContainerKey = uint64(0xffffffffffff0000) // or math.MaxUint64 - (1<<16 - 1), or 18446744073709486080
|
|
|
|
// NilInside checks if the provided iface is nil or
|
|
// contains a nil pointer, slice, array, map, or channel.
|
|
func NilInside(iface interface{}) bool {
|
|
if iface == nil {
|
|
return true
|
|
}
|
|
switch reflect.TypeOf(iface).Kind() {
|
|
case reflect.Ptr, reflect.Slice, reflect.Array, reflect.Map, reflect.Chan:
|
|
return reflect.ValueOf(iface).IsNil()
|
|
}
|
|
return false
|
|
}
|
|
|
|
//////////////////////////////////
|
|
// helper utility functions
|
|
|
|
func highbits(v uint64) uint64 { return v >> 16 }
|
|
func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) }
|
|
|
|
// called by Holder.hasRoaringData()
|
|
func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) {
|
|
|
|
var info roaring.BitmapInfo
|
|
_ = info
|
|
var f *os.File
|
|
f, err = os.Open(path)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
var fi os.FileInfo
|
|
fi, err = f.Stat()
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// Memory map the file.
|
|
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "mmapping")
|
|
return
|
|
}
|
|
defer func() {
|
|
err = syscall.Munmap(data)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "roaringFragmentHasData: munmap failed")
|
|
}
|
|
err = f.Close()
|
|
if err != nil {
|
|
err = errors.Wrap(err, "roaringFragmentHasData f.Close() in defer")
|
|
}
|
|
}()
|
|
|
|
// Attach the mmap file to the bitmap.
|
|
var rbm *roaring.Bitmap
|
|
rbm, _, err = roaring.InspectBinary(data, true, &info)
|
|
if err != nil {
|
|
err = errors.Wrap(err, "inspecting")
|
|
return
|
|
}
|
|
|
|
if info.ContainerCount > 0 {
|
|
return true, nil
|
|
}
|
|
if info.Ops > 0 {
|
|
return true, nil
|
|
}
|
|
|
|
citer, found := rbm.Containers.Iterator(0)
|
|
_ = found
|
|
|
|
for citer.Next() {
|
|
return true, nil
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// GetLoopProgress returns the estimated remaining time to iterate through some
|
|
// items as well as the loop completion percentage with the following
|
|
// parameters:
|
|
// the start time, the current time, the iteration, and the number of items
|
|
func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) (remaining time.Duration, pctDone float64) {
|
|
itemsLeft := total - (iteration + 1)
|
|
avgItemTime := float64(now.Sub(start)) / float64(iteration+1)
|
|
pctDone = (float64(iteration+1) / float64(total)) * 100
|
|
return time.Duration(avgItemTime * float64(itemsLeft)), pctDone
|
|
}
|
|
|
|
// FormatTimestampNano returns the string representation of a timestamp given:
|
|
// an epoch value, base, and time unit
|
|
func FormatTimestampNano(value, base int64, timeUnit string) string {
|
|
return time.Unix(0, (value+base)*TimeUnitNanos(timeUnit)).UTC().Format(time.RFC3339Nano)
|
|
}
|