featurebase/util.go

// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

// util.go: a place for generic, reusable utilities.

import (
	"os"
	"reflect"
	"syscall"
	"time"

	"github.com/molecula/featurebase/v2/roaring"
	"github.com/pkg/errors"
)

// LeftShifted16MaxContainerKey is 0xffffffffffff0000. It is similar
// to the roaring.maxContainerKey  0x0000ffffffffffff, but
// shifted 16 bits to the left so its domain is the full [0, 2^64) bit space.
// It is used to match the semantics of the roaring.OffsetRange() API.
// This is the maximum endx value for Tx.OffsetRange(), because the lowbits,
// as in the roaring.OffsetRange(), are not allowed to be set.
// It is used in Tx.RoaringBitamp() to obtain the full contents of a fragment
// from a call from tx.OffsetRange() by requesting [0, LeftShifted16MaxContainerKey)
// with an offset of 0.
const LeftShifted16MaxContainerKey = uint64(0xffffffffffff0000) // or math.MaxUint64 - (1<<16 - 1), or 18446744073709486080

// NilInside checks if the provided iface is nil or
// contains a nil pointer, slice, array, map, or channel.
func NilInside(iface interface{}) bool {
	if iface == nil {
		return true
	}
	switch reflect.TypeOf(iface).Kind() {
	case reflect.Ptr, reflect.Slice, reflect.Array, reflect.Map, reflect.Chan:
		return reflect.ValueOf(iface).IsNil()
	}
	return false
}

//////////////////////////////////
// helper utility functions

func highbits(v uint64) uint64 { return v >> 16 }
func lowbits(v uint64) uint16  { return uint16(v & 0xFFFF) }

// called by Holder.hasRoaringData()
func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) {

	var info roaring.BitmapInfo
	_ = info
	var f *os.File
	f, err = os.Open(path)
	if err != nil {
		return
	}

	var fi os.FileInfo
	fi, err = f.Stat()
	if err != nil {
		return
	}

	// Memory map the file.
	data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		err = errors.Wrap(err, "mmapping")
		return
	}
	defer func() {
		err = syscall.Munmap(data)
		if err != nil {
			err = errors.Wrap(err, "roaringFragmentHasData: munmap failed")
		}
		err = f.Close()
		if err != nil {
			err = errors.Wrap(err, "roaringFragmentHasData f.Close() in defer")
		}
	}()

	// Attach the mmap file to the bitmap.
	var rbm *roaring.Bitmap
	rbm, _, err = roaring.InspectBinary(data, true, &info)
	if err != nil {
		err = errors.Wrap(err, "inspecting")
		return
	}

	if info.ContainerCount > 0 {
		return true, nil
	}
	if info.Ops > 0 {
		return true, nil
	}

	citer, found := rbm.Containers.Iterator(0)
	_ = found

	for citer.Next() {
		return true, nil
	}

	return
}

// GetLoopProgress returns the estimated remaining time to iterate through some items
// as well as the loop completion percentage with the following parameters:
// the start time, the current time, the iteration, and the number of items
func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) (remaining time.Duration, pctDone float64) {
	itemsLeft := total - (iteration + 1)
	avgItemTime := float64(now.Sub(start)) / float64(iteration+1)
	pctDone = (float64(iteration+1) / float64(total)) * 100
	return time.Duration(avgItemTime * float64(itemsLeft)), pctDone
}