mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-05 16:15:56 +00:00
e7f65cf7be
close files after using them if global max is passed. I originally implemented this without the global count—just always closing files when done with them, and reopening for new writes. This was crazy slow for that one test that uses mustSetBits in a big loop. I modified the test to use importRoaring and everything worked better (though much more slowly). After adding the global counter, I ran the tests with that one test using mustSetBits again, and the performance was similar to master. After completing this PR, I ran the tests with the max limit set to 5—they still passed but were much slower.
41 lines
1.1 KiB
Go
41 lines
1.1 KiB
Go
package syswrap
|
|
|
|
import (
|
|
"os"
|
|
"sync"
|
|
"sync/atomic"
|
|
)
|
|
|
|
var fileCount uint64
|
|
|
|
// maxFileCount is the soft limit on the number of open files. syswrap.OpenFile
|
|
// will warn when this limit is passed.
|
|
var maxFileCount uint64 = 500000
|
|
var fileMu sync.RWMutex
|
|
|
|
func SetMaxFileCount(max uint64) {
|
|
fileMu.Lock()
|
|
maxFileCount = max
|
|
fileMu.Unlock()
|
|
}
|
|
|
|
// OpenFile passes the arguments along to os.OpenFile while incrementing a
|
|
// counter. If the counter is above the maximum, it returns mustClose true to
|
|
// signal the calling function that it should not keep the file open
|
|
// indefinitely. Files opened with this function should be closed by
|
|
// syswrap.CloseFile.
|
|
func OpenFile(name string, flag int, perm os.FileMode) (file *os.File, mustClose bool, err error) {
|
|
file, err = os.OpenFile(name, flag, perm)
|
|
fileMu.RLock()
|
|
defer fileMu.RUnlock()
|
|
if newCount := atomic.AddUint64(&fileCount, 1); newCount > maxFileCount {
|
|
mustClose = true
|
|
}
|
|
return file, mustClose, err
|
|
}
|
|
|
|
// CloseFile decrements the global count of open files and closes the file.
|
|
func CloseFile(f *os.File) error {
|
|
atomic.AddUint64(&fileCount, ^uint64(0)) // decrement
|
|
return f.Close()
|
|
}
|