scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

improve ramdisk config in Makefile, use it for everything in tests

Browse source 
So, we were special-casing creating a ramdisk, and setting a special
environment variable for it, for boltdb translate files, to improve
performance.

But actually, etcd and test cluster data and so on all go in $TMPDIR,
and if you move all of those also into a ram disk, you get way better
performance. But 2GB may not be enough for that.

So!

We unify on $TMPDIR, we bump the default size to 4GB, we make the
size configurable, and we stop using the name RAMDISK. This should
improve performance on MacOS significantly for `make test` and
things like it, and also simplifies our lives by not having a special
case for the boltdb translate files.

Also change the environment variable names used in our CI config.
(I don't see where we mount the ramdisks, but I think that's happening
in our setup.)
This commit is contained in:
Seebs 2023-02-21 11:38:09 -06:00 • committed by seebs
parent df7e813f01
commit 8724eb09b0
3 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitlab/.gitlab-ci.yml
View file

@ -216,7 +216,7 @@ run go tests race:
script:
- echo "Running featurebase race tests..."
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
- RAMDISK=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ }
- TMPDIR=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ }
tags:
- docker
@ -232,7 +232,7 @@ run go tests:
script:
- echo "Running featurebase unit tests..."
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
- RAMDISK=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
- TMPDIR=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
artifacts:
paths:
- coverage.out

16
Makefile
View file

@ -24,6 +24,9 @@ BUILD_TAGS +=
TEST_TAGS = roaringparanoia
TEST_TIMEOUT=10m
RACE_TEST_TIMEOUT=10m
# size in GB to use for ramdisk, ?= so you can override it with env
# 4GB is not enough for `make test`, 8GB usually is.
RAMDISK_SIZE ?= 8
export GO111MODULE=on
export GOPRIVATE=github.com/molecula
@ -77,13 +80,18 @@ testvsub:
echo; echo "999 done testing subpkg $$pkg"; \
done
# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/mnt/ramdisk
# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
# them with TMPDIR=/mnt/ramdisk.
ramdisk-linux:
mount -o size=2G -t tmpfs none /mnt/ramdisk
mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/Volumes/RAMDisk
# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
# them with TMPDIR=/Volumes/RAMDisk. This is more important on
# OS X than it is on Linux, because there's performance issues
# with fsync on OS X that can make the SSD slow down to moving-platters
# drive speeds. Oops.
ramdisk-osx:
diskutil erasevolume HFS+ 'RAMDisk' `hdiutil attach -nobrowse -nomount ram://4194304`
diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
detach-ramdisk-osx:
hdiutil detach /Volumes/RAMDisk

2
translate.go
View file

@ -289,7 +289,7 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition
fname = fname[:10]
}
tf, err := os.CreateTemp(os.Getenv("RAMDISK"), fmt.Sprintf("bolt-i%s-f%s-%d-%d-", iname, fname, partitionID, partitionN))
tf, err := os.CreateTemp("", fmt.Sprintf("bolt-i%s-f%s-%d-%d-", iname, fname, partitionID, partitionN))
if err != nil {
return nil, errors.Wrap(err, "making temp file for boltdb key translation")
}