From b5fb9aad8434eed5e4e5383b9093129caf523a2c Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 20 Jan 2022 13:10:15 -0600 Subject: [PATCH] bump test timeouts ridiculously gitlab CI runs as much as 5x slower sometimes during business hours, resulting in tests failing due to 10-11 minute timeouts that would succeed in under 2-3 minutes outside of business hours. to allow us to do anything at all, let's just set that to half an hour, and 90 minutes for `go test -race`. Concern: It's possible there's a timeout that's a gitlab CI configuration thing involved too, because we see some go test timeout panics, but we also see some weird messages about SIGQUIT at 11 minutes, which isn't the go test timeout, so we may need to address that too. Note that we're changing the Makefile, and also the config for the gitlab CI passes, which don't use the Makefile. The Makefile changes are just to be careful and avoid retriggering this later. We may want to revert these if we get the other issues fixed. --- .gitlab/.gitlab-ci.yml | 10 +++++----- Makefile | 16 +++++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 793e70aae..09987cadd 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -66,7 +66,7 @@ run go tests: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests..." - - go test ./... + - go test -timeout=30m ./... tags: - aws @@ -77,7 +77,7 @@ run go tests race: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase race tests..." - - go test -race -timeout=30m ./... + - go test -race -timeout=90m ./... tags: - aws @@ -88,10 +88,10 @@ run go tests shardwidth22: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase race tests..." - - go test -tags=shardwidth22 ./... + - go test -timeout=30m -tags=shardwidth22 ./... tags: - aws - + # we do coverage reporting from the future tests because the json # output is very difficult to human-read. The alternative would be to # run the regular tests twice and also run the future tests. @@ -103,7 +103,7 @@ run go tests future: script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) - - go test -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out + - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out artifacts: paths: - coverage.out diff --git a/Makefile b/Makefile index 8545c0cb8..540c1f9b7 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia UNAME := $(shell uname -s) +TEST_TIMEOUT=30m +RACE_TEST_TIMEOUT=90m ifeq ($(UNAME), Darwin) IS_MACOS:=1 else @@ -45,11 +47,11 @@ version: # Run test suite test: - $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v + $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) # Run test suite with race flag test-race: - CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v + CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v testv: topt testvsub @@ -64,7 +66,7 @@ testvsub: set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ - $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \ + $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) || break; \ echo; echo "999 done testing subpkg $$i"; \ cd ..; \ done @@ -73,7 +75,7 @@ testvsub-race: set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ - CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \ + CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) || break; \ echo; echo "999 done testing subpkg $$i -race"; \ cd ..; \ done @@ -248,20 +250,20 @@ pilosa-fsck: # Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./... + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout $(TEST_TIMEOUT) ./... # Must use bash in order to -o pipefail; otherwise the tee will hide red tests. # run top tests, not subdirs. print summary red/green after. # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt. topt: mv log.topt.roar log.topt.roar.prev || true - $(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout 60m -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar + $(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout $(RACE_TEST_TIMEOUT) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l topt-race: mv log.topt.race log.topt.race.prev || true - $(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout 60m -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race + $(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout $(RACE_TEST_TIMEOUT) -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l