From 20d23cfcb70af8e846b604c4ddc02252df7fbbee Mon Sep 17 00:00:00 2001 From: jacob Date: Tue, 4 Apr 2023 13:35:49 -0500 Subject: [PATCH] update ci behavior --- .gitlab/.gitlab-ci.yml | 15 +++++++++++---- Dockerfile-fbsql-darwin | 4 +++- cli/cli_kafka_integration_test.go | 18 +++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a9b6dc13d..7f46f7fe2 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -286,10 +286,10 @@ run go tests race: script: - echo "Running featurebase race tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - SKIP_LIST=TestKafkaRunner - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID + - export SKIP_INTEGRATION_TEST=true - mkdir -p $TMPDIR - - go test -skip ${SKIP_LIST} -race -v -timeout=10m ${PKG_LIST//,/ } + - go test -race -v -timeout=10m ${PKG_LIST//,/ } after_script: - rm -rf /mnt/ramdisk/test-$CI_JOB_ID tags: @@ -317,10 +317,10 @@ run go tests: script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -) - - SKIP_LIST=TestKafkaRunner - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID + - export SKIP_INTEGRATION_TEST=true - mkdir -p $TMPDIR - - go test -skip ${SKIP_LIST} -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } + - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ } after_script: - rm -rf /mnt/ramdisk/test-$CI_JOB_ID artifacts: @@ -368,12 +368,19 @@ run go tests cli kafka integration: KAFKA_RUNNER_TEST_FEATUREBASEGRPC_HOST: pilosa:20101 KAFKA_RUNNER_TEST_KAFKA_HOST: kafka:9092 KAFKA_RUNNER_TEST_REGISTRY_HOST: schema-registry:8081 + rules: + - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")' script: - echo "running fbsql integration tests" - cd ./idk/ - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make start-all - cd .. - go test -coverprofile=coverage-cli-kafka-integration.out -run -timeout=10m TestKafkaRunner ./cli + after_script: + - rm -rf /mnt/ramdisk/test-$CI_JOB_ID + artifacts: + paths: + - coverage-cli-kafka-integration.out # idk tests run go tests idk race: diff --git a/Dockerfile-fbsql-darwin b/Dockerfile-fbsql-darwin index 5a7116e69..448f0858c 100644 --- a/Dockerfile-fbsql-darwin +++ b/Dockerfile-fbsql-darwin @@ -76,4 +76,6 @@ COPY --from=builder /featurebase/fbsql /usr/local/bin/ # Verify that the linker can find everything. FROM runner AS linkcheck -RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi \ No newline at end of file +RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi + +FROM runner \ No newline at end of file diff --git a/cli/cli_kafka_integration_test.go b/cli/cli_kafka_integration_test.go index 37197dd09..c8ca6ec8c 100644 --- a/cli/cli_kafka_integration_test.go +++ b/cli/cli_kafka_integration_test.go @@ -73,7 +73,7 @@ type testQuery struct { // A slice of KafkaRunnerTest structs that will be used in TestKafkaRunner test // function. var kafkaRunnerTests = []kafkaRunnerTest{ - { // id keys + { // id keys json ConfigFile: "config00.toml", DataFile: "data00.json", Tests: []testQuery{ @@ -84,7 +84,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ }, CreateTableStmt: "(_id ID, name String, age Int, hobbies StringSet)", }, - { // string keys + { // string keys json ConfigFile: "config01.toml", DataFile: "data00.json", Tests: []testQuery{ @@ -95,7 +95,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ }, CreateTableStmt: "(_id String, name String, age Int, hobbies StringSet)", }, - { // two string keys + { // two string keys json ConfigFile: "config02.toml", DataFile: "data00.json", Tests: []testQuery{ @@ -106,7 +106,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ }, CreateTableStmt: "(_id String, id String, name String, age Int, hobbies StringSet)", }, - { // string, id, and int + { // string, id, and int json ConfigFile: "config03.toml", DataFile: "data00.json", Tests: []testQuery{ @@ -117,7 +117,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ }, CreateTableStmt: "(_id String, id id, name String, age Int, hobbies StringSet)", }, - { // missing values + { // missing values json ConfigFile: "config05.toml", DataFile: "data02.json", Tests: []testQuery{ @@ -128,7 +128,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ }, CreateTableStmt: "(_id String, name String, age Int, hobbies StringSet)", }, - { // string keys + { // string keys avro ConfigFile: "config04.toml", DataFile: "data01.json", SchemaFile: "schema01.json", @@ -140,7 +140,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ }, CreateTableStmt: "(_id string, string_string string, string_bytes string, pk2 stringset, stringset_bytes stringset, idset_long idset, decimal_double decimal(2), timestamp_bytes_ts timestamp, idset_longarray idset, dateint_bytes_ts int, bools stringset, stringset_string stringset, stringset_stringarray stringset, idset_int idset, timestamp_bytes_int timestamp, int_long int, id_long id, id_int id, idset_intarray idset, decimal_float decimal(2), bools-exists stringset, stringset_bytesarray stringset, int_int int, decimal_bytes decimal(2), pk1 stringset)", }, - { // id keys + { // id keys avro ConfigFile: "config06.toml", DataFile: "data03.json", SchemaFile: "schema02.json", @@ -152,7 +152,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ }, CreateTableStmt: "(_id id, string_string string, idset_longarray idset, stringset_stringarray stringset, int_int int, bools stringset, bools-exists stringset)", }, - { // compound keys + { // compound keys avro ConfigFile: "config07.toml", DataFile: "data01.json", SchemaFile: "schema01.json", @@ -175,7 +175,7 @@ var kafkaRunnerTests = []kafkaRunnerTest{ // 5. Runs the cli.Command // 6. Confirms that the data was written to FeatureBase as expected func TestKafkaRunner(t *testing.T) { - if testing.Short() { + if testing.Short() || os.Getenv("SKIP_INTEGRATION_TEST") == "true" { t.Skip("skipping integration test") }