add reproducer for FB-1270 into smoke tests

.gitlab/.gitlab-ci.yml

@@ -376,14 +376,18 @@ smoke test:
     - chmod 700 /root/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
     - apt update && apt -y install jq wget
-    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
     - export PATH=$PATH:/usr/local/go/bin
     - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
     - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    # download datagen for FB-1270 repro test. TODO replace w/ locally built datagen once we merge IDK into FB
+    - aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
+    - chmod +x ./datagen_linux_arm64
   script:
     - ./qa/scripts/setupSmokeTest.sh
     - ./qa/scripts/testSmokeTest.sh
+    - ./qa/scripts/test_FB-1270_repro.sh
   after_script:
     - ./qa/scripts/teardownSmokeTest.sh
   needs:

qa/scripts/test_FB-1270_repro.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/utilCluster.sh
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+# install librdkafka ... workaround until we have static datagen builds for arm
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo yum -y install librdkafka" 
+
+echo "Copying tests to remote"
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/FB-1270_repro/test.sh ec2-user@${INGESTNODE0}:/data
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen_linux_arm64 ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+# run 1270 repro
+echo "Running smoke test..."
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/; ./test.sh ${DATANODE0}:10101" 
+SMOKETESTRESULT=$?
+
+
+if (( $SMOKETESTRESULT != 0 )) 
+then 
+    echo "FB-1270 test complete with test failures"
+else
+    echo "FB-1270 test complete"
+fi
+
+exit $SMOKETESTRESULT 

qa/testcases/FB-1270_repro/test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+for i in {41..44}; do
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    echo "ROUND $i"
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    /data/datagen_linux_arm64 -s texas_health --pilosa.index thr --end-at 1048575 --pilosa.batch-size 1048576 --concurrency 1 --seed=$i --pilosa.hosts=$1
+    curl -XPOST $1/index/thr/query -d 'Delete(Row(timestamp<2020-12-13T00:32:42Z)'
+done