Merge pull request #1995 from molecula/sup-194

FB-1287 - fix viewTimePart to account for bad strings

.gitlab/.gitlab-ci.yml

@@ -387,7 +387,7 @@ smoke test:
   script:
     - ./qa/scripts/setupSmokeTest.sh
     - ./qa/scripts/testSmokeTest.sh
-    - ./qa/scripts/test_FB-1270_repro.sh
+    - ./qa/scripts/bug_repro_tests.sh
   after_script:
     - ./qa/scripts/teardownSmokeTest.sh
   needs:

qa/scripts/bug_repro_tests.sh

@@ -17,7 +17,7 @@ echo "using DATANODE0 ${DATANODE0}"
 ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo yum -y install librdkafka" 
 
 echo "Copying tests to remote"
-scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/FB-1270_repro/test.sh ec2-user@${INGESTNODE0}:/data
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/bug-repros/ ec2-user@${INGESTNODE0}:/data
 scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen_linux_arm64 ec2-user@${INGESTNODE0}:/data
 if (( $? != 0 )) 
 then 
@@ -25,17 +25,17 @@ then
     exit 1
 fi
 
-# run 1270 repro
-echo "Running smoke test..."
-ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/; ./test.sh ${DATANODE0}:10101" 
+# run all repros
+echo "Running smoke tests..."
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/bug-repros; ./run-all.sh ${DATANODE0}:10101"
 SMOKETESTRESULT=$?
 
 
 if (( $SMOKETESTRESULT != 0 )) 
 then 
-    echo "FB-1270 test complete with test failures"
+    echo "smoke tests complete with test failures"
 else
-    echo "FB-1270 test complete"
+    echo "smoke tests complete"
 fi
 
 exit $SMOKETESTRESULT 

qa/scripts/utilCluster.sh

@@ -179,6 +179,7 @@ setupIngestNode() {
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U pytest"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U json"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install -y jq"
 }
 
 setupDataNodes() {

qa/testcases/bug-repros/README

@@ -0,0 +1,17 @@
+adding a test case to the bug repros directory??? no problem!!! its a snap!!!
+
+just make a shell script that does the test you want and name it some thing like:
+
+fb42069-test.sh
+
+this will get picked up by run-all.sh and get run automatically!!!
+
+
+# THINGS TO NOTE
+- the datanode0 ip will be passed to your script in $1 via qa/scripts/bug_repro_tests.sh
+
+# ENTHUSIASM
+WOW
+
+shout out to computers for making our lives easier! :) 👍
+

qa/testcases/bug-repros/fb-1287-datagen.yaml

@@ -0,0 +1,57 @@
+fields:
+  - name: "id"
+    type: uint
+    step: 1
+    distribution: "sequential"
+    min: 1
+    max: 200
+  - name: "segid"
+    type: "int" # (default IntField)
+    min: 0
+    max: 3
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+  - name: "ts"
+    type: "timestamp"
+    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2022-01-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "1ms"
+    max_step_duration: "200ms"
+  - name: "lastupdated"
+    type: "timestamp"
+    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2022-01-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "1ms"
+    max_step_duration: "200ms"
+  - name: "slice"
+    type: "uint-set" # (default IDArrayField)
+    min: 0
+    max: 35000
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+    min_num: 1
+    max_num: 50
+
+idk_params:
+  primary_key_config:
+    field: "id"
+  fields:
+    segid:
+      - type: "ID"
+    ts:
+      - type: "RecordTime"
+        layout: "2006-01-02T15:04:05Z"
+        epoch: 1970-01-01T00:00:00.0Z
+        name: "na"
+      - type: "Timestamp"
+        layout: "2006-01-02T15:04:05Z"
+        epoch: 1970-01-01T00:00:00.0Z
+        name: "last_update"
+        granularity: "s"
+    slice:
+      - type: "IDArray"
+        time_quantum: "D"

qa/testcases/bug-repros/fb-1287-test.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+/data/datagen_linux_arm64 -s custom --custom-config=./fb-1287-datagen.yaml --pilosa.index=fb1287 --pilosa.batch-size=100 --pilosa.hosts=$1
+
+# if we get an error, exit 1
+if [[ $( curl $1/index/fb1287/query -d 'Rows(segid,from="2022-01-02T15:04",to="2022-04-02T15:04")' | jq '.error' ) != "null" ]]; then
+    exit 1;
+else
+    exit 0;
+fi

qa/testcases/bug-repros/run-all.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+for file in `ls *-test.sh`; do
+    echo "running $file";
+    ./$file "$@"
+done

time.go

@@ -410,7 +410,7 @@ func minMaxViews(views []string, q TimeQuantum) (min string, max string) {
 	// Sort the list of views.
 	sort.Strings(views)
 
-	// Determine the least significant quantum and set that as the
+	// Determine the least precise quantum and set that as the
 	// number of string characters to compare against.
 	var chars int
 	if q.HasYear() {
@@ -499,5 +499,9 @@ func timeOfView(v string, adj bool) (time.Time, error) {
 // e.g. the view "string_201901" would return "201901".
 func viewTimePart(v string) string {
 	parts := strings.Split(v, "_")
+	if _, err := strconv.Atoi(parts[len(parts)-1]); err != nil {
+		// it's not a number!
+		return ""
+	}
 	return parts[len(parts)-1]
 }

time_internal_test.go

@@ -398,3 +398,15 @@ func TestParsePartialTime(t *testing.T) {
 	}
 
 }
+
+func TestViewTimePart(t *testing.T) {
+	for input, want := range map[string]string{
+		"standard":         "",
+		"standard_1234567": "1234567",
+		"standard1234567":  "",
+	} {
+		if got := viewTimePart(input); got != want {
+			t.Errorf("expected %v got %v", want, got)
+		}
+	}
+}