From 7de69df4241e7c0f25369c8ede56a191df0195ee Mon Sep 17 00:00:00 2001
From: Todd Gruben <todd@molecula.com>
Date: Mon, 15 Nov 2021 10:54:13 -0600
Subject: [PATCH 1/2] added protection against trailing slash

---
 cmd/roaring-migrate/main.go | 3 +++
 cover-everything.sh         | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go
index 627d56ef2..e12348e56 100644
--- a/cmd/roaring-migrate/main.go
+++ b/cmd/roaring-migrate/main.go
@@ -243,6 +243,9 @@ func copyFile(src, dest string) error {
 }
 
 func Migrate(dataDir, backupPath string) error {
+	if strings.HasSuffix(dataDir, "/") {
+		dataDir = dataDir[:len(dataDir)-1]
+	}
 	err := os.MkdirAll(backupPath, 0777)
 	if err != nil {
 		return err
diff --git a/cover-everything.sh b/cover-everything.sh
index f34640cfb..500c9013b 100755
--- a/cover-everything.sh
+++ b/cover-everything.sh
@@ -2,7 +2,8 @@
 
 # actually get test coverage for every single package and subpackage
 # very slow but oh well what are you gonna do, not test things?  
+# note it skips the roaring migrate
 echo "mode: atomic" > coverage.out
-for pkg in $(go list all | grep featurebase); do
+for pkg in $(go list all | grep featurebase | grep -v roaring-migrate); do
     go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg;
     tail -n +2 pkgcoverage.out >> coverage.out; done

From f0a6ebae8f58636ff889f6e0e15ff42cd2be8cf0 Mon Sep 17 00:00:00 2001
From: Souhaila Noor <souhaila.noor@molecula.com>
Date: Mon, 15 Nov 2021 14:51:57 -0600
Subject: [PATCH 2/2] ingest and delete for samsung workflow

---
 .gitlab/ingestWorkload.sh | 88 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100755 .gitlab/ingestWorkload.sh

diff --git a/.gitlab/ingestWorkload.sh b/.gitlab/ingestWorkload.sh
new file mode 100755
index 000000000..2502dba2a
--- /dev/null
+++ b/.gitlab/ingestWorkload.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+# To run:
+# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag}
+
+function delete_field {
+    if (($INITIALIZE == 0));
+    then 
+        curl -XDELETE $HOST/index/$INDEX/field/$FIELD
+    fi
+}
+
+# Script to replicate samsung workload of deleting and re-ingesting fields every night 
+# outline delete and re-ingest workload 
+function ingest_int_field { 
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_time_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_set_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+# path for featurebase binary
+FEATUREBASE_PATH=$1
+shift
+
+# path for directory with csv directory files for all fields to be ingested 
+CSV_DIR_PATH=$1
+shift 
+
+# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
+INITIALIZE=$1
+shift 
+
+# get a list of csv files in the directory 
+CSV_FILES=`ls $CSV_DIR_PATH/*.csv`
+
+# featurebase host 
+HOST="localhost:10101"
+# assign index name 
+INDEX="samsung"
+if (($INITIALIZE == 1));
+then 
+    curl -XPOST $HOST/index/$INDEX
+fi
+
+# perform delete and re-ingest for all fields 
+for CSV_FILE in ${CSV_FILES[@]}
+    do 
+        # get field name from csv file path 
+        FIELD="$(basename $CSV_FILE .csv)"
+        if [[ "$FIELD" == *"age"* ]];
+        then 
+            MAX=100 
+            echo $CSV_FILE $MAX
+            ingest_int_field
+        elif [[ "$FIELD" == *"identifier"* ]];
+        then 
+            MAX=$((2**63 - 1)) # compute max value for 64bit
+            echo $CSV_FILE $MAX
+            ingest_int_field
+        elif [[ "$FIELD" == *"ip"* ]];
+        then 
+            MAX=$((2**31 - 1)) # compute max value for 32bit
+            echo $CSV_FILE $MAX
+            ingest_int_field
+        elif [[ "$FIELD" == *"time"* ]];
+        then 
+            echo $CSV_FILE
+            ingest_time_field
+        else
+            echo $CSV_FILE
+            ingest_set_field
+        fi
+    done
+
+
+