Merge pull request #1775 from molecula/generate-test-data-FB-971

generate csv files that simulate Samsung's data
2026-09-07 09:05:55 +00:00 · 2021-11-22 15:22:01 -08:00 · 2021-11-22 15:22:01 -08:00 · 769a750cdf
commit 769a750cdf
parent 3789a8dd05 11f1cd3689
8 changed files with 441 additions and 10 deletions
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -218,9 +218,17 @@ deploy node for linux amd64:
    - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE)
    - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
    - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
-    - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json
+    - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json
    - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') 
    - echo $INSTANCE_ID > linux-amd64-instance.txt
+    - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data
+    - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE)
+    - echo $PUBLIC_IP
+    - scp  -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:.
+    - scp  -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$PUBLIC_IP:.
+    - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2
+  needs:
+    - job: build for linux amd64
  artifacts:
    paths:
      - linux-amd64-instance.txt
@ -274,4 +282,4 @@ terminate node for linux amd64:
  needs:
    - job: deploy node for linux amd64
    - job: initiate featurebase linux amd64 in node
-  when: always 
+  when: always 
--- a/qa/scripts/cloud-init.sh
+++ b/qa/scripts/cloud-init.sh
--- a/qa/scripts/configureFeatureBase.json
+++ b/qa/scripts/configureFeatureBase.json
--- a/qa/scripts/featurebase.conf
+++ b/qa/scripts/featurebase.conf
--- a/qa/scripts/featurebase.service
+++ b/qa/scripts/featurebase.service
--- a/qa/scripts/ingestWorkload.sh
+++ b/qa/scripts/ingestWorkload.sh
@ -1,7 +1,7 @@
 #!/usr/bin/env bash

 # To run:
-# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag}
+# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag}

 function delete_field {
    if (($INITIALIZE == 0));
@ -34,6 +34,10 @@ function ingest_set_field {
 FEATUREBASE_PATH=$1
 shift

+# featurebase host & port
+HOST=$1
+shift
+
 # path for directory with csv directory files for all fields to be ingested 
 CSV_DIR_PATH=$1
 shift 
@ -45,8 +49,6 @@ shift
 # get a list of csv files in the directory 
 CSV_FILES=`ls $CSV_DIR_PATH/*.csv`

-# featurebase host 
-HOST="localhost:10101"
 # assign index name 
 INDEX="samsung"
 if (($INITIALIZE == 1));
@ -62,24 +64,19 @@ for CSV_FILE in ${CSV_FILES[@]}
        if [[ "$FIELD" == *"age"* ]];
        then 
            MAX=100 
-            echo $CSV_FILE $MAX
            ingest_int_field
        elif [[ "$FIELD" == *"identifier"* ]];
        then 
            MAX=$((2**63 - 1)) # compute max value for 64bit
-            echo $CSV_FILE $MAX
            ingest_int_field
        elif [[ "$FIELD" == *"ip"* ]];
        then 
            MAX=$((2**31 - 1)) # compute max value for 32bit
-            echo $CSV_FILE $MAX
            ingest_int_field
        elif [[ "$FIELD" == *"time"* ]];
        then 
-            echo $CSV_FILE
            ingest_time_field
        else
-            echo $CSV_FILE
            ingest_set_field
        fi
    done
--- a/qa/simulacraData/simulacra_data.go
+++ b/qa/simulacraData/simulacra_data.go
@ -0,0 +1,357 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971
+// INPUT: none
+// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data)
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG",
+	"ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS",
+	"BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE",
+	"CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE",
+	"DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK",
+	"FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD",
+	"GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL",
+	"IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR",
+	"KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO",
+	"MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ",
+	"MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL",
+	"OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT",
+	"REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM",
+	"SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL",
+	"TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT",
+	"VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE",
+}
+
+const totalRecords int = 200000000
+
+func main() {
+	if err := GenerateAgeField(totalRecords); err != nil {
+		log.Fatalf("unable to generate age field: %v", err)
+	}
+
+	if err := GenerateIPField(totalRecords); err != nil {
+		log.Fatalf("unable to generate IP field: %v", err)
+	}
+
+	if err := GenerateArbIdField(totalRecords); err != nil {
+		log.Fatalf("unable to generate indentifier field: %v", err)
+	}
+
+	if err := GenerateOptInField(totalRecords); err != nil {
+		log.Fatalf("unable to generate opt in field: %v", err)
+	}
+
+	if err := GenerateCountryField(totalRecords); err != nil {
+		log.Fatalf("unable to generate country field: %v", err)
+	}
+
+	if err := GenerateTimeField(totalRecords); err != nil {
+		log.Fatalf("unable to generate time field: %v", err)
+	}
+}
+
+func GenerateAgeField(requestedRecords int) error {
+	csvFile, err0 := os.Create("age.csv")
+	if err0 != nil {
+		return errors.Wrap(err0, "unable to create age.csv")
+	}
+
+	writer := bufio.NewWriter(csvFile)
+
+	for i := 0; i < requestedRecords; i++ {
+		if rand.Intn(100) <= 20 {
+			_, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n")
+			if err != nil {
+				return errors.Wrap(err, "unable to write to age.csv")
+			}
+
+			if rand.Intn(10) == 1 {
+				_, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n")
+				if err != nil {
+					return errors.Wrap(err, "unable to write to age.csv")
+
+				}
+			}
+
+		}
+	}
+	err1 := writer.Flush()
+	if err1 != nil {
+		return errors.Wrap(err1, "unable to flush writer")
+	}
+
+	err2 := csvFile.Close()
+	if err2 != nil {
+		return errors.Wrap(err2, "unable to close age.csv")
+	}
+	return nil
+}
+
+func GenerateIPField(requestedRecords int) error {
+	csvFile, err0 := os.Create("ip.csv")
+	if err0 != nil {
+		return errors.Wrap(err0, "unable to create ip.csv")
+	}
+
+	writer := bufio.NewWriter(csvFile)
+
+	for i := 0; i < requestedRecords; i++ {
+		_, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n")
+		if err != nil {
+			return errors.Wrap(err, "unable to write to ip.csv")
+		}
+
+		if rand.Intn(10) == 1 {
+			_, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n")
+			if err != nil {
+				return errors.Wrap(err, "unable to write to ip.csv")
+			}
+		}
+	}
+
+	err1 := writer.Flush()
+	if err1 != nil {
+		return errors.Wrap(err1, "unable to flush writer")
+	}
+
+	err2 := csvFile.Close()
+	if err2 != nil {
+		return errors.Wrap(err2, "unable to close ip.csv")
+	}
+	return nil
+
+}
+
+func GenerateArbIdField(requestedRecords int) error {
+	csvFile, err0 := os.Create("identifier.csv")
+	if err0 != nil {
+		return errors.Wrap(err0, "unable to create identifier.csv")
+	}
+
+	writer := bufio.NewWriter(csvFile)
+
+	for i := 0; i < requestedRecords; i++ {
+		_, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n")
+		if err != nil {
+			return errors.Wrap(err, "unable to write to identifier.csv")
+		}
+
+		if rand.Intn(10) == 1 {
+			_, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n")
+			if err != nil {
+				return errors.Wrap(err, "unable to write to identifier.csv")
+			}
+		}
+	}
+
+	err1 := writer.Flush()
+	if err1 != nil {
+		return errors.Wrap(err1, "unable to flush writer")
+	}
+
+	err2 := csvFile.Close()
+	if err2 != nil {
+		return errors.Wrap(err2, "unable to close identifier.csv")
+	}
+	return nil
+
+}
+
+func GenerateTimeField(requestedRecords int) error {
+	csvFile, err0 := os.Create("time.csv")
+	if err0 != nil {
+		return errors.Wrap(err0, "unable to create time.csv")
+	}
+
+	writer := bufio.NewWriter(csvFile)
+
+	r := rand.New(rand.NewSource(rand.Int63()))
+	s := 1.01
+	v := 1.01
+	var imax uint64 = 4000000
+
+	distro := rand.NewZipf(r, s, v, imax)
+
+	for i := 0; i < requestedRecords; i++ {
+		value := distro.Uint64()
+		urlString := "https://www.test.com/" + strconv.FormatUint(value, 10)
+
+		date := generateDate()
+
+		_, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n")
+		if err != nil {
+			return errors.Wrap(err, "unable to write to time.csv")
+		}
+
+		if rand.Intn(5) == 1 {
+			date = generateDate()
+			_, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n")
+			if err != nil {
+				return errors.Wrap(err, "unable to write to time.csv")
+			}
+		}
+
+		if rand.Intn(10) == 1 {
+			value := distro.Uint64()
+			urlString = "https://www.test.com/" + strconv.FormatUint(value, 10)
+			date = generateDate()
+			_, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n")
+			if err != nil {
+				return errors.Wrap(err, "unable to write to time.csv")
+			}
+		}
+
+	}
+	err1 := writer.Flush()
+	if err1 != nil {
+		return errors.Wrap(err1, "unable to flush writer")
+	}
+
+	err2 := csvFile.Close()
+	if err2 != nil {
+		return errors.Wrap(err2, "unable to close time.csv")
+	}
+	return nil
+
+}
+
+func GenerateOptInField(requestedRecords int) error {
+	csvFile, err0 := os.Create("optin.csv")
+	if err0 != nil {
+		return errors.Wrap(err0, "unable to create optin.csv")
+	}
+
+	writer := bufio.NewWriter(csvFile)
+
+	var optString string
+
+	for i := 0; i < requestedRecords; i++ {
+
+		if rand.Intn(20) == 1 {
+			optString = "1"
+		} else {
+			optString = "0"
+		}
+
+		_, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n")
+		if err != nil {
+			return errors.Wrap(err, "unable to write to optin.csv")
+		}
+
+		if rand.Intn(10) == 1 {
+			if optString == "0" {
+				optString = "1"
+				_, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n")
+				if err != nil {
+					return errors.Wrap(err, "unable to write to optin.csv")
+				}
+			} else {
+				optString = "0"
+				_, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n")
+				if err != nil {
+					return errors.Wrap(err, "unable to write to optin.csv")
+				}
+			}
+		}
+	}
+
+	err1 := writer.Flush()
+	if err1 != nil {
+		return errors.Wrap(err1, "unable to flush writer")
+	}
+
+	err2 := csvFile.Close()
+	if err2 != nil {
+		return errors.Wrap(err2, "unable to close optin.csv")
+	}
+	return nil
+
+}
+
+func GenerateCountryField(requestedRecords int) error {
+	csvFile, err0 := os.Create("country.csv")
+	if err0 != nil {
+		return errors.Wrap(err0, "unable to create country.csv")
+	}
+
+	writer := bufio.NewWriter(csvFile)
+
+	// populate countries
+	r := rand.New(rand.NewSource(rand.Int63()))
+	s := 1.01
+	v := 1.01
+	var imax uint64 = 245
+
+	distro := rand.NewZipf(r, s, v, imax)
+
+	for i := 0; i < requestedRecords; i++ {
+		if rand.Intn(2) == 1 {
+			value := distro.Uint64()
+			country := countryList[value]
+			_, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n")
+			if err != nil {
+				return errors.Wrap(err, "unable to write to country.csv")
+			}
+
+			if rand.Intn(10) == 1 {
+				value = distro.Uint64()
+				country = countryList[value]
+				_, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n")
+				if err != nil {
+					return errors.Wrap(err, "unable to write to country.csv")
+				}
+			}
+		}
+
+	}
+
+	err1 := writer.Flush()
+	if err1 != nil {
+		return errors.Wrap(err1, "unable to flush writer")
+	}
+
+	err2 := csvFile.Close()
+	if err2 != nil {
+		return errors.Wrap(err2, "unable to close country.csv")
+	}
+	return nil
+
+}
+
+func generateDate() string {
+	// Nov 22, 2021 = 1637616960
+	// twenty years =  631138520
+	// date min 	= 1006478440
+	date := rand.Int63n(631138520) + 1006478440
+
+	t := time.Unix(date, 0)
+	YMDstring := t.Format("2006-01-02T15:04")
+
+	return YMDstring
+}
--- a/qa/simulacraData/simulacra_data_test.go
+++ b/qa/simulacraData/simulacra_data_test.go
@ -0,0 +1,69 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"testing"
+)
+
+const testRecords int = 1000
+
+func TestAge(t *testing.T) {
+	err := GenerateAgeField(testRecords)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestIP(t *testing.T) {
+	err := GenerateIPField(testRecords)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestIndentifer(t *testing.T) {
+	err := GenerateArbIdField(testRecords)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestOptIn(t *testing.T) {
+	err := GenerateOptInField(testRecords)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestCountry(t *testing.T) {
+	err := GenerateCountryField(testRecords)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestTime(t *testing.T) {
+	err := GenerateTimeField(testRecords)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestGenerateDate(t *testing.T) {
+	testDate := generateDate()
+	if testDate == "" {
+		t.Fatalf("error generating date")
+	}
+}