From 9302775fd86d027839c10f7f3a22b64e10428385 Mon Sep 17 00:00:00 2001 From: "Kasey C. Rodgers" <49999391+kcrodgers24@users.noreply.github.com> Date: Wed, 17 Nov 2021 08:21:57 -0800 Subject: [PATCH 01/15] generate csv files that simulate Samsung's data --- .gitlab/simulacraData.go | 238 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 .gitlab/simulacraData.go diff --git a/.gitlab/simulacraData.go b/.gitlab/simulacraData.go new file mode 100644 index 000000000..db4fd2012 --- /dev/null +++ b/.gitlab/simulacraData.go @@ -0,0 +1,238 @@ +// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 +// INPUT: none +// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) + +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "strconv" +) + +var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", + "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", + "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", + "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", + "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", + "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", + "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", + "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", + "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", + "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", + "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", + "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", + "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", + "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", + "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", + "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", +} + +const totalRecords int = 200000000 + +func main() { + ageField() + ipField() + arbIdField() + optInField() + countryField() + timeField() +} + +func ageField() { + csvFile, err := os.Create("age.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(100) <= 20 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") + } + } + } + writer.Flush() + csvFile.Close() + +} + +func ipField() { + csvFile, err := os.Create("ip.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func arbIdField() { + csvFile, err := os.Create("identifier.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func timeField() { + csvFile, err := os.Create("time.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 4000000 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + value := distro.Uint64() + urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) + + date := generateDate() + + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + + if rand.Intn(5) == 1 { + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + if rand.Intn(10) == 1 { + value := distro.Uint64() + urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + } + + writer.Flush() + csvFile.Close() + +} + +func optInField() { + csvFile, err := os.Create("optin.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + var optString string + + for i := 0; i < totalRecords; i++ { + + if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% + optString = "1" + } else { + optString = "0" + } + + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if rand.Intn(10) == 1 { + if optString == "0" { + optString = "1" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } else { + optString = "0" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } + } + } + + writer.Flush() + csvFile.Close() + +} + +func countryField() { + csvFile, err := os.Create("country.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + //populate countries + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 245 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(2) == 1 { + value := distro.Uint64() + country := countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + + if rand.Intn(10) == 1 { + value = distro.Uint64() + country = countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + } + } + + } + + writer.Flush() + csvFile.Close() + +} + +func generateDate() string { + // generates a random date in format YYYY-MM-DDT00:00 + // does not include leap days or populata the hour/minute time + year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 + month := rand.Intn(12) + 1 + day := rand.Intn(30) + 1 + + if month == 2 && day >= 29 { + day = 28 + } + if month == 4 && day == 31 { + day = 30 + } + if month == 6 && day == 31 { + day = 30 + } + if month == 11 && day == 31 { + day = 30 + } + + monthFix := fmt.Sprintf("%02d", month) + dayFix := fmt.Sprintf("%02d", day) + YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" + + return YMDstring +} From 442d109814ab58fd108b9d42bca4fd760ba798bd Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 18 Nov 2021 10:28:49 -0600 Subject: [PATCH 02/15] created qa folder --- .gitlab/.gitlab-ci.yml | 6 +++--- {.gitlab => qa/scripts}/cloud-init.sh | 0 {.gitlab => qa/scripts}/configureFeatureBase.json | 0 {.gitlab => qa/scripts}/featurebase.conf | 0 {.gitlab => qa/scripts}/featurebase.service | 0 {.gitlab => qa/scripts}/ingestWorkload.sh | 0 {.gitlab => qa/simulacraData}/simulacraData.go | 0 7 files changed, 3 insertions(+), 3 deletions(-) rename {.gitlab => qa/scripts}/cloud-init.sh (100%) rename {.gitlab => qa/scripts}/configureFeatureBase.json (100%) rename {.gitlab => qa/scripts}/featurebase.conf (100%) rename {.gitlab => qa/scripts}/featurebase.service (100%) rename {.gitlab => qa/scripts}/ingestWorkload.sh (100%) rename {.gitlab => qa/simulacraData}/simulacraData.go (100%) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ccd710889..82710bb4d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -225,15 +225,15 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec3-user@$PUBLIC_IP:. + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 artifacts: diff --git a/.gitlab/cloud-init.sh b/qa/scripts/cloud-init.sh similarity index 100% rename from .gitlab/cloud-init.sh rename to qa/scripts/cloud-init.sh diff --git a/.gitlab/configureFeatureBase.json b/qa/scripts/configureFeatureBase.json similarity index 100% rename from .gitlab/configureFeatureBase.json rename to qa/scripts/configureFeatureBase.json diff --git a/.gitlab/featurebase.conf b/qa/scripts/featurebase.conf similarity index 100% rename from .gitlab/featurebase.conf rename to qa/scripts/featurebase.conf diff --git a/.gitlab/featurebase.service b/qa/scripts/featurebase.service similarity index 100% rename from .gitlab/featurebase.service rename to qa/scripts/featurebase.service diff --git a/.gitlab/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh similarity index 100% rename from .gitlab/ingestWorkload.sh rename to qa/scripts/ingestWorkload.sh diff --git a/.gitlab/simulacraData.go b/qa/simulacraData/simulacraData.go similarity index 100% rename from .gitlab/simulacraData.go rename to qa/simulacraData/simulacraData.go From d40eefb36499f2bf4a26d5ccd911092faec799e2 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 18 Nov 2021 12:06:07 -0600 Subject: [PATCH 03/15] added host:port as a command line input --- qa/scripts/ingestWorkload.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/qa/scripts/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh index 2502dba2a..23d11dd32 100755 --- a/qa/scripts/ingestWorkload.sh +++ b/qa/scripts/ingestWorkload.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # To run: -# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag} +# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag} function delete_field { if (($INITIALIZE == 0)); @@ -34,6 +34,10 @@ function ingest_set_field { FEATUREBASE_PATH=$1 shift +# featurebase host & port +HOST=$1 +shift + # path for directory with csv directory files for all fields to be ingested CSV_DIR_PATH=$1 shift @@ -45,8 +49,6 @@ shift # get a list of csv files in the directory CSV_FILES=`ls $CSV_DIR_PATH/*.csv` -# featurebase host -HOST="localhost:10101" # assign index name INDEX="samsung" if (($INITIALIZE == 1)); @@ -62,24 +64,19 @@ for CSV_FILE in ${CSV_FILES[@]} if [[ "$FIELD" == *"age"* ]]; then MAX=100 - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"identifier"* ]]; then MAX=$((2**63 - 1)) # compute max value for 64bit - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"ip"* ]]; then MAX=$((2**31 - 1)) # compute max value for 32bit - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"time"* ]]; then - echo $CSV_FILE ingest_time_field else - echo $CSV_FILE ingest_set_field fi done From 14050c567c8f535da21edaa77abaa928849cd587 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 13:15:44 -0800 Subject: [PATCH 04/15] corrects naming convention and age range --- qa/simulacraData/simulacra_data.go | 238 +++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 qa/simulacraData/simulacra_data.go diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go new file mode 100644 index 000000000..e09769a74 --- /dev/null +++ b/qa/simulacraData/simulacra_data.go @@ -0,0 +1,238 @@ +// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 +// INPUT: none +// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) + +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "strconv" +) + +var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", + "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", + "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", + "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", + "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", + "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", + "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", + "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", + "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", + "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", + "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", + "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", + "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", + "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", + "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", + "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", +} + +const totalRecords int = 200000000 + +func main() { + ageField() + ipField() + arbIdField() + optInField() + countryField() + timeField() +} + +func ageField() { + csvFile, err := os.Create("age.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(100) <= 20 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + } + } + } + writer.Flush() + csvFile.Close() + +} + +func ipField() { + csvFile, err := os.Create("ip.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func arbIdField() { + csvFile, err := os.Create("identifier.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func timeField() { + csvFile, err := os.Create("time.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 4000000 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + value := distro.Uint64() + urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) + + date := generateDate() + + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + + if rand.Intn(5) == 1 { + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + if rand.Intn(10) == 1 { + value := distro.Uint64() + urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + } + + writer.Flush() + csvFile.Close() + +} + +func optInField() { + csvFile, err := os.Create("optin.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + var optString string + + for i := 0; i < totalRecords; i++ { + + if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% + optString = "1" + } else { + optString = "0" + } + + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if rand.Intn(10) == 1 { + if optString == "0" { + optString = "1" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } else { + optString = "0" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } + } + } + + writer.Flush() + csvFile.Close() + +} + +func countryField() { + csvFile, err := os.Create("country.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + //populate countries + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 245 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(2) == 1 { + value := distro.Uint64() + country := countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + + if rand.Intn(10) == 1 { + value = distro.Uint64() + country = countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + } + } + + } + + writer.Flush() + csvFile.Close() + +} + +func generateDate() string { + // generates a random date in format YYYY-MM-DDT00:00 + // does not include leap days or populata the hour/minute time + year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 + month := rand.Intn(12) + 1 + day := rand.Intn(30) + 1 + + if month == 2 && day >= 29 { + day = 28 + } + if month == 4 && day == 31 { + day = 30 + } + if month == 6 && day == 31 { + day = 30 + } + if month == 11 && day == 31 { + day = 30 + } + + monthFix := fmt.Sprintf("%02d", month) + dayFix := fmt.Sprintf("%02d", day) + YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" + + return YMDstring +} From 90ace6b970c3e5cc3d450530c00eeff6bd110365 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 13:47:23 -0800 Subject: [PATCH 05/15] deletes duplicate file and adds license header --- qa/simulacraData/simulacraData.go | 238 ----------------------------- qa/simulacraData/simulacra_data.go | 14 ++ 2 files changed, 14 insertions(+), 238 deletions(-) delete mode 100644 qa/simulacraData/simulacraData.go diff --git a/qa/simulacraData/simulacraData.go b/qa/simulacraData/simulacraData.go deleted file mode 100644 index db4fd2012..000000000 --- a/qa/simulacraData/simulacraData.go +++ /dev/null @@ -1,238 +0,0 @@ -// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 -// INPUT: none -// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) - -package main - -import ( - "bufio" - "fmt" - "log" - "math/rand" - "os" - "strconv" -) - -var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", - "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", - "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", - "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", - "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", - "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", - "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", - "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", - "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", - "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", - "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", - "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", - "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", - "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", - "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", - "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", -} - -const totalRecords int = 200000000 - -func main() { - ageField() - ipField() - arbIdField() - optInField() - countryField() - timeField() -} - -func ageField() { - csvFile, err := os.Create("age.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - for i := 0; i < totalRecords; i++ { - if rand.Intn(100) <= 20 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") - } - } - } - writer.Flush() - csvFile.Close() - -} - -func ipField() { - csvFile, err := os.Create("ip.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") - } - } - - writer.Flush() - csvFile.Close() - -} - -func arbIdField() { - csvFile, err := os.Create("identifier.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") - } - } - - writer.Flush() - csvFile.Close() - -} - -func timeField() { - csvFile, err := os.Create("time.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - r := rand.New(rand.NewSource(rand.Int63())) - s := 1.01 - v := 1.01 - var imax uint64 = 4000000 - - distro := rand.NewZipf(r, s, v, imax) - - for i := 0; i < totalRecords; i++ { - value := distro.Uint64() - urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) - - date := generateDate() - - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") - - if rand.Intn(5) == 1 { - date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") - } - - if rand.Intn(10) == 1 { - value := distro.Uint64() - urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) - date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") - } - - } - - writer.Flush() - csvFile.Close() - -} - -func optInField() { - csvFile, err := os.Create("optin.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - var optString string - - for i := 0; i < totalRecords; i++ { - - if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% - optString = "1" - } else { - optString = "0" - } - - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") - if rand.Intn(10) == 1 { - if optString == "0" { - optString = "1" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") - } else { - optString = "0" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") - } - } - } - - writer.Flush() - csvFile.Close() - -} - -func countryField() { - csvFile, err := os.Create("country.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - //populate countries - r := rand.New(rand.NewSource(rand.Int63())) - s := 1.01 - v := 1.01 - var imax uint64 = 245 - - distro := rand.NewZipf(r, s, v, imax) - - for i := 0; i < totalRecords; i++ { - if rand.Intn(2) == 1 { - value := distro.Uint64() - country := countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") - - if rand.Intn(10) == 1 { - value = distro.Uint64() - country = countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") - } - } - - } - - writer.Flush() - csvFile.Close() - -} - -func generateDate() string { - // generates a random date in format YYYY-MM-DDT00:00 - // does not include leap days or populata the hour/minute time - year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 - month := rand.Intn(12) + 1 - day := rand.Intn(30) + 1 - - if month == 2 && day >= 29 { - day = 28 - } - if month == 4 && day == 31 { - day = 30 - } - if month == 6 && day == 31 { - day = 30 - } - if month == 11 && day == 31 { - day = 30 - } - - monthFix := fmt.Sprintf("%02d", month) - dayFix := fmt.Sprintf("%02d", day) - YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" - - return YMDstring -} diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index e09769a74..6e3dffc7e 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 // INPUT: none // OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) From 27b73d2b7d7baf8acd2ae241a1400c48a30c6b49 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 14:27:19 -0800 Subject: [PATCH 06/15] corrects typo on line 229 --- qa/simulacraData/simulacra_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 6e3dffc7e..d227f921e 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -226,7 +226,7 @@ func countryField() { func generateDate() string { // generates a random date in format YYYY-MM-DDT00:00 - // does not include leap days or populata the hour/minute time + // does not include leap days or populate the hour/minute time year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 month := rand.Intn(12) + 1 day := rand.Intn(30) + 1 From ce13bb78ede558ff170aaea8abcee0a24e0b7a9c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 18 Nov 2021 17:09:52 -0600 Subject: [PATCH 07/15] fixed typo --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f94cbcc0e..8294d2a3e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -233,7 +233,7 @@ deploy node for linux amd64: - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec3-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$PUBLIC_IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 From b5e5df8b58ba17356837fc3a87a012788676c374 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 16:10:49 -0800 Subject: [PATCH 08/15] add error checking --- qa/simulacraData/simulacra_data.go | 83 ++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 15 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index d227f921e..0ca7e2a16 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -61,14 +61,23 @@ func ageField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) for i := 0; i < totalRecords; i++ { if rand.Intn(100) <= 20 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + log.Fatalf("failed to write to age.csv %s", err) } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + log.Fatalf("failed to write to age.csv %s", err) + } + } + } } writer.Flush() @@ -81,12 +90,20 @@ func ipField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + log.Fatalf("failed to write to ip.csv %s", err) + } + if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + log.Fatalf("failed to write to ip.csv %s", err) + } } } @@ -100,12 +117,20 @@ func arbIdField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + log.Fatalf("failed to write to identifier.csv %s", err) + } + if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + log.Fatalf("failed to write to identifier.csv %s", err) + } } } @@ -119,6 +144,7 @@ func timeField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) r := rand.New(rand.NewSource(rand.Int63())) @@ -134,18 +160,27 @@ func timeField() { date := generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + log.Fatalf("failed to write to time.csv %s", err) + } if rand.Intn(5) == 1 { date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + log.Fatalf("failed to write to time.csv %s", err) + } } if rand.Intn(10) == 1 { value := distro.Uint64() urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + log.Fatalf("failed to write to time.csv %s", err) + } } } @@ -160,6 +195,7 @@ func optInField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) var optString string @@ -172,14 +208,24 @@ func optInField() { optString = "0" } - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to optin.csv %s", err) + } + if rand.Intn(10) == 1 { if optString == "0" { optString = "1" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to optin.csv %s", err) + } } else { optString = "0" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to optin.csv %s", err) + } } } } @@ -194,6 +240,7 @@ func countryField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) //populate countries @@ -208,12 +255,18 @@ func countryField() { if rand.Intn(2) == 1 { value := distro.Uint64() country := countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to country.csv %s", err) + } if rand.Intn(10) == 1 { value = distro.Uint64() country = countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to country.csv %s", err) + } } } From dc9c2a842aac63bf409e8c3d0d7f3c6f9ce734a8 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Fri, 19 Nov 2021 09:38:12 -0800 Subject: [PATCH 09/15] make error handling better --- qa/simulacraData/simulacra_data.go | 163 ++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 48 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 0ca7e2a16..f5741201a 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -21,7 +21,6 @@ package main import ( "bufio" "fmt" - "log" "math/rand" "os" "strconv" @@ -48,18 +47,40 @@ var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "AL const totalRecords int = 200000000 func main() { - ageField() - ipField() - arbIdField() - optInField() - countryField() - timeField() + err0 := ageField() + if err0 != nil { + fmt.Printf("unable to generate age field: %v", err0) + } + + err1 := ipField() + if err1 != nil { + fmt.Printf("unable to generate IPfield: %v", err1) + } + + err2 := arbIdField() + if err2 != nil { + fmt.Printf("unable to generate indentifier field: %v", err2) + } + + err3 := optInField() + if err3 != nil { + fmt.Printf("unable to generate opt in field: %v", err3) + } + err4 := countryField() + if err4 != nil { + fmt.Printf("unable to generate country field: %v", err4) + } + + err5 := timeField() + if err5 != nil { + fmt.Printf("unable to generate time field: %v", err5) + } } -func ageField() { +func ageField() error { csvFile, err := os.Create("age.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -68,27 +89,34 @@ func ageField() { if rand.Intn(100) <= 20 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - log.Fatalf("failed to write to age.csv %s", err) + return err } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - log.Fatalf("failed to write to age.csv %s", err) + return err } } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func ipField() { +func ipField() error { csvFile, err := os.Create("ip.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -96,26 +124,34 @@ func ipField() { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - log.Fatalf("failed to write to ip.csv %s", err) + return err } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - log.Fatalf("failed to write to ip.csv %s", err) + return err } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func arbIdField() { +func arbIdField() error { csvFile, err := os.Create("identifier.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -123,26 +159,34 @@ func arbIdField() { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - log.Fatalf("failed to write to identifier.csv %s", err) + return err } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - log.Fatalf("failed to write to identifier.csv %s", err) + return err } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func timeField() { +func timeField() error { csvFile, err := os.Create("time.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -162,14 +206,14 @@ func timeField() { _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - log.Fatalf("failed to write to time.csv %s", err) + return err } if rand.Intn(5) == 1 { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - log.Fatalf("failed to write to time.csv %s", err) + return err } } @@ -179,21 +223,28 @@ func timeField() { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - log.Fatalf("failed to write to time.csv %s", err) + return err } } } + err1 := writer.Flush() + if err1 != nil { + return err1 + } - writer.Flush() - csvFile.Close() + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func optInField() { +func optInField() error { csvFile, err := os.Create("optin.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -202,7 +253,7 @@ func optInField() { for i := 0; i < totalRecords; i++ { - if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% + if rand.Intn(20) == 1 { optString = "1" } else { optString = "0" @@ -210,7 +261,7 @@ func optInField() { _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to optin.csv %s", err) + return err } if rand.Intn(10) == 1 { @@ -218,32 +269,40 @@ func optInField() { optString = "1" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to optin.csv %s", err) + return err } } else { optString = "0" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to optin.csv %s", err) + return err } } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func countryField() { +func countryField() error { csvFile, err := os.Create("country.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) - //populate countries + // populate countries r := rand.New(rand.NewSource(rand.Int63())) s := 1.01 v := 1.01 @@ -257,7 +316,7 @@ func countryField() { country := countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to country.csv %s", err) + return err } if rand.Intn(10) == 1 { @@ -265,22 +324,30 @@ func countryField() { country = countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to country.csv %s", err) + return err } } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } func generateDate() string { // generates a random date in format YYYY-MM-DDT00:00 // does not include leap days or populate the hour/minute time - year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 + year := strconv.Itoa(rand.Intn(20) + 2002) // year range 2002-2021 month := rand.Intn(12) + 1 day := rand.Intn(30) + 1 From 4f5f27f5138962d922e5bcfc611310d5ebc2f3ea Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Fri, 19 Nov 2021 11:47:32 -0800 Subject: [PATCH 10/15] make error handling best --- qa/simulacraData/simulacra_data.go | 93 +++++++++++++++--------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index f5741201a..14f40b42f 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -24,6 +24,8 @@ import ( "math/rand" "os" "strconv" + + "github.com/pkg/errors" ) var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", @@ -48,7 +50,7 @@ const totalRecords int = 200000000 func main() { err0 := ageField() - if err0 != nil { + if err0 == nil { fmt.Printf("unable to generate age field: %v", err0) } @@ -78,9 +80,9 @@ func main() { } func ageField() error { - csvFile, err := os.Create("age.csv") - if err != nil { - return err + csvFile, err0 := os.Create("age.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create age.csv") } writer := bufio.NewWriter(csvFile) @@ -89,13 +91,14 @@ func ageField() error { if rand.Intn(100) <= 20 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to age.csv") } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to age.csv") + } } @@ -103,20 +106,20 @@ func ageField() error { } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close age.csv") } return nil } func ipField() error { - csvFile, err := os.Create("ip.csv") - if err != nil { - return err + csvFile, err0 := os.Create("ip.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create ip.csv") } writer := bufio.NewWriter(csvFile) @@ -124,34 +127,34 @@ func ipField() error { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to ip.csv") } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to ip.csv") } } } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close ip.csv") } return nil } func arbIdField() error { - csvFile, err := os.Create("identifier.csv") - if err != nil { - return err + csvFile, err0 := os.Create("identifier.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create identifier.csv") } writer := bufio.NewWriter(csvFile) @@ -159,34 +162,34 @@ func arbIdField() error { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to identifier.csv") } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to identifier.csv") } } } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close identifier.csv") } return nil } func timeField() error { - csvFile, err := os.Create("time.csv") - if err != nil { - return err + csvFile, err0 := os.Create("time.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create time.csv") } writer := bufio.NewWriter(csvFile) @@ -206,14 +209,14 @@ func timeField() error { _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to time.csv") } if rand.Intn(5) == 1 { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to time.csv") } } @@ -223,28 +226,28 @@ func timeField() error { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to time.csv") } } } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close time.csv") } return nil } func optInField() error { - csvFile, err := os.Create("optin.csv") - if err != nil { - return err + csvFile, err0 := os.Create("optin.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create optin.csv") } writer := bufio.NewWriter(csvFile) @@ -261,7 +264,7 @@ func optInField() error { _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to optin.csv") } if rand.Intn(10) == 1 { @@ -269,13 +272,13 @@ func optInField() error { optString = "1" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to optin.csv") } } else { optString = "0" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to optin.csv") } } } @@ -283,21 +286,21 @@ func optInField() error { err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close optin.csv") } return nil } func countryField() error { - csvFile, err := os.Create("country.csv") - if err != nil { - return err + csvFile, err0 := os.Create("country.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create country.csv") } writer := bufio.NewWriter(csvFile) @@ -316,7 +319,7 @@ func countryField() error { country := countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to country.csv") } if rand.Intn(10) == 1 { @@ -324,7 +327,7 @@ func countryField() error { country = countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to country.csv") } } } @@ -333,12 +336,12 @@ func countryField() error { err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close country.csv") } return nil From 97c4ee670b5f4e2925c207902c66f9446666e144 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 10:43:13 -0800 Subject: [PATCH 11/15] add tests --- qa/simulacraData/simulacra_data_test.go | 94 +++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 qa/simulacraData/simulacra_data_test.go diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go new file mode 100644 index 000000000..54e785b0c --- /dev/null +++ b/qa/simulacraData/simulacra_data_test.go @@ -0,0 +1,94 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "fmt" + "testing" +) + +func TestAge(t *testing.T) { + err := ageField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestIP(t *testing.T) { + err := ipField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestIndentifer(t *testing.T) { + err := arbIdField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestOptIn(t *testing.T) { + err := optInField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestCountry(t *testing.T) { + err := countryField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestTime(t *testing.T) { + err := timeField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestGenerateDate(t *testing.T) { + testDate := generateDate() + if testDate == "" { + fmt.Printf("error generating date") + } +} + +// func TestReadAge(t *testing.T) { +// csvfile, err := os.Open("age.csv") +// if err != nil { +// fmt.Printf("%v", err) +// } + +// r := csv.NewReader(csvfile) +// records, err := r.ReadAll() +// if err != nil { +// fmt.Printf("%v", err) +// } + +// for i := 0; i < len(records); i++ { + +// age, err := strconv.Atoi(records[i][1]) +// if err != nil { +// fmt.Printf("%v", err) +// } + +// if age > 100 || age < 13 { +// fmt.Printf("age outside of allowed range") +// } +// fmt.Println(age) +// } +// } From 2385fda2593297593b89b0249cffcb5e87e54b1a Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 10:45:12 -0800 Subject: [PATCH 12/15] add tests, remove commented code --- qa/simulacraData/simulacra_data_test.go | 26 ------------------------- 1 file changed, 26 deletions(-) diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index 54e785b0c..0620e54bc 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -66,29 +66,3 @@ func TestGenerateDate(t *testing.T) { fmt.Printf("error generating date") } } - -// func TestReadAge(t *testing.T) { -// csvfile, err := os.Open("age.csv") -// if err != nil { -// fmt.Printf("%v", err) -// } - -// r := csv.NewReader(csvfile) -// records, err := r.ReadAll() -// if err != nil { -// fmt.Printf("%v", err) -// } - -// for i := 0; i < len(records); i++ { - -// age, err := strconv.Atoi(records[i][1]) -// if err != nil { -// fmt.Printf("%v", err) -// } - -// if age > 100 || age < 13 { -// fmt.Printf("age outside of allowed range") -// } -// fmt.Println(age) -// } -// } From 06be6c0981524ce19a9f6bc5af39ffd307698474 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 11:06:38 -0800 Subject: [PATCH 13/15] use log.Fatal in lieu of fmt.Print --- qa/simulacraData/simulacra_data.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 14f40b42f..656c02211 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -21,6 +21,7 @@ package main import ( "bufio" "fmt" + "log" "math/rand" "os" "strconv" @@ -51,31 +52,31 @@ const totalRecords int = 200000000 func main() { err0 := ageField() if err0 == nil { - fmt.Printf("unable to generate age field: %v", err0) + log.Fatalf("unable to generate age field: %v", err0) } err1 := ipField() if err1 != nil { - fmt.Printf("unable to generate IPfield: %v", err1) + log.Fatalf("unable to generate IPfield: %v", err1) } err2 := arbIdField() if err2 != nil { - fmt.Printf("unable to generate indentifier field: %v", err2) + log.Fatalf("unable to generate indentifier field: %v", err2) } err3 := optInField() if err3 != nil { - fmt.Printf("unable to generate opt in field: %v", err3) + log.Fatalf("unable to generate opt in field: %v", err3) } err4 := countryField() if err4 != nil { - fmt.Printf("unable to generate country field: %v", err4) + log.Fatalf("unable to generate country field: %v", err4) } err5 := timeField() if err5 != nil { - fmt.Printf("unable to generate time field: %v", err5) + log.Fatalf("unable to generate time field: %v", err5) } } From 49289f2237dafce5ad7fa96e3a7e2b68286e2895 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 13:53:41 -0800 Subject: [PATCH 14/15] replace fmt.Print with t.Fatal, and improve date generator --- qa/simulacraData/simulacra_data.go | 28 +++++++------------------ qa/simulacraData/simulacra_data_test.go | 15 +++++++------ 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 656c02211..1e1656a95 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -25,6 +25,7 @@ import ( "math/rand" "os" "strconv" + "time" "github.com/pkg/errors" ) @@ -349,28 +350,13 @@ func countryField() error { } func generateDate() string { - // generates a random date in format YYYY-MM-DDT00:00 - // does not include leap days or populate the hour/minute time - year := strconv.Itoa(rand.Intn(20) + 2002) // year range 2002-2021 - month := rand.Intn(12) + 1 - day := rand.Intn(30) + 1 + // Nov 22, 2021 = 1637616960 + // twenty years = 631138520 + // date min = 1006478440 + date := rand.Int63n(631138520) + 1006478440 - if month == 2 && day >= 29 { - day = 28 - } - if month == 4 && day == 31 { - day = 30 - } - if month == 6 && day == 31 { - day = 30 - } - if month == 11 && day == 31 { - day = 30 - } - - monthFix := fmt.Sprintf("%02d", month) - dayFix := fmt.Sprintf("%02d", day) - YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" + t := time.Unix(date, 0) + YMDstring := t.Format("2006-01-02T15:04") return YMDstring } diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index 0620e54bc..17dc8b623 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -14,55 +14,54 @@ package main import ( - "fmt" "testing" ) func TestAge(t *testing.T) { err := ageField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestIP(t *testing.T) { err := ipField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestIndentifer(t *testing.T) { err := arbIdField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestOptIn(t *testing.T) { err := optInField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestCountry(t *testing.T) { err := countryField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestTime(t *testing.T) { err := timeField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestGenerateDate(t *testing.T) { testDate := generateDate() if testDate == "" { - fmt.Printf("error generating date") + t.Fatalf("error generating date") } } From 50e7cda07fd6f83264575d4d7c686189f45ebf18 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 14:53:17 -0800 Subject: [PATCH 15/15] modify functions to accept a requested num of records --- qa/simulacraData/simulacra_data.go | 57 +++++++++++-------------- qa/simulacraData/simulacra_data_test.go | 14 +++--- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 1e1656a95..e94c619d8 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -51,37 +51,32 @@ var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "AL const totalRecords int = 200000000 func main() { - err0 := ageField() - if err0 == nil { - log.Fatalf("unable to generate age field: %v", err0) + if err := GenerateAgeField(totalRecords); err != nil { + log.Fatalf("unable to generate age field: %v", err) } - err1 := ipField() - if err1 != nil { - log.Fatalf("unable to generate IPfield: %v", err1) + if err := GenerateIPField(totalRecords); err != nil { + log.Fatalf("unable to generate IP field: %v", err) } - err2 := arbIdField() - if err2 != nil { - log.Fatalf("unable to generate indentifier field: %v", err2) + if err := GenerateArbIdField(totalRecords); err != nil { + log.Fatalf("unable to generate indentifier field: %v", err) } - err3 := optInField() - if err3 != nil { - log.Fatalf("unable to generate opt in field: %v", err3) - } - err4 := countryField() - if err4 != nil { - log.Fatalf("unable to generate country field: %v", err4) + if err := GenerateOptInField(totalRecords); err != nil { + log.Fatalf("unable to generate opt in field: %v", err) } - err5 := timeField() - if err5 != nil { - log.Fatalf("unable to generate time field: %v", err5) + if err := GenerateCountryField(totalRecords); err != nil { + log.Fatalf("unable to generate country field: %v", err) + } + + if err := GenerateTimeField(totalRecords); err != nil { + log.Fatalf("unable to generate time field: %v", err) } } -func ageField() error { +func GenerateAgeField(requestedRecords int) error { csvFile, err0 := os.Create("age.csv") if err0 != nil { return errors.Wrap(err0, "unable to create age.csv") @@ -89,7 +84,7 @@ func ageField() error { writer := bufio.NewWriter(csvFile) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { if rand.Intn(100) <= 20 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { @@ -118,7 +113,7 @@ func ageField() error { return nil } -func ipField() error { +func GenerateIPField(requestedRecords int) error { csvFile, err0 := os.Create("ip.csv") if err0 != nil { return errors.Wrap(err0, "unable to create ip.csv") @@ -126,7 +121,7 @@ func ipField() error { writer := bufio.NewWriter(csvFile) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { return errors.Wrap(err, "unable to write to ip.csv") @@ -153,7 +148,7 @@ func ipField() error { } -func arbIdField() error { +func GenerateArbIdField(requestedRecords int) error { csvFile, err0 := os.Create("identifier.csv") if err0 != nil { return errors.Wrap(err0, "unable to create identifier.csv") @@ -161,7 +156,7 @@ func arbIdField() error { writer := bufio.NewWriter(csvFile) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { return errors.Wrap(err, "unable to write to identifier.csv") @@ -188,7 +183,7 @@ func arbIdField() error { } -func timeField() error { +func GenerateTimeField(requestedRecords int) error { csvFile, err0 := os.Create("time.csv") if err0 != nil { return errors.Wrap(err0, "unable to create time.csv") @@ -203,7 +198,7 @@ func timeField() error { distro := rand.NewZipf(r, s, v, imax) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { value := distro.Uint64() urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) @@ -246,7 +241,7 @@ func timeField() error { } -func optInField() error { +func GenerateOptInField(requestedRecords int) error { csvFile, err0 := os.Create("optin.csv") if err0 != nil { return errors.Wrap(err0, "unable to create optin.csv") @@ -256,7 +251,7 @@ func optInField() error { var optString string - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { if rand.Intn(20) == 1 { optString = "1" @@ -299,7 +294,7 @@ func optInField() error { } -func countryField() error { +func GenerateCountryField(requestedRecords int) error { csvFile, err0 := os.Create("country.csv") if err0 != nil { return errors.Wrap(err0, "unable to create country.csv") @@ -315,7 +310,7 @@ func countryField() error { distro := rand.NewZipf(r, s, v, imax) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { if rand.Intn(2) == 1 { value := distro.Uint64() country := countryList[value] diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index 17dc8b623..38462f5e5 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -17,43 +17,45 @@ import ( "testing" ) +const testRecords int = 1000 + func TestAge(t *testing.T) { - err := ageField() + err := GenerateAgeField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestIP(t *testing.T) { - err := ipField() + err := GenerateIPField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestIndentifer(t *testing.T) { - err := arbIdField() + err := GenerateArbIdField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestOptIn(t *testing.T) { - err := optInField() + err := GenerateOptInField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestCountry(t *testing.T) { - err := countryField() + err := GenerateCountryField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestTime(t *testing.T) { - err := timeField() + err := GenerateTimeField(testRecords) if err != nil { t.Fatalf("%v", err) }