diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ba793e62e..4fe913596 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -218,9 +218,17 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - echo $INSTANCE_ID > linux-amd64-instance.txt + - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data + - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - echo $PUBLIC_IP + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$PUBLIC_IP:. + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 + needs: + - job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt @@ -274,4 +282,4 @@ terminate node for linux amd64: needs: - job: deploy node for linux amd64 - job: initiate featurebase linux amd64 in node - when: always \ No newline at end of file + when: always diff --git a/.gitlab/cloud-init.sh b/qa/scripts/cloud-init.sh similarity index 100% rename from .gitlab/cloud-init.sh rename to qa/scripts/cloud-init.sh diff --git a/.gitlab/configureFeatureBase.json b/qa/scripts/configureFeatureBase.json similarity index 100% rename from .gitlab/configureFeatureBase.json rename to qa/scripts/configureFeatureBase.json diff --git a/.gitlab/featurebase.conf b/qa/scripts/featurebase.conf similarity index 100% rename from .gitlab/featurebase.conf rename to qa/scripts/featurebase.conf diff --git a/.gitlab/featurebase.service b/qa/scripts/featurebase.service similarity index 100% rename from .gitlab/featurebase.service rename to qa/scripts/featurebase.service diff --git a/.gitlab/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh similarity index 87% rename from .gitlab/ingestWorkload.sh rename to qa/scripts/ingestWorkload.sh index 2502dba2a..23d11dd32 100755 --- a/.gitlab/ingestWorkload.sh +++ b/qa/scripts/ingestWorkload.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # To run: -# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag} +# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag} function delete_field { if (($INITIALIZE == 0)); @@ -34,6 +34,10 @@ function ingest_set_field { FEATUREBASE_PATH=$1 shift +# featurebase host & port +HOST=$1 +shift + # path for directory with csv directory files for all fields to be ingested CSV_DIR_PATH=$1 shift @@ -45,8 +49,6 @@ shift # get a list of csv files in the directory CSV_FILES=`ls $CSV_DIR_PATH/*.csv` -# featurebase host -HOST="localhost:10101" # assign index name INDEX="samsung" if (($INITIALIZE == 1)); @@ -62,24 +64,19 @@ for CSV_FILE in ${CSV_FILES[@]} if [[ "$FIELD" == *"age"* ]]; then MAX=100 - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"identifier"* ]]; then MAX=$((2**63 - 1)) # compute max value for 64bit - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"ip"* ]]; then MAX=$((2**31 - 1)) # compute max value for 32bit - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"time"* ]]; then - echo $CSV_FILE ingest_time_field else - echo $CSV_FILE ingest_set_field fi done diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go new file mode 100644 index 000000000..e94c619d8 --- /dev/null +++ b/qa/simulacraData/simulacra_data.go @@ -0,0 +1,357 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 +// INPUT: none +// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) + +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "strconv" + "time" + + "github.com/pkg/errors" +) + +var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", + "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", + "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", + "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", + "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", + "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", + "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", + "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", + "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", + "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", + "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", + "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", + "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", + "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", + "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", + "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", +} + +const totalRecords int = 200000000 + +func main() { + if err := GenerateAgeField(totalRecords); err != nil { + log.Fatalf("unable to generate age field: %v", err) + } + + if err := GenerateIPField(totalRecords); err != nil { + log.Fatalf("unable to generate IP field: %v", err) + } + + if err := GenerateArbIdField(totalRecords); err != nil { + log.Fatalf("unable to generate indentifier field: %v", err) + } + + if err := GenerateOptInField(totalRecords); err != nil { + log.Fatalf("unable to generate opt in field: %v", err) + } + + if err := GenerateCountryField(totalRecords); err != nil { + log.Fatalf("unable to generate country field: %v", err) + } + + if err := GenerateTimeField(totalRecords); err != nil { + log.Fatalf("unable to generate time field: %v", err) + } +} + +func GenerateAgeField(requestedRecords int) error { + csvFile, err0 := os.Create("age.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create age.csv") + } + + writer := bufio.NewWriter(csvFile) + + for i := 0; i < requestedRecords; i++ { + if rand.Intn(100) <= 20 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to age.csv") + } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to age.csv") + + } + } + + } + } + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close age.csv") + } + return nil +} + +func GenerateIPField(requestedRecords int) error { + csvFile, err0 := os.Create("ip.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create ip.csv") + } + + writer := bufio.NewWriter(csvFile) + + for i := 0; i < requestedRecords; i++ { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to ip.csv") + } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to ip.csv") + } + } + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close ip.csv") + } + return nil + +} + +func GenerateArbIdField(requestedRecords int) error { + csvFile, err0 := os.Create("identifier.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create identifier.csv") + } + + writer := bufio.NewWriter(csvFile) + + for i := 0; i < requestedRecords; i++ { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to identifier.csv") + } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to identifier.csv") + } + } + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close identifier.csv") + } + return nil + +} + +func GenerateTimeField(requestedRecords int) error { + csvFile, err0 := os.Create("time.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create time.csv") + } + + writer := bufio.NewWriter(csvFile) + + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 4000000 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < requestedRecords; i++ { + value := distro.Uint64() + urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) + + date := generateDate() + + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to time.csv") + } + + if rand.Intn(5) == 1 { + date = generateDate() + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to time.csv") + } + } + + if rand.Intn(10) == 1 { + value := distro.Uint64() + urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) + date = generateDate() + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to time.csv") + } + } + + } + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close time.csv") + } + return nil + +} + +func GenerateOptInField(requestedRecords int) error { + csvFile, err0 := os.Create("optin.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create optin.csv") + } + + writer := bufio.NewWriter(csvFile) + + var optString string + + for i := 0; i < requestedRecords; i++ { + + if rand.Intn(20) == 1 { + optString = "1" + } else { + optString = "0" + } + + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to optin.csv") + } + + if rand.Intn(10) == 1 { + if optString == "0" { + optString = "1" + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to optin.csv") + } + } else { + optString = "0" + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to optin.csv") + } + } + } + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close optin.csv") + } + return nil + +} + +func GenerateCountryField(requestedRecords int) error { + csvFile, err0 := os.Create("country.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create country.csv") + } + + writer := bufio.NewWriter(csvFile) + + // populate countries + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 245 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < requestedRecords; i++ { + if rand.Intn(2) == 1 { + value := distro.Uint64() + country := countryList[value] + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to country.csv") + } + + if rand.Intn(10) == 1 { + value = distro.Uint64() + country = countryList[value] + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + return errors.Wrap(err, "unable to write to country.csv") + } + } + } + + } + + err1 := writer.Flush() + if err1 != nil { + return errors.Wrap(err1, "unable to flush writer") + } + + err2 := csvFile.Close() + if err2 != nil { + return errors.Wrap(err2, "unable to close country.csv") + } + return nil + +} + +func generateDate() string { + // Nov 22, 2021 = 1637616960 + // twenty years = 631138520 + // date min = 1006478440 + date := rand.Int63n(631138520) + 1006478440 + + t := time.Unix(date, 0) + YMDstring := t.Format("2006-01-02T15:04") + + return YMDstring +} diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go new file mode 100644 index 000000000..38462f5e5 --- /dev/null +++ b/qa/simulacraData/simulacra_data_test.go @@ -0,0 +1,69 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "testing" +) + +const testRecords int = 1000 + +func TestAge(t *testing.T) { + err := GenerateAgeField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestIP(t *testing.T) { + err := GenerateIPField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestIndentifer(t *testing.T) { + err := GenerateArbIdField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestOptIn(t *testing.T) { + err := GenerateOptInField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestCountry(t *testing.T) { + err := GenerateCountryField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestTime(t *testing.T) { + err := GenerateTimeField(testRecords) + if err != nil { + t.Fatalf("%v", err) + } +} + +func TestGenerateDate(t *testing.T) { + testDate := generateDate() + if testDate == "" { + t.Fatalf("error generating date") + } +}