From 14050c567c8f535da21edaa77abaa928849cd587 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 13:15:44 -0800 Subject: [PATCH] corrects naming convention and age range --- qa/simulacraData/simulacra_data.go | 238 +++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 qa/simulacraData/simulacra_data.go diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go new file mode 100644 index 000000000..e09769a74 --- /dev/null +++ b/qa/simulacraData/simulacra_data.go @@ -0,0 +1,238 @@ +// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 +// INPUT: none +// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) + +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "strconv" +) + +var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", + "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", + "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", + "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", + "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", + "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", + "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", + "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", + "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", + "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", + "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", + "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", + "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", + "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", + "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", + "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", +} + +const totalRecords int = 200000000 + +func main() { + ageField() + ipField() + arbIdField() + optInField() + countryField() + timeField() +} + +func ageField() { + csvFile, err := os.Create("age.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(100) <= 20 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + } + } + } + writer.Flush() + csvFile.Close() + +} + +func ipField() { + csvFile, err := os.Create("ip.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func arbIdField() { + csvFile, err := os.Create("identifier.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func timeField() { + csvFile, err := os.Create("time.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 4000000 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + value := distro.Uint64() + urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) + + date := generateDate() + + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + + if rand.Intn(5) == 1 { + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + if rand.Intn(10) == 1 { + value := distro.Uint64() + urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + } + + writer.Flush() + csvFile.Close() + +} + +func optInField() { + csvFile, err := os.Create("optin.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + var optString string + + for i := 0; i < totalRecords; i++ { + + if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% + optString = "1" + } else { + optString = "0" + } + + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if rand.Intn(10) == 1 { + if optString == "0" { + optString = "1" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } else { + optString = "0" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } + } + } + + writer.Flush() + csvFile.Close() + +} + +func countryField() { + csvFile, err := os.Create("country.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + //populate countries + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 245 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(2) == 1 { + value := distro.Uint64() + country := countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + + if rand.Intn(10) == 1 { + value = distro.Uint64() + country = countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + } + } + + } + + writer.Flush() + csvFile.Close() + +} + +func generateDate() string { + // generates a random date in format YYYY-MM-DDT00:00 + // does not include leap days or populata the hour/minute time + year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 + month := rand.Intn(12) + 1 + day := rand.Intn(30) + 1 + + if month == 2 && day >= 29 { + day = 28 + } + if month == 4 && day == 31 { + day = 30 + } + if month == 6 && day == 31 { + day = 30 + } + if month == 11 && day == 31 { + day = 30 + } + + monthFix := fmt.Sprintf("%02d", month) + dayFix := fmt.Sprintf("%02d", day) + YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" + + return YMDstring +}