diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3f1b4eb0a..25a31899d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -279,8 +279,7 @@ gauntlet: - docker - fbsmoke rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' before_script: - apt-get update && apt-get install -y gnupg software-properties-common curl git - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index 24576b132..31ef6f49d 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -17,6 +17,11 @@ echo "using INGESTNODE0 ${INGESTNODE0}" DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') echo "using DATANODE0 ${DATANODE0}" +echo "Writing config.py file..." +cat << EOT > config.py +datanode0="${DATANODE0}" +EOT +mv config.py ./qa/testcases/smoketest/config.py echo "Copying tests to remote" scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${INGESTNODE0}:/data diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index f62e07208..769e84a49 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -180,6 +180,7 @@ setupIngestNode() { ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown -R ec2-user /data" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U pytest" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests" + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U json" } setupDataNodes() { diff --git a/qa/testcases/smoketest/config.py b/qa/testcases/smoketest/config.py new file mode 100644 index 000000000..42f642ee2 --- /dev/null +++ b/qa/testcases/smoketest/config.py @@ -0,0 +1 @@ +datanode0="10.0.1.16" diff --git a/qa/testcases/smoketest/test_smoke.py b/qa/testcases/smoketest/test_smoke.py index 08d266dc5..81ad0e273 100644 --- a/qa/testcases/smoketest/test_smoke.py +++ b/qa/testcases/smoketest/test_smoke.py @@ -1,7 +1,60 @@ -# content of test_smoke.py -def inc(x): - return x + 1 +import config +import json +import requests -def test_answer(): - assert inc(3) == 1 \ No newline at end of file +createIndex = { 'options': { 'keys': False } } +createField = { 'options': { 'type': 'int', 'min': 0, 'max': 100000 } } + +def setup_module(module): + data_to_send = json.dumps(createIndex).encode("utf-8") + response = requests.post("http://" + config.datanode0 + ":10101/index/user", data = data_to_send) + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + resp_body = response.json() + assert resp_body['success'] == True + + data_to_send = json.dumps(createField).encode("utf-8") + response = requests.post("http://" + config.datanode0 + ":10101/index/user/field/stats", data = data_to_send) + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + resp_body = response.json() + assert resp_body['success'] == True + + +def teardown_module(module): + response = requests.delete("http://" + config.datanode0 + ":10101/index/user") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + resp_body = response.json() + assert resp_body['success'] == True + + +def test_api_is_responding(): + response = requests.get("http://" + config.datanode0 + ":10101/status") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + resp_body = response.json() + assert resp_body['state'] == "NORMAL" + + +def test_get_index_api(): + response = requests.get("http://" + config.datanode0 + ":10101/index/user") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + resp_body = response.json() + assert resp_body['name'] == "user" + + +def test_set_and_read_query_api(): + response = requests.post("http://" + config.datanode0 + ":10101/index/user/query", data = "Set(10, stats=1)") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + resp_body = response.json() + assert resp_body['results'][0] == True + + response = requests.post("http://" + config.datanode0 + ":10101/index/user/query", data = "Row(stats=1)") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + resp_body = response.json() + assert resp_body['results'][0]['columns'][0] == 10 \ No newline at end of file diff --git a/qa/tf/ci/smoketest/terraform.tfstate.backup b/qa/tf/ci/smoketest/terraform.tfstate.backup index b4998f14f..bc199358f 100644 --- a/qa/tf/ci/smoketest/terraform.tfstate.backup +++ b/qa/tf/ci/smoketest/terraform.tfstate.backup @@ -1,16 +1,16 @@ { "version": 4, "terraform_version": "1.1.2", - "serial": 203, + "serial": 220, "lineage": "0f5e8a05-0e94-e86f-f384-26086bd40585", "outputs": { "cluster_prefix": { - "value": "smoke-BzP3aSw62HwEPFS", + "value": "gauntlet-wFQOOzXB51R3ebr", "type": "string" }, "data_node_ips": { "value": [ - "10.0.1.185" + "10.0.1.16" ], "type": [ "tuple", @@ -25,7 +25,7 @@ }, "ingest_ips": { "value": [ - "3.144.237.6" + "3.142.172.86" ], "type": [ "tuple", @@ -47,7 +47,7 @@ "schema_version": 0, "attributes": { "architecture": "arm64", - "arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2::image/ami-088e1f338c3b87d1a", "block_device_mappings": [ { "device_name": "/dev/xvda", @@ -55,7 +55,7 @@ "delete_on_termination": "true", "encrypted": "false", "iops": "0", - "snapshot_id": "snap-0617b00e90bae012b", + "snapshot_id": "snap-0f9ae89577e61b172", "throughput": "0", "volume_size": "8", "volume_type": "gp2" @@ -64,8 +64,8 @@ "virtual_name": "" } ], - "creation_date": "2021-12-01T19:36:11.000Z", - "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2", + "creation_date": "2022-01-05T21:55:03.000Z", + "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211223.0 arm64 HVM gp2", "ena_support": true, "executable_users": null, "filter": [ @@ -89,14 +89,14 @@ } ], "hypervisor": "xen", - "id": "ami-0b09f36be67d32fff", - "image_id": "ami-0b09f36be67d32fff", - "image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "id": "ami-088e1f338c3b87d1a", + "image_id": "ami-088e1f338c3b87d1a", + "image_location": "amazon/amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", "image_owner_alias": "amazon", "image_type": "machine", "kernel_id": null, "most_recent": true, - "name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "name": "amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", "name_regex": null, "owner_id": "137112412989", "owners": [ @@ -109,7 +109,7 @@ "ramdisk_id": null, "root_device_name": "/dev/xvda", "root_device_type": "ebs", - "root_snapshot_id": "snap-0617b00e90bae012b", + "root_snapshot_id": "snap-0f9ae89577e61b172", "sriov_net_support": "simple", "state": "available", "state_reason": { @@ -134,16 +134,16 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:iam::977373308795:instance-profile/smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", - "create_date": "2022-01-09T17:47:01Z", - "id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", - "name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "arn": "arn:aws:iam::977373308795:instance-profile/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "create_date": "2022-01-12T15:00:15Z", + "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", "name_prefix": null, "path": "/", - "role": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "tags": null, "tags_all": {}, - "unique_id": "AIPA6HD75E55XTIU7KRKL" + "unique_id": "AIPA6HD75E55WG6AVJLA4" }, "sensitive_attributes": [], "private": "bnVsbA==", @@ -163,12 +163,12 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:iam::977373308795:role/smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "arn": "arn:aws:iam::977373308795:role/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", - "create_date": "2022-01-09T17:46:58Z", + "create_date": "2022-01-12T15:00:12Z", "description": "", "force_detach_policies": false, - "id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "inline_policy": [ { "name": "ec2_read_all", @@ -177,13 +177,13 @@ ], "managed_policy_arns": [], "max_session_duration": 3600, - "name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "name_prefix": "", "path": "/", "permissions_boundary": null, "tags": null, "tags_all": {}, - "unique_id": "AROA6HD75E55YXA4Z4XK2" + "unique_id": "AROA6HD75E55YDDESCKUN" }, "sensitive_attributes": [], "private": "bnVsbA==" @@ -201,8 +201,8 @@ "index_key": 0, "schema_version": 1, "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-077d0596e47712614", + "ami": "ami-088e1f338c3b87d1a", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-03a8897456f88b2a4", "associate_public_ip_address": false, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -225,7 +225,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-05eb623ed8991e3ec", + "volume_id": "vol-0028ea6c90c8ed849", "volume_size": 100, "volume_type": "gp3" } @@ -240,14 +240,14 @@ "get_password_data": false, "hibernation": false, "host_id": null, - "iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", - "id": "i-077d0596e47712614", + "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "id": "i-03a8897456f88b2a4", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.large", "ipv6_address_count": 0, "ipv6_addresses": [], - "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "launch_template": [], "metadata_options": [ { @@ -262,9 +262,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-076ed20f66a9f3552", - "private_dns": "ip-10-0-1-185.us-east-2.compute.internal", - "private_ip": "10.0.1.185", + "primary_network_interface_id": "eni-0deaddc1bee29d648", + "private_dns": "ip-10-0-1-16.us-east-2.compute.internal", + "private_ip": "10.0.1.16", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -276,7 +276,7 @@ "kms_key_id": "", "tags": null, "throughput": 125, - "volume_id": "vol-03c02dd3fdafd0aa0", + "volume_id": "vol-00eb233b46ec8746f", "volume_size": 20, "volume_type": "gp3" } @@ -286,13 +286,13 @@ "source_dest_check": true, "subnet_id": "subnet-050b1219d78f2db1b", "tags": { - "Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0", - "Prefix": "smoke-BzP3aSw62HwEPFS", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tags_all": { - "Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0", - "Prefix": "smoke-BzP3aSw62HwEPFS", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tenancy": "default", @@ -301,7 +301,7 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-06b14c4162509fb5f" + "sg-060f8471c4271acb8" ] }, "sensitive_attributes": [], @@ -327,8 +327,8 @@ "index_key": 0, "schema_version": 1, "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0ca325881e8a1dd8a", + "ami": "ami-088e1f338c3b87d1a", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-012b6f6ad4a0295a8", "associate_public_ip_address": true, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -337,7 +337,7 @@ "capacity_reservation_target": [] } ], - "cpu_core_count": 8, + "cpu_core_count": 2, "cpu_threads_per_core": 1, "credit_specification": [], "disable_api_termination": false, @@ -351,7 +351,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-0803614106f4d6033", + "volume_id": "vol-0278897f78cb4f99e", "volume_size": 100, "volume_type": "gp3" } @@ -366,14 +366,14 @@ "get_password_data": false, "hibernation": false, "host_id": null, - "iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", - "id": "i-0ca325881e8a1dd8a", + "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "id": "i-012b6f6ad4a0295a8", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", - "instance_type": "c6g.2xlarge", + "instance_type": "m6g.large", "ipv6_address_count": 0, "ipv6_addresses": [], - "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "launch_template": [], "metadata_options": [ { @@ -388,11 +388,11 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-06216f96b73bcf92d", - "private_dns": "ip-10-0-101-105.us-east-2.compute.internal", - "private_ip": "10.0.101.105", + "primary_network_interface_id": "eni-0bb5e02c328f3c371", + "private_dns": "ip-10-0-101-66.us-east-2.compute.internal", + "private_ip": "10.0.101.66", "public_dns": "", - "public_ip": "3.144.237.6", + "public_ip": "3.142.172.86", "root_block_device": [ { "delete_on_termination": true, @@ -402,7 +402,7 @@ "kms_key_id": "", "tags": null, "throughput": 125, - "volume_id": "vol-070f8a9022e93f9ee", + "volume_id": "vol-073cdccfd75958e27", "volume_size": 20, "volume_type": "gp3" } @@ -412,13 +412,13 @@ "source_dest_check": true, "subnet_id": "subnet-066b4b922b54e51a2", "tags": { - "Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0", - "Prefix": "smoke-BzP3aSw62HwEPFS", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "ingest_node" }, "tags_all": { - "Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0", - "Prefix": "smoke-BzP3aSw62HwEPFS", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "ingest_node" }, "tenancy": "default", @@ -427,7 +427,7 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-062ddd221720047f4" + "sg-07e67c395f920042c" ] }, "sensitive_attributes": [], @@ -452,12 +452,12 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/smoke-BzP3aSw62HwEPFS-gitlab-ci", + "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", - "id": "smoke-BzP3aSw62HwEPFS-gitlab-ci", - "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "id": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "key_name_prefix": "", - "key_pair_id": "key-01ca26b2795043890", + "key_pair_id": "key-0a49a5ef950bc7f0c", "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", "tags": null, "tags_all": {} @@ -477,7 +477,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-06b14c4162509fb5f", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-060f8471c4271acb8", "description": "Allow featurebase inbound traffic", "egress": [ { @@ -496,7 +496,7 @@ "to_port": 0 } ], - "id": "sg-06b14c4162509fb5f", + "id": "sg-060f8471c4271acb8", "ingress": [ { "cidr_blocks": [ @@ -595,7 +595,7 @@ "to_port": 55432 } ], - "name": "smoke-BzP3aSw62HwEPFS-allow_featurebase", + "name": "gauntlet-wFQOOzXB51R3ebr-allow_featurebase", "name_prefix": "", "owner_id": "977373308795", "revoke_rules_on_delete": false, @@ -623,7 +623,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-062ddd221720047f4", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-07e67c395f920042c", "description": "Allow ingest inbound traffic", "egress": [ { @@ -642,7 +642,7 @@ "to_port": 0 } ], - "id": "sg-062ddd221720047f4", + "id": "sg-07e67c395f920042c", "ingress": [ { "cidr_blocks": [ @@ -688,7 +688,7 @@ "to_port": -1 } ], - "name": "smoke-BzP3aSw62HwEPFS-allow_ingest", + "name": "gauntlet-wFQOOzXB51R3ebr-allow_ingest", "name_prefix": "", "owner_id": "977373308795", "revoke_rules_on_delete": false,