scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into bearer-conversion-squashed

Browse source
This commit is contained in:
reesporte 2022-01-14 12:32:06 -06:00
commit 9f37cf7b48
9 changed files with 137 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitlab/.gitlab-ci.yml
View file

@ -279,8 +279,7 @@ gauntlet:
- docker
- fbsmoke
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -

1
client/ingest_api_batch_test.go
View file

@ -133,6 +133,7 @@ func TestIngestAPIBatchAdd(t *testing.T) {
}
func TestIngestAPIBatch(t *testing.T) {
t.Skip("causing sporadic CI failures... on my list to debug, but this code doesn't affect anyone's production anyhow (jaffee)")
c := test.MustRunCluster(t, 3)
defer c.Close()

1
field.go
View file

@ -1032,6 +1032,7 @@ func (f *Field) createViewIfNotExistsBase(cvm *CreateViewMessage) (*view, bool,
func (f *Field) newView(path, name string) *view {
view := newView(f.holder, path, f.index, f.name, name, f.options)
view.idx = f.idx
view.fld = f
view.stats = f.Stats
view.broadcaster = f.broadcaster
return view

5
qa/scripts/testSmokeTest.sh
View file

@ -17,6 +17,11 @@ echo "using INGESTNODE0 ${INGESTNODE0}"
DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
echo "using DATANODE0 ${DATANODE0}"
echo "Writing config.py file..."
cat << EOT > config.py
datanode0="${DATANODE0}"
EOT
mv config.py ./qa/testcases/smoketest/config.py
echo "Copying tests to remote"
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${INGESTNODE0}:/data

1
qa/scripts/utilCluster.sh
View file

@ -180,6 +180,7 @@ setupIngestNode() {
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown -R ec2-user /data"
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U pytest"
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests"
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U json"
}
setupDataNodes() {

1
qa/testcases/smoketest/config.py Normal file
View file

@ -0,0 +1 @@
datanode0="10.0.1.16"

63
qa/testcases/smoketest/test_smoke.py
View file

@ -1,7 +1,60 @@
# content of test_smoke.py
def inc(x):
return x + 1
import config
import json
import requests
def test_answer():
assert inc(3) == 1
createIndex = { 'options': { 'keys': False } }
createField = { 'options': { 'type': 'int', 'min': 0, 'max': 100000 } }
def setup_module(module):
data_to_send = json.dumps(createIndex).encode("utf-8")
response = requests.post("http://" + config.datanode0 + ":10101/index/user", data = data_to_send)
assert response.status_code == 200
assert response.headers["Content-Type"] == "application/json"
resp_body = response.json()
assert resp_body['success'] == True
data_to_send = json.dumps(createField).encode("utf-8")
response = requests.post("http://" + config.datanode0 + ":10101/index/user/field/stats", data = data_to_send)
assert response.status_code == 200
assert response.headers["Content-Type"] == "application/json"
resp_body = response.json()
assert resp_body['success'] == True
def teardown_module(module):
response = requests.delete("http://" + config.datanode0 + ":10101/index/user")
assert response.status_code == 200
assert response.headers["Content-Type"] == "application/json"
resp_body = response.json()
assert resp_body['success'] == True
def test_api_is_responding():
response = requests.get("http://" + config.datanode0 + ":10101/status")
assert response.status_code == 200
assert response.headers["Content-Type"] == "application/json"
resp_body = response.json()
assert resp_body['state'] == "NORMAL"
def test_get_index_api():
response = requests.get("http://" + config.datanode0 + ":10101/index/user")
assert response.status_code == 200
assert response.headers["Content-Type"] == "application/json"
resp_body = response.json()
assert resp_body['name'] == "user"
def test_set_and_read_query_api():
response = requests.post("http://" + config.datanode0 + ":10101/index/user/query", data = "Set(10, stats=1)")
assert response.status_code == 200
assert response.headers["Content-Type"] == "application/json"
resp_body = response.json()
assert resp_body['results'][0] == True
response = requests.post("http://" + config.datanode0 + ":10101/index/user/query", data = "Row(stats=1)")
assert response.status_code == 200
assert response.headers["Content-Type"] == "application/json"
resp_body = response.json()
assert resp_body['results'][0]['columns'][0] == 10

134
qa/tf/ci/smoketest/terraform.tfstate.backup
View file

@ -1,16 +1,16 @@
{
"version": 4,
"terraform_version": "1.1.2",
"serial": 203,
"serial": 220,
"lineage": "0f5e8a05-0e94-e86f-f384-26086bd40585",
"outputs": {
"cluster_prefix": {
"value": "smoke-BzP3aSw62HwEPFS",
"value": "gauntlet-wFQOOzXB51R3ebr",
"type": "string"
},
"data_node_ips": {
"value": [
"10.0.1.185"
"10.0.1.16"
],
"type": [
"tuple",
@ -25,7 +25,7 @@
},
"ingest_ips": {
"value": [
"3.144.237.6"
"3.142.172.86"
],
"type": [
"tuple",
@ -47,7 +47,7 @@
"schema_version": 0,
"attributes": {
"architecture": "arm64",
"arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff",
"arn": "arn:aws:ec2:us-east-2::image/ami-088e1f338c3b87d1a",
"block_device_mappings": [
{
"device_name": "/dev/xvda",
@ -55,7 +55,7 @@
"delete_on_termination": "true",
"encrypted": "false",
"iops": "0",
"snapshot_id": "snap-0617b00e90bae012b",
"snapshot_id": "snap-0f9ae89577e61b172",
"throughput": "0",
"volume_size": "8",
"volume_type": "gp2"
@ -64,8 +64,8 @@
"virtual_name": ""
}
],
"creation_date": "2021-12-01T19:36:11.000Z",
"description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2",
"creation_date": "2022-01-05T21:55:03.000Z",
"description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211223.0 arm64 HVM gp2",
"ena_support": true,
"executable_users": null,
"filter": [
@ -89,14 +89,14 @@
}
],
"hypervisor": "xen",
"id": "ami-0b09f36be67d32fff",
"image_id": "ami-0b09f36be67d32fff",
"image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2",
"id": "ami-088e1f338c3b87d1a",
"image_id": "ami-088e1f338c3b87d1a",
"image_location": "amazon/amzn2-ami-hvm-2.0.20211223.0-arm64-gp2",
"image_owner_alias": "amazon",
"image_type": "machine",
"kernel_id": null,
"most_recent": true,
"name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2",
"name": "amzn2-ami-hvm-2.0.20211223.0-arm64-gp2",
"name_regex": null,
"owner_id": "137112412989",
"owners": [
@ -109,7 +109,7 @@
"ramdisk_id": null,
"root_device_name": "/dev/xvda",
"root_device_type": "ebs",
"root_snapshot_id": "snap-0617b00e90bae012b",
"root_snapshot_id": "snap-0f9ae89577e61b172",
"sriov_net_support": "simple",
"state": "available",
"state_reason": {
@ -134,16 +134,16 @@
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:iam::977373308795:instance-profile/smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile",
"create_date": "2022-01-09T17:47:01Z",
"id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile",
"name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile",
"arn": "arn:aws:iam::977373308795:instance-profile/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
"create_date": "2022-01-12T15:00:15Z",
"id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
"name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
"name_prefix": null,
"path": "/",
"role": "smoke-BzP3aSw62HwEPFS-fb_cluster_node",
"role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
"tags": null,
"tags_all": {},
"unique_id": "AIPA6HD75E55XTIU7KRKL"
"unique_id": "AIPA6HD75E55WG6AVJLA4"
},
"sensitive_attributes": [],
"private": "bnVsbA==",
@ -163,12 +163,12 @@
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:iam::977373308795:role/smoke-BzP3aSw62HwEPFS-fb_cluster_node",
"arn": "arn:aws:iam::977373308795:role/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
"assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}",
"create_date": "2022-01-09T17:46:58Z",
"create_date": "2022-01-12T15:00:12Z",
"description": "",
"force_detach_policies": false,
"id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node",
"id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
"inline_policy": [
{
"name": "ec2_read_all",
@ -177,13 +177,13 @@
],
"managed_policy_arns": [],
"max_session_duration": 3600,
"name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node",
"name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
"name_prefix": "",
"path": "/",
"permissions_boundary": null,
"tags": null,
"tags_all": {},
"unique_id": "AROA6HD75E55YXA4Z4XK2"
"unique_id": "AROA6HD75E55YDDESCKUN"
},
"sensitive_attributes": [],
"private": "bnVsbA=="
@ -201,8 +201,8 @@
"index_key": 0,
"schema_version": 1,
"attributes": {
"ami": "ami-0b09f36be67d32fff",
"arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-077d0596e47712614",
"ami": "ami-088e1f338c3b87d1a",
"arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-03a8897456f88b2a4",
"associate_public_ip_address": false,
"availability_zone": "us-east-2a",
"capacity_reservation_specification": [
@ -225,7 +225,7 @@
"snapshot_id": "",
"tags": {},
"throughput": 125,
"volume_id": "vol-05eb623ed8991e3ec",
"volume_id": "vol-0028ea6c90c8ed849",
"volume_size": 100,
"volume_type": "gp3"
}
@ -240,14 +240,14 @@
"get_password_data": false,
"hibernation": false,
"host_id": null,
"iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile",
"id": "i-077d0596e47712614",
"iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
"id": "i-03a8897456f88b2a4",
"instance_initiated_shutdown_behavior": "stop",
"instance_state": "running",
"instance_type": "m6g.large",
"ipv6_address_count": 0,
"ipv6_addresses": [],
"key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci",
"key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
"launch_template": [],
"metadata_options": [
{
@ -262,9 +262,9 @@
"password_data": "",
"placement_group": "",
"placement_partition_number": null,
"primary_network_interface_id": "eni-076ed20f66a9f3552",
"private_dns": "ip-10-0-1-185.us-east-2.compute.internal",
"private_ip": "10.0.1.185",
"primary_network_interface_id": "eni-0deaddc1bee29d648",
"private_dns": "ip-10-0-1-16.us-east-2.compute.internal",
"private_ip": "10.0.1.16",
"public_dns": "",
"public_ip": "",
"root_block_device": [
@ -276,7 +276,7 @@
"kms_key_id": "",
"tags": null,
"throughput": 125,
"volume_id": "vol-03c02dd3fdafd0aa0",
"volume_id": "vol-00eb233b46ec8746f",
"volume_size": 20,
"volume_type": "gp3"
}
@ -286,13 +286,13 @@
"source_dest_check": true,
"subnet_id": "subnet-050b1219d78f2db1b",
"tags": {
"Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0",
"Prefix": "smoke-BzP3aSw62HwEPFS",
"Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0",
"Prefix": "gauntlet-wFQOOzXB51R3ebr",
"Role": "cluster_node"
},
"tags_all": {
"Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0",
"Prefix": "smoke-BzP3aSw62HwEPFS",
"Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0",
"Prefix": "gauntlet-wFQOOzXB51R3ebr",
"Role": "cluster_node"
},
"tenancy": "default",
@ -301,7 +301,7 @@
"user_data_base64": null,
"volume_tags": null,
"vpc_security_group_ids": [
"sg-06b14c4162509fb5f"
"sg-060f8471c4271acb8"
]
},
"sensitive_attributes": [],
@ -327,8 +327,8 @@
"index_key": 0,
"schema_version": 1,
"attributes": {
"ami": "ami-0b09f36be67d32fff",
"arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0ca325881e8a1dd8a",
"ami": "ami-088e1f338c3b87d1a",
"arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-012b6f6ad4a0295a8",
"associate_public_ip_address": true,
"availability_zone": "us-east-2a",
"capacity_reservation_specification": [
@ -337,7 +337,7 @@
"capacity_reservation_target": []
}
],
"cpu_core_count": 8,
"cpu_core_count": 2,
"cpu_threads_per_core": 1,
"credit_specification": [],
"disable_api_termination": false,
@ -351,7 +351,7 @@
"snapshot_id": "",
"tags": {},
"throughput": 125,
"volume_id": "vol-0803614106f4d6033",
"volume_id": "vol-0278897f78cb4f99e",
"volume_size": 100,
"volume_type": "gp3"
}
@ -366,14 +366,14 @@
"get_password_data": false,
"hibernation": false,
"host_id": null,
"iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile",
"id": "i-0ca325881e8a1dd8a",
"iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
"id": "i-012b6f6ad4a0295a8",
"instance_initiated_shutdown_behavior": "stop",
"instance_state": "running",
"instance_type": "c6g.2xlarge",
"instance_type": "m6g.large",
"ipv6_address_count": 0,
"ipv6_addresses": [],
"key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci",
"key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
"launch_template": [],
"metadata_options": [
{
@ -388,11 +388,11 @@
"password_data": "",
"placement_group": "",
"placement_partition_number": null,
"primary_network_interface_id": "eni-06216f96b73bcf92d",
"private_dns": "ip-10-0-101-105.us-east-2.compute.internal",
"private_ip": "10.0.101.105",
"primary_network_interface_id": "eni-0bb5e02c328f3c371",
"private_dns": "ip-10-0-101-66.us-east-2.compute.internal",
"private_ip": "10.0.101.66",
"public_dns": "",
"public_ip": "3.144.237.6",
"public_ip": "3.142.172.86",
"root_block_device": [
{
"delete_on_termination": true,
@ -402,7 +402,7 @@
"kms_key_id": "",
"tags": null,
"throughput": 125,
"volume_id": "vol-070f8a9022e93f9ee",
"volume_id": "vol-073cdccfd75958e27",
"volume_size": 20,
"volume_type": "gp3"
}
@ -412,13 +412,13 @@
"source_dest_check": true,
"subnet_id": "subnet-066b4b922b54e51a2",
"tags": {
"Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0",
"Prefix": "smoke-BzP3aSw62HwEPFS",
"Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0",
"Prefix": "gauntlet-wFQOOzXB51R3ebr",
"Role": "ingest_node"
},
"tags_all": {
"Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0",
"Prefix": "smoke-BzP3aSw62HwEPFS",
"Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0",
"Prefix": "gauntlet-wFQOOzXB51R3ebr",
"Role": "ingest_node"
},
"tenancy": "default",
@ -427,7 +427,7 @@
"user_data_base64": null,
"volume_tags": null,
"vpc_security_group_ids": [
"sg-062ddd221720047f4"
"sg-07e67c395f920042c"
]
},
"sensitive_attributes": [],
@ -452,12 +452,12 @@
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/smoke-BzP3aSw62HwEPFS-gitlab-ci",
"arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
"fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69",
"id": "smoke-BzP3aSw62HwEPFS-gitlab-ci",
"key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci",
"id": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
"key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
"key_name_prefix": "",
"key_pair_id": "key-01ca26b2795043890",
"key_pair_id": "key-0a49a5ef950bc7f0c",
"public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL",
"tags": null,
"tags_all": {}
@ -477,7 +477,7 @@
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-06b14c4162509fb5f",
"arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-060f8471c4271acb8",
"description": "Allow featurebase inbound traffic",
"egress": [
{
@ -496,7 +496,7 @@
"to_port": 0
}
],
"id": "sg-06b14c4162509fb5f",
"id": "sg-060f8471c4271acb8",
"ingress": [
{
"cidr_blocks": [
@ -595,7 +595,7 @@
"to_port": 55432
}
],
"name": "smoke-BzP3aSw62HwEPFS-allow_featurebase",
"name": "gauntlet-wFQOOzXB51R3ebr-allow_featurebase",
"name_prefix": "",
"owner_id": "977373308795",
"revoke_rules_on_delete": false,
@ -623,7 +623,7 @@
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-062ddd221720047f4",
"arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-07e67c395f920042c",
"description": "Allow ingest inbound traffic",
"egress": [
{
@ -642,7 +642,7 @@
"to_port": 0
}
],
"id": "sg-062ddd221720047f4",
"id": "sg-07e67c395f920042c",
"ingress": [
{
"cidr_blocks": [
@ -688,7 +688,7 @@
"to_port": -1
}
],
"name": "smoke-BzP3aSw62HwEPFS-allow_ingest",
"name": "gauntlet-wFQOOzXB51R3ebr-allow_ingest",
"name_prefix": "",
"owner_id": "977373308795",
"revoke_rules_on_delete": false,

3
view.go
View file

@ -40,6 +40,7 @@ type view struct {
holder *Holder
idx *Index
fld *Field
fieldType string
cacheType string
@ -363,7 +364,7 @@ func (v *view) notifyIfNewShard(shard uint64) {
}
func (v *view) newFragment(shard uint64) *fragment {
fld := v.idx.Field(v.field)
fld := v.fld
spec := fragSpec{
index: v.idx,
field: fld,