From eb06bb50ae460cbf0185c6e89301ad8989b90b5a Mon Sep 17 00:00:00 2001 From: Fletcher Haynes Date: Fri, 2 Sep 2022 13:23:39 -0700 Subject: [PATCH] Updated code to latest version for open-sourcing. --- .circleci/config.yml | 195 - .github/ISSUE_TEMPLATE.md | 16 - .github/PULL_REQUEST_TEMPLATE.md | 28 - CHANGELOG.md | 715 - CODE_OF_CONDUCT.md | 133 + CONTRIBUTING.md | 177 - Dockerfile | 51 +- Dockerfile-clustertests | 26 +- Dockerfile-clustertests-client | 35 + LICENSE => LICENSE-2.0.txt | 0 Makefile | 337 +- NOTES | 26 - NOTICE | 60 +- README.md | 115 +- api.go | 2708 +- api/client/grpc.go | 203 + api_test.go | 1782 +- apimethod_string.go | 40 +- attr.go | 205 - attr_test.go | 201 - audit.go | 13 + audit_internal_test.go | 40 + audit_test.go | 95 + authn/authenticate.go | 443 + authn/authenticate_internal_test.go | 766 + authz/authorization.go | 131 + authz/authorization_test.go | 305 + boltdb/attrstore.go | 423 - boltdb/translate.go | 577 +- boltdb/translate_internal_test.go | 107 + boltdb/translate_test.go | 385 +- broadcast.go | 71 +- bsi.go | 284 + bsi_test.go | 160 + cache.go | 344 +- cache_test.go | 86 +- catcher.go | 248 + client.go | 174 - client/README.md | 85 + client/batch.go | 1778 + client/batch_test.go | 1841 + client/client.go | 1781 + client/client_it_test.go | 893 + client/client_test.go | 210 + client/cluster.go | 88 + client/cluster_test.go | 52 + client/csv/csv.go | 182 + client/csv/csv_it_test.go | 49 + client/csv/csv_test.go | 255 + client/doc.go | 56 + client/docs/data-model-queries.md | 149 + client/docs/server-interaction.md | 160 + client/docs/tracing.md | 111 + client/egpool/egpool.go | 111 + client/egpool/egpool_test.go | 38 + client/error.go | 26 + client/ingest_api_batch.go | 145 + client/ingest_api_batch_test.go | 305 + client/logimport.go | 33 + client/logimport_test.go | 143 + client/metrics.go | 28 + client/orm.go | 1586 + client/orm_test.go | 1235 + client/record.go | 63 + client/record_test.go | 71 + client/response.go | 496 + client/response_test.go | 273 + client/shardnodes.go | 54 + client/tracer.go | 77 + client/validate.go | 42 + client/validate_test.go | 61 + client/version.go | 9 + cluster.go | 2738 +- cluster_internal_test.go | 879 +- cmd.go | 25 +- cmd/auth_token.go | 30 + cmd/backup.go | 38 + cmd/badloader/badloader.go | 138 + cmd/check.go | 46 - cmd/check_test.go | 36 - cmd/chksum.go | 31 + cmd/config.go | 20 +- cmd/doc.go | 16 +- cmd/export.go | 26 +- cmd/export_test.go | 20 +- cmd/featurebase-parse-sql/main.go | 48 + cmd/featurebase/main.go | 23 + cmd/featurebase/main_test.go | 13 + cmd/generate_config.go | 18 +- cmd/import.go | 71 +- cmd/import_test.go | 27 +- cmd/inspect.go | 49 - cmd/inspect_test.go | 42 - cmd/keygen.go | 29 + cmd/pilosa-bench/main.go | 346 + cmd/pilosa/main.go | 33 - cmd/rbf.go | 148 + cmd/restore.go | 42 + cmd/roaring-migrate/ctim_darwin.go | 16 + cmd/roaring-migrate/ctim_linux.go | 14 + cmd/roaring-migrate/main.go | 481 + cmd/roaring-migrate/main_test.go | 56 + cmd/roaring-migrate/testdata/data-dir/.id | 1 + .../testdata/data-dir/.startup.log | 1 + .../testdata/data-dir/.topology | 2 + .../testdata/data-dir/idalloc.db | Bin 0 -> 16384 bytes .../testdata/data-dir/repository/.data | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/.meta | 1 + .../data-dir/repository/_exists/.data | Bin 0 -> 32768 bytes .../data-dir/repository/_exists/.meta | Bin 0 -> 23 bytes .../testdata/data-dir/repository/_exists/keys | Bin 0 -> 32768 bytes .../_exists/views/standard/fragments/222 | Bin 0 -> 21 bytes .../views/standard/fragments/222.cache | Bin 0 -> 3 bytes .../testdata/data-dir/repository/_keys/0 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/1 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/10 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/100 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/101 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/102 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/103 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/104 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/105 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/106 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/107 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/108 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/109 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/11 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/110 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/111 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/112 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/113 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/114 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/115 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/116 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/117 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/118 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/119 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/12 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/120 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/121 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/122 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/123 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/124 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/125 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/126 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/127 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/128 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/129 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/13 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/130 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/131 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/132 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/133 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/134 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/135 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/136 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/137 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/138 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/139 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/14 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/140 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/141 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/142 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/143 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/144 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/145 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/146 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/147 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/148 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/149 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/15 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/150 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/151 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/152 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/153 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/154 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/155 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/156 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/157 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/158 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/159 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/16 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/160 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/161 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/162 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/163 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/164 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/165 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/166 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/167 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/168 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/169 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/17 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/170 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/171 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/172 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/173 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/174 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/175 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/176 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/177 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/178 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/179 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/18 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/180 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/181 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/182 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/183 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/184 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/185 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/186 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/187 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/188 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/189 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/19 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/190 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/191 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/192 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/193 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/194 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/195 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/196 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/197 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/198 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/199 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/2 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/20 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/200 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/201 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/202 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/203 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/204 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/205 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/206 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/207 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/208 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/209 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/21 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/210 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/211 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/212 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/213 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/214 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/215 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/216 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/217 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/218 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/219 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/22 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/220 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/221 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/222 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/223 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/224 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/225 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/226 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/227 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/228 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/229 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/23 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/230 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/231 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/232 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/233 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/234 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/235 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/236 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/237 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/238 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/239 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/24 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/240 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/241 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/242 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/243 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/244 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/245 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/246 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/247 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/248 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/249 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/25 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/250 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/251 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/252 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/253 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/254 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/255 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/26 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/27 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/28 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/29 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/3 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/30 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/31 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/32 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/33 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/34 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/35 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/36 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/37 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/38 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/39 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/4 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/40 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/41 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/42 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/43 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/44 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/45 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/46 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/47 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/48 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/49 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/5 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/50 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/51 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/52 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/53 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/54 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/55 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/56 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/57 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/58 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/59 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/6 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/60 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/61 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/62 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/63 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/64 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/65 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/66 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/67 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/68 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/69 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/7 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/70 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/71 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/72 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/73 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/74 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/75 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/76 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/77 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/78 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/79 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/8 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/80 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/81 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/82 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/83 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/84 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/85 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/86 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/87 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/88 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/89 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/9 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/90 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/91 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/92 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/93 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/94 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/95 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/96 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/97 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/98 | Bin 0 -> 32768 bytes .../testdata/data-dir/repository/_keys/99 | Bin 0 -> 32768 bytes .../data-dir/repository/language/.data | Bin 0 -> 32768 bytes .../data-dir/repository/language/.meta | Bin 0 -> 23 bytes .../data-dir/repository/language/keys | Bin 0 -> 32768 bytes .../language/views/standard/fragments/222 | Bin 0 -> 21 bytes .../views/standard/fragments/222.cache | 2 + .../data-dir/repository/stargazer/.data | Bin 0 -> 32768 bytes .../data-dir/repository/stargazer/.meta | Bin 0 -> 23 bytes .../data-dir/repository/stargazer/keys | Bin 0 -> 32768 bytes cmd/root.go | 58 +- cmd/root_test.go | 40 +- cmd/server.go | 70 +- cmd/server_test.go | 237 +- cmd/slurp/slurp.go | 254 + const_amd64.go | 8 + const_other.go | 10 + ctl/auth_token.go | 274 + ctl/auth_token_internal_test.go | 191 + ctl/backup.go | 526 + ctl/check.go | 135 - ctl/check_test.go | 108 - ctl/chksum.go | 138 + ctl/common.go | 76 +- ctl/config.go | 20 +- ctl/config_test.go | 18 +- ctl/doc.go | 16 +- ctl/export.go | 20 +- ctl/export_test.go | 22 +- ctl/generate_config.go | 20 +- ctl/generate_config_test.go | 16 +- ctl/import.go | 242 +- ctl/import_test.go | 288 +- ctl/inspect.go | 110 - ctl/inspect_test.go | 60 - ctl/keygen.go | 31 + ctl/main_test.go | 31 + ctl/rbf_check.go | 56 + ctl/rbf_check_test.go | 34 + ctl/rbf_dump.go | 99 + ctl/rbf_page.go | 128 + ctl/rbf_pages.go | 141 + ctl/rbf_pages_test.go | 53 + ctl/restore.go | 476 + ctl/server.go | 145 +- ctl/server_test.go | 18 +- ctl/testdata/certs/README.md | 12 + ctl/testdata/certs/localhost.crt | 25 + ctl/testdata/certs/localhost.csr | 16 + ctl/testdata/certs/localhost.key | 27 + ctl/testdata/certs/pilosa-ca.crl | 16 + ctl/testdata/certs/pilosa-ca.crt | 29 + ctl/testdata/certs/pilosa-ca.key | 51 + ctl/testdata/permissions.yaml | 4 + .../rbf-check/err-invalid-page-type/data | Bin 0 -> 40960 bytes .../rbf-check/err-invalid-page-type/wal | 0 ctl/testdata/rbf-check/ok/data | Bin 0 -> 32768 bytes ctl/testdata/rbf-check/ok/wal | 0 .../rbf-pages/err-invalid-page-type/data | Bin 0 -> 40960 bytes .../rbf-pages/err-invalid-page-type/wal | 0 ctl/testdata/rbf-pages/ok/data | Bin 0 -> 32768 bytes ctl/testdata/rbf-pages/ok/wal | 0 ctl/util.go | 56 + dbshard.go | 704 + dbshard_internal_test.go | 298 + dbshard_test.go | 99 + debugstats/stats.go | 128 + debugstats/stats_test.go | 42 + delete_test.go | 277 + diagnostics.go | 38 +- diagnostics_internal_test.go | 24 +- disco/disco.go | 429 + disco/hasher.go | 39 + disco/node.go | 135 + disco/noder.go | 96 + disco/snapshot.go | 294 + doc.go | 16 +- docs/README.md | 5 - docs/administration.md | 327 - docs/api-reference.md | 352 - docs/architecture.md | 25 - docs/client-libraries.md | 18 - docs/configuration.md | 638 - docs/console.md | 58 - docs/data-model.md | 205 - docs/examples.md | 221 - docs/faq.md | 43 - docs/getting-started.md | 970 - docs/glossary.md | 77 - docs/installation.md | 382 - docs/introduction.md | 19 - docs/pdk.md | 74 - docs/query-language.md | 943 - docs/tutorials.md | 779 - encoding/proto/proto.go | 1819 +- encoding/proto/proto_test.go | 177 + enterprise/COPYING | 661 - enterprise/enterprise.go | 23 - etcd/config_gen.go | 71 + etcd/embed.go | 1114 + etcd/embed_test.go | 93 + etcd/enterprise_cluster.go | 12 + etcd/external.go | 94 + etcd/external_test.go | 41 + etcd/fake_test.go | 126 + etcd/leasedkv.go | 245 + etcd/leasedkv_test.go | 140 + etcd/plg_cluster.go | 12 + event.go | 20 +- executor.go | 8365 +- executor_internal_test.go | 692 +- executor_test.go | 8957 +- field.go | 1924 +- field_internal_test.go | 707 +- field_test.go | 219 +- filesystem.go | 30 + fragment.go | 3467 +- fragment_internal_test.go | 4047 +- gc.go | 16 +- gcnotify/gcnotify.go | 18 +- generator/slice.go | 39 + gid.go | 164 + go.mod | 192 +- go.sum | 1662 +- gopsutil/systeminfo.go | 43 +- gopsutil/systeminfo_test.go | 20 +- gossip/gossip.go | 614 - hack.go | 82 + handler.go | 386 +- hash/blake3.go | 95 + hash/blake3_test.go | 78 + holder.go | 1887 +- holder_internal_test.go | 389 +- holder_test.go | 526 +- http/client.go | 1280 - http/client_test.go | 1089 - http/error.go | 26 - http/handler.go | 1718 - http/handler_internal_test.go | 180 - http/handler_test.go | 42 - http/translator_test.go | 153 - http_handler.go | 3967 + http_handler_internal_test.go | 1060 + http_handler_test.go | 924 + http/translator.go => http_translator.go | 74 +- http_translator_test.go | 148 + idalloc.go | 577 + idalloc_test.go | 202 + idk/.cloud-env.template | 4 + idk/Dockerfile | 47 + idk/Dockerfile-fakeIDP | 7 + idk/Dockerfile-test | 24 + idk/Dockerfile-wait | 8 + idk/Makefile | 330 + idk/README.md | 124 + idk/api/codec.go | 84 + idk/api/codec_test.go | 112 + idk/api/schema.go | 183 + idk/api/schema_test.go | 139 + idk/api/source.go | 477 + idk/api/source_test.go | 225 + idk/bankgen/put.go | 315 + idk/bankgen/put_test.go | 26 + idk/cmd/bankgen/main.go | 35 + idk/cmd/bankgen/main_test.go | 129 + idk/cmd/datagen/main.go | 52 + idk/cmd/datagen/main_test.go | 174 + idk/cmd/ingester/README.md | 12 + idk/cmd/ingester/main.go | 187 + idk/cmd/kafkagen/main.go | 30 + idk/cmd/kafkagen/main_test.go | 90 + idk/cmd/kafkaput/main.go | 34 + idk/cmd/kafkaput/main_test.go | 103 + idk/cmd/molecula-consumer-csv/main.go | 48 + idk/cmd/molecula-consumer-csv/main_test.go | 207 + idk/cmd/molecula-consumer-github/README.md | 4 + idk/cmd/molecula-consumer-github/main.go | 435 + .../molecula-consumer-kafka-delete/main.go | 32 + .../main_test.go | 192 + idk/cmd/molecula-consumer-kafka-sasl/main.go | 36 + .../molecula-consumer-kafka-sasl/readme.md | 67 + .../molecula-consumer-kafka-static/main.go | 32 + idk/cmd/molecula-consumer-kafka/main.go | 37 + idk/cmd/molecula-consumer-kafka/main_test.go | 206 + idk/cmd/molecula-consumer-kinesis/main.go | 50 + idk/cmd/molecula-consumer-sql/main.go | 38 + idk/cmd/molecula-consumer-sql/main_test.go | 151 + idk/common/configMap.go | 191 + idk/csv/cmd.go | 138 + idk/csv/csv_test.go | 653 + idk/csv/source.go | 187 + idk/datagen/README.md | 39 + idk/datagen/all-field-types.go | 319 + idk/datagen/bank.go | 311 + idk/datagen/claim.go | 128 + idk/datagen/cmd.go | 598 + idk/datagen/common.go | 34 + idk/datagen/custom.go | 949 + idk/datagen/custom_test.go | 309 + idk/datagen/customer.go | 106 + idk/datagen/customer_segmentation.go | 603 + idk/datagen/customer_segmentation_test.go | 27 + idk/datagen/equipment.data.go | 269 + idk/datagen/equipment.go | 282 + idk/datagen/example.go | 100 + idk/datagen/gen/constants.data.go | 65 + idk/datagen/gen/gen.go | 154 + idk/datagen/gen/gen_test.go | 87 + idk/datagen/hobbies.data.go | 114 + idk/datagen/item.go | 131 + idk/datagen/kitchen-sink-keyed.go | 163 + idk/datagen/kitchen-sink.go | 139 + idk/datagen/locations.data.go | 104535 +++++++++++++++ idk/datagen/network.go | 248 + idk/datagen/shared.go | 28 + idk/datagen/sites.data.go | 37 + idk/datagen/sites.go | 211 + idk/datagen/sizing.go | 124 + idk/datagen/skills.data.go | 218 + idk/datagen/stringpk.go | 94 + idk/datagen/testdata/custom.yaml | 119 + .../testdata/file_with_line_delimited_values | 5 + idk/datagen/testdata/gender.txt | 5 + idk/datagen/testdata/schema.json | 37 + idk/datagen/testdata/t_keys.yaml | 48 + idk/datagen/timeseries.go | 188 + idk/datagen/titles.data.go | 5005 + idk/datagen/transactions.go | 207 + idk/datagen/transactions_scenario_1.go | 203 + idk/datagen/uscities.data.go | 28342 ++++ idk/datagen/warranty.go | 148 + idk/datagen/zip_codes.data.go | 42728 ++++++ idk/docker-compose.yml | 136 + idk/docker-sasl/client-ssl-test.conf | 7 + idk/docker-sasl/jaas_kafka_broker.conf | 30 + idk/docker-sasl/jaas_zookeeper_server.conf | 7 + .../ssl_keys/broker_kafkaBroker_cert-file | 17 + .../ssl_keys/broker_kafkaBroker_cert-signed | 20 + .../broker_kafkaBroker_server.keystore.jks | Bin 0 -> 3936 bytes .../broker_kafkaBroker_server.truststore.jks | Bin 0 -> 917 bytes idk/docker-sasl/ssl_keys/ca-cert | 20 + idk/docker-sasl/ssl_keys/ca-cert.key | 30 + idk/docker-sasl/ssl_keys/ca-cert.srl | 1 + idk/docker-sasl/ssl_keys/client-ssl-test.conf | 14 + .../ssl_keys/client_kafkaClient_cert-file | 17 + .../ssl_keys/client_kafkaClient_cert-signed | 20 + .../ssl_keys/client_kafkaClient_client.key | 30 + .../client_kafkaClient_client.keystore.jks | Bin 0 -> 3935 bytes .../ssl_keys/client_kafkaClient_client.pem | 20 + .../ssl_keys/client_kafkaClient_client.req | 17 + .../client_kafkaClient_client.truststore.jks | Bin 0 -> 917 bytes idk/docker-sasl/ssl_keys/credentials | 1 + idk/docker-sasl/ssl_keys/gen-ssl-certs.sh | 181 + .../ssl_keys/jaas_kafka_broker.conf | 30 + idk/dup.go | 16 + idk/dup_arm64.go | 17 + idk/fakeidp/go.mod | 5 + idk/fakeidp/go.sum | 2 + idk/fakeidp/server.go | 44 + idk/file_with_line_delimited_values | 5 + idk/header.go | 671 + idk/header_test.go | 423 + idk/idallocator.go | 462 + idk/idallocator_test.go | 349 + idk/idktest/idktest.go | 76 + idk/idktest/mocks/kinesis.go | 2470 + idk/idktest/mocks/s3.go | 7835 ++ idk/idktest/mocks/sqs.go | 1662 + idk/ingest.go | 2059 + idk/ingest_test.go | 1532 + idk/interfaces.go | 1306 + idk/interfaces_test.go | 437 + idk/internal/reader.go | 225 + idk/internal/reader_test.go | 84 + idk/internal/s3.go | 100 + idk/internal/write.go | 111 + idk/kafka/cmd.go | 53 + idk/kafka/cmd_delete_test.go | 270 + idk/kafka/cmd_test.go | 682 + idk/kafka/csrc/csrc.go | 133 + idk/kafka/csrc/csrc_integration_test.go | 48 + idk/kafka/put.go | 193 + idk/kafka/putsource.go | 821 + idk/kafka/putsource_test.go | 309 + idk/kafka/source.go | 1134 + idk/kafka/source_test.go | 1159 + idk/kafka/testdata/schemas/bigschema.json | 139 + idk/kafka/testdata/schemas/decimal.json | 8 + idk/kafka/testdata/schemas/delete.json | 19 + idk/kafka/testdata/schemas/delete_string.json | 30 + idk/kafka/testdata/schemas/fieldisrecord.json | 14 + idk/kafka/testdata/schemas/floatscale.json | 8 + idk/kafka/testdata/schemas/ids.json | 8 + idk/kafka/testdata/schemas/notarecord.json | 4 + idk/kafka/testdata/schemas/othertypes.json | 16 + idk/kafka/testdata/schemas/simple.json | 9 + idk/kafka/testdata/schemas/stringtypes.json | 10 + idk/kafka/testdata/schemas/timestamp.json | 8 + idk/kafka/testdata/schemas/unions.json | 12 + idk/kafka_sasl/cmd.go | 72 + idk/kafka_sasl/cmd_test.go | 1068 + idk/kafka_sasl/source.go | 338 + idk/kafka_sasl/source_test.go | 300 + idk/kafka_static/cmd.go | 55 + idk/kafka_static/cmd_test.go | 733 + idk/kafka_static/source.go | 271 + idk/kafka_static/source_test.go | 368 + idk/kafka_static/testdata/Flat.json | 3 + idk/kafka_static/testdata/Lookup.json | 16 + .../testdata/LookupDuplicate.json | 23 + idk/kafka_static/testdata/LookupId.json | 16 + idk/kafka_static/testdata/TestFieldTypes.json | 53 + idk/kafka_static/testdata/Tree.json | 8 + idk/kafkagen/cmd.go | 176 + idk/kinesis/cmd.go | 85 + idk/kinesis/cmd_test.go | 12 + idk/kinesis/logger.go | 234 + idk/kinesis/logger_test.go | 540 + idk/kinesis/reader.go | 389 + idk/kinesis/reader_test.go | 406 + idk/kinesis/source.go | 251 + idk/kinesis/source_test.go | 78 + idk/kinesis/testdata/header.json | 22 + idk/kinesis/testdata/invalid_offsets.json | 1 + idk/kinesis/testdata/offsets.json | 17 + idk/metrics.go | 8 + idk/pilosa-sec-test.conf | 27 + idk/postgres.go | 337 + idk/progress.go | 37 + idk/reingest_test.sh | 29 + idk/sample.csv | 9 + idk/sql/odbc.go | 6 + idk/sql/source.go | 240 + idk/sql/source_test.go | 32 + idk/test_postgres.go | 41 + idk/testdata/certs/README.md | 12 + idk/testdata/certs/localhost.crt | 25 + idk/testdata/certs/localhost.csr | 16 + idk/testdata/certs/localhost.key | 27 + idk/testdata/certs/pilosa-ca.crl | 16 + idk/testdata/certs/pilosa-ca.crt | 29 + idk/testdata/certs/pilosa-ca.key | 51 + idk/testdata/featurebase.conf | 383 + idk/testdata/permissions.yaml | 4 + idk/testenv/certs/ca.crl | 16 + idk/testenv/certs/ca.crt | 28 + idk/testenv/certs/ca.key | 51 + idk/testenv/certs/localhost.crt | 25 + idk/testenv/certs/localhost.csr | 16 + idk/testenv/certs/localhost.key | 27 + idk/testenv/certs/pilosa-tls.crt | 25 + idk/testenv/certs/pilosa-tls.csr | 16 + idk/testenv/certs/pilosa-tls.key | 27 + idk/testenv/certs/theclient.crt | 25 + idk/testenv/certs/theclient.csr | 16 + idk/testenv/certs/theclient.key | 27 + idk/tls.go | 234 + idk/util.go | 13 + idk/util_test.go | 71 + idk/version.go | 4 + idk/wait.sh | 26 + index.go | 859 +- index_internal_test.go | 81 +- index_test.go | 188 +- ingest/codec.go | 1171 + ingest/codec_test.go | 979 + ingest/doc.go | 18 + ingest/op.go | 827 + ingest/op_test.go | 276 + ingest/shard.go | 3 + ingest/sort.go | 233 + ingest/sort_test.go | 219 + ingest/translate_test.go | 125 + ingest/update.go | 244 + ingest/vec.go | 123 + ingest/vec_test.go | 102 + ingest_test.go | 450 + ingest_testdata/bool.tc | 17 + ingest_testdata/expect_errors.tc | 101 + ingest_testdata/keyed.tc | 32 + ingest_testdata/sample.tc | 99 + install/featurebase.conf | 364 + install/featurebase.debian.service | 13 + install/featurebase.redhat.service | 18 + install/postinstall.sh | 2 + install/preinstall.sh | 12 + install/preremove.sh | 2 + install/test_installation.Dockerfile | 18 + install/test_installation.sh | 29 + internal/LICENSE | 202 - internal/clustertests/Dockerfile | 3 - internal/clustertests/Dockerfile-fakeIDP | 7 + internal/clustertests/cluster_test.go | 471 +- .../docker-compose-replication2.yml | 12 +- internal/clustertests/docker-compose.yml | 72 +- internal/clustertests/fakeidp/go.mod | 5 + internal/clustertests/fakeidp/go.sum | 2 + internal/clustertests/fakeidp/server.go | 44 + internal/clustertests/pause_node_test.go | 421 + .../clustertests/testdata/certs/README.md | 12 + .../clustertests/testdata/certs/localhost.crt | 25 + .../clustertests/testdata/certs/localhost.csr | 16 + .../clustertests/testdata/certs/localhost.key | 27 + .../clustertests/testdata/certs/pilosa-ca.crl | 16 + .../clustertests/testdata/certs/pilosa-ca.crt | 29 + .../clustertests/testdata/certs/pilosa-ca.key | 51 + .../clustertests/testdata/featurebase.conf | 361 + .../clustertests/testdata/permissions.yaml | 4 + internal/private.pb.go | 7635 -- internal/public.pb.go | 5225 - internal/public.proto | 122 - internal/test/querygenerator.go | 18 +- internal/test/querygenerator_test.go | 18 +- internal_client.go | 2625 + internal_client_test.go | 1678 + iterator.go | 18 +- iterator_internal_test.go | 16 +- lattice/.gitignore | 16 + lattice/Caddyfile | 3 + lattice/Dockerfile | 14 + lattice/Makefile | 16 + lattice/README.md | 22 + lattice/package.json | 70 + lattice/public/favicon.ico | Bin 0 -> 15406 bytes lattice/public/favicon.png | Bin 0 -> 8182 bytes lattice/public/favicon.svg | 7 + lattice/public/index.html | 43 + lattice/public/manifest.json | 15 + lattice/src/App.module.scss | 15 + lattice/src/App.test.tsx | 9 + lattice/src/App.tsx | 37 + lattice/src/App/AuthFlow/AuthFlow.module.scss | 56 + lattice/src/App/AuthFlow/SignInButton.tsx | 20 + lattice/src/App/AuthFlow/SignOutButton.tsx | 21 + lattice/src/App/AuthFlow/Signin.tsx | 30 + lattice/src/App/AuthFlow/index.ts | 1 + .../ClusterHealth/ClusterHealth.module.scss | 24 + .../App/Home/ClusterHealth/ClusterHealth.tsx | 159 + .../ClusterInfo/ClusterInfo.module.scss | 17 + .../ClusterHealth/ClusterInfo/ClusterInfo.tsx | 20 + .../Home/ClusterHealth/ClusterInfo/index.ts | 1 + .../ImportStatus/ImportStatus.module.scss | 29 + .../ImportStatus/ImportStatus.tsx | 284 + .../ClusterHealth/ImportStatus/helpers.ts | 8 + .../Home/ClusterHealth/ImportStatus/index.ts | 1 + .../ClusterHealth/Metrics/Metrics.module.scss | 119 + .../Home/ClusterHealth/Metrics/Metrics.tsx | 334 + .../App/Home/ClusterHealth/Metrics/index.ts | 1 + .../ClusterHealth/Metrics/priorityMetrics.ts | 10 + .../Home/ClusterHealth/Node/Node.module.scss | 160 + .../src/App/Home/ClusterHealth/Node/Node.tsx | 159 + .../src/App/Home/ClusterHealth/Node/index.ts | 1 + .../App/Home/ClusterHealth/Node/nodeInfo.ts | 10 + .../App/Home/ClusterHealth/Node/nodeStatus.ts | 31 + .../App/Home/ClusterHealth/clusterStatus.ts | 27 + lattice/src/App/Home/ClusterHealth/index.ts | 1 + lattice/src/App/Home/Home.module.scss | 3 + lattice/src/App/Home/Home.tsx | 22 + .../NodeIndicator/NodeIndicator.module.scss | 28 + .../NodeIndicator/NodeIndicator.tsx | 62 + .../Home/QueryHistory/NodeIndicator/index.ts | 1 + .../QueryHistory/QueryHistory.module.scss | 28 + .../App/Home/QueryHistory/QueryHistory.tsx | 106 + .../QueryItem/QueryItem.module.scss | 87 + .../Home/QueryHistory/QueryItem/QueryItem.tsx | 87 + .../App/Home/QueryHistory/QueryItem/index.ts | 1 + lattice/src/App/Home/QueryHistory/index.ts | 1 + .../src/App/Home/QueryHistory/nodeColors.ts | 10 + .../Transaction/Transaction.module.scss | 76 + .../Transactions/Transaction/Transaction.tsx | 113 + .../Home/Transactions/Transaction/index.ts | 1 + .../Transactions/Transaction/utils.test.ts | 12 + .../Home/Transactions/Transaction/utils.ts | 27 + .../Transactions/Transactions.module.scss | 85 + .../App/Home/Transactions/Transactions.tsx | 178 + lattice/src/App/Home/Transactions/index.ts | 1 + lattice/src/App/Home/index.ts | 1 + .../MoleculaTable/MoleculaTable.module.scss | 140 + .../MoleculaTable/MoleculaTable.tsx | 234 + .../App/MoleculaTables/MoleculaTable/index.ts | 1 + .../MoleculaTables/MoleculaTables.module.scss | 83 + .../src/App/MoleculaTables/MoleculaTables.tsx | 138 + .../MoleculaTablesContainer.tsx | 52 + lattice/src/App/MoleculaTables/index.ts | 2 + lattice/src/App/NotFound/NotFound.tsx | 10 + lattice/src/App/NotFound/index.tsx | 1 + .../src/App/Query/Console/Console.module.scss | 92 + lattice/src/App/Query/Console/Console.tsx | 760 + lattice/src/App/Query/Console/helpers.ts | 151 + lattice/src/App/Query/Console/index.ts | 1 + lattice/src/App/Query/Query.module.scss | 108 + lattice/src/App/Query/Query.tsx | 253 + lattice/src/App/Query/QueryContainer.tsx | 144 + .../QueryResults/QueryResults.module.scss | 57 + .../App/Query/QueryResults/QueryResults.tsx | 154 + lattice/src/App/Query/QueryResults/index.ts | 1 + .../RecentQueries/RecentQueries.module.scss | 28 + .../App/Query/RecentQueries/RecentQueries.tsx | 72 + lattice/src/App/Query/RecentQueries/index.ts | 1 + lattice/src/App/Query/index.ts | 2 + lattice/src/App/Query/splitjs.scss | 32 + .../ColumnSelector/ColumnSelector.module.scss | 20 + .../ColumnSelector/ColumnSelector.tsx | 212 + .../App/QueryBuilder/ColumnSelector/index.ts | 1 + .../CountBuilder/CountBuilder.tsx | 36 + .../App/QueryBuilder/CountBuilder/index.ts | 1 + .../ExtractBuilder/ExtractBuilder.module.scss | 21 + .../ExtractBuilder/ExtractBuilder.tsx | 92 + .../App/QueryBuilder/ExtractBuilder/index.ts | 1 + .../GroupByBuilder/GroupByBuilder.module.scss | 32 + .../GroupByBuilder/GroupByBuilder.tsx | 183 + .../App/QueryBuilder/GroupByBuilder/index.ts | 1 + .../GroupByChart/GroupByChart.tsx | 128 + .../App/QueryBuilder/GroupByChart/index.ts | 1 + .../GroupBySort/GroupBySort.module.scss | 8 + .../QueryBuilder/GroupBySort/GroupBySort.tsx | 125 + .../src/App/QueryBuilder/GroupBySort/index.ts | 1 + .../App/QueryBuilder/QueryBuilder.module.scss | 52 + lattice/src/App/QueryBuilder/QueryBuilder.tsx | 413 + .../QueryBuilderContainer.module.scss | 66 + .../QueryBuilder/QueryBuilderContainer.tsx | 326 + .../RowCall/RowCall.module.scss | 102 + .../RowCallBuilder/RowCall/RowCall.tsx | 308 + .../RowCallBuilder/RowCall/helpers.ts | 52 + .../RowCallBuilder/RowCall/index.ts | 1 + .../RowCallBuilder/RowCallBuilder.module.scss | 40 + .../RowCallBuilder/RowCallBuilder.tsx | 177 + .../App/QueryBuilder/RowCallBuilder/index.ts | 1 + .../SavedQueries/SavedQueries.tsx | 92 + .../App/QueryBuilder/SavedQueries/index.ts | 1 + lattice/src/App/QueryBuilder/index.ts | 2 + lattice/src/App/QueryBuilder/rowTypes.ts | 22 + lattice/src/App/QueryBuilder/utils.test.ts | 835 + lattice/src/App/QueryBuilder/utils.ts | 199 + lattice/src/Main.tsx | 58 + lattice/src/assets/bg-pattern.png | Bin 0 -> 67342 bytes lattice/src/assets/darkTheme/MoleculaBug.svg | 1 + lattice/src/assets/darkTheme/MoleculaLogo.svg | 1 + lattice/src/assets/lightTheme/MoleculaBug.svg | 1 + .../src/assets/lightTheme/MoleculaLogo.svg | 1 + lattice/src/assets/m-bug-alt.svg | 16 + lattice/src/index.scss | 83 + lattice/src/index.tsx | 21 + lattice/src/lattice.config.js | 4 + lattice/src/proto/pilosa_pb.d.ts | 611 + lattice/src/proto/pilosa_pb.js | 4699 + lattice/src/proto/pilosa_pb_service.d.ts | 191 + lattice/src/proto/pilosa_pb_service.js | 407 + lattice/src/react-app-env.d.ts | 1 + lattice/src/serviceWorker.tsx | 135 + .../src/services/__mocks__/eventServices.tsx | 12 + lattice/src/services/baseURL.ts | 10 + lattice/src/services/eventServices.tsx | 57 + lattice/src/services/grpcServices.tsx | 28 + lattice/src/services/useAuth.test.tsx | 96 + lattice/src/services/useAuth.tsx | 81 + lattice/src/setupTests.ts | 5 + lattice/src/shared/Animations/Animations.tsx | 135 + lattice/src/shared/Animations/index.tsx | 1 + lattice/src/shared/Block/Block.module.scss | 20 + lattice/src/shared/Block/Block.tsx | 32 + lattice/src/shared/Block/index.ts | 1 + .../shared/DataTable/DataTable.module.scss | 62 + lattice/src/shared/DataTable/DataTable.tsx | 163 + lattice/src/shared/DataTable/index.ts | 1 + lattice/src/shared/Header/Header.module.scss | 42 + lattice/src/shared/Header/Header.tsx | 67 + lattice/src/shared/Header/index.tsx | 1 + lattice/src/shared/Nav/Nav.module.scss | 54 + lattice/src/shared/Nav/Nav.tsx | 57 + lattice/src/shared/Nav/index.tsx | 1 + lattice/src/shared/Pager/Pager.module.scss | 47 + lattice/src/shared/Pager/Pager.tsx | 81 + lattice/src/shared/Pager/index.ts | 1 + .../src/shared/PrivateRoute/PrivateRoute.tsx | 33 + lattice/src/shared/Select/Select.module.scss | 3 + lattice/src/shared/Select/Select.tsx | 65 + lattice/src/shared/Select/index.tsx | 1 + lattice/src/shared/SortBy/SortBy.module.scss | 3 + lattice/src/shared/SortBy/SortBy.tsx | 44 + lattice/src/shared/SortBy/index.ts | 1 + .../StatusIndicator.module.scss | 81 + .../StatusIndicator/StatusIndicator.tsx | 42 + lattice/src/shared/StatusIndicator/index.ts | 1 + .../ThemeToggle/ThemeToggle.module.scss | 46 + .../src/shared/ThemeToggle/ThemeToggle.tsx | 22 + lattice/src/shared/ThemeToggle/index.tsx | 1 + lattice/src/shared/Toggle/Toggle.module.scss | 64 + lattice/src/shared/Toggle/Toggle.tsx | 29 + lattice/src/shared/Toggle/index.ts | 1 + lattice/src/shared/utils/formatBytes.test.ts | 21 + lattice/src/shared/utils/formatBytes.ts | 17 + .../src/shared/utils/formatDuration.test.ts | 16 + lattice/src/shared/utils/formatDuration.ts | 10 + .../src/shared/utils/formatTableCell.test.tsx | 77 + lattice/src/shared/utils/formatTableCell.tsx | 22 + lattice/src/theme/baseTheme.tsx | 96 + lattice/src/theme/darkTheme.tsx | 85 + lattice/src/theme/index.tsx | 3 + lattice/src/theme/lightTheme.tsx | 75 + lattice/tsconfig.json | 31 + lattice/tslint.json | 25 + lattice/yarn.lock | 13864 ++ license.exceptions | 26 + like.go | 223 + like_test.go | 234 + logger/filewriter.go | 108 + logger/filewriter_test.go | 163 + logger/logger.go | 253 +- lru/lru.go | 22 - main_test.go | 31 + metrics.go | 54 + mock/mock.go | 16 +- mock/translator.go | 56 +- monitor/monitor.go | 108 + uri.go => net/uri.go | 71 +- .../uri_internal_test.go | 40 +- nfpm.yaml | 47 + internal/internal.go => pb/pb.go | 18 +- pb/private.pb.go | 13400 ++ {internal => pb}/private.proto | 100 +- pb/public.pb.go | 12712 ++ pb/public.proto | 262 + pg/cancel.go | 125 + pg/cancel_test.go | 66 + pg/io.go | 237 + pg/lookerToFeaturebase.md | 652 + pg/lookerToPostgres.md | 677 + pg/message/io.go | 112 + pg/message/message.go | 532 + pg/pgtest/handler.go | 113 + pg/pgtest/memnet.go | 59 + pg/pgtest/server.go | 118 + pg/pgtest/tls.go | 89 + pg/protocol.go | 1102 + pg/query.go | 120 + pg/server.go | 133 + pg/server_test.go | 283 + pg/type.go | 63 + pilosa.go | 136 +- pilosa_internal_test.go | 66 +- pilosa_test.go | 22 +- planner.go | 1330 + planner_test.go | 531 + pprof.go | 97 + pql/ast.go | 927 +- pql/ast_test.go | 280 +- pql/decimal.go | 522 + pql/decimal_internal_test.go | 90 + pql/decimal_test.go | 250 + pql/doc.go | 16 +- pql/parser.go | 113 +- pql/parser_test.go | 177 +- pql/pql.peg | 116 +- pql/pql.peg.go | 5061 +- pql/pqlpeg_test.go | 333 +- pql/token.go | 42 +- prometheus/prometheus.go | 69 +- prometheus/prometheus_test.go | 24 +- proto/interface.go | 315 + proto/pilosa.pb.go | 1731 + proto/pilosa.proto | 134 + proto/vdsm/vdsm.pb.go | 1210 + proto/vdsm/vdsm.proto | 89 + qa/fakeidp/go.mod | 5 + qa/fakeidp/go.sum | 2 + qa/fakeidp/server.go | 65 + qa/scripts/auth-smoke/setup.sh | 91 + qa/scripts/auth-smoke/teardown.sh | 7 + qa/scripts/auth-smoke/test.sh | 37 + qa/scripts/auth-smoke/tests/README | 16 + qa/scripts/auth-smoke/tests/run-all.sh | 8 + qa/scripts/auth-smoke/tests/sup218-test.sh | 33 + .../auth-smoke/tests/sup218_datagen.yaml | 61 + qa/scripts/backupRestoreTest.sh | 112 + qa/scripts/backup_test_datagen.yaml | 61 + qa/scripts/bug_repro_tests.sh | 41 + qa/scripts/gitlabCleanupBuild.sh | 37 + qa/scripts/perf/able/able.yaml | 96 + qa/scripts/perf/able/ableRun.sh | 16 + qa/scripts/perf/able/ableSetup.sh | 95 + qa/scripts/perf/able/ableTeardown.sh | 7 + qa/scripts/perf/able/ableTest.sh | 66 + qa/scripts/perf/able/generateTestData.sh | 33 + .../perf/able/highcardinalitygroupby.js | 11 + qa/scripts/perf/able/values/education.txt | 6 + qa/scripts/perf/able/values/gender.txt | 5 + qa/scripts/perf/able/values/hobbies.txt | 642 + qa/scripts/perf/able/values/income.txt | 7 + .../able/values/opendns-top-domains-10K.txt | 10000 ++ .../perf/able/values/political_parties.txt | 7 + qa/scripts/runSmokeTest.sh | 16 + qa/scripts/setupBackupRestoreGauntlet.sh | 89 + qa/scripts/setupSmokeTest.sh | 89 + qa/scripts/setupTLS.sh | 84 + qa/scripts/teardownBackupRestoreGauntlet.sh | 7 + qa/scripts/teardownSmokeTest.sh | 7 + qa/scripts/testBackupRestoreGauntlet.sh | 56 + qa/scripts/testSmokeTest.sh | 48 + qa/simulacraData/simulacra_data.go | 372 + qa/simulacraData/simulacra_data_test.go | 65 + qa/testcases/bug-repros/README | 17 + qa/testcases/bug-repros/fb-1287-datagen.yaml | 57 + qa/testcases/bug-repros/fb-1287-test.sh | 10 + qa/testcases/bug-repros/fb-1332-datagen.yaml | 57 + qa/testcases/bug-repros/fb-1332.sh | 36 + qa/testcases/bug-repros/fb-1371-datagen.yaml | 57 + qa/testcases/bug-repros/fb-1371-test.sh | 25 + qa/testcases/bug-repros/run-all.sh | 8 + qa/testcases/smoketest/config.py | 1 + qa/testcases/smoketest/test_smoke.py | 60 + qa/tf/.modules/featurebase-cluster/README.md | 37 + .../featurebase-cluster/cloud-init.sh | 4 + qa/tf/.modules/featurebase-cluster/main.tf | 376 + qa/tf/.modules/featurebase-cluster/outputs.tf | 15 + .../.modules/featurebase-cluster/provider.tf | 11 + .../.modules/featurebase-cluster/variables.tf | 129 + qa/tf/README.md | 16 + qa/tf/ci/auth-smoke/main.tf | 15 + qa/tf/ci/auth-smoke/outputs.tf | 19 + qa/tf/ci/auth-smoke/provider.tf | 4 + qa/tf/ci/auth-smoke/tf.auto.tfvars | 2 + qa/tf/ci/auth-smoke/variables.tf | 15 + qa/tf/ci/deletetest/delete_cloud_init.sh | 20 + qa/tf/ci/deletetest/main.tf | 19 + qa/tf/ci/deletetest/outputs.tf | 19 + qa/tf/ci/deletetest/provider.tf | 4 + qa/tf/ci/deletetest/tf.auto.tfvars | 2 + qa/tf/ci/deletetest/variables.tf | 15 + qa/tf/ci/smoketest/main.tf | 16 + qa/tf/ci/smoketest/outputs.tf | 19 + qa/tf/ci/smoketest/provider.tf | 4 + qa/tf/ci/smoketest/tf.auto.tfvars | 2 + qa/tf/ci/smoketest/variables.tf | 15 + qa/tf/gauntlet/backuprestore/README.md | 35 + qa/tf/gauntlet/backuprestore/main.tf | 19 + qa/tf/gauntlet/backuprestore/outputs.tf | 19 + qa/tf/gauntlet/backuprestore/provider.tf | 4 + qa/tf/gauntlet/backuprestore/tf.auto.tfvars | 2 + qa/tf/gauntlet/backuprestore/variables.tf | 14 + qa/tf/perf/able/main.tf | 19 + qa/tf/perf/able/outputs.tf | 19 + qa/tf/perf/able/provider.tf | 4 + qa/tf/perf/able/tf.auto.tfvars | 2 + qa/tf/perf/able/variables.tf | 14 + qa/tf/perf/delete/main.tf | 19 + qa/tf/perf/delete/outputs.tf | 19 + qa/tf/perf/delete/provider.tf | 4 + qa/tf/perf/delete/tf.auto.tfvars | 2 + qa/tf/perf/delete/variables.tf | 14 + rbf.go | 557 + rbf/README.md | 206 + rbf/array.go | 73 + rbf/cfg/cfg.go | 78 + rbf/cfg/os.go | 19 + rbf/cfg/os_386.go | 13 + rbf/cursor.go | 1501 + rbf/cursor_internal_test.go | 512 + rbf/cursor_test.go | 1310 + rbf/cursorx.go | 301 + rbf/db.go | 909 + rbf/db_test.go | 600 + rbf/dot.go | 99 + rbf/helpers_test.go | 96 + rbf/ingest_test.go | 157 + rbf/page_map.go | 910 + rbf/rbf.go | 852 + rbf/rbf/testdata/check/bad-freelist/data | Bin 0 -> 24576 bytes rbf/rbf/testdata/check/bad-freelist/wal | 0 rbf/rbf_test.go | 192 + rbf/testdata/check/bad-bitmap/data | Bin 0 -> 32768 bytes rbf/testdata/check/bad-bitmap/wal | 0 rbf/testdata/check/bad-freelist/data | Bin 0 -> 32768 bytes rbf/testdata/check/bad-freelist/wal | 0 rbf/tx.go | 2529 + rbf/tx_test.go | 1334 + rbf/util.go | 216 + rbf/util_test.go | 176 + roaring/add.go | 849 + roaring/add_test.go | 405 + roaring/benchpretty/main.go | 116 + roaring/btree.go | 9 + roaring/btree_test.go | 39 +- roaring/container_archetypes.go | 193 + roaring/container_stash.go | 742 +- roaring/containers_btree.go | 63 +- roaring/containers_slice.go | 97 +- roaring/containers_test.go | 246 +- roaring/filter.go | 1424 + roaring/filter_internal_test.go | 504 + roaring/fuzz_test.go | 15 +- roaring/fuzzer.go | 27 +- roaring/inst.go | 17 +- roaring/naive.go | 16 +- roaring/naive_test.go | 21 +- roaring/nop_inst.go | 17 +- roaring/printutil.go | 76 + roaring/printutil_test.go | 80 + roaring/roaring.go | 3696 +- roaring/roaring_container_test.go | 99 + roaring/roaring_helpers_test.go | 175 +- roaring/roaring_internal_test.go | 2123 +- roaring/roaring_nop_paranoia.go | 24 +- roaring/roaring_nop_sentinel.go | 17 +- roaring/roaring_nop_stats.go | 17 +- roaring/roaring_paranoia.go | 32 +- roaring/roaring_sentinel.go | 17 +- roaring/roaring_stats.go | 19 +- roaring/roaring_test.go | 919 +- roaring/unmarshal_binary.go | 415 +- row.go | 314 +- row_test.go | 91 +- scripts/bench_read.sh | 49 + scripts/bench_write.sh | 42 + scripts/etc/gloat/gh.1d.yml | 9 + scripts/etc/gloat/gh.1m.yml | 9 + scripts/etc/gloat/gh.1w.yml | 9 + scripts/etc/gloat/gh.issues.autogenerate.yml | 9 + scripts/etc/gloat/gh.issues.keyed.yml | 9 + scripts/etc/gloat/gh.issues.unkeyed.yml | 9 + scripts/etc/gloat/query.count-keyed.yml | 11 + scripts/etc/gloat/query.count.yml | 11 + scripts/etc/gloat/query.difference.yml | 11 + scripts/etc/gloat/query.groupby.yml | 11 + scripts/etc/gloat/query.intersect.yml | 11 + scripts/etc/gloat/query.row-bsi.yml | 11 + scripts/etc/gloat/query.row-range.yml | 11 + scripts/etc/gloat/query.row.yml | 11 + scripts/etc/gloat/query.topk.yml | 11 + scripts/etc/gloat/query.union.yml | 11 + scripts/etc/gloat/query.xor.yml | 11 + scripts/populate_query_db.keyed.sh | 28 + scripts/populate_query_db.sh | 28 + serializer.go | 48 + server.go | 1077 +- server/cluster_test.go | 573 +- server/config.go | 502 +- server/config_internal_test.go | 508 +- server/config_test.go | 34 +- server/default.go | 24 - server/dup.go | 16 + server/dup_arm64.go | 16 + server/enterprise.go | 21 - server/grpc.go | 1823 + server/grpc_internal_test.go | 169 + server/grpc_test.go | 2051 + server/handler_test.go | 990 +- server/pg.go | 665 + server/pg_internal_test.go | 51 + server/pg_test.go | 314 + server/release.go | 24 - server/server.go | 585 +- server/server_internal_test.go | 40 + server/server_test.go | 1109 +- server/setup_logger.go | 49 - server/setup_logger_arm64.go | 47 - server/sql.go | 39 + server/testdata/certs/README.md | 2 +- server/testdata/certs/localhost.crt | 46 +- server/testdata/certs/localhost.csr | 27 +- server/testdata/certs/localhost.key | 50 +- server/testdata/certs/pilosa-ca.crl | 26 +- server/testdata/certs/pilosa-ca.crt | 52 +- server/testdata/certs/pilosa-ca.key | 98 +- server/tlsconfig.go | 48 +- server/tlsconfig_test.go | 60 + server/trial.go | 75 + server_internal_test.go | 73 +- server_test.go | 280 + shardwidth/16.go | 17 +- shardwidth/17.go | 17 +- shardwidth/18.go | 17 +- shardwidth/19.go | 17 +- shardwidth/20.go | 17 +- shardwidth/21.go | 17 +- shardwidth/22.go | 17 +- shardwidth/23.go | 17 +- shardwidth/24.go | 17 +- shardwidth/25.go | 17 +- shardwidth/26.go | 17 +- shardwidth/27.go | 17 +- shardwidth/28.go | 17 +- shardwidth/29.go | 17 +- shardwidth/30.go | 17 +- shardwidth/31.go | 17 +- shardwidth/32.go | 17 +- shardwidth/helper.go | 58 + shardwidth/helper_test.go | 91 + short_txkey/txkey.go | 196 + short_txkey/txkey_test.go | 79 + sql/column.go | 100 + sql/ddl.go | 53 + sql/extract.go | 1218 + sql/handler_test.go | 120 + sql/mapper.go | 107 + sql/mapper_test.go | 382 + sql/mask.go | 389 + sql/model.go | 129 + sql/query.go | 291 + sql/reduce.go | 430 + sql/reduce_test.go | 108 + sql/router.go | 163 + sql/select.go | 868 + sql/show.go | 136 + sql2/ast.go | 3717 + sql2/ast_test.go | 1149 + sql2/parser.go | 2948 + sql2/parser_test.go | 3299 + sql2/scanner.go | 350 + sql2/scanner_test.go | 168 + sql2/token.go | 544 + sql2/token_test.go | 15 + sql2/walk.go | 903 + statik/.gitignore | 1 + statik/filesystem.go | 26 + stats/stats.go | 23 +- stats/stats_test.go | 168 +- statsd/statsd.go | 53 +- statsd/statsd_test.go | 24 +- stattx.go | 547 + storage/config.go | 28 + syswrap/mmap.go | 25 +- syswrap/os.go | 20 +- task/doc.go | 43 + task/pool.go | 152 + task/pool_test.go | 432 + test/cluster.go | 616 +- test/disco.go | 82 + test/field.go | 92 +- test/handler.go | 16 +- test/holder.go | 167 +- test/index.go | 55 +- test/logger.go | 48 - test/pilosa.go | 420 +- test/pilosa_test.go | 38 +- test/transaction.go | 34 + testdata/certs/localhost.crt | 26 + testdata/certs/localhost.key | 28 + testdata/schema.json | 424 + testdata/timeRegressionSchema.json | 32 + testhook/auditor.go | 158 + testhook/auditor_test.go | 319 + testhook/cleanup1.13.go | 36 + testhook/cleanup1.14.go | 17 + testhook/hook.go | 125 + testhook/registry.go | 512 + time.go | 295 +- time_internal_test.go | 222 +- toml/toml.go | 16 +- tracing/opentracing/opentracing.go | 27 +- tracing/tracing.go | 108 +- tracker.go | 191 + tracker_test.go | 69 + transaction.go | 466 + transaction_test.go | 317 + translate.go | 416 +- translator_test.go | 652 +- tx.go | 180 + tx_internal_test.go | 134 + tx_test.go | 246 + txfactory.go | 697 + txfactory_internal_test.go | 20 + txkey/txkey.go | 247 + txkey/txkey_test.go | 126 + util.go | 121 + util_test.go | 92 + utils_internal_test.go | 471 +- version.go | 70 +- view.go | 659 +- view_internal_test.go | 52 +- vprint/vprint.go | 152 + 1338 files changed, 452272 insertions(+), 49527 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/ISSUE_TEMPLATE.md delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md delete mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile-clustertests-client rename LICENSE => LICENSE-2.0.txt (100%) delete mode 100644 NOTES create mode 100644 api/client/grpc.go delete mode 100644 attr.go delete mode 100644 attr_test.go create mode 100644 audit.go create mode 100644 audit_internal_test.go create mode 100644 audit_test.go create mode 100644 authn/authenticate.go create mode 100644 authn/authenticate_internal_test.go create mode 100644 authz/authorization.go create mode 100644 authz/authorization_test.go delete mode 100644 boltdb/attrstore.go create mode 100644 boltdb/translate_internal_test.go create mode 100644 bsi.go create mode 100644 bsi_test.go create mode 100644 catcher.go delete mode 100644 client.go create mode 100644 client/README.md create mode 100644 client/batch.go create mode 100644 client/batch_test.go create mode 100644 client/client.go create mode 100644 client/client_it_test.go create mode 100644 client/client_test.go create mode 100644 client/cluster.go create mode 100644 client/cluster_test.go create mode 100644 client/csv/csv.go create mode 100644 client/csv/csv_it_test.go create mode 100644 client/csv/csv_test.go create mode 100644 client/doc.go create mode 100644 client/docs/data-model-queries.md create mode 100644 client/docs/server-interaction.md create mode 100644 client/docs/tracing.md create mode 100644 client/egpool/egpool.go create mode 100644 client/egpool/egpool_test.go create mode 100644 client/error.go create mode 100644 client/ingest_api_batch.go create mode 100644 client/ingest_api_batch_test.go create mode 100644 client/logimport.go create mode 100644 client/logimport_test.go create mode 100644 client/metrics.go create mode 100644 client/orm.go create mode 100644 client/orm_test.go create mode 100644 client/record.go create mode 100644 client/record_test.go create mode 100644 client/response.go create mode 100644 client/response_test.go create mode 100644 client/shardnodes.go create mode 100644 client/tracer.go create mode 100644 client/validate.go create mode 100644 client/validate_test.go create mode 100644 client/version.go create mode 100644 cmd/auth_token.go create mode 100644 cmd/backup.go create mode 100644 cmd/badloader/badloader.go delete mode 100644 cmd/check.go delete mode 100644 cmd/check_test.go create mode 100644 cmd/chksum.go create mode 100644 cmd/featurebase-parse-sql/main.go create mode 100644 cmd/featurebase/main.go create mode 100644 cmd/featurebase/main_test.go delete mode 100644 cmd/inspect.go delete mode 100644 cmd/inspect_test.go create mode 100644 cmd/keygen.go create mode 100644 cmd/pilosa-bench/main.go delete mode 100644 cmd/pilosa/main.go create mode 100644 cmd/rbf.go create mode 100644 cmd/restore.go create mode 100644 cmd/roaring-migrate/ctim_darwin.go create mode 100644 cmd/roaring-migrate/ctim_linux.go create mode 100644 cmd/roaring-migrate/main.go create mode 100644 cmd/roaring-migrate/main_test.go create mode 100644 cmd/roaring-migrate/testdata/data-dir/.id create mode 100644 cmd/roaring-migrate/testdata/data-dir/.startup.log create mode 100644 cmd/roaring-migrate/testdata/data-dir/.topology create mode 100644 cmd/roaring-migrate/testdata/data-dir/idalloc.db create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/.data create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/.meta create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_exists/.data create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_exists/.meta create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_exists/keys create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222.cache create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/0 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/1 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/10 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/100 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/101 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/102 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/103 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/104 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/105 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/106 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/107 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/108 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/109 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/11 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/110 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/111 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/112 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/113 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/114 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/115 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/116 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/117 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/118 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/119 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/12 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/120 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/121 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/122 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/123 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/124 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/125 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/126 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/127 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/128 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/129 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/13 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/130 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/131 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/132 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/133 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/134 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/135 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/136 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/137 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/138 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/139 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/14 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/140 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/141 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/142 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/143 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/144 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/145 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/146 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/147 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/148 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/149 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/15 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/150 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/151 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/152 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/153 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/154 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/155 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/156 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/157 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/158 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/159 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/16 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/160 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/161 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/162 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/163 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/164 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/165 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/166 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/167 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/168 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/169 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/17 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/170 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/171 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/172 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/173 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/174 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/175 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/176 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/177 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/178 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/179 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/18 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/180 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/181 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/182 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/183 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/184 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/185 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/186 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/187 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/188 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/189 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/19 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/190 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/191 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/192 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/193 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/194 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/195 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/196 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/197 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/198 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/199 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/2 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/20 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/200 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/201 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/202 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/203 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/204 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/205 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/206 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/207 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/208 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/209 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/21 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/210 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/211 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/212 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/213 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/214 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/215 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/216 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/217 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/218 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/219 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/22 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/220 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/221 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/222 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/223 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/224 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/225 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/226 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/227 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/228 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/229 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/23 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/230 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/231 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/232 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/233 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/234 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/235 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/236 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/237 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/238 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/239 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/24 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/240 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/241 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/242 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/243 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/244 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/245 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/246 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/247 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/248 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/249 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/25 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/250 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/251 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/252 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/253 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/254 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/255 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/26 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/27 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/28 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/29 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/3 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/30 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/31 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/32 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/33 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/34 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/35 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/36 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/37 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/38 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/39 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/4 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/40 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/41 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/42 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/43 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/44 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/45 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/46 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/47 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/48 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/49 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/5 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/50 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/51 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/52 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/53 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/54 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/55 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/56 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/57 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/58 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/59 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/6 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/60 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/61 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/62 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/63 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/64 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/65 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/66 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/67 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/68 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/69 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/7 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/70 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/71 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/72 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/73 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/74 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/75 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/76 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/77 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/78 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/79 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/8 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/80 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/81 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/82 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/83 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/84 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/85 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/86 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/87 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/88 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/89 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/9 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/90 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/91 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/92 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/93 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/94 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/95 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/96 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/97 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/98 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/_keys/99 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/language/.data create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/language/.meta create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/language/keys create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222 create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222.cache create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.data create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.meta create mode 100644 cmd/roaring-migrate/testdata/data-dir/repository/stargazer/keys create mode 100644 cmd/slurp/slurp.go create mode 100644 const_amd64.go create mode 100644 const_other.go create mode 100644 ctl/auth_token.go create mode 100644 ctl/auth_token_internal_test.go create mode 100644 ctl/backup.go delete mode 100644 ctl/check.go delete mode 100644 ctl/check_test.go create mode 100644 ctl/chksum.go delete mode 100644 ctl/inspect.go delete mode 100644 ctl/inspect_test.go create mode 100644 ctl/keygen.go create mode 100644 ctl/main_test.go create mode 100644 ctl/rbf_check.go create mode 100644 ctl/rbf_check_test.go create mode 100644 ctl/rbf_dump.go create mode 100644 ctl/rbf_page.go create mode 100644 ctl/rbf_pages.go create mode 100644 ctl/rbf_pages_test.go create mode 100644 ctl/restore.go create mode 100644 ctl/testdata/certs/README.md create mode 100644 ctl/testdata/certs/localhost.crt create mode 100644 ctl/testdata/certs/localhost.csr create mode 100644 ctl/testdata/certs/localhost.key create mode 100644 ctl/testdata/certs/pilosa-ca.crl create mode 100644 ctl/testdata/certs/pilosa-ca.crt create mode 100644 ctl/testdata/certs/pilosa-ca.key create mode 100644 ctl/testdata/permissions.yaml create mode 100644 ctl/testdata/rbf-check/err-invalid-page-type/data create mode 100644 ctl/testdata/rbf-check/err-invalid-page-type/wal create mode 100644 ctl/testdata/rbf-check/ok/data create mode 100644 ctl/testdata/rbf-check/ok/wal create mode 100644 ctl/testdata/rbf-pages/err-invalid-page-type/data create mode 100644 ctl/testdata/rbf-pages/err-invalid-page-type/wal create mode 100644 ctl/testdata/rbf-pages/ok/data create mode 100644 ctl/testdata/rbf-pages/ok/wal create mode 100644 ctl/util.go create mode 100644 dbshard.go create mode 100644 dbshard_internal_test.go create mode 100644 dbshard_test.go create mode 100644 debugstats/stats.go create mode 100644 debugstats/stats_test.go create mode 100644 delete_test.go create mode 100644 disco/disco.go create mode 100644 disco/hasher.go create mode 100644 disco/node.go create mode 100644 disco/noder.go create mode 100644 disco/snapshot.go delete mode 100644 docs/README.md delete mode 100644 docs/administration.md delete mode 100644 docs/api-reference.md delete mode 100644 docs/architecture.md delete mode 100644 docs/client-libraries.md delete mode 100644 docs/configuration.md delete mode 100644 docs/console.md delete mode 100644 docs/data-model.md delete mode 100644 docs/examples.md delete mode 100644 docs/faq.md delete mode 100644 docs/getting-started.md delete mode 100644 docs/glossary.md delete mode 100644 docs/installation.md delete mode 100644 docs/introduction.md delete mode 100644 docs/pdk.md delete mode 100644 docs/query-language.md delete mode 100644 docs/tutorials.md create mode 100644 encoding/proto/proto_test.go delete mode 100644 enterprise/COPYING delete mode 100644 enterprise/enterprise.go create mode 100644 etcd/config_gen.go create mode 100644 etcd/embed.go create mode 100644 etcd/embed_test.go create mode 100644 etcd/enterprise_cluster.go create mode 100644 etcd/external.go create mode 100644 etcd/external_test.go create mode 100644 etcd/fake_test.go create mode 100644 etcd/leasedkv.go create mode 100644 etcd/leasedkv_test.go create mode 100644 etcd/plg_cluster.go create mode 100644 filesystem.go create mode 100644 generator/slice.go create mode 100644 gid.go delete mode 100644 gossip/gossip.go create mode 100644 hack.go create mode 100644 hash/blake3.go create mode 100644 hash/blake3_test.go delete mode 100644 http/client.go delete mode 100644 http/client_test.go delete mode 100644 http/error.go delete mode 100644 http/handler.go delete mode 100644 http/handler_internal_test.go delete mode 100644 http/handler_test.go delete mode 100644 http/translator_test.go create mode 100644 http_handler.go create mode 100644 http_handler_internal_test.go create mode 100644 http_handler_test.go rename http/translator.go => http_translator.go (57%) create mode 100644 http_translator_test.go create mode 100644 idalloc.go create mode 100644 idalloc_test.go create mode 100644 idk/.cloud-env.template create mode 100644 idk/Dockerfile create mode 100644 idk/Dockerfile-fakeIDP create mode 100644 idk/Dockerfile-test create mode 100644 idk/Dockerfile-wait create mode 100644 idk/Makefile create mode 100644 idk/README.md create mode 100644 idk/api/codec.go create mode 100644 idk/api/codec_test.go create mode 100644 idk/api/schema.go create mode 100644 idk/api/schema_test.go create mode 100644 idk/api/source.go create mode 100644 idk/api/source_test.go create mode 100644 idk/bankgen/put.go create mode 100644 idk/bankgen/put_test.go create mode 100644 idk/cmd/bankgen/main.go create mode 100644 idk/cmd/bankgen/main_test.go create mode 100644 idk/cmd/datagen/main.go create mode 100644 idk/cmd/datagen/main_test.go create mode 100644 idk/cmd/ingester/README.md create mode 100644 idk/cmd/ingester/main.go create mode 100644 idk/cmd/kafkagen/main.go create mode 100644 idk/cmd/kafkagen/main_test.go create mode 100644 idk/cmd/kafkaput/main.go create mode 100644 idk/cmd/kafkaput/main_test.go create mode 100644 idk/cmd/molecula-consumer-csv/main.go create mode 100644 idk/cmd/molecula-consumer-csv/main_test.go create mode 100644 idk/cmd/molecula-consumer-github/README.md create mode 100644 idk/cmd/molecula-consumer-github/main.go create mode 100644 idk/cmd/molecula-consumer-kafka-delete/main.go create mode 100644 idk/cmd/molecula-consumer-kafka-delete/main_test.go create mode 100644 idk/cmd/molecula-consumer-kafka-sasl/main.go create mode 100644 idk/cmd/molecula-consumer-kafka-sasl/readme.md create mode 100644 idk/cmd/molecula-consumer-kafka-static/main.go create mode 100644 idk/cmd/molecula-consumer-kafka/main.go create mode 100644 idk/cmd/molecula-consumer-kafka/main_test.go create mode 100644 idk/cmd/molecula-consumer-kinesis/main.go create mode 100644 idk/cmd/molecula-consumer-sql/main.go create mode 100644 idk/cmd/molecula-consumer-sql/main_test.go create mode 100644 idk/common/configMap.go create mode 100644 idk/csv/cmd.go create mode 100644 idk/csv/csv_test.go create mode 100644 idk/csv/source.go create mode 100644 idk/datagen/README.md create mode 100644 idk/datagen/all-field-types.go create mode 100644 idk/datagen/bank.go create mode 100644 idk/datagen/claim.go create mode 100644 idk/datagen/cmd.go create mode 100644 idk/datagen/common.go create mode 100644 idk/datagen/custom.go create mode 100644 idk/datagen/custom_test.go create mode 100644 idk/datagen/customer.go create mode 100644 idk/datagen/customer_segmentation.go create mode 100644 idk/datagen/customer_segmentation_test.go create mode 100644 idk/datagen/equipment.data.go create mode 100644 idk/datagen/equipment.go create mode 100644 idk/datagen/example.go create mode 100644 idk/datagen/gen/constants.data.go create mode 100644 idk/datagen/gen/gen.go create mode 100644 idk/datagen/gen/gen_test.go create mode 100644 idk/datagen/hobbies.data.go create mode 100644 idk/datagen/item.go create mode 100644 idk/datagen/kitchen-sink-keyed.go create mode 100644 idk/datagen/kitchen-sink.go create mode 100644 idk/datagen/locations.data.go create mode 100644 idk/datagen/network.go create mode 100644 idk/datagen/shared.go create mode 100644 idk/datagen/sites.data.go create mode 100644 idk/datagen/sites.go create mode 100644 idk/datagen/sizing.go create mode 100644 idk/datagen/skills.data.go create mode 100644 idk/datagen/stringpk.go create mode 100644 idk/datagen/testdata/custom.yaml create mode 100644 idk/datagen/testdata/file_with_line_delimited_values create mode 100644 idk/datagen/testdata/gender.txt create mode 100644 idk/datagen/testdata/schema.json create mode 100644 idk/datagen/testdata/t_keys.yaml create mode 100644 idk/datagen/timeseries.go create mode 100644 idk/datagen/titles.data.go create mode 100644 idk/datagen/transactions.go create mode 100644 idk/datagen/transactions_scenario_1.go create mode 100644 idk/datagen/uscities.data.go create mode 100644 idk/datagen/warranty.go create mode 100644 idk/datagen/zip_codes.data.go create mode 100644 idk/docker-compose.yml create mode 100644 idk/docker-sasl/client-ssl-test.conf create mode 100644 idk/docker-sasl/jaas_kafka_broker.conf create mode 100644 idk/docker-sasl/jaas_zookeeper_server.conf create mode 100644 idk/docker-sasl/ssl_keys/broker_kafkaBroker_cert-file create mode 100644 idk/docker-sasl/ssl_keys/broker_kafkaBroker_cert-signed create mode 100644 idk/docker-sasl/ssl_keys/broker_kafkaBroker_server.keystore.jks create mode 100644 idk/docker-sasl/ssl_keys/broker_kafkaBroker_server.truststore.jks create mode 100644 idk/docker-sasl/ssl_keys/ca-cert create mode 100644 idk/docker-sasl/ssl_keys/ca-cert.key create mode 100644 idk/docker-sasl/ssl_keys/ca-cert.srl create mode 100644 idk/docker-sasl/ssl_keys/client-ssl-test.conf create mode 100644 idk/docker-sasl/ssl_keys/client_kafkaClient_cert-file create mode 100644 idk/docker-sasl/ssl_keys/client_kafkaClient_cert-signed create mode 100644 idk/docker-sasl/ssl_keys/client_kafkaClient_client.key create mode 100644 idk/docker-sasl/ssl_keys/client_kafkaClient_client.keystore.jks create mode 100644 idk/docker-sasl/ssl_keys/client_kafkaClient_client.pem create mode 100644 idk/docker-sasl/ssl_keys/client_kafkaClient_client.req create mode 100644 idk/docker-sasl/ssl_keys/client_kafkaClient_client.truststore.jks create mode 100644 idk/docker-sasl/ssl_keys/credentials create mode 100644 idk/docker-sasl/ssl_keys/gen-ssl-certs.sh create mode 100644 idk/docker-sasl/ssl_keys/jaas_kafka_broker.conf create mode 100644 idk/dup.go create mode 100644 idk/dup_arm64.go create mode 100644 idk/fakeidp/go.mod create mode 100644 idk/fakeidp/go.sum create mode 100644 idk/fakeidp/server.go create mode 100644 idk/file_with_line_delimited_values create mode 100644 idk/header.go create mode 100644 idk/header_test.go create mode 100644 idk/idallocator.go create mode 100644 idk/idallocator_test.go create mode 100644 idk/idktest/idktest.go create mode 100644 idk/idktest/mocks/kinesis.go create mode 100644 idk/idktest/mocks/s3.go create mode 100644 idk/idktest/mocks/sqs.go create mode 100644 idk/ingest.go create mode 100644 idk/ingest_test.go create mode 100644 idk/interfaces.go create mode 100644 idk/interfaces_test.go create mode 100644 idk/internal/reader.go create mode 100644 idk/internal/reader_test.go create mode 100644 idk/internal/s3.go create mode 100644 idk/internal/write.go create mode 100644 idk/kafka/cmd.go create mode 100644 idk/kafka/cmd_delete_test.go create mode 100644 idk/kafka/cmd_test.go create mode 100644 idk/kafka/csrc/csrc.go create mode 100644 idk/kafka/csrc/csrc_integration_test.go create mode 100644 idk/kafka/put.go create mode 100644 idk/kafka/putsource.go create mode 100644 idk/kafka/putsource_test.go create mode 100644 idk/kafka/source.go create mode 100644 idk/kafka/source_test.go create mode 100644 idk/kafka/testdata/schemas/bigschema.json create mode 100644 idk/kafka/testdata/schemas/decimal.json create mode 100644 idk/kafka/testdata/schemas/delete.json create mode 100644 idk/kafka/testdata/schemas/delete_string.json create mode 100644 idk/kafka/testdata/schemas/fieldisrecord.json create mode 100644 idk/kafka/testdata/schemas/floatscale.json create mode 100644 idk/kafka/testdata/schemas/ids.json create mode 100644 idk/kafka/testdata/schemas/notarecord.json create mode 100644 idk/kafka/testdata/schemas/othertypes.json create mode 100644 idk/kafka/testdata/schemas/simple.json create mode 100644 idk/kafka/testdata/schemas/stringtypes.json create mode 100644 idk/kafka/testdata/schemas/timestamp.json create mode 100644 idk/kafka/testdata/schemas/unions.json create mode 100644 idk/kafka_sasl/cmd.go create mode 100644 idk/kafka_sasl/cmd_test.go create mode 100644 idk/kafka_sasl/source.go create mode 100644 idk/kafka_sasl/source_test.go create mode 100644 idk/kafka_static/cmd.go create mode 100644 idk/kafka_static/cmd_test.go create mode 100644 idk/kafka_static/source.go create mode 100644 idk/kafka_static/source_test.go create mode 100644 idk/kafka_static/testdata/Flat.json create mode 100644 idk/kafka_static/testdata/Lookup.json create mode 100644 idk/kafka_static/testdata/LookupDuplicate.json create mode 100644 idk/kafka_static/testdata/LookupId.json create mode 100644 idk/kafka_static/testdata/TestFieldTypes.json create mode 100644 idk/kafka_static/testdata/Tree.json create mode 100644 idk/kafkagen/cmd.go create mode 100644 idk/kinesis/cmd.go create mode 100644 idk/kinesis/cmd_test.go create mode 100644 idk/kinesis/logger.go create mode 100644 idk/kinesis/logger_test.go create mode 100644 idk/kinesis/reader.go create mode 100644 idk/kinesis/reader_test.go create mode 100644 idk/kinesis/source.go create mode 100644 idk/kinesis/source_test.go create mode 100644 idk/kinesis/testdata/header.json create mode 100644 idk/kinesis/testdata/invalid_offsets.json create mode 100644 idk/kinesis/testdata/offsets.json create mode 100644 idk/metrics.go create mode 100644 idk/pilosa-sec-test.conf create mode 100644 idk/postgres.go create mode 100644 idk/progress.go create mode 100755 idk/reingest_test.sh create mode 100644 idk/sample.csv create mode 100644 idk/sql/odbc.go create mode 100644 idk/sql/source.go create mode 100644 idk/sql/source_test.go create mode 100644 idk/test_postgres.go create mode 100644 idk/testdata/certs/README.md create mode 100644 idk/testdata/certs/localhost.crt create mode 100644 idk/testdata/certs/localhost.csr create mode 100644 idk/testdata/certs/localhost.key create mode 100644 idk/testdata/certs/pilosa-ca.crl create mode 100644 idk/testdata/certs/pilosa-ca.crt create mode 100644 idk/testdata/certs/pilosa-ca.key create mode 100644 idk/testdata/featurebase.conf create mode 100644 idk/testdata/permissions.yaml create mode 100644 idk/testenv/certs/ca.crl create mode 100644 idk/testenv/certs/ca.crt create mode 100644 idk/testenv/certs/ca.key create mode 100644 idk/testenv/certs/localhost.crt create mode 100644 idk/testenv/certs/localhost.csr create mode 100644 idk/testenv/certs/localhost.key create mode 100644 idk/testenv/certs/pilosa-tls.crt create mode 100644 idk/testenv/certs/pilosa-tls.csr create mode 100644 idk/testenv/certs/pilosa-tls.key create mode 100644 idk/testenv/certs/theclient.crt create mode 100644 idk/testenv/certs/theclient.csr create mode 100644 idk/testenv/certs/theclient.key create mode 100644 idk/tls.go create mode 100644 idk/util.go create mode 100644 idk/util_test.go create mode 100644 idk/version.go create mode 100755 idk/wait.sh create mode 100644 ingest/codec.go create mode 100644 ingest/codec_test.go create mode 100644 ingest/doc.go create mode 100644 ingest/op.go create mode 100644 ingest/op_test.go create mode 100644 ingest/shard.go create mode 100644 ingest/sort.go create mode 100644 ingest/sort_test.go create mode 100644 ingest/translate_test.go create mode 100644 ingest/update.go create mode 100644 ingest/vec.go create mode 100644 ingest/vec_test.go create mode 100644 ingest_test.go create mode 100644 ingest_testdata/bool.tc create mode 100644 ingest_testdata/expect_errors.tc create mode 100644 ingest_testdata/keyed.tc create mode 100644 ingest_testdata/sample.tc create mode 100644 install/featurebase.conf create mode 100644 install/featurebase.debian.service create mode 100644 install/featurebase.redhat.service create mode 100644 install/postinstall.sh create mode 100755 install/preinstall.sh create mode 100755 install/preremove.sh create mode 100644 install/test_installation.Dockerfile create mode 100644 install/test_installation.sh delete mode 100644 internal/LICENSE delete mode 100644 internal/clustertests/Dockerfile create mode 100644 internal/clustertests/Dockerfile-fakeIDP create mode 100644 internal/clustertests/fakeidp/go.mod create mode 100644 internal/clustertests/fakeidp/go.sum create mode 100644 internal/clustertests/fakeidp/server.go create mode 100644 internal/clustertests/pause_node_test.go create mode 100644 internal/clustertests/testdata/certs/README.md create mode 100644 internal/clustertests/testdata/certs/localhost.crt create mode 100644 internal/clustertests/testdata/certs/localhost.csr create mode 100644 internal/clustertests/testdata/certs/localhost.key create mode 100644 internal/clustertests/testdata/certs/pilosa-ca.crl create mode 100644 internal/clustertests/testdata/certs/pilosa-ca.crt create mode 100644 internal/clustertests/testdata/certs/pilosa-ca.key create mode 100644 internal/clustertests/testdata/featurebase.conf create mode 100644 internal/clustertests/testdata/permissions.yaml delete mode 100644 internal/private.pb.go delete mode 100644 internal/public.pb.go delete mode 100644 internal/public.proto create mode 100644 internal_client.go create mode 100644 internal_client_test.go create mode 100644 lattice/.gitignore create mode 100644 lattice/Caddyfile create mode 100644 lattice/Dockerfile create mode 100644 lattice/Makefile create mode 100644 lattice/README.md create mode 100644 lattice/package.json create mode 100755 lattice/public/favicon.ico create mode 100644 lattice/public/favicon.png create mode 100644 lattice/public/favicon.svg create mode 100644 lattice/public/index.html create mode 100644 lattice/public/manifest.json create mode 100644 lattice/src/App.module.scss create mode 100644 lattice/src/App.test.tsx create mode 100644 lattice/src/App.tsx create mode 100644 lattice/src/App/AuthFlow/AuthFlow.module.scss create mode 100644 lattice/src/App/AuthFlow/SignInButton.tsx create mode 100644 lattice/src/App/AuthFlow/SignOutButton.tsx create mode 100644 lattice/src/App/AuthFlow/Signin.tsx create mode 100644 lattice/src/App/AuthFlow/index.ts create mode 100644 lattice/src/App/Home/ClusterHealth/ClusterHealth.module.scss create mode 100644 lattice/src/App/Home/ClusterHealth/ClusterHealth.tsx create mode 100644 lattice/src/App/Home/ClusterHealth/ClusterInfo/ClusterInfo.module.scss create mode 100644 lattice/src/App/Home/ClusterHealth/ClusterInfo/ClusterInfo.tsx create mode 100644 lattice/src/App/Home/ClusterHealth/ClusterInfo/index.ts create mode 100644 lattice/src/App/Home/ClusterHealth/ImportStatus/ImportStatus.module.scss create mode 100644 lattice/src/App/Home/ClusterHealth/ImportStatus/ImportStatus.tsx create mode 100644 lattice/src/App/Home/ClusterHealth/ImportStatus/helpers.ts create mode 100644 lattice/src/App/Home/ClusterHealth/ImportStatus/index.ts create mode 100644 lattice/src/App/Home/ClusterHealth/Metrics/Metrics.module.scss create mode 100644 lattice/src/App/Home/ClusterHealth/Metrics/Metrics.tsx create mode 100644 lattice/src/App/Home/ClusterHealth/Metrics/index.ts create mode 100644 lattice/src/App/Home/ClusterHealth/Metrics/priorityMetrics.ts create mode 100644 lattice/src/App/Home/ClusterHealth/Node/Node.module.scss create mode 100644 lattice/src/App/Home/ClusterHealth/Node/Node.tsx create mode 100644 lattice/src/App/Home/ClusterHealth/Node/index.ts create mode 100644 lattice/src/App/Home/ClusterHealth/Node/nodeInfo.ts create mode 100644 lattice/src/App/Home/ClusterHealth/Node/nodeStatus.ts create mode 100644 lattice/src/App/Home/ClusterHealth/clusterStatus.ts create mode 100644 lattice/src/App/Home/ClusterHealth/index.ts create mode 100644 lattice/src/App/Home/Home.module.scss create mode 100644 lattice/src/App/Home/Home.tsx create mode 100644 lattice/src/App/Home/QueryHistory/NodeIndicator/NodeIndicator.module.scss create mode 100644 lattice/src/App/Home/QueryHistory/NodeIndicator/NodeIndicator.tsx create mode 100644 lattice/src/App/Home/QueryHistory/NodeIndicator/index.ts create mode 100644 lattice/src/App/Home/QueryHistory/QueryHistory.module.scss create mode 100644 lattice/src/App/Home/QueryHistory/QueryHistory.tsx create mode 100644 lattice/src/App/Home/QueryHistory/QueryItem/QueryItem.module.scss create mode 100644 lattice/src/App/Home/QueryHistory/QueryItem/QueryItem.tsx create mode 100644 lattice/src/App/Home/QueryHistory/QueryItem/index.ts create mode 100644 lattice/src/App/Home/QueryHistory/index.ts create mode 100644 lattice/src/App/Home/QueryHistory/nodeColors.ts create mode 100644 lattice/src/App/Home/Transactions/Transaction/Transaction.module.scss create mode 100644 lattice/src/App/Home/Transactions/Transaction/Transaction.tsx create mode 100644 lattice/src/App/Home/Transactions/Transaction/index.ts create mode 100644 lattice/src/App/Home/Transactions/Transaction/utils.test.ts create mode 100644 lattice/src/App/Home/Transactions/Transaction/utils.ts create mode 100644 lattice/src/App/Home/Transactions/Transactions.module.scss create mode 100644 lattice/src/App/Home/Transactions/Transactions.tsx create mode 100644 lattice/src/App/Home/Transactions/index.ts create mode 100644 lattice/src/App/Home/index.ts create mode 100644 lattice/src/App/MoleculaTables/MoleculaTable/MoleculaTable.module.scss create mode 100644 lattice/src/App/MoleculaTables/MoleculaTable/MoleculaTable.tsx create mode 100644 lattice/src/App/MoleculaTables/MoleculaTable/index.ts create mode 100644 lattice/src/App/MoleculaTables/MoleculaTables.module.scss create mode 100644 lattice/src/App/MoleculaTables/MoleculaTables.tsx create mode 100644 lattice/src/App/MoleculaTables/MoleculaTablesContainer.tsx create mode 100644 lattice/src/App/MoleculaTables/index.ts create mode 100644 lattice/src/App/NotFound/NotFound.tsx create mode 100644 lattice/src/App/NotFound/index.tsx create mode 100644 lattice/src/App/Query/Console/Console.module.scss create mode 100644 lattice/src/App/Query/Console/Console.tsx create mode 100644 lattice/src/App/Query/Console/helpers.ts create mode 100644 lattice/src/App/Query/Console/index.ts create mode 100644 lattice/src/App/Query/Query.module.scss create mode 100644 lattice/src/App/Query/Query.tsx create mode 100644 lattice/src/App/Query/QueryContainer.tsx create mode 100644 lattice/src/App/Query/QueryResults/QueryResults.module.scss create mode 100644 lattice/src/App/Query/QueryResults/QueryResults.tsx create mode 100644 lattice/src/App/Query/QueryResults/index.ts create mode 100644 lattice/src/App/Query/RecentQueries/RecentQueries.module.scss create mode 100644 lattice/src/App/Query/RecentQueries/RecentQueries.tsx create mode 100644 lattice/src/App/Query/RecentQueries/index.ts create mode 100644 lattice/src/App/Query/index.ts create mode 100644 lattice/src/App/Query/splitjs.scss create mode 100644 lattice/src/App/QueryBuilder/ColumnSelector/ColumnSelector.module.scss create mode 100644 lattice/src/App/QueryBuilder/ColumnSelector/ColumnSelector.tsx create mode 100644 lattice/src/App/QueryBuilder/ColumnSelector/index.ts create mode 100644 lattice/src/App/QueryBuilder/CountBuilder/CountBuilder.tsx create mode 100644 lattice/src/App/QueryBuilder/CountBuilder/index.ts create mode 100644 lattice/src/App/QueryBuilder/ExtractBuilder/ExtractBuilder.module.scss create mode 100644 lattice/src/App/QueryBuilder/ExtractBuilder/ExtractBuilder.tsx create mode 100644 lattice/src/App/QueryBuilder/ExtractBuilder/index.ts create mode 100644 lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.module.scss create mode 100644 lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx create mode 100644 lattice/src/App/QueryBuilder/GroupByBuilder/index.ts create mode 100644 lattice/src/App/QueryBuilder/GroupByChart/GroupByChart.tsx create mode 100644 lattice/src/App/QueryBuilder/GroupByChart/index.ts create mode 100644 lattice/src/App/QueryBuilder/GroupBySort/GroupBySort.module.scss create mode 100644 lattice/src/App/QueryBuilder/GroupBySort/GroupBySort.tsx create mode 100644 lattice/src/App/QueryBuilder/GroupBySort/index.ts create mode 100644 lattice/src/App/QueryBuilder/QueryBuilder.module.scss create mode 100644 lattice/src/App/QueryBuilder/QueryBuilder.tsx create mode 100644 lattice/src/App/QueryBuilder/QueryBuilderContainer.module.scss create mode 100644 lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx create mode 100644 lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/RowCall.module.scss create mode 100644 lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/RowCall.tsx create mode 100644 lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/helpers.ts create mode 100644 lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/index.ts create mode 100644 lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.module.scss create mode 100644 lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx create mode 100644 lattice/src/App/QueryBuilder/RowCallBuilder/index.ts create mode 100644 lattice/src/App/QueryBuilder/SavedQueries/SavedQueries.tsx create mode 100644 lattice/src/App/QueryBuilder/SavedQueries/index.ts create mode 100644 lattice/src/App/QueryBuilder/index.ts create mode 100644 lattice/src/App/QueryBuilder/rowTypes.ts create mode 100644 lattice/src/App/QueryBuilder/utils.test.ts create mode 100644 lattice/src/App/QueryBuilder/utils.ts create mode 100644 lattice/src/Main.tsx create mode 100644 lattice/src/assets/bg-pattern.png create mode 100644 lattice/src/assets/darkTheme/MoleculaBug.svg create mode 100644 lattice/src/assets/darkTheme/MoleculaLogo.svg create mode 100644 lattice/src/assets/lightTheme/MoleculaBug.svg create mode 100644 lattice/src/assets/lightTheme/MoleculaLogo.svg create mode 100644 lattice/src/assets/m-bug-alt.svg create mode 100644 lattice/src/index.scss create mode 100644 lattice/src/index.tsx create mode 100644 lattice/src/lattice.config.js create mode 100644 lattice/src/proto/pilosa_pb.d.ts create mode 100644 lattice/src/proto/pilosa_pb.js create mode 100644 lattice/src/proto/pilosa_pb_service.d.ts create mode 100644 lattice/src/proto/pilosa_pb_service.js create mode 100644 lattice/src/react-app-env.d.ts create mode 100644 lattice/src/serviceWorker.tsx create mode 100644 lattice/src/services/__mocks__/eventServices.tsx create mode 100644 lattice/src/services/baseURL.ts create mode 100644 lattice/src/services/eventServices.tsx create mode 100644 lattice/src/services/grpcServices.tsx create mode 100644 lattice/src/services/useAuth.test.tsx create mode 100644 lattice/src/services/useAuth.tsx create mode 100644 lattice/src/setupTests.ts create mode 100644 lattice/src/shared/Animations/Animations.tsx create mode 100644 lattice/src/shared/Animations/index.tsx create mode 100644 lattice/src/shared/Block/Block.module.scss create mode 100644 lattice/src/shared/Block/Block.tsx create mode 100644 lattice/src/shared/Block/index.ts create mode 100644 lattice/src/shared/DataTable/DataTable.module.scss create mode 100644 lattice/src/shared/DataTable/DataTable.tsx create mode 100644 lattice/src/shared/DataTable/index.ts create mode 100644 lattice/src/shared/Header/Header.module.scss create mode 100644 lattice/src/shared/Header/Header.tsx create mode 100644 lattice/src/shared/Header/index.tsx create mode 100644 lattice/src/shared/Nav/Nav.module.scss create mode 100644 lattice/src/shared/Nav/Nav.tsx create mode 100644 lattice/src/shared/Nav/index.tsx create mode 100644 lattice/src/shared/Pager/Pager.module.scss create mode 100644 lattice/src/shared/Pager/Pager.tsx create mode 100644 lattice/src/shared/Pager/index.ts create mode 100644 lattice/src/shared/PrivateRoute/PrivateRoute.tsx create mode 100644 lattice/src/shared/Select/Select.module.scss create mode 100644 lattice/src/shared/Select/Select.tsx create mode 100644 lattice/src/shared/Select/index.tsx create mode 100644 lattice/src/shared/SortBy/SortBy.module.scss create mode 100644 lattice/src/shared/SortBy/SortBy.tsx create mode 100644 lattice/src/shared/SortBy/index.ts create mode 100644 lattice/src/shared/StatusIndicator/StatusIndicator.module.scss create mode 100644 lattice/src/shared/StatusIndicator/StatusIndicator.tsx create mode 100644 lattice/src/shared/StatusIndicator/index.ts create mode 100644 lattice/src/shared/ThemeToggle/ThemeToggle.module.scss create mode 100644 lattice/src/shared/ThemeToggle/ThemeToggle.tsx create mode 100644 lattice/src/shared/ThemeToggle/index.tsx create mode 100644 lattice/src/shared/Toggle/Toggle.module.scss create mode 100644 lattice/src/shared/Toggle/Toggle.tsx create mode 100644 lattice/src/shared/Toggle/index.ts create mode 100644 lattice/src/shared/utils/formatBytes.test.ts create mode 100644 lattice/src/shared/utils/formatBytes.ts create mode 100644 lattice/src/shared/utils/formatDuration.test.ts create mode 100644 lattice/src/shared/utils/formatDuration.ts create mode 100644 lattice/src/shared/utils/formatTableCell.test.tsx create mode 100644 lattice/src/shared/utils/formatTableCell.tsx create mode 100644 lattice/src/theme/baseTheme.tsx create mode 100644 lattice/src/theme/darkTheme.tsx create mode 100644 lattice/src/theme/index.tsx create mode 100644 lattice/src/theme/lightTheme.tsx create mode 100644 lattice/tsconfig.json create mode 100644 lattice/tslint.json create mode 100644 lattice/yarn.lock create mode 100644 license.exceptions create mode 100644 like.go create mode 100644 like_test.go create mode 100644 logger/filewriter.go create mode 100644 logger/filewriter_test.go create mode 100644 main_test.go create mode 100644 metrics.go create mode 100644 monitor/monitor.go rename uri.go => net/uri.go (76%) rename uri_internal_test.go => net/uri_internal_test.go (80%) create mode 100644 nfpm.yaml rename internal/internal.go => pb/pb.go (64%) create mode 100644 pb/private.pb.go rename {internal => pb}/private.proto (62%) create mode 100644 pb/public.pb.go create mode 100644 pb/public.proto create mode 100644 pg/cancel.go create mode 100644 pg/cancel_test.go create mode 100644 pg/io.go create mode 100644 pg/lookerToFeaturebase.md create mode 100644 pg/lookerToPostgres.md create mode 100644 pg/message/io.go create mode 100644 pg/message/message.go create mode 100644 pg/pgtest/handler.go create mode 100644 pg/pgtest/memnet.go create mode 100644 pg/pgtest/server.go create mode 100644 pg/pgtest/tls.go create mode 100644 pg/protocol.go create mode 100644 pg/query.go create mode 100644 pg/server.go create mode 100644 pg/server_test.go create mode 100644 pg/type.go create mode 100644 planner.go create mode 100644 planner_test.go create mode 100644 pprof.go create mode 100644 pql/decimal.go create mode 100644 pql/decimal_internal_test.go create mode 100644 pql/decimal_test.go create mode 100644 proto/interface.go create mode 100644 proto/pilosa.pb.go create mode 100644 proto/pilosa.proto create mode 100644 proto/vdsm/vdsm.pb.go create mode 100644 proto/vdsm/vdsm.proto create mode 100644 qa/fakeidp/go.mod create mode 100644 qa/fakeidp/go.sum create mode 100644 qa/fakeidp/server.go create mode 100755 qa/scripts/auth-smoke/setup.sh create mode 100755 qa/scripts/auth-smoke/teardown.sh create mode 100755 qa/scripts/auth-smoke/test.sh create mode 100644 qa/scripts/auth-smoke/tests/README create mode 100755 qa/scripts/auth-smoke/tests/run-all.sh create mode 100755 qa/scripts/auth-smoke/tests/sup218-test.sh create mode 100644 qa/scripts/auth-smoke/tests/sup218_datagen.yaml create mode 100755 qa/scripts/backupRestoreTest.sh create mode 100644 qa/scripts/backup_test_datagen.yaml create mode 100755 qa/scripts/bug_repro_tests.sh create mode 100755 qa/scripts/gitlabCleanupBuild.sh create mode 100644 qa/scripts/perf/able/able.yaml create mode 100755 qa/scripts/perf/able/ableRun.sh create mode 100755 qa/scripts/perf/able/ableSetup.sh create mode 100755 qa/scripts/perf/able/ableTeardown.sh create mode 100755 qa/scripts/perf/able/ableTest.sh create mode 100755 qa/scripts/perf/able/generateTestData.sh create mode 100644 qa/scripts/perf/able/highcardinalitygroupby.js create mode 100644 qa/scripts/perf/able/values/education.txt create mode 100644 qa/scripts/perf/able/values/gender.txt create mode 100644 qa/scripts/perf/able/values/hobbies.txt create mode 100644 qa/scripts/perf/able/values/income.txt create mode 100644 qa/scripts/perf/able/values/opendns-top-domains-10K.txt create mode 100644 qa/scripts/perf/able/values/political_parties.txt create mode 100755 qa/scripts/runSmokeTest.sh create mode 100755 qa/scripts/setupBackupRestoreGauntlet.sh create mode 100755 qa/scripts/setupSmokeTest.sh create mode 100755 qa/scripts/setupTLS.sh create mode 100755 qa/scripts/teardownBackupRestoreGauntlet.sh create mode 100755 qa/scripts/teardownSmokeTest.sh create mode 100755 qa/scripts/testBackupRestoreGauntlet.sh create mode 100755 qa/scripts/testSmokeTest.sh create mode 100644 qa/simulacraData/simulacra_data.go create mode 100644 qa/simulacraData/simulacra_data_test.go create mode 100644 qa/testcases/bug-repros/README create mode 100644 qa/testcases/bug-repros/fb-1287-datagen.yaml create mode 100755 qa/testcases/bug-repros/fb-1287-test.sh create mode 100644 qa/testcases/bug-repros/fb-1332-datagen.yaml create mode 100755 qa/testcases/bug-repros/fb-1332.sh create mode 100644 qa/testcases/bug-repros/fb-1371-datagen.yaml create mode 100755 qa/testcases/bug-repros/fb-1371-test.sh create mode 100755 qa/testcases/bug-repros/run-all.sh create mode 100644 qa/testcases/smoketest/config.py create mode 100644 qa/testcases/smoketest/test_smoke.py create mode 100644 qa/tf/.modules/featurebase-cluster/README.md create mode 100644 qa/tf/.modules/featurebase-cluster/cloud-init.sh create mode 100644 qa/tf/.modules/featurebase-cluster/main.tf create mode 100644 qa/tf/.modules/featurebase-cluster/outputs.tf create mode 100644 qa/tf/.modules/featurebase-cluster/provider.tf create mode 100644 qa/tf/.modules/featurebase-cluster/variables.tf create mode 100644 qa/tf/README.md create mode 100644 qa/tf/ci/auth-smoke/main.tf create mode 100644 qa/tf/ci/auth-smoke/outputs.tf create mode 100644 qa/tf/ci/auth-smoke/provider.tf create mode 100644 qa/tf/ci/auth-smoke/tf.auto.tfvars create mode 100644 qa/tf/ci/auth-smoke/variables.tf create mode 100644 qa/tf/ci/deletetest/delete_cloud_init.sh create mode 100644 qa/tf/ci/deletetest/main.tf create mode 100644 qa/tf/ci/deletetest/outputs.tf create mode 100644 qa/tf/ci/deletetest/provider.tf create mode 100644 qa/tf/ci/deletetest/tf.auto.tfvars create mode 100644 qa/tf/ci/deletetest/variables.tf create mode 100644 qa/tf/ci/smoketest/main.tf create mode 100644 qa/tf/ci/smoketest/outputs.tf create mode 100644 qa/tf/ci/smoketest/provider.tf create mode 100644 qa/tf/ci/smoketest/tf.auto.tfvars create mode 100644 qa/tf/ci/smoketest/variables.tf create mode 100644 qa/tf/gauntlet/backuprestore/README.md create mode 100644 qa/tf/gauntlet/backuprestore/main.tf create mode 100644 qa/tf/gauntlet/backuprestore/outputs.tf create mode 100644 qa/tf/gauntlet/backuprestore/provider.tf create mode 100644 qa/tf/gauntlet/backuprestore/tf.auto.tfvars create mode 100644 qa/tf/gauntlet/backuprestore/variables.tf create mode 100644 qa/tf/perf/able/main.tf create mode 100644 qa/tf/perf/able/outputs.tf create mode 100644 qa/tf/perf/able/provider.tf create mode 100644 qa/tf/perf/able/tf.auto.tfvars create mode 100644 qa/tf/perf/able/variables.tf create mode 100644 qa/tf/perf/delete/main.tf create mode 100644 qa/tf/perf/delete/outputs.tf create mode 100644 qa/tf/perf/delete/provider.tf create mode 100644 qa/tf/perf/delete/tf.auto.tfvars create mode 100644 qa/tf/perf/delete/variables.tf create mode 100644 rbf.go create mode 100644 rbf/README.md create mode 100644 rbf/array.go create mode 100644 rbf/cfg/cfg.go create mode 100644 rbf/cfg/os.go create mode 100644 rbf/cfg/os_386.go create mode 100644 rbf/cursor.go create mode 100644 rbf/cursor_internal_test.go create mode 100644 rbf/cursor_test.go create mode 100644 rbf/cursorx.go create mode 100644 rbf/db.go create mode 100644 rbf/db_test.go create mode 100644 rbf/dot.go create mode 100644 rbf/helpers_test.go create mode 100644 rbf/ingest_test.go create mode 100644 rbf/page_map.go create mode 100644 rbf/rbf.go create mode 100644 rbf/rbf/testdata/check/bad-freelist/data create mode 100644 rbf/rbf/testdata/check/bad-freelist/wal create mode 100644 rbf/rbf_test.go create mode 100644 rbf/testdata/check/bad-bitmap/data create mode 100644 rbf/testdata/check/bad-bitmap/wal create mode 100644 rbf/testdata/check/bad-freelist/data create mode 100644 rbf/testdata/check/bad-freelist/wal create mode 100644 rbf/tx.go create mode 100644 rbf/tx_test.go create mode 100644 rbf/util.go create mode 100644 rbf/util_test.go create mode 100644 roaring/add.go create mode 100644 roaring/add_test.go create mode 100644 roaring/benchpretty/main.go create mode 100644 roaring/container_archetypes.go create mode 100644 roaring/filter.go create mode 100644 roaring/filter_internal_test.go create mode 100644 roaring/printutil.go create mode 100644 roaring/printutil_test.go create mode 100644 roaring/roaring_container_test.go create mode 100755 scripts/bench_read.sh create mode 100755 scripts/bench_write.sh create mode 100644 scripts/etc/gloat/gh.1d.yml create mode 100644 scripts/etc/gloat/gh.1m.yml create mode 100644 scripts/etc/gloat/gh.1w.yml create mode 100644 scripts/etc/gloat/gh.issues.autogenerate.yml create mode 100644 scripts/etc/gloat/gh.issues.keyed.yml create mode 100644 scripts/etc/gloat/gh.issues.unkeyed.yml create mode 100644 scripts/etc/gloat/query.count-keyed.yml create mode 100644 scripts/etc/gloat/query.count.yml create mode 100644 scripts/etc/gloat/query.difference.yml create mode 100644 scripts/etc/gloat/query.groupby.yml create mode 100644 scripts/etc/gloat/query.intersect.yml create mode 100644 scripts/etc/gloat/query.row-bsi.yml create mode 100644 scripts/etc/gloat/query.row-range.yml create mode 100644 scripts/etc/gloat/query.row.yml create mode 100644 scripts/etc/gloat/query.topk.yml create mode 100644 scripts/etc/gloat/query.union.yml create mode 100644 scripts/etc/gloat/query.xor.yml create mode 100755 scripts/populate_query_db.keyed.sh create mode 100755 scripts/populate_query_db.sh create mode 100644 serializer.go delete mode 100644 server/default.go create mode 100644 server/dup.go create mode 100644 server/dup_arm64.go delete mode 100644 server/enterprise.go create mode 100644 server/grpc.go create mode 100644 server/grpc_internal_test.go create mode 100644 server/grpc_test.go create mode 100644 server/pg.go create mode 100644 server/pg_internal_test.go create mode 100644 server/pg_test.go delete mode 100644 server/release.go create mode 100644 server/server_internal_test.go delete mode 100644 server/setup_logger.go delete mode 100644 server/setup_logger_arm64.go create mode 100644 server/sql.go create mode 100644 server/tlsconfig_test.go create mode 100644 server/trial.go create mode 100644 server_test.go create mode 100644 shardwidth/helper.go create mode 100644 shardwidth/helper_test.go create mode 100644 short_txkey/txkey.go create mode 100644 short_txkey/txkey_test.go create mode 100644 sql/column.go create mode 100644 sql/ddl.go create mode 100644 sql/extract.go create mode 100644 sql/handler_test.go create mode 100644 sql/mapper.go create mode 100644 sql/mapper_test.go create mode 100644 sql/mask.go create mode 100644 sql/model.go create mode 100644 sql/query.go create mode 100644 sql/reduce.go create mode 100644 sql/reduce_test.go create mode 100644 sql/router.go create mode 100644 sql/select.go create mode 100644 sql/show.go create mode 100644 sql2/ast.go create mode 100644 sql2/ast_test.go create mode 100644 sql2/parser.go create mode 100644 sql2/parser_test.go create mode 100644 sql2/scanner.go create mode 100644 sql2/scanner_test.go create mode 100644 sql2/token.go create mode 100644 sql2/token_test.go create mode 100644 sql2/walk.go create mode 100644 statik/.gitignore create mode 100644 statik/filesystem.go create mode 100644 stattx.go create mode 100644 storage/config.go create mode 100644 task/doc.go create mode 100644 task/pool.go create mode 100644 task/pool_test.go create mode 100644 test/disco.go delete mode 100644 test/logger.go create mode 100644 test/transaction.go create mode 100644 testdata/certs/localhost.crt create mode 100644 testdata/certs/localhost.key create mode 100644 testdata/schema.json create mode 100644 testdata/timeRegressionSchema.json create mode 100644 testhook/auditor.go create mode 100644 testhook/auditor_test.go create mode 100644 testhook/cleanup1.13.go create mode 100644 testhook/cleanup1.14.go create mode 100644 testhook/hook.go create mode 100644 testhook/registry.go create mode 100644 tracker.go create mode 100644 tracker_test.go create mode 100644 transaction.go create mode 100644 transaction_test.go create mode 100644 tx.go create mode 100644 tx_internal_test.go create mode 100644 tx_test.go create mode 100644 txfactory.go create mode 100644 txfactory_internal_test.go create mode 100644 txkey/txkey.go create mode 100644 txkey/txkey_test.go create mode 100644 util.go create mode 100644 util_test.go create mode 100644 vprint/vprint.go diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 5384c556d..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,195 +0,0 @@ -version: 2 -defaults: &defaults - working_directory: /go/src/github.com/pilosa/pilosa - docker: - - image: circleci/golang:1.13 - environment: - GO111MODULE: "on" -fast-checkout: &fast-checkout - attach_workspace: - at: . -jobs: - setup: - <<: *defaults - steps: - - checkout - - restore_cache: - keys: - - mod-cache-{{ checksum "go.sum" }} - - run: "go mod download" - - save_cache: - key: mod-cache-{{ checksum "go.sum" }} - paths: - - /go/pkg/mod/ - - persist_to_workspace: - root: . - paths: "*" - check-license-headers: - <<: *defaults - steps: - - *fast-checkout - - run: make check-license-headers - linter: - <<: *defaults - steps: - - *fast-checkout - - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0 - - run: sudo cp bin/golangci-lint /usr/local/bin/ - - run: make golangci-lint - test-build-arm: - <<: *defaults - steps: - - *fast-checkout - - run: make build GOOS=linux GOARCH=arm GOARM=5 - - run: make build GOOS=linux GOARCH=arm GOARM=6 - - run: make build GOOS=linux GOARCH=arm GOARM=7 - - run: make build GOOS=linux GOARCH=arm64 - test-golang-1.13: &base-test - <<: *defaults - steps: - - *fast-checkout - - run: sudo apt-get install lsof - - run: make test - test-golang-1.13-shard22: - <<: *defaults - steps: - - *fast-checkout - - run: sudo apt-get install lsof - - run: make test SHARD_WIDTH=22 - test-golang-1.13-race: - <<: *defaults - steps: - - *fast-checkout - - run: sudo apt-get install lsof - - run: - command: make test TESTFLAGS="-race -v -timeout=30m" - no_output_timeout: 30m - test-golang-1.13-386: - <<: *base-test - environment: - GO111MODULE: "on" - GOARCH: 386 - test-golang-1.13-enterprise: - <<: *defaults - steps: - - *fast-checkout - - run: sudo apt-get install lsof - - run: make test ENTERPRISE=1 - test-golang-1.12: - <<: *defaults - docker: - - image: circleci/golang:1.12 - steps: - - *fast-checkout - - run: sudo apt-get install lsof - - run: make test - test-golang-1.11: - <<: *defaults - docker: - - image: circleci/golang:1.11 - steps: - - *fast-checkout - - run: sudo apt-get install lsof - - run: make test - cluster-tests: - <<: *defaults - steps: - - *fast-checkout - - setup_remote_docker - - run: make clustertests-build - prerelease: - <<: *base-test - steps: - - *fast-checkout - - run: make prerelease - - store_artifacts: - path: build - - persist_to_workspace: - root: . - paths: build - release: - <<: *defaults - steps: - - *fast-checkout - - run: make release - - store_artifacts: - path: build - - persist_to_workspace: - root: . - paths: build - prerelease-upload: - docker: - - image: circleci/python:2.7-jessie - steps: - - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - - *fast-checkout - - run: sudo pip install awscli - - run: make prerelease-upload - dockerhub-upload: - <<: *defaults - steps: - - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - - *fast-checkout - - setup_remote_docker - - run: make docker - - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master - - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: docker push pilosa/pilosa:master -workflows: - version: 2 - test: - jobs: - - setup - - linter: - requires: - - setup - - check-license-headers: - requires: - - setup - - test-build-arm: - requires: - - setup - - test-golang-1.13-enterprise: - requires: - - setup - - test-golang-1.13-race: - requires: - - setup - - test-golang-1.13-386: - requires: - - setup - - test-golang-1.13: - requires: - - setup - - test-golang-1.12: - requires: - - setup - - test-golang-1.11: - requires: - - setup - - cluster-tests: - requires: - - setup - - prerelease: - requires: - - linter - - check-license-headers - - test-golang-1.13 - - release: - requires: - - linter - - check-license-headers - - test-golang-1.13 - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ - - prerelease-upload: - requires: - - prerelease - - dockerhub-upload: - requires: - - linter - - check-license-headers - - test-golang-1.13 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 05043a418..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,16 +0,0 @@ -For bugs, please provide the following: - -### What's going wrong? - -### What was expected? - -### Steps to reproduce the behavior - -### Information about your environment (OS/architecture, CPU, RAM, cluster/solo, configuration, etc.) - - -For feature requests, please provide the following: - -### Description - -### Success criteria (What criteria will consider this ticket closeable?) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index ceb76f05f..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,28 +0,0 @@ -## Overview - -[Describe what this pull request addresses.] - -Fixes # - -## Pull request checklist - -- [ ] I have read the [contributing guide](https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md). -- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). -- [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs). -- [ ] I have resolved any merge conflicts. -- [ ] I have included tests that cover my changes. -- [ ] All new and existing tests pass. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. -- [ ] Add appropriate changelog label to PR (if applicable). - -## Code review checklist -This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for. - -- [ ] Ensure that any changes to external docs have been included in this pull request. -- [ ] If the changes require that minor/major versions need to be updated, tag the PR appropriately. -- [ ] Ensure the new code is [properly commented](https://github.com/golang/go/wiki/CodeReviewComments#doc-comments) and follows [Idiomatic Go](https://dmitri.shuralyov.com/idiomatic-go). -- [ ] Check that tests have been written and that they cover the new functionality. -- [ ] Run tests and ensure they pass. -- [ ] Build and run the code, performing any applicable integration testing. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. -- [ ] Make sure PR is tagged with appropriate changelog label. diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 57de1f746..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,715 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) -and this project adheres to [Semantic Versioning](http://semver.org/). - -## [1.4.0] - 2019-09-17 - -This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions. - -**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release. - -### Added -- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028)) -- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029)) -- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016)) -- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023)) -- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004)) -- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983)) -- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992)) -- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939)) -- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950)) -- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956)) - -### Changed -- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064)) -- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001)) -- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994)) -- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993)) -- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978)) -- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902)) -- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966)) - -### Fixed -- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050)) -- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034)) -- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033)) -- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026)) -- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021)) -- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019)) -- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017)) -- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012)) -- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000)) -- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997)) -- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975)) -- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973)) -- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970)) -- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954)) -- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951)) -- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947)) - -### Performance -- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988)) -- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048)) -- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041)) -- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024)) -- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974)) - -### Removed -- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996)) - -## [1.3.1] - 2019-05-01 - -This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions. - -### Fixed -- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964)) - -## [1.3.0] - 2019-04-16 - -This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions. - -### Added -- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940)) -- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921)) -- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915)) -- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859)) -- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910)) -- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906)) -- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903)) -- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881)) -- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761)) -- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832)) -- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839)) -- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831)) -- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763)) -- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820)) -- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818)) -- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811)) - -### Fixed -- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944)) -- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943)) -- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938)) -- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936)) -- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931)) -- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876)) -- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863)) -- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887)) -- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848)) -- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846)) -- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842)) -- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834)) -- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835)) -- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836)) -- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826)) -- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827)) -- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822)) -- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802)) - -### Changed -- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924)) -- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937)) -- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909)) -- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900)) -- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616)) -- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804)) -- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851)) -- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830)) - -### Performance -- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916)) -- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901)) -- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897)) -- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892)) -- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871)) -- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865)) -- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814)) -- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803)) - -## [1.2.0] - 2018-12-20 - -This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. - -### Added - -- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) -- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774)) -- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) -- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) -- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) -- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) -- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) -- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) -- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) -- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717)) -- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) -- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) -- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) -- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) -- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) -- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) -- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666)) -- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) -- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) -- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) -- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) -- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) -- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) -- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738)) -- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) -- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) -- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) -- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628)) - -### Changed - -- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) -- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) -- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) -- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) -- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) -- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) -- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) -- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) - -### Performance - -- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) -- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741)) - -### Fixed - -- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795)) -- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) -- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) -- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) -- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782)) -- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) -- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) -- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) -- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) -- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) -- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) -- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) -- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) -- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) -- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729)) -- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) -- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) -- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) -- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711)) -- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674)) -- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675)) -- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) -- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) -- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) -- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) -- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) -- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) -- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) -- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) -- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) -- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) -- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) -- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620)) -- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) - - -## [1.1.0] - 2018-08-21 - -This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. - -### Added - -- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610)) -- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608)) -- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601)) -- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582)) -- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584)) -- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527)) -- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524)) -- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532)) - -### Fixed - -- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607)) -- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586)) -- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603)) -- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602)) -- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) - -## [1.0.2] - 2018-08-01 - -This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions. - -### Fixed - -- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551)) -- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547)) -- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541)) -- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542)) -- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533)) -- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502)) -- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555)) - -## [1.0.1] - 2018-07-11 - -This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions. - -### Fixed - -- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491)) -- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485)) -- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486)) -- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483)) -- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493)) - -## [1.0.0] - 2018-07-09 - -This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions. - -### Added - -- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337)) -- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327)) - -### Changed - -- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466)) -- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297)) -- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389)) -- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452)) -- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426)) -- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424)) -- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461)) -- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451)) -- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407)) -- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463)) -- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355)) -- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365)) -- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333)) -- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326)) -- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311)) -- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454)) - -### Removed - -- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397)) -- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462)) -- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396)) -- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363)) -- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347)) -- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346)) -- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341)) -- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335)) -- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332)) -- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468)) - -### Fixed - -- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423)) -- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401)) -- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) -- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) - -## [0.10.0] - 2018-05-15 - -This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions. - -### Added - -- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285)) -- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236)) - -### Changed - -- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265)) - -### Removed - -- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286)) -- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284)) -- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238)) -- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255)) - -### Fixed - -- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269)) -- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279)) -- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278)) -- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268)) -- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266)) -- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242)) -- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234)) -- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253)) - -## [0.9.0] - 2018-05-04 - -This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions. - -*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* - -### Added - -- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) -- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) -- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) -- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) -- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138)) -- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132)) -- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109)) -- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) -- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) -- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) -- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) -- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) -- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) -- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) -- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) - -### Changed - -- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163)) -- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166)) -- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158)) -- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148)) -- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797)) -- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115)) -- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099)) -- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077)) -- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032)) -- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) -- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) -- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) -- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) -- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) -- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) -- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) -- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) - -### Removed - -- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) -- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) -- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) -- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) -- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) -- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) -- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) -- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) -- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) - -### Fixed - -- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143)) -- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139)) -- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110)) -- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102)) -- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105)) -- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101)) -- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) -- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) -- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) -- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) -- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) -- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) - -### Deprecated - -- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) - -### Performance - -- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) - -## [0.8.8] - 2018-02-19 - -This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions. - -### Fixed - -- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118)) - -## [0.8.7] - 2018-02-12 - -This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions. - -### Fixed - -- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111)) - -## [0.8.6] - 2018-02-09 - -This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions. - -### Fixed - -- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106) -- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083) - -## [0.8.5] - 2018-01-18 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions. - -### Fixed - -- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061)) - -## [0.8.4] - 2018-01-10 - -This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions. - -### Fixed - -- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038)) -- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033)) - -### Performance - -- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046)) -- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042)) - -## [0.8.3] - 2017-12-12 - -This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions. - -### Fixed - -- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000)) - -## [0.8.2] - 2017-12-05 - -This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions. - -### Fixed - -- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994)) - -## [0.8.1] - 2017-11-15 - -This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions. - -### Fixed - -- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969)) -- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968)) - -## [0.8.0] - 2017-11-15 - -This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions. - -### Added - -- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895)) -- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933)) -- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938)) -- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942)) -- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867)) -- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898)) -- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889)) -- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881)) -- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955)) - -### Changed - -- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943)) -- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884)) -- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892)) -- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945)) -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Fixed - -- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941)) -- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934)) -- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919)) -- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887)) -- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882)) -- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949)) - -### Performance - -- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.2] - 2017-11-15 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion. - -### Changed - -- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920)) - -## [0.7.1] - 2017-10-09 - -This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions. - -### Changed - -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Performance - -- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.0] - 2017-10-03 - -This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions. - -### Added - -- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856)) -- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785)) -- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840)) -- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791)) -- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755)) -- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778)) -- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861)) -- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847)) -- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789)) -- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837)) -- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831)) -- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830)) -- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806)) -- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783)) - -### Fixed - -- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860)) -- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814)) -- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795)) -- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794)) -- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779)) -- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773)) - -### Deprecated - -- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839)) - -### Performance - -- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859)) -- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841)) -- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842)) -- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848)) -- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846)) -- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849)) -- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820)) -- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812)) - -## [0.6.0] - 2017-08-11 - -This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions. - -### Added - -- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758)) - -### Changed - -- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750)) - -### Fixed - -- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759)) -- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674)) - -## [0.5.0] - 2017-08-02 - -This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions. - -### Added - -- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748)) -- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646)) -- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745)) -- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741)) -- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636)) -- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635)) -- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629)) -- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600)) -- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606)) -- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721)) -- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702)) -- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670)) - -### Changed - -- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747)) -- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744)) -- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699)) -- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675)) - -### Fixed - -- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729)) -- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678)) - -## [0.4.0] - 2017-06-08 - -This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions. - -*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).* - -### Added -- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580)) -- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586)) -- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592)) -- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571)) -- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560)) -- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537)) -- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541)) -- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563)) -- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551)) -- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540)) -- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546)) -- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535)) -- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533)) -- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510)) - -### Changed -- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520)) -- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578)) -- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573)) -- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524)) -- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525)) - -### Fixed -- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605)) -- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594)) -- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599)) -- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597)) -- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589)) -- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565)) -- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559)) -- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555)) -- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554)) -- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548)) -- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528)) -- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503)) -- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507)) -- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462)) - -### Performance -- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590)) -- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577)) -- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562)) - -[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD -[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 -[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 -[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 -[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 -[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 -[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9 -[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 -[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 -[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1 -[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..9daea3c77 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,133 @@ + +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +community@featurebase.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 8e4b45e4e..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,177 +0,0 @@ -# Contributing to Pilosa - -The workflow components of these instructions apply to all Pilosa repositories. - -## Reporting a bug - -If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]. - -## Submitting a feature request - -Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4] -which starts by [opening a new issue][1] that describes the new feature proposal. - -## Making code contributions - -Before you start working on new features, you should [open a new issue][1] to let others know what -you're doing, otherwise you run the risk of duplicating effort. This also -gives others an opportunity to provide input for your feature. - -If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6]. - - -### Development Environment - -- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information. - -- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). - -- Fork the [Pilosa repository][2] to your own account. - -- It will be easier to follow these instructions if you: - - ```sh - export GH_USERNAME= - ``` - -- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa: - - ```sh - mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ - git clone https://github.com/pilosa/pilosa.git - ``` - -- `cd` to your pilosa directory: - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - ``` - -- Install Pilosa command line tools: - - ```sh - make install - ``` - - Running `pilosa` should now run a Pilosa instance. - -- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git - ``` - -### Makefile - -Pilosa includes a Makefile that automates several tasks: - -- Install Pilosa: - - ```sh - make install - ``` - -- Install build dependencies: - - ```sh - make install-build-deps - ``` - -- Create the vendor directory: - - ```sh - make vendor - ``` - -- Run the test suite: - - ```sh - make test - ``` - -- View the coverage report: - - ```sh - make cover-viz - ``` - -- Clear the `vendor/` and `build/` directories: - - ```sh - make clean - ``` - -- Create release tarballs: - - ```sh - make release - ``` - -- Regenerate protocol buffer files in `internal/`: - - ```sh - make generate-protoc - ``` - -- Create tagged Docker image: - - ```sh - make docker - ``` - -- Run tests inside Docker container: - - ```sh - make docker-test - ``` - -Additional commands are available in the `Makefile`. - -### Submitting code changes - -- Before starting to work on a task, sync your branch with the upstream: - - ```sh - git checkout master - git pull - ``` - -- Create a local feature branch: - - ```sh - git checkout -b something-amazing - ``` - -- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/). - -- Make sure that you've written tests for your new feature, and then run the tests: - - ```sh - make test - ``` - -- Verify that your pull request is applied to the latest version of code on github: - - ```sh - git checkout master - git pull - git checkout something-amazing - git rebase master - ``` - -- Push to your fork: - - ```sh - git push -u $GH_USERNAME something-amazing:something-amazing - ``` - -- Submit a [pull request][3] - - -[1]: https://github.com/pilosa/pilosa/issues/new -[2]: https://github.com/pilosa/pilosa -[3]: https://github.com/pilosa/pilosa/compare/ -[4]: https://github.com/pilosa/general/blob/master/proposal.md -[5]: https://github.com/pilosa/pilosa/issues -[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer diff --git a/Dockerfile b/Dockerfile index 0dcf9c4a1..25ea9c3f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,55 @@ -FROM golang:1.13.0 as builder +ARG GO_VERSION=latest -COPY . pilosa +####################### +### Lattice builder ### +####################### -RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a" +FROM moleculacorp/nodejs:latest as lattice-builder +WORKDIR /lattice -FROM alpine:3.9.4 +COPY lattice/package.json ./ +COPY lattice/yarn.lock ./ +RUN yarn install -LABEL maintainer "dev@pilosa.com" +COPY lattice ./ +RUN yarn build + +###################### +### Pilosa builder ### +###################### + +FROM golang:${GO_VERSION} as pilosa-builder +ARG MAKE_FLAGS +WORKDIR /pilosa + +RUN go get github.com/rakyll/statik + +COPY . ./ +COPY --from=lattice-builder /lattice/build /lattice +RUN /go/bin/statik -src=/lattice -dest=/pilosa + +RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS} + +##################### +### Pilosa runner ### +##################### + +FROM alpine:3.13.2 as runner + +LABEL maintainer "dev@molecula.com" RUN apk add --no-cache curl jq -COPY --from=builder /go/bin/pilosa /pilosa +COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 VOLUME /data -ENTRYPOINT ["/pilosa"] -CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] +ENV PILOSA_DATA_DIR /data +ENV PILOSA_BIND 0.0.0.0:10101 +ENV PILOSA_BIND_GRPC 0.0.0.0:20101 + +ENTRYPOINT ["/featurebase"] +CMD ["server"] diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 53621b9ab..98cd2a2a6 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -1,29 +1,35 @@ # This Dockerfile is used for cluster testing - it produces a much larger image # and includes all of Go as well as some utilities. -FROM golang:1.11 +FROM golang:1.19 LABEL maintainer "dev@pilosa.com" -COPY . /go/src/github.com/pilosa/pilosa/ +COPY . /go/src/github.com/molecula/featurebase/ -RUN cd /go/src/github.com/pilosa/pilosa \ - && GO111MODULE=on make vendor - -RUN cd /go/src/github.com/pilosa/pilosa \ - && CGO_ENABLED=0 make install FLAGS="-a" # download pumba for fault injection ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba RUN chmod +x /pumba -RUN cp /go/bin/pilosa /pilosa +# add docker client to pause/unpause nodes +RUN apt update +RUN apt install -y docker.io + +# add docker-compose so tests can use it for stuff +ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose +RUN chmod +x /usr/local/bin/docker-compose + +# generate an instrumented binary to allow for calculating code coverage for clustertests +# the entrypoint for the binary is TestRunMain, which is wrapper for main +RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \ + go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \ + cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 VOLUME /data ENTRYPOINT ["bash", "-c"] -CMD ["/pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] +CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] diff --git a/Dockerfile-clustertests-client b/Dockerfile-clustertests-client new file mode 100644 index 000000000..85e10a5f5 --- /dev/null +++ b/Dockerfile-clustertests-client @@ -0,0 +1,35 @@ +# This Dockerfile is used for cluster testing - it produces a much larger image +# and includes all of Go as well as some utilities. + +FROM golang:1.19 + +LABEL maintainer "dev@pilosa.com" + +COPY . /go/src/github.com/molecula/featurebase/ + +# download pumba for fault injection +ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba +RUN chmod +x /pumba + +# add docker client to pause/unpause nodes +RUN apt update +RUN apt install -y docker.io + +# add docker-compose so tests can use it for stuff +ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose +RUN chmod +x /usr/local/bin/docker-compose + +RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \ + go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \ + cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase + + +COPY NOTICE /NOTICE + +COPY ./internal/clustertests /go/src/github.com/molecula/featurebase/internal/clustertests + +EXPOSE 10101 +VOLUME /data + +ENTRYPOINT ["bash", "-c"] +CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] diff --git a/LICENSE b/LICENSE-2.0.txt similarity index 100% rename from LICENSE rename to LICENSE-2.0.txt diff --git a/Makefile b/Makefile index 969624d90..8f474f1e7 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,34 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH) -BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) +VARIANT = Molecula +GO=go +GOOS=$(shell $(GO) env GOOS) +GOARCH=$(shell $(GO) env GOARCH) +VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 -LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Enterprise=$(if $(ENTERPRISE_ENABLED),1)" -GO_VERSION=latest -ENTERPRISE ?= 0 -ENTERPRISE_ENABLED = $(subst 0,,$(ENTERPRISE)) -RELEASE ?= 0 -RELEASE_ENABLED = $(subst 0,,$(RELEASE)) -BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise) -BUILD_TAGS += $(if $(RELEASE_ENABLED),release) +COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) +LDFLAGS="-X github.com/molecula/featurebase/v3.Version=$(VERSION) -X github.com/molecula/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v3.Variant=$(VARIANT) -X github.com/molecula/featurebase/v3.Commit=$(COMMIT) -X github.com/molecula/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)" +GO_VERSION=1.19 +DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) -LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ") +TEST_TAGS = roaringparanoia +UNAME := $(shell uname -s) +TEST_TIMEOUT=30m +RACE_TEST_TIMEOUT=90m +ifeq ($(UNAME), Darwin) + IS_MACOS:=1 +else + IS_MACOS:=0 +endif + export GO111MODULE=on +export GOPRIVATE=github.com/molecula +export CGO_ENABLED=0 # Run tests and compile Pilosa default: test build @@ -25,17 +36,57 @@ default: test build # Remove build directories clean: rm -rf vendor build - + rm -f *.rpm *.deb + # Set up vendor directory using `go mod vendor` vendor: go.mod - go mod vendor + $(GO) mod vendor + +version: + @echo $(VERSION) + +# We build a list of packages that omits the IDK packages because the IDK +# packages require fancy environment setup. +GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk") # Run test suite test: - go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) + $(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) + +# Run test suite with race flag +test-race: + CGO_ENABLED=1 $(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v + +testv: testvsub + +testv-race: testvsub-race + +# testvsub: run go test -v in sub-directories in "local mode" with incremental output, +# avoiding go -test ./... "package list mode" which doesn't give output +# until the test run finishes. Package list mode makes it hard to +# find which test is hung/deadlocked. +# +testvsub: + @set -e; for pkg in $(GOPACKAGES); do \ + if [ $${pkg:0:38} == "github.com/molecula/featurebase/v3/idk" ]; then \ + echo; echo "___ skipping subpkg $$pkg"; \ + continue; \ + fi; \ + echo; echo "___ testing subpkg $$pkg"; \ + $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \ + echo; echo "999 done testing subpkg $$pkg"; \ + done + + +testvsub-race: + @set -e; for pkg in $(GOPACKAGES); do \ + echo; echo "___ testing subpkg $$pkg"; \ + CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \ + echo; echo "999 done testing subpkg $$pkg"; \ + done bench: - go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) + $(GO) test $(GOPACKAGES) -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) # Run test suite with coverage enabled cover: @@ -44,19 +95,27 @@ cover: # Run test suite with coverage enabled and view coverage results in browser cover-viz: cover - go tool cover -html=build/coverage.out + $(GO) tool cover -html=build/coverage.out # Compile Pilosa build: - go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase # Create a single release build under the build directory release-build: - $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE README.md build/pilosa-$(VERSION_ID) - $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID)) - tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ - @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase" + cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID) + tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ + @echo Created release build: build/featurebase-$(VERSION_ID).tar.gz + +test-release-build: docker-build + mv build/featurebase-$(VERSION_ID).tar.gz install/ + cd install && docker build -t featurebase:test_installation \ + -f test_installation.Dockerfile \ + --build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz . + mv install/featurebase-$(VERSION_ID).tar.gz build/ + docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + featurebase:test_installation # Error out if there are untracked changes in Git check-clean: @@ -64,78 +123,196 @@ ifndef SKIP_CHECK_CLEAN $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) endif -# Create release build tarballs for all supported platforms. Linux compilation happens under Docker. -release: check-clean +# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release` +release: check-clean generate-statik-docker $(MAKE) release-build GOOS=darwin GOARCH=amd64 - $(MAKE) release-build GOOS=darwin GOARCH=amd64 ENTERPRISE=1 + $(MAKE) release-build GOOS=darwin GOARCH=arm64 $(MAKE) release-build GOOS=linux GOARCH=amd64 - $(MAKE) release-build GOOS=linux GOARCH=amd64 ENTERPRISE=1 - $(MAKE) release-build GOOS=linux GOARCH=386 - $(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1 + $(MAKE) release-build GOOS=linux GOARCH=arm64 +# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI. +release-sans-ui: check-clean + rm -f statik/statik.go + $(MAKE) release-build GOOS=darwin GOARCH=amd64 + $(MAKE) release-build GOOS=darwin GOARCH=arm64 + $(MAKE) release-build GOOS=linux GOARCH=amd64 + $(MAKE) release-build GOOS=linux GOARCH=arm64 -# try (e.g.) internal/clustertests/docker-compose-replication2.yml -DOCKER_COMPOSE=internal/clustertests/docker-compose.yml +package: + go build -o featurebase ./cmd/featurebase + GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb + GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm + +# We allow setting a custom docker-compose "project". Multiple of the +# same docker-compose environment can exist simultaneously as long as +# they use different projects (the project name is prepended to +# container names and such). This is useful in a CI environment where +# we might be running multiple instances of the tests concurrently. +PROJECT ?= clustertests +DOCKER_COMPOSE = docker-compose -p $(PROJECT) # Run cluster integration tests using docker. Requires docker daemon to be -# running. This will catch changes to internal/clustertests/*.go, but if you -# make changes to Pilosa, you'll want to run clustertests-build to rebuild the -# pilosa image. -clustertests: - docker-compose -f $(DOCKER_COMPOSE) down - docker-compose -f $(DOCKER_COMPOSE) build client1 - docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 +# running and docker-compose to be installed. +clustertests: vendor + $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down + $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build + $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3 + PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1 + $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down - -# Like clustertests, but rebuilds all images. -clustertests-build: - docker-compose -f $(DOCKER_COMPOSE) down - docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build - -# Create prerelease builds -prerelease: - $(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\) - $(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release) - -prerelease-upload: - aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read +# Run the cluster tests with authentication enabled +AUTH_ARGS="-c /go/src/github.com/molecula/featurebase/internal/clustertests/testdata/featurebase.conf" +authclustertests: vendor + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3 + PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1 + CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down # Install Pilosa install: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase + +# Install the single-node PLG version of FeatureBase +plg: + $(GO) build -tags='plg $(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase + +install-bench: + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench + +# Build the lattice assets +build-lattice: + docker build -t lattice:build ./lattice + export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast - go generate github.com/pilosa/pilosa/v2/internal + $(GO) generate github.com/molecula/featurebase/v3/pb + +# `go generate` statik assets (lattice UI) +generate-statik: build-lattice require-statik + $(GO) generate github.com/molecula/featurebase/v3/statik + +# `go generate` statik assets (lattice UI) in Docker +generate-statik-docker: build-lattice + docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f" # `go generate` stringers generate-stringer: - go generate github.com/pilosa/pilosa/v2 + $(GO) generate github.com/molecula/featurebase/v3 generate-pql: require-peg cd pql && peg -inline pql.peg && cd .. +generate-proto-grpc: require-protoc require-protoc-gen-go + protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto + protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto + # TODO: Modify above commands and remove the below mv if possible. + # See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt + # I couldn't get it to work during development - Cody + cp -r proto/github.com/molecula/featurebase/v3/proto/ proto/ + rm -rf proto/github.com + # `go generate` all needed packages -generate: generate-protoc generate-stringer generate-pql +generate: generate-protoc generate-statik generate-stringer generate-pql + +# Create release using Docker +docker-release: + $(MAKE) docker-build GOOS=linux GOARCH=amd64 + $(MAKE) docker-build GOOS=linux GOARCH=arm64 + $(MAKE) docker-build GOOS=darwin GOARCH=amd64 + $(MAKE) docker-build GOOS=darwin GOARCH=arm64 + +# Build a release in Docker +docker-build: vendor + docker build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \ + --target pilosa-builder \ + --tag featurebase:build . + docker create --name featurebase-build featurebase:build + mkdir -p build/featurebase-$(VERSION_ID) + docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID) + docker rm featurebase-build + tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ # Create Docker image from Dockerfile -docker: - docker build -t "pilosa:$(VERSION)" . - @echo Created docker image: pilosa:$(VERSION) +docker-image: vendor + docker build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \ + --tag featurebase:$(VERSION) . + @echo Created docker image: featurebase:$(VERSION) -# Compile Pilosa inside Docker container -docker-build: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Create docker image (alias) +docker: docker-image # alias + +# Tag and push a Docker image +docker-tag-push: vendor + docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET) + docker push $(DOCKER_TARGET) + @echo Pushed docker image: $(DOCKER_TARGET) + +# These commands (docker-idk and docker-idk-tag-push) +# are designed to be used in CI. +# docker-idk builds idk docker images and tags them - intended for use in CI. +docker-idk: vendor + docker build \ + -f idk/Dockerfile \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \ + --tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) . + @echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) +# docker-idk-tag-push pushes tagged docker images to the GitLab container +# registry - intended for use in CI. +docker-idk-tag-push: + docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) + @echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) + +# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory. +pilosa-keydump: + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump + +# Install diagnostic pilosa-chk tool for string translations and fragment checksums. +pilosa-chk: + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk + +pilosa-fsck: + cd ./cmd/pilosa-fsck && make install && make release # Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./... + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout $(TEST_TIMEOUT) $(GOPACKAGES) + +# Must use bash in order to -o pipefail; otherwise the tee will hide red tests. +# run top tests, not subdirs. print summary red/green after. +# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt. +topt: + mv log.topt.roar log.topt.roar.prev || true + $(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout $(RACE_TEST_TIMEOUT) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar + @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l + @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l + +topt-race: + mv log.topt.race log.topt.race.prev || true + $(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout $(RACE_TEST_TIMEOUT) -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race + @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l + @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l # Run golangci-lint golangci-lint: require-golangci-lint - golangci-lint run + golangci-lint run --timeout 3m --skip-files '.*\.peg\.go' + +# Alias +linter: golangci-lint + +# Better alias +ocd: golangci-lint # Run gometalinter with custom flags +# Note the "./..." in gometalinter is still allowed, because we do want +# linting to reach IDK pagkages. gometalinter: require-gometalinter vendor GO111MODULE=off gometalinter --vendor --disable-all \ --deadline=300s \ @@ -158,13 +335,6 @@ gometalinter: require-gometalinter vendor --exclude "^pql/pql.peg.go" \ ./... -# Verify that all Go files have license header -check-license-headers: SHELL:=/bin/bash -check-license-headers: - @! find . -name '*.go' | grep -v '^./vendor' | while read fn;\ - do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \ - grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise - ###################### # Build dependencies # ###################### @@ -175,24 +345,37 @@ require-%: $(info Verified build dependency "$*" is installed.),\ $(error Build dependency "$*" not installed. To install, try `make install-$*`)) -install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg +install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg + +install-statik: + go install github.com/rakyll/statik@latest install-stringer: - GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer + GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer install-protoc-gen-gofast: - GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast + GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast + +install-protoc-gen-go: + GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html + @echo On mac, brew install protobuf seems to work. + @echo As of the commit that added this line, protoc-gen-gofast was at 226206f39bd7, and the protoc version in use was: + @echo $$ protoc --version + @echo libprotoc 3.19.4 install-peg: - GO111MODULE=off go get github.com/pointlander/peg + GO111MODULE=off $(GO) get github.com/pointlander/peg install-golangci-lint: - GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint + GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint install-gometalinter: - GO111MODULE=off go get -u github.com/alecthomas/gometalinter + GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter GO111MODULE=off gometalinter --install - GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode + GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode + +test-external-lookup: + $(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN) diff --git a/NOTES b/NOTES deleted file mode 100644 index 8da55fe11..000000000 --- a/NOTES +++ /dev/null @@ -1,26 +0,0 @@ - - Index Column - ┌───────────▼────────────────────────────┐ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - Row──▶0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - F ▶│0000000000000000000000000000000000000000│ - i ││0000000000000000000000000000000000000000│ - e ││0000000000000000000000000000000000000000│ - l ││0000000000000000000000000000000000000000│ - d ▶│0000000000000000000000000000000000000000│ - └────────────────────────────────────────┘ - ▲───────────▲ - Shard - - -Fragment=intersection of field & shard diff --git a/NOTICE b/NOTICE index 2d655865e..cb62dfdca 100644 --- a/NOTICE +++ b/NOTICE @@ -1,44 +1,12 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Enterprise Edition software license -=================================== - -Files contained under the directory `enterprise` are subject to the following -license notice (Full license included in the file `COPYING`): - - Copyright (C) 2018 Pilosa Corp. All rights reserved. - - Pilosa Enterprise Edition is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Pilosa Enterprise Edition is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Pilosa Enterprise Edition. If not, see . +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= -The file /pilosa/lru/lru.go contains a redistribution of lru +The file /lru/lru.go contains a redistribution of lru (github.com/golang/groupcache/lru); the license follows: Copyright 2013 Google Inc. @@ -115,3 +83,27 @@ The file /server/tlsconfig.go contains a modified redistribution of bridge CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows: + + The MIT License (MIT) + + Copyright (c) 2015 Nick Galbreath + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/README.md b/README.md index d2daf4391..167dc3b0e 100644 --- a/README.md +++ b/README.md @@ -1,80 +1,85 @@ -

- - - -

+# FeatureBase -[![CircleCI](https://circleci.com/gh/pilosa/pilosa/tree/master.svg?style=shield)](https://circleci.com/gh/pilosa/pilosa/tree/master) -[![GoDoc](https://godoc.org/github.com/pilosa/pilosa?status.svg)](https://godoc.org/github.com/pilosa/pilosa) -[![Go Report Card](https://goreportcard.com/badge/github.com/pilosa/pilosa)](https://goreportcard.com/report/github.com/pilosa/pilosa) -[![license](https://img.shields.io/github/license/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/blob/master/LICENSE) -[![CLA Assistant](https://cla-assistant.io/readme/badge/pilosa/pilosa)](https://cla-assistant.io/pilosa/pilosa) -[![GitHub release](https://img.shields.io/github/release/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/releases) +## Pilosa is now FeatureBase -## An open source, distributed bitmap index. -- [Docs](#docs) -- [Getting Started](#getting-started) -- [Data Model](#data-model) -- [Query Language](#query-language) -- [Client Libraries](#client-libraries) -- [Get Support](#get-support) -- [Contributing](#contributing) +As of September 7, 2022, the Pilosa project is now FeatureBase. The core of the project remains the same: FeatureBase is the first real-time distributed database built entirely on bitmaps. (More information about updated capabilities and improvements below.) -Want to contribute? One of the easiest ways is to [tell us how you're using (or want to use) Pilosa](https://github.com/pilosa/pilosa/issues/1074). We learn from every discussion! - -## Docs - -See our [Documentation](https://www.pilosa.com/docs/) for information about installing and working with Pilosa. +FeatureBase delivers low-latency query results, regardless of throughput or query volumes, on fresh data with extreme efficiency. It works because bitmaps are faster, simpler, and far more I/O efficient than traditional column-oriented data formats. With FeatureBase, you can ingest data from batch data sources (e.g. S3, CSV, Snowflake, BigQuery, etc.) and/or streaming data sources (e.g. Kafka/Confluent, Kinesis, Pulsar). +For more information about FeatureBase, please visit [www.featurebase.com][HomePage]. ## Getting Started -1. [Install Pilosa](https://www.pilosa.com/docs/installation/). +### Build FeatureBase Server from source -2. [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration: +0. Install go. Ensure that your shell's search path includes the go/bin directory. +1. Clone the FeatureBase repository (or download as zip). +2. In the featurebase directory, run `make install` to compile the FeatureBase server binary. By default, it will be installed in the go/bin directory. +3. In the idk directory, run `make install` to compile the ingester binaries. By default, they will be installed in the go/bin directory. +4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed. +5. Run `curl localhost:10101/status` to verify the server is running and accessible. - ```shell - pilosa server - ``` - - and verify that it's running: - - ```shell - curl localhost:10101/nodes - ``` +### Ingest Data and Query -3. Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities. +1. Run +``` +molecula-consumer-csv \ + --index repository \ + --header "language__ID_F,project_id__ID_F" \ + --id-field project_id \ + --batch-size 1000 \ + --files example.csv +``` +This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about ingesting into FeatureBase: [https://docs.featurebase.com/data-ingestion/enterprise/ingesters][Ingest] -## Data Model +2. Query your data. +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'Row(example=5)' +``` +Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL]. -Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) works. +### Data Model +Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. +Data Model Guide: [https://docs.featurebase.com/data-modeling-guide/data-modeling][DataModel] -## Query Language +### More Information -You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL). +Installation:[https://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase][Install] +Configuration: [https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration][Config] -## Client Libraries +## Community -There are supported libraries for the following languages: -- [Go](https://www.pilosa.com/docs/client-libraries/#go) -- [Java](https://www.pilosa.com/docs/client-libraries/#java) -- [Python](https://www.pilosa.com/docs/client-libraries/#python) +You can email us at comminuty@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community]. -## Licenses +Chat with us: [https://discord.gg/bKAP5CEY][Discord] -The core Pilosa code base and all default builds (referred to as Pilosa Community Edition) are licensed completely under the Apache License, Version 2.0. -If you build Pilosa with the `enterprise` build tag (Pilosa Enterprise Edition), then that build will include features licensed under the GNU Affero General -Public License (AGPL). Enterprise code is located entirely in the [github.com/pilosa/pilosa/enterprise](https://github.com/pilosa/pilosa/tree/master/enterprise) -directory. See [github.com/pilosa/pilosa/NOTICE](https://github.com/pilosa/pilosa/blob/master/NOTICE) and -[github.com/pilosa/pilosa/LICENSE](https://github.com/pilosa/pilosa/blob/master/LICENSE) for more information about Pilosa licenses. +## What's Changed Since the Pilosa Days? -## Get Support +A lot has changed since the days of Pilosa. This list highlights some new capabilites included in FeatureBase. We have also made signficant improvements to the performance, scalability, and stability of the FeatureBase product. -There are [several channels](https://www.pilosa.com/community/#support) available for you to reach out to us for support. The Slack channel (#pilosa in the [Golang](https://invite.slack.golangbridge.org/) team) is the most active. +* Query Languages: FeatureBase supports Pilosa Query Language (PQL), as well as SQL +* Stream and Batch Ingest: Combine real-time data streams with batch historical data and act on it within milliseconds. +* Mutable: Perform inserts, updates, and deletes at scale, in real time and on-the-fly. This is key for meeting data compliance requirements, and for reflecting the constantly-changing nature of high-volume data. +* Multi-Valued Set Fields: Store multiple comma-delimited values within a single field while *increasing* query performance of counts, TopKs, etc. +* Time Quantums: Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to YMD, ranged Row queries down to the granularity of a day are supported. +* RBF storage backend: this is a new compressed bitmap format which improves performance in a number of ways: ACID support on a per shard basis, prevents issues with the number of open files, reduces memory allocation and lock contention for reads, provides more consistent garbage collection, and allows backups to run concurrently with writes. However, because of this change, Pilosa backup files cannot be restored into FeatureBase. -## Contributing +## License -Pilosa is an open source project. Please see our [Contributing Guide](CONTRIBUTING.md) for information about how to get involved. +FeatureBase is licensed under the [Apache License, Version 2.0][License] + +[Community]: https://www.featurebase.com/community +[Config]: https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration +[DataModel]: https://docs.featurebase.com/data-modeling-guide/data-modeling +[Discord]: https://discord.gg/bKAP5CEY +[HomePage]: https://www.featurebase.com +[Ingest]: https://docs.featurebase.com/data-ingestion/enterprise/ingesters +[Install]: https://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase +[License]: http://www.apache.org/licenses/LICENSE-2.0 +[PQL]: https://docs.featurebase.com/data-querying/pql/introduction +[SQL]: https://docs.featurebase.com/data-querying/sql diff --git a/api.go b/api.go index ef535b879..986340986 100644 --- a/api.go +++ b/api.go @@ -1,38 +1,37 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 //go:generate stringer -type=apiMethod package pilosa import ( + "bufio" + "bytes" "context" "encoding/binary" "encoding/csv" "fmt" "io" "io/ioutil" + "math" "net/url" + "os" + "runtime" + "sort" "strconv" "strings" "sync" "time" - "github.com/pilosa/pilosa/v2/pql" - "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/stats" - "github.com/pilosa/pilosa/v2/tracing" + "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/ingest" + "github.com/molecula/featurebase/v3/rbf" + + //"github.com/molecula/featurebase/v3/pg" + "github.com/molecula/featurebase/v3/pql" + "github.com/molecula/featurebase/v3/roaring" + "github.com/molecula/featurebase/v3/stats" + "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -40,9 +39,13 @@ import ( // API provides the top level programmatic interface to Pilosa. It is usually // wrapped by a handler which provides an external interface (e.g. HTTP). type API struct { + mu sync.Mutex + closed bool // protected by mu + holder *Holder cluster *cluster server *Server + tracker *queryTracker importWorkersWG sync.WaitGroup importWorkerPoolSize int @@ -51,6 +54,10 @@ type API struct { Serializer Serializer } +func (api *API) Holder() *Holder { + return api.holder +} + // apiOption is a functional option type for pilosa.API type apiOption func(*API) error @@ -93,16 +100,36 @@ func NewAPI(opts ...apiOption) (*API, error) { }() } + api.tracker = newQueryTracker(api.server.queryHistoryLength) + return api, nil } +// Setter for API options. +func (api *API) SetAPIOptions(opts ...apiOption) error { + for _, opt := range opts { + err := opt(api) + if err != nil { + return errors.Wrap(err, "setting API option") + } + } + return nil +} + // validAPIMethods specifies the api methods that are valid for each // cluster state. -var validAPIMethods = map[string]map[apiMethod]struct{}{ - ClusterStateStarting: methodsCommon, - ClusterStateNormal: appendMap(methodsCommon, methodsNormal), - ClusterStateDegraded: appendMap(methodsCommon, methodsNormal), - ClusterStateResizing: appendMap(methodsCommon, methodsResizing), +var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{ + disco.ClusterStateStarting: methodsCommon, + disco.ClusterStateNormal: appendMap(methodsCommon, methodsNormal), + // Ideally, this would be just `appendMap(methodsCommon, methodsDegraded)`, + // but in an attempt to reduce the influence that state (determined by etcd) + // has on a node under load, this is set to effectively allow all requests + // in a DEGRADED state. + disco.ClusterStateDegraded: appendMap(methodsCommon, methodsNormal), + // Ideally, this would be just `methodsCommon`, but in an attempt to reduce + // the influence that state (determined by etcd) has on a node under load, + // this is set to effectively allow all requests in a DOWN state. + disco.ClusterStateDown: appendMap(methodsCommon, methodsNormal), } func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { @@ -117,7 +144,10 @@ func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { } func (api *API) validate(f apiMethod) error { - state := api.cluster.State() + state, err := api.cluster.State() + if err != nil { + return errors.Wrap(err, "getting cluster state") + } if _, ok := validAPIMethods[state][f]; ok { return nil } @@ -126,13 +156,27 @@ func (api *API) validate(f apiMethod) error { // Close closes the api and waits for it to shutdown. func (api *API) Close() error { + // only close once + api.mu.Lock() + defer api.mu.Unlock() + if api.closed { + return nil + } + api.closed = true + close(api.importWork) api.importWorkersWG.Wait() + api.tracker.Stop() return nil } +func (api *API) Txf() *TxFactory { + return api.holder.Txf() +} + // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + start := time.Now() span, ctx := tracing.StartSpanFromContext(ctx, "API.Query") defer span.Finish() @@ -140,22 +184,39 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return QueryResponse{}, errors.Wrap(err, "validating api method") } + if !req.Remote { + defer api.tracker.Finish(api.tracker.Start(req.Query, req.SQLQuery, api.server.nodeID, req.Index, start)) + } + + return api.query(ctx, req) +} + +// query provides query functionality for internal use, without tracing, validation, or tracking +func (api *API) query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { return QueryResponse{}, errors.Wrap(err, "parsing") } + + // TODO can we get rid of exec options and pass the QueryRequest directly to executor? execOpts := &execOptions{ - Remote: req.Remote, - ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat. - ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. - ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. + Remote: req.Remote, + Profile: req.Profile, + PreTranslated: req.PreTranslated, + EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request + MaxMemory: req.MaxMemory, } resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { return QueryResponse{}, errors.Wrap(err, "executing") } - return resp, nil + // Check for an error embedded in the response. + if resp.Err != nil { + err = errors.Wrap(resp.Err, "executing") + } + + return resp, err } // CreateIndex makes a new Pilosa index. @@ -167,21 +228,20 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return nil, errors.Wrap(err, "validating api method") } + // Populate the create index message. + cim := &CreateIndexMessage{ + Index: indexName, + CreatedAt: timestamp(), + Meta: options, + } + // Create index. - index, err := api.holder.CreateIndex(indexName, options) + index, err := api.holder.CreateIndexAndBroadcast(ctx, cim) if err != nil { return nil, errors.Wrap(err, "creating index") } - // Send the create index message to all nodes. - err = api.server.SendSync( - &CreateIndexMessage{ - Index: indexName, - Meta: &options, - }) - if err != nil { - return nil, errors.Wrap(err, "sending CreateIndex message") - } - api.holder.Stats.Count("createIndex", 1, 1.0) + + api.holder.Stats.Count(MetricCreateIndex, 1, 1.0) return index, nil } @@ -222,10 +282,17 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { Index: indexName, }) if err != nil { - api.server.logger.Printf("problem sending DeleteIndex message: %s", err) + api.server.logger.Errorf("problem sending DeleteIndex message: %s", err) return errors.Wrap(err, "sending DeleteIndex message") } - api.holder.Stats.Count("deleteIndex", 1, 1.0) + // Delete ids allocated for index if any present + snap := api.cluster.NewSnapshot() + if snap.IsPrimaryFieldTranslationNode(api.NodeID()) { + if err := api.holder.ida.reset(indexName); err != nil { + return errors.Wrap(err, "deleting id allocation for index") + } + } + api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0) return nil } @@ -240,13 +307,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str return nil, errors.Wrap(err, "validating api method") } - // Apply functional options. - fo := FieldOptions{} - for _, opt := range opts { - err := opt(&fo) - if err != nil { - return nil, NewBadRequestError(errors.Wrap(err, "applying option")) - } + // Apply and validate functional options. + fo, err := newFieldOptions(opts...) + if err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "applying option")) } // Find index. @@ -255,27 +319,62 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str return nil, newNotFoundError(ErrIndexNotFound, indexName) } + // Populate the create field message. + cfm := &CreateFieldMessage{ + Index: indexName, + Field: fieldName, + CreatedAt: timestamp(), + Meta: fo, + } + // Create field. field, err := index.CreateField(fieldName, opts...) if err != nil { return nil, errors.Wrap(err, "creating field") } - // Send the create field message to all nodes. - err = api.server.SendSync( - &CreateFieldMessage{ - Index: indexName, - Field: fieldName, - Meta: &fo, - }) - if err != nil { - api.server.logger.Printf("problem sending CreateField message: %s", err) + // Send the create field message to all nodes. We do this *outside* the + // CreateField logic so we're not blocking on it. + if err := api.holder.sendOrSpool(cfm); err != nil { return nil, errors.Wrap(err, "sending CreateField message") } - api.holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + + api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return field, nil } +// FieldUpdate represents a change to a field. The thinking is to only +// support changing one field option at a time to keep the +// implementation sane. At time of writing, only TTL is supported. +type FieldUpdate struct { + Option string `json:"option"` + Value string `json:"value"` +} + +func (api *API) UpdateField(ctx context.Context, indexName, fieldName string, update FieldUpdate) error { + // Find index. + index := api.holder.Index(indexName) + if index == nil { + return newNotFoundError(ErrIndexNotFound, indexName) + } + + cfm, err := index.UpdateField(ctx, fieldName, update) + if err != nil { + return errors.Wrap(err, "updating field") + } + + if err := index.UpdateFieldLocal(cfm, update); err != nil { + return errors.Wrap(err, "updating field locally") + } + + // broadcast field update + err = api.holder.sendOrSpool(&UpdateFieldMessage{ + CreateFieldMessage: *cfm, + Update: update, + }) + return errors.Wrap(err, "sending UpdateField message") +} + // Field retrieves the named field. func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Field") @@ -305,6 +404,7 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) { type importJob struct { ctx context.Context + qcx *Qcx req *ImportRoaringRequest shard uint64 field *Field @@ -313,29 +413,85 @@ type importJob struct { func importWorker(importWork chan importJob) { for j := range importWork { - err := func() error { + err := func() (err0 error) { for viewName, viewData := range j.req.Views { + // The logic here corresponds to the logic in fragment.cleanViewName(). + // Unfortunately, the logic in that method is not completely exclusive + // (i.e. an "other" view named with format YYYYMMDD would be handled + // incorrectly). One way to address this would be to change the logic + // overall so there weren't conflicts. For now, we just + // rely on the field type to inform the intended view name. if viewName == "" { viewName = viewStandard - } else { + } else if j.field.Type() == FieldTypeTime { viewName = fmt.Sprintf("%s_%s", viewStandard, viewName) } if len(viewData) == 0 { return fmt.Errorf("no data to import for view: %s", viewName) } - fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) - if fileMagic == roaring.MagicNumber { // if pilosa roaring format - if err := j.field.importRoaring(j.ctx, viewData, j.shard, viewName, j.req.Clear); err != nil { - return errors.Wrap(err, "importing pilosa roaring") + + // TODO: deprecate ImportRoaringRequest.Clear, but + // until we do, we need to check its value to provide + // backward compatibility. + doAction := j.req.Action + if doAction == "" { + if j.req.Clear { + doAction = RequestActionClear + } else { + doAction = RequestActionSet } - } else { - // must make a copy of data to operate on locally on standard roaring format. - // field.importRoaring changes the standard roaring run format to pilosa roaring - data := make([]byte, len(viewData)) - copy(data, viewData) - if err := j.field.importRoaring(j.ctx, data, j.shard, viewName, j.req.Clear); err != nil { - return errors.Wrap(err, "importing standard roaring") + } + + if err := func() (err1 error) { + tx, finisher, err := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard}) + if err != nil { + return err } + defer finisher(&err1) + + var doClear bool + switch doAction { + case RequestActionOverwrite: + err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block) + if err != nil { + return errors.Wrap(err, "importing roaring as overwrite") + } + case RequestActionClear: + doClear = true + fallthrough + case RequestActionSet: + fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) + data := viewData + if fileMagic != roaring.MagicNumber { + // if the view data arrives is in the "standard" roaring format, we must + // make a copy of data in order allow for the conversion to the pilosa roaring run format + // in field.importRoaring + data = make([]byte, len(viewData)) + copy(data, viewData) + } + if j.req.UpdateExistence { + if ef := j.field.idx.existenceField(); ef != nil { + existence, err := combineForExistence(data) + if err != nil { + return errors.Wrap(err, "merging existence on roaring import") + } + + err = ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + if err != nil { + return errors.Wrap(err, "updating existence on roaring import") + } + } + } + + err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + + if err != nil { + return errors.Wrap(err, "importing standard roaring") + } + } + return nil + }(); err != nil { + return err } } return nil @@ -348,6 +504,24 @@ func importWorker(importWork chan importJob) { } } +// combineForExistence unions all rows in the fragment to be imported into a single row to update the existence field. TODO: It would probably be more efficient to only unmarshal the input data once, and use the calculated existence Bitmap directly rather than returning it to bytes, but most of our ingest paths update existence separately, so it's more important that this just be obviously correct at the moment. +func combineForExistence(inputRoaringData []byte) ([]byte, error) { + rowSize := uint64(1 << shardVsContainerExponent) + rit, err := roaring.NewRoaringIterator(inputRoaringData) + if err != nil { + return nil, err + } + bm := roaring.NewBitmap() + err = bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + + _, err = bm.WriteTo(buf) + return buf.Bytes(), err +} + // ImportRoaring is a low level interface for importing data to Pilosa when // extremely high throughput is desired. The data must be encoded in a // particular way which may be unintuitive (discussed below). The data is merged @@ -365,34 +539,38 @@ func importWorker(importWork chan importJob) { // (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all // of the rows in this shard of this field concatenated together in one long // bitmap. -func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error) { +func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err0 error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring") span.LogKV("index", indexName, "field", fieldName) defer span.Finish() - if err = api.validate(apiField); err != nil { + if err := api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } - nodes := api.cluster.shardNodes(indexName, shard) - - field := api.holder.Field(indexName, fieldName) - if field == nil { - return newNotFoundError(ErrFieldNotFound, fieldName) + index, field, err := api.indexField(indexName, fieldName, shard) + if index == nil || field == nil { + return err } - // only set and time fields are supported - if field.Type() != FieldTypeSet && field.Type() != FieldTypeTime { - return NewBadRequestError(errors.New("roaring import is only supported for set and time fields")) + if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return newPreconditionFailedError(err) } + qcx := api.Txf().NewQcx() + defer qcx.Abort() + + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + + nodes := snap.ShardNodes(indexName, shard) errCh := make(chan error, len(nodes)) - for _, node := range nodes { node := node if node.ID == api.server.nodeID { api.importWork <- importJob{ ctx: ctx, + qcx: qcx, req: req, shard: shard, field: field, @@ -412,9 +590,11 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, for { select { case <-ctx.Done(): + // defered tx.Rollback() happens automatically here. return ctx.Err() case nodeErr := <-errCh: if nodeErr != nil { + // defered tx.Rollback() happens automatically here. return nodeErr } maxNode++ @@ -422,7 +602,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, // Exit once all nodes are processed. if maxNode == len(nodes) { - return nil + return qcx.Finish() } } } @@ -456,10 +636,10 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str Field: fieldName, }) if err != nil { - api.server.logger.Printf("problem sending DeleteField message: %s", err) + api.server.logger.Errorf("problem sending DeleteField message: %s", err) return errors.Wrap(err, "sending DeleteField message") } - api.holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return nil } @@ -488,10 +668,10 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str ShardID: shardID, }) if err != nil { - api.server.logger.Printf("problem sending DeleteAvailableShard message: %s", err) + api.server.logger.Errorf("problem sending DeleteAvailableShard message: %s", err) return errors.Wrap(err, "sending DeleteAvailableShard message") } - api.holder.Stats.CountWithCustomTags("deleteAvailableShard", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return nil } @@ -505,9 +685,12 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin return errors.Wrap(err, "validating api method") } + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + // Validate that this handler owns the shard. - if !api.cluster.ownsShard(api.Node().ID, indexName, shard) { - api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName) + if !snap.OwnsShard(api.NodeID(), indexName, shard) { + api.server.logger.Errorf("node %s does not own shard %d of index %s", api.NodeID(), shard, indexName) return ErrClusterDoesNotOwnShard } @@ -529,6 +712,10 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin return ErrFragmentNotFound } + // Obtain transaction + tx := index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Shard: shard}) + defer tx.Rollback() + // Wrap writer with a CSV writer. cw := csv.NewWriter(w) @@ -540,8 +727,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin var colStr string var err error - if field.keys() { - if rowStr, err = field.translateStore.TranslateID(rowID); err != nil { + if field.Keys() { + // TODO: handle case: field.ForeignIndex + if rowStr, err = field.TranslateStore().TranslateID(rowID); err != nil { return errors.Wrap(err, "translating row") } } else { @@ -549,7 +737,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } if index.Keys() { - if colStr, err = index.translateStore.TranslateID(columnID); err != nil { + if store := index.TranslateStore(snap.IDToShardPartition(indexName, columnID)); store == nil { + return errors.Wrap(err, "partition does not exist") + } else if colStr, err = store.TranslateID(columnID); err != nil { return errors.Wrap(err, "translating column") } } else { @@ -561,20 +751,19 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } // Iterate over each column. - if err := f.forEachBit(fn); err != nil { + if err := f.forEachBit(tx, fn); err != nil { return errors.Wrap(err, "writing CSV") } // Ensure data is flushed. cw.Flush() - span.LogKV("n", n) - + tx.Rollback() return nil } // ShardNodes returns the node and all replicas which should contain a shard's data. -func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) { +func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*disco.Node, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.ShardNodes") defer span.Finish() @@ -582,13 +771,31 @@ func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) return nil, errors.Wrap(err, "validating api method") } - return api.cluster.shardNodes(indexName, shard), nil + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + + return snap.ShardNodes(indexName, shard), nil +} + +// PartitionNodes returns the node and all replicas which should contain a partition key data. +func (api *API) PartitionNodes(ctx context.Context, partitionID int) ([]*disco.Node, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.PartitionNodes") + defer span.Finish() + + if err := api.validate(apiPartitionNodes); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + + return snap.PartitionNodes(partitionID), nil } // FragmentBlockData is an endpoint for internal usage. It is not guaranteed to // return anything useful. Currently it returns protobuf encoded row and column // ids from a "block" which is a subdivision of a fragment. -func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { +func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) (_ []byte, err error) { span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlockData") defer span.Finish() @@ -612,7 +819,10 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } var resp = BlockDataResponse{} - resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) + resp.RowIDs, resp.ColumnIDs, err = f.blockData(int(req.Block)) + if err != nil { + return nil, err + } // Encode response. buf, err := api.Serializer.Marshal(&resp) @@ -639,8 +849,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewNa } // Retrieve blocks. - blocks := f.Blocks() - return blocks, nil + return f.Blocks() } // FragmentData returns all data in the specified fragment. @@ -660,21 +869,126 @@ func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName return f, nil } +type RedirectError struct { + HostPort string + error string +} + +func (r RedirectError) Error() string { + return r.error +} + +// TranslateData returns all translation data in the specified partition. +func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData") + defer span.Finish() + + if err := api.validate(apiTranslateData); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + // Retrieve index from holder. + idx := api.holder.Index(indexName) + if idx == nil { + return nil, newNotFoundError(ErrIndexNotFound, indexName) + } + + // Find the node that can service the request. + snap := api.cluster.NewSnapshot() + nodes := snap.PartitionNodes(partition) + var upNode *disco.Node + for _, node := range nodes { + // we all UNKNOWN state here because we often mistakenly think + // a node is not up under heavy load, but prefer STARTED if we + // find one. + if node.State == disco.NodeStateStarted { + upNode = node + break + } else if node.State == disco.NodeStateUnknown { + if upNode != nil { + upNode = node + } + } + } + + // If there is no upNode, then we can't service the request. + if upNode == nil { + return nil, fmt.Errorf("can't get translate data, no nodes available for partition %d", partition) + } + + // If we're not the upNode, we need to redirect to it. + if upNode.ID != api.server.NodeID() { + return nil, RedirectError{ + HostPort: upNode.URI.HostPort(), + error: fmt.Sprintf("can't translate data, this node(%s) does not partition %d", api.server.uri, partition), + } + } + + // We are the upNode! + store := idx.TranslateStore(partition) + if store == nil { + return nil, ErrTranslateStoreNotFound + } + + return store, nil +} + +// FieldTranslateData returns all translation data in the specified field. +func (api *API) FieldTranslateData(ctx context.Context, indexName, fieldName string) (io.WriterTo, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.FieldTranslateData") + defer span.Finish() + if err := api.validate(apiFieldTranslateData); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + // Retrieve index from holder. + idx := api.holder.Index(indexName) + if idx == nil { + return nil, newNotFoundError(ErrIndexNotFound, indexName) + } + + // Retrieve field from index. + field := idx.Field(fieldName) + if field == nil { + return nil, newNotFoundError(ErrFieldNotFound, fieldName) + } + + // Retrieve translatestore from holder. + store := field.TranslateStore() + if store == nil { + return nil, ErrTranslateStoreNotFound + } + return store, nil +} + // Hosts returns a list of the hosts in the cluster including their ID, -// URL, and which is the coordinator. -func (api *API) Hosts(ctx context.Context) []*Node { +// URL, and which is the primary. +func (api *API) Hosts(ctx context.Context) []*disco.Node { span, _ := tracing.StartSpanFromContext(ctx, "API.Hosts") defer span.Finish() return api.cluster.Nodes() } -// Node gets the ID, URI and coordinator status for this particular node. -func (api *API) Node() *Node { - node := api.server.node() - return &node +// Node gets the ID, URI and primary status for this particular node. +func (api *API) Node() *disco.Node { + return api.server.node() } -// RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. +// NodeID gets the ID alone, so it doesn't have to do a complete lookup +// of the node, searching by its ID, to return the ID it searched for. +func (api *API) NodeID() string { + return api.server.nodeID +} + +// PrimaryNode returns the primary node for the cluster. +func (api *API) PrimaryNode() *disco.Node { + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + return snap.PrimaryFieldTranslationNode() +} + +// RecalculateCaches forces all TopN caches to be updated. +// This is done internally within a TopN query, but a user may want to do it ahead of time? func (api *API) RecalculateCaches(ctx context.Context) error { span, _ := tracing.StartSpanFromContext(ctx, "API.RecalculateCaches") defer span.Finish() @@ -714,18 +1028,46 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { return errors.Wrap(err, "deserializing cluster message") } - // Forward the error message. + // Forward the message. if err := api.server.receiveMessage(msg); err != nil { - return errors.Wrap(err, "receiving message") + return MessageProcessingError{err} } return nil } +// MessageProcessingError is an error indicating that a cluster message could not be processed. +type MessageProcessingError struct { + Err error +} + +func (err MessageProcessingError) Error() string { + return "processing message: " + err.Err.Error() +} + +// Cause allows the error to be unwrapped. +func (err MessageProcessingError) Cause() error { + return err.Err +} + +// Unwrap allows the error to be unwrapped. +func (err MessageProcessingError) Unwrap() error { + return err.Err +} + // Schema returns information about each index in Pilosa including which fields // they contain. -func (api *API) Schema(ctx context.Context) []*IndexInfo { +func (api *API) Schema(ctx context.Context, withViews bool) ([]*IndexInfo, error) { + if err := api.validate(apiSchema); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() + + if withViews { + return api.holder.Schema() + } + return api.holder.limitedSchema() } @@ -743,17 +1085,170 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } - if !remote { - nodes := api.cluster.Nodes() - for i, node := range nodes { - err := api.server.defaultClient.PostSchema(ctx, &node.URI, s, true) - if err != nil { - return errors.Wrapf(err, "forwarding post schema to node %d of %d", i+1, len(nodes)) - } + err := api.holder.applySchema(s) + if err != nil { + return errors.Wrap(err, "applying schema") + } + + return nil +} + +// applyOneIngestSchema applies a single ingestSpec, which specifies operations on +// a single index and possibly fields. If it is successful, it returns the name +// of the index and an empty slice (if it created the index), or the name of the +// index and a slice of the fields within that index that it created. If it +// is unsuccessful, it tries to delete whatever it created. +// +// The intended idiom is that if the returned list of fields isn't empty, the index +// already existed and only those fields need to be cleaned up in the event of +// a later error, but if the list of fields is empty, the entire index was new, +// and should be cleaned up, in which case there's no need to track or delete +// the specific fields separately. +func (api *API) ApplyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) { + if api.PrimaryNode().ID != api.NodeID() { + return nil, nil, RedirectError{ + HostPort: api.PrimaryNode().URI.Normalize(), + error: "request made to non-primary node", } } - return api.holder.applySchema(s) + // create index + indexName := schema.IndexName + var createdFields []string + var useKeys bool + switch schema.PrimaryKeyType { + case "string": + useKeys = true + case "uint": + useKeys = false + default: + return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType) + } + opts := IndexOptions{ + Keys: useKeys, + TrackExistence: true, + } + createdIndex := false + + // We check this up here because, if there's at least one field but we don't know what to do with + // it, we will necessarily fail, which means we'd delete the index anyway, so there's no point in + // trying to create it. We don't care about this if there's no fields specified. + if len(schema.Fields) > 0 { + switch schema.FieldAction { + case "create", "ensure", "require": + // do nothing + case "": + schema.FieldAction = schema.IndexAction + default: + return nil, nil, fmt.Errorf("invalid field-action %q, expecting create/ensure/require", schema.FieldAction) + } + } + + switch schema.IndexAction { + case "ensure", "require": + index, err = api.Index(ctx, indexName) + if err != nil { + if _, ok := err.(NotFoundError); !ok { + return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err) + } else { + err = nil + } + } + if index != nil { + existingOpts := index.Options() + if existingOpts != opts { + return nil, nil, fmt.Errorf("index %q options mismatch: schema %#v, existing %#v", indexName, opts, existingOpts) + } + break + } + if schema.IndexAction == "require" { + return nil, nil, fmt.Errorf("index %q does not exist", indexName) + } + fallthrough + case "create": + index, err = api.CreateIndex(ctx, indexName, opts) + if err != nil { + return nil, nil, err + } + createdIndex = true + default: + return nil, nil, fmt.Errorf("invalid index-action %q, need create/ensure/require", schema.IndexAction) + } + + // Now we might have an index, so we need our cleanup code. + defer func() { + if err == nil { + return + } + if createdIndex { + err := api.DeleteIndex(ctx, indexName) + if err != nil { + + api.server.logger.Printf("trying to undo failed index %q creation: %v", indexName, err) + } + return + } + for _, field := range createdFields { + err := api.DeleteField(ctx, indexName, field) + if err != nil { + api.server.logger.Printf("trying to undo failed field %q creation in index %q: %v", field, indexName, err) + } + } + }() + + // create all the fields specified in the index + for _, fSpec := range schema.Fields { + fieldName := fSpec.FieldName + opt := fieldSpecToFieldOption(fSpec) + err = opt.validate() + if err != nil { + return nil, nil, err + } + switch schema.FieldAction { + case "ensure", "require": + field, schemaErr := api.Field(ctx, indexName, fieldName) + if schemaErr != nil { + // NotFoundError is fine + if _, ok := schemaErr.(NotFoundError); !ok { + return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err) + } + } + if field != nil { + existing := field.Options() + if opt.Type != existing.Type { + return nil, nil, fmt.Errorf("existing field %q is %q, not %q", fieldName, existing.Type, opt.Type) + } + if ((opt.Keys != nil) && *opt.Keys) != existing.Keys { + if existing.Keys { + return nil, nil, fmt.Errorf("existing field %q in %q uses keys", fieldName, indexName) + } else { + return nil, nil, fmt.Errorf("existing field %q in %q doesn't use keys", fieldName, indexName) + } + } + // TODO: verify compatibility of other field opts, this is sorta hard + break + } + if schema.FieldAction == "require" { + return nil, nil, fmt.Errorf("field %q does not exist in %q", fieldName, indexName) + } + fallthrough + case "create": + fos := fieldOptionsToFunctionalOpts(opt) + _, err = api.CreateField(ctx, indexName, fieldName, fos...) + if err != nil { + return nil, nil, fmt.Errorf("creating field %q in %q: %v", fieldName, indexName, err) + } + createdFields = append(createdFields, fieldName) + } + } + + // we don't report the fields back, so we can distinguish "created index" + // from "created fields within index" + if createdIndex { + createdFields = nil + } + + return index, createdFields, nil } // Views returns the views in the given field. @@ -807,92 +1302,69 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri View: viewName, }) if err != nil { - api.server.logger.Printf("problem sending DeleteView message: %s", err) + api.server.logger.Errorf("problem sending DeleteView message: %s", err) } return errors.Wrap(err, "sending DeleteView message") } -// IndexAttrDiff determines the local column attribute data blocks which differ from those provided. -func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { - span, _ := tracing.StartSpanFromContext(ctx, "API.IndexAttrDiff") +// IndexShardSnapshot returns a reader that contains the contents of an RBF snapshot for an index/shard. +func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard uint64) (io.ReadCloser, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.IndexShardSnapshot") defer span.Finish() - if err := api.validate(apiIndexAttrDiff); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - // Retrieve index from holder. + // Find index. index := api.holder.Index(indexName) if index == nil { return nil, newNotFoundError(ErrIndexNotFound, indexName) } - // Retrieve local blocks. - localBlocks, err := index.ColumnAttrStore().Blocks() + // Start transaction. + tx := index.holder.txf.NewTx(Txo{Index: index, Shard: shard}) + + // Ensure transaction is an RBF transaction. + rtx, ok := tx.(*RBFTx) + if !ok { + tx.Rollback() + return nil, fmt.Errorf("snapshot not available for %q storage", tx.Type()) + } + + r, err := rtx.SnapshotReader() if err != nil { - return nil, errors.Wrap(err, "getting blocks") + tx.Rollback() + return nil, err } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range attrBlocks(localBlocks).Diff(blocks) { - // Retrieve block data. - m, err := index.ColumnAttrStore().BlockData(blockID) - if err != nil { - return nil, errors.Wrap(err, "getting block") - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v - } - } - return attrs, nil + return &txReadCloser{tx: tx, Reader: r}, nil } -// FieldAttrDiff determines the local row attribute data blocks which differ from those provided. -func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { - span, _ := tracing.StartSpanFromContext(ctx, "API.FieldAttrDiff") - defer span.Finish() +var _ io.ReadCloser = (*txReadCloser)(nil) - if err := api.validate(apiFieldAttrDiff); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - // Retrieve index from holder. - f := api.holder.Field(indexName, fieldName) - if f == nil { - return nil, newNotFoundError(ErrFieldNotFound, fieldName) - } - - // Retrieve local blocks. - localBlocks, err := f.RowAttrStore().Blocks() - if err != nil { - return nil, errors.Wrap(err, "getting blocks") - } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range attrBlocks(localBlocks).Diff(blocks) { - // Retrieve block data. - m, err := f.RowAttrStore().BlockData(blockID) - if err != nil { - return nil, errors.Wrap(err, "getting block") - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v - } - } - return attrs, nil +// txReadCloser wraps a reader to close a tx on close. +type txReadCloser struct { + io.Reader + tx Tx } -// ImportOptions holds the options for the API.Import method. +func (r *txReadCloser) Close() error { + r.tx.Rollback() + return nil +} + +// ImportOptions holds the options for the API.Import +// method. +// +// TODO(2.0) we have entirely missed the point of functional options +// by exporting this structure. If it needs to be exported for some +// reason, we should consider not using functional options here which +// just adds complexity. type ImportOptions struct { Clear bool IgnoreKeyCheck bool + Presorted bool + fullySorted bool // format-aware sorting, internal use only please. + + // test Tx atomicity if > 0 + SimPowerLossAfter int } // ImportOption is a functional option type for API.Import. @@ -916,8 +1388,112 @@ func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption { } } -// Import bulk imports data into a particular index,field,shard. -func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error { +func OptImportOptionsPresorted(b bool) ImportOption { + return func(o *ImportOptions) error { + o.Presorted = b + return nil + } +} + +var ErrAborted = fmt.Errorf("error: update was aborted") + +func (api *API) ImportAtomicRecord(ctx context.Context, qcx *Qcx, req *AtomicRecord, opts ...ImportOption) error { + + simPowerLoss := false + lossAfter := -1 + var opt ImportOptions + for _, setter := range opts { + if setter != nil { + err := setter(&opt) + if err != nil { + return errors.Wrap(err, "ImportAtomicRecord ImportOptions") + } + } + } + if opt.SimPowerLossAfter > 0 { + simPowerLoss = true + lossAfter = opt.SimPowerLossAfter + } + + idx, err := api.Index(ctx, req.Index) + if err != nil { + return errors.Wrap(err, "getting index") + } + + // the whole point is to run this part of the import atomically. + // Begin that Tx now! + qcx.StartAtomicWriteTx(Txo{Write: writable, Index: idx, Shard: req.Shard}) + tot := 0 + + options, err := setUpImportOptions(opts...) + if err != nil { + return errors.Wrap(err, "setting up import options") + } + + // BSIs (Values) + for _, ivr := range req.Ivr { + tot++ + if simPowerLoss && tot > lossAfter { + return ErrAborted + } + subOpts := *options + subOpts.Clear = ivr.Clear + err = api.ImportValueWithTx(ctx, qcx, ivr, &subOpts) + if err != nil { + return errors.Wrap(err, "ImportAtomicRecord ImportValueWithTx") + } + } + + // other bits, non-BSI + for _, ir := range req.Ir { + tot++ + if simPowerLoss && tot > lossAfter { + return ErrAborted + } + subOpts := *options + subOpts.Clear = ir.Clear + err := api.ImportWithTx(ctx, qcx, ir, &subOpts) + if err != nil { + return errors.Wrap(err, "ImportAtomicRecord ImportWithTx") + } + } + return nil +} + +func addClearToImportOptions(opts []ImportOption) []ImportOption { + var opt ImportOptions + for _, o := range opts { + // check for side-effect of setting io.Clear; that is + // how we know it is present. + _ = o(&opt) + if opt.Clear { + // we already have the clear flag set, so nothing more to do. + return opts + } + } + // no clear flag being set, add that option now. + return append(opts, OptImportOptionsClear(true)) +} + +// Import does the top-level importing. +func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) { + if req.Clear { + opts = addClearToImportOptions(opts) + } + // Set up import options. + options, err := setUpImportOptions(opts...) + if err != nil { + return errors.Wrap(err, "setting up import options") + } + err = api.ImportWithTx(ctx, qcx, req, options) + if err != nil { + return err + } + return nil +} + +// ImportWithTx bulk imports data into a particular index,field,shard. +func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error { span, _ := tracing.StartSpanFromContext(ctx, "API.Import") defer span.Finish() @@ -925,110 +1501,238 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp return errors.Wrap(err, "validating api method") } - // Set up import options. - options, err := setUpImportOptions(opts...) - if err != nil { - return errors.Wrap(err, "setting up import options") - } - - index, field, err := api.indexField(req.Index, req.Field, req.Shard) + idx, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") } + if err := req.ValidateWithTimestamp(idx.CreatedAt(), field.CreatedAt()); err != nil { + return errors.Wrap(err, "validating import value request") + } + + span.LogKV( + "index", req.Index, + "field", req.Field) + // Unless explicitly ignoring key validation (meaning keys have been - // translated to ids in a previous step at the coordinator node), then + // translated to ids in a previous step at the primary node), then // check to see if keys need translation. if !options.IgnoreKeyCheck { // Translate row keys. - if field.keys() { + if field.Keys() { + span.LogKV("rowKeys", true) if len(req.RowIDs) != 0 { return errors.New("row ids cannot be used because field uses string keys") } - if req.RowIDs, err = field.translateStore.TranslateKeys(req.RowKeys); err != nil { - return errors.Wrap(err, "translating rows") + if req.RowIDs, err = api.cluster.translateFieldKeys(ctx, field, req.RowKeys, true); err != nil { + return errors.Wrapf(err, "translating field keys") } + } else if len(req.RowKeys) != 0 { + return errors.New("value keys cannot be used because field uses integer IDs") } // Translate column keys. - if index.Keys() { + if idx.Keys() { + span.LogKV("columnKeys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } - if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil { + if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys, true); err != nil { return errors.Wrap(err, "translating columns") } - } - - // For translated data, map the columnIDs to shards. If - // this node does not own the shard, forward to the node that does. - if index.Keys() || field.keys() { - m := make(map[uint64][]Bit) - - for i, colID := range req.ColumnIDs { - shard := colID / ShardWidth - if _, ok := m[shard]; !ok { - m[shard] = make([]Bit, 0) - } - bit := Bit{ - RowID: req.RowIDs[i], - ColumnID: colID, - } - if len(req.Timestamps) > 0 { - bit.Timestamp = req.Timestamps[i] - } - m[shard] = append(m[shard], bit) - } - - // Signal to the receiving nodes to ignore checking for key translation. - opts = append(opts, OptImportOptionsIgnoreKeyCheck(true)) - - var eg errgroup.Group - for shard, bits := range m { - // TODO: if local node owns this shard we don't need to go through the client - shard := shard - bits := bits - eg.Go(func() error { - return api.server.defaultClient.Import(ctx, req.Index, req.Field, shard, bits, opts...) - }) - } - return eg.Wait() + // mark this request as having an unknown shard, meaning it will + // be sorted and served out to multiple nodes. + req.Shard = ^uint64(0) + } else if len(req.ColumnKeys) != 0 { + return errors.New("record keys cannot be used because field uses integer IDs") } } + // if you specify a shard of ^0, we try to split this out. If we did any + // key translation, we set it to ^0 already above. + if req.Shard == ^uint64(0) { + reqs := req.SortToShards() + + // Signal to the receiving nodes to ignore checking for key translation. + options.IgnoreKeyCheck = true + + var eg errgroup.Group + guard := make(chan struct{}, runtime.NumCPU()) // only run as many goroutines as CPUs available + for _, subReq := range reqs { + // TODO: if local node owns this shard we don't need to go through the client + guard <- struct{}{} // would block if guard channel is already filled + subReq := subReq + eg.Go(func() error { + err := api.server.defaultClient.Import(ctx, qcx, subReq, options) + <-guard + return err + }) + } + return eg.Wait() + } + + // otherwise, this has to be a shard that we have, and everything has + // to be for that shard. + // Validate shard ownership. if err := api.validateShardOwnership(req.Index, req.Shard); err != nil { return errors.Wrap(err, "validating shard ownership") } - // Convert timestamps to time.Time. - timestamps := make([]*time.Time, len(req.Timestamps)) - for i, ts := range req.Timestamps { - if ts == 0 { - continue + var timestamps []int64 + for _, v := range req.Timestamps { + if v != 0 { + timestamps = req.Timestamps + break } - t := time.Unix(0, ts).UTC() - timestamps[i] = &t } // Import columnIDs into existence field. if !options.Clear { - if err := importExistenceColumns(index, req.ColumnIDs); err != nil { - api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + if err := importExistenceColumns(qcx, idx, req.ColumnIDs, req.Shard); err != nil { + api.server.logger.Errorf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return err + } + if err != nil { return errors.Wrap(err, "importing existence columns") } } // Import into fragment. - err = field.Import(req.RowIDs, req.ColumnIDs, timestamps, opts...) + err = field.Import(qcx, req.RowIDs, req.ColumnIDs, timestamps, req.Shard, options) if err != nil { - api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + api.server.logger.Errorf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing") } - return errors.Wrap(err, "importing") + return errors.Wrap(err, "committing") } -// ImportValue bulk imports values into a particular field. -func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error { +// ImportRoaringShard transactionally imports roaring-encoded data +// across many fields in a single shard. It can both set and clear +// bits and updates caches/bitDepth as appropriate, although only the +// bitmap parts happen truly transactionally. +func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error { + index, err := api.Index(ctx, indexName) + if err != nil { + return errors.Wrap(err, "getting index") + } + + // we really only need a Tx, but getting a Qcx so that there's only one path for getting a Tx + qcx := api.Txf().NewQcx() + qcx.write = true + tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard}) + if err != nil { + return errors.Wrap(err, "getting Tx") + } + defer qcx.Finish() + var err1 error + defer finisher(&err1) + + if !req.Remote { + return errors.New("forwarding unimplemented on this endpoint") + } + + for _, viewUpdate := range req.Views { + field := index.Field(viewUpdate.Field) + if field == nil { + err1 = errors.Errorf("no field named '%s' found.", viewUpdate.Field) + return err1 + } + + fieldType := field.Options().Type + if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil { + return err1 + } + + view, err := field.createViewIfNotExists(viewUpdate.View) + if err != nil { + err1 = errors.Wrap(err, "getting view") + return err1 + } + + frag, err := view.CreateFragmentIfNotExists(shard) + if err != nil { + err1 = errors.Wrap(err, "getting fragment") + return err1 + } + + switch fieldType { + case FieldTypeSet, FieldTypeTime: + if !viewUpdate.ClearRecords { + err1 = frag.ImportRoaringClearAndSet(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + } else { + err1 = frag.ImportRoaringSingleValued(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + } + case FieldTypeInt, FieldTypeTimestamp, FieldTypeDecimal: + err1 = frag.ImportRoaringBSI(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + case FieldTypeMutex, FieldTypeBool: + err1 = frag.ImportRoaringSingleValued(ctx, tx, viewUpdate.Clear, viewUpdate.Set) + default: + err1 = errors.Errorf("field type %s is not supported", fieldType) + } + if err1 != nil { + return err1 + } + + // need to update field/bsiGroup bitDepth value if this is an int-like field. + // + // TODO get rid of cached bitDepth entirely because the fact + // that we have to do this is weird and since this state isn't + // in RBF might have transactional issues. + if len(field.bsiGroups) > 0 { + maxRowID, _, err := frag.maxRow(tx, nil) + if err != nil { + err1 = errors.Wrapf(err, "getting fragment max row id") + return err1 + } + var bd uint64 + if maxRowID+1 > bsiOffsetBit { + bd = maxRowID + 1 - bsiOffsetBit + } + field.cacheBitDepth(bd) // updating bitDepth shouldn't harm anything even if we roll back... only might make some ops slightly more inefficient + } + } + + return nil +} + +func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error { + // TODO wouldn't hurt to have consolidated logic somewhere for validating view names. + switch fieldType { + case FieldTypeSet, FieldTypeTime: + if viewUpdate.View == "" { + viewUpdate.View = "standard" + } + // add 'standard_' if we just have a time... this is how IDK works by default + if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) { + viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View) + } + case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: + if viewUpdate.View == "" { + viewUpdate.View = "bsig_" + viewUpdate.Field + } else if viewUpdate.View != "bsig_"+viewUpdate.Field { + return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType)) + } + } + return nil +} + +// ImportValue is a wrapper around the common code in ImportValueWithTx, which +// currently just translates req.Clear into a clear ImportOption. +func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error { + if req.Clear { + opts = addClearToImportOptions(opts) + } + // Set up import options. + options, err := setUpImportOptions(opts...) + if err != nil { + return errors.Wrap(err, "setting up import options") + } + return api.ImportValueWithTx(ctx, qcx, req, options) +} + +// ImportValueWithTx bulk imports values into a particular field. +func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) (err0 error) { span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue") defer span.Finish() @@ -1036,90 +1740,463 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "validating api method") } - // Set up import options. - options, err := setUpImportOptions(opts...) - if err != nil { - return errors.Wrap(err, "setting up import options") + numCols := len(req.ColumnIDs) + len(req.ColumnKeys) + numVals := len(req.Values) + len(req.FloatValues) + len(req.TimestampValues) + len(req.StringValues) + if numCols != numVals { + return errors.New(fmt.Sprintf("number of columns (%v) and number of values (%v) do not match", numCols, numVals)) + } + if numCols == 0 { + return nil } - index, field, err := api.indexField(req.Index, req.Field, req.Shard) + idx, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("getting index '%v' and field '%v'; shard=%v", req.Index, req.Field, req.Shard)) + } + + if err := req.ValidateWithTimestamp(idx.CreatedAt(), field.CreatedAt()); err != nil { + return errors.Wrap(err, "validating import value request") + } + + idx, field, err = api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") } - + span.LogKV( + "index", req.Index, + "field", req.Field) // Unless explicitly ignoring key validation (meaning keys have been - // translate to ids in a previous step at the coordinator node), then + // translate to ids in a previous step at the primary node), then // check to see if keys need translation. if !options.IgnoreKeyCheck { // Translate column keys. - if index.Keys() { + if idx.Keys() { + span.LogKV("columnKeys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } - if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil { + if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys, true); err != nil { return errors.Wrap(err, "translating columns") } + req.Shard = math.MaxUint64 + } - // For translated data, map the columnIDs to shards. If - // this node does not own the shard, forward to the node that does. - m := make(map[uint64][]FieldValue) - - for i, colID := range req.ColumnIDs { - shard := colID / ShardWidth - if _, ok := m[shard]; !ok { - m[shard] = make([]FieldValue, 0) - } - m[shard] = append(m[shard], FieldValue{ - Value: req.Values[i], - ColumnID: colID, - }) + // Translate values when the field uses keys (for example, when + // the field has a ForeignIndex with keys). + if field.Keys() { + // Perform translation. + span.LogKV("rowKeys", true) + uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues, true) + if err != nil { + return err } - // Signal to the receiving nodes to ignore checking for key translation. - opts = append(opts, OptImportOptionsIgnoreKeyCheck(true)) - - var eg errgroup.Group - for shard, vals := range m { - // TODO: if local node owns this shard we don't need to go through the client - shard := shard - vals := vals - eg.Go(func() error { - return api.server.defaultClient.ImportValue(ctx, req.Index, req.Field, shard, vals, opts...) - }) + // Because the BSI field supports negative values, we have to + // convert the uint64 keys to a slice of int64. + ints := make([]int64, len(uints)) + for i := range uints { + ints[i] = int64(uints[i]) } - return eg.Wait() + req.Values = ints } } - // Validate shard ownership. - if err := api.validateShardOwnership(req.Index, req.Shard); err != nil { - return errors.Wrap(err, "validating shard ownership") + if !options.Presorted { + // horrible hackery: we implement a secondary key so we can + // get a stable sort without using sort.Stable + req.scratch = make([]int, len(req.ColumnIDs)) + for i := range req.scratch { + req.scratch[i] = i + } + sort.Sort(req) + // don't keep that list around since we don't need it anymore + req.scratch = nil } - // Import columnIDs into existence field. - if !options.Clear { - if err := importExistenceColumns(index, req.ColumnIDs); err != nil { - api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) - return errors.Wrap(err, "importing existence columns") + // if we're importing into a specific shard + if req.Shard != math.MaxUint64 { + // Check that column IDs match the stated shard. + shard := req.ColumnIDs[0] / ShardWidth + if s2 := req.ColumnIDs[len(req.ColumnIDs)-1] / ShardWidth; (shard != s2) || (shard != req.Shard) { + return errors.Errorf("shard %d specified, but import spans shards %d to %d", req.Shard, shard, s2) + } + // Validate shard ownership. TODO - we should forward to the + // correct node rather than barfing here. + if err := api.validateShardOwnership(req.Index, req.Shard); err != nil { + return errors.Wrap(err, "validating shard ownership") + } + // Import columnIDs into existence field. + if !options.Clear { + if err := importExistenceColumns(qcx, idx, req.ColumnIDs, shard); err != nil { + api.server.logger.Errorf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing existence columns") + } + } + + // Import into fragment. + if len(req.Values) > 0 { + err = field.importValue(qcx, req.ColumnIDs, req.Values, shard, options) + if err != nil { + api.server.logger.Errorf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + } + } else if len(req.TimestampValues) > 0 { + err = field.importTimestampValue(qcx, req.ColumnIDs, req.TimestampValues, shard, options) + if err != nil { + api.server.logger.Errorf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + } + } else if len(req.FloatValues) > 0 { + err = field.importFloatValue(qcx, req.ColumnIDs, req.FloatValues, shard, options) + if err != nil { + api.server.logger.Errorf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + } + } + return errors.Wrap(err, "importing value") + + } // end if req.Shard != math.MaxUint64 + options.IgnoreKeyCheck = true + start := 0 + shard := req.ColumnIDs[0] / ShardWidth + var eg errgroup.Group + guard := make(chan struct{}, runtime.NumCPU()) // only run as many goroutines as CPUs available + for i, colID := range req.ColumnIDs { + if colID/ShardWidth != shard { + subreq := &ImportValueRequest{ + Index: req.Index, + Field: req.Field, + Shard: shard, + ColumnIDs: req.ColumnIDs[start:i], + } + if req.Values != nil { + subreq.Values = req.Values[start:i] + } else if req.FloatValues != nil { + subreq.FloatValues = req.FloatValues[start:i] + } + guard <- struct{}{} // would block if guard channel is already filled + eg.Go(func() error { + err := api.server.defaultClient.ImportValue(ctx, qcx, subreq, options) + <-guard + return err + }) + start = i + shard = colID / ShardWidth } } - - // Import into fragment. - err = field.importValue(req.ColumnIDs, req.Values, options) + subreq := &ImportValueRequest{ + Index: req.Index, + Field: req.Field, + Shard: shard, + ColumnIDs: req.ColumnIDs[start:], + } + if req.Values != nil { + subreq.Values = req.Values[start:] + } else if req.FloatValues != nil { + subreq.FloatValues = req.FloatValues[start:] + } + eg.Go(func() error { + // TODO we should elevate the logic for figuring out which + // node(s) to send to into API instead of having those details + // in the client implementation. + return api.server.defaultClient.ImportValue(ctx, qcx, subreq, options) + }) + err = eg.Wait() if err != nil { - api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return err } - return errors.Wrap(err, "importing") + return nil } -func importExistenceColumns(index *Index, columnIDs []uint64) error { +// ingestNodeOperationsForFields does the actual work of applying operations +// to a given index with a map of known fields and an already-parsed +// ShardedRequest. This is used locally on the node that first receives +// the request, after it does the parsing, and on other nodes because the +// format they get is already that rather than JSON, so it's the common +// path *after* key translation and sorting into shards. +func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error { + eg, ctx := errgroup.WithContext(ctx) + for shard, ops := range req.Ops { + // create new local copies of these values so the goroutine uses these + // copies, and doesn't read the actual loop variables, which are being + // changed by the loop. + shard, ops := shard, ops + eg.Go(func() error { + return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) + }) + } + return eg.Wait() +} + +// IngestNodeOperations handles protobuf-formatted data which does not need +// key translation and is applicable to this specific node. +func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error { + index := api.holder.Index(indexName) + if index == nil { + api.server.logger.Errorf("ingest: no such index %q", indexName) + return newNotFoundError(ErrIndexNotFound, indexName) + } + fields := index.Fields() + knownFields := map[string]*Field{} + for _, field := range fields { + knownFields[field.name] = field + } + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req) +} + +// IngestOperations handles JSON-formatted data which may need key translation +// and may be for any or all nodes. +func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations") + defer span.Finish() + + if api.PrimaryNode().ID != api.NodeID() { + return RedirectError{ + HostPort: api.PrimaryNode().URI.Normalize(), + error: "request made to non-primary node", + } + } + + if err := api.validate(apiIngestOperations); err != nil { + return errors.Wrap(err, "validating api method") + } + + // Find the Index. + index := api.holder.Index(indexName) + if index == nil { + api.server.logger.Errorf("ingest: no such index %q", indexName) + return newNotFoundError(ErrIndexNotFound, indexName) + } + fields := index.Fields() + var indexKeys ingest.KeyTranslator + if index.Keys() { + indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName) + } + codec, err := ingest.NewJSONCodec(indexKeys) + if err != nil { + return errors.Wrap(err, "creating JSON codec") + } + knownFields := map[string]*Field{} + for _, field := range fields { + var keys ingest.KeyTranslator + if field.usesKeys { + keys = newIngestKeyTranslatorFromStore(field.translateStore) + } + knownFields[field.name] = field + switch field.Type() { + case "set": + if err = codec.AddSetField(field.name, keys); err != nil { + return fmt.Errorf("adding set field to codec: %w", err) + } + case "time": + if err = codec.AddTimeQuantumField(field.name, keys); err != nil { + return fmt.Errorf("adding time quantum field to codec: %w", err) + } + case "mutex": + if err = codec.AddMutexField(field.name, keys); err != nil { + return fmt.Errorf("adding mutex field to codec: %w", err) + } + case "bool": + if err = codec.AddBoolField(field.name); err != nil { + return fmt.Errorf("adding bool field to codec: %w", err) + } + case "int": + if err = codec.AddIntField(field.name, keys); err != nil { + return fmt.Errorf("adding int field to codec: %w", err) + } + case "decimal": + if err = codec.AddDecimalField(field.name, field.options.Scale); err != nil { + return fmt.Errorf("adding decimal field to codec: %w", err) + } + case "timestamp": + if err = codec.AddTimestampField(field.name, field.options.TimeUnit, field.options.Base); err != nil { + return fmt.Errorf("adding timestamp field to codec: %w", err) + } + default: + return fmt.Errorf("unhandled field type %q", field.Type()) + } + } + req, err := codec.Parse(stream) + if err != nil { + return errors.Wrap(err, "parsing input data") + } + sharded, err := codec.RequestByShard(req) + if err != nil { + return errors.Wrap(err, "sharding input data") + } + // now that we have this, let's assign the shards to nodes + snap := api.cluster.NewSnapshot() + // oh hey an easy case: we're presumably the only node + if len(snap.Nodes) == 1 { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + } + // Created new ShardedRequest objects for every node, giving each of them + // all the shards that apply to them. + byNode := make(map[string]*ingest.ShardedRequest) + for shard, ops := range sharded.Ops { + nodes := snap.ShardNodes(indexName, shard) + for _, node := range nodes { + forThisShard := byNode[node.ID] + if forThisShard == nil { + // Create new ShardedRequest for the target node, with its op map + // mapping this shard to the ops for this shard. + byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} + continue + } + // Add this shard to the existing ShardedRequest's Ops map. Note that + // we don't have to worry about overwrites; we can't have seen this + // shard before, because we're in a range loop on a map where the shard + // is the key. + forThisShard.Ops[shard] = ops + } + } + eg, ctx := errgroup.WithContext(ctx) + for _, node := range snap.Nodes { + node := node + sharded := byNode[node.ID] + // Sometimes, there's nothing for a specific node. + if sharded == nil { + continue + } + if node.ID == api.NodeID() { + eg.Go(func() error { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + }) + } else { + eg.Go(func() error { + return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded) + }) + } + } + return eg.Wait() +} + +// applyOperations applies a set of operations to one specific shard. +func (api *API) applyOperations(ctx context.Context, qcx *Qcx, index *Index, shard uint64, fields map[string]*Field, ops []*ingest.Operation) error { + // For each operation, we may have a set of records/fields to clear, and then + // also a set of fields to set/remove specific bits in. + opts := &ImportOptions{Presorted: true, IgnoreKeyCheck: true, fullySorted: true} + for _, op := range ops { + // ClearRecordIDs should exist only for delete, clear, and write. For clear and write, + // we'll have a list of fields, for delete, it should be all the fields. + if len(op.ClearRecordIDs) > 0 { + // anonymous func lets us defer a finisher from any of the inner error returns + err := func() (e0 error) { + // WARNING: Depends on GetTx being per-shard/index, not per-field. + tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard}) + if err != nil { + return fmt.Errorf("getting Tx: %w", err) + } + defer finisher(&e0) + // For a delete, we don't look at the fields the codec was defined with, + // We delete from the existence field unconditionally and other fields + // if we know they exist. + if op.OpType == ingest.OpDelete { + err = clearExistenceColumns(tx, index, op.ClearRecordIDs, shard) + if err != nil { + return fmt.Errorf("clearing existence columns: %w", err) + } + for name, field := range fields { + if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil { + return fmt.Errorf("clearing field %q: %w", name, err) + } + } + return nil + } + // clear things that we need to wipe out, whether it's because + // this is a Clear op, or because it's a write op that + // specifies clears for the fields it's going to write to. + if len(op.ClearFields) > 0 { + for _, fieldName := range op.ClearFields { + field, ok := fields[fieldName] + if !ok { + return fmt.Errorf("can't find a field named %q", fieldName) + } + if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil { + return fmt.Errorf("clearing record IDs: %w", err) + } + } + } + return nil + }() + if err != nil { + return err + } + } + opts.Clear = (op.OpType == ingest.OpRemove) + // for "set" and "write" ops, we'll be setting bits, for + // "remove" ops we'll be clearing them, and for "clear" ops + // there shouldn't be anything here. + for fieldName, fieldOp := range op.FieldOps { + field, ok := fields[fieldName] + if !ok { + return fmt.Errorf("can't find a field named %q", fieldName) + } + var err error + err = importExistenceColumns(qcx, index, fieldOp.RecordIDs, shard) + if err != nil { + return errors.Wrap(err, "importing existence columns") + } + switch field.Type() { + case "set", "time", "mutex", "bool": + err = field.Import(qcx, fieldOp.Values, fieldOp.RecordIDs, fieldOp.Signed, shard, opts) + case "int", "timestamp", "decimal": + err = field.importValue(qcx, fieldOp.RecordIDs, fieldOp.Signed, shard, opts) + default: + err = fmt.Errorf("unhandled field type %q", field.Type()) + } + if err != nil { + return err + } + } + } + return nil +} + +func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error { ef := index.existenceField() if ef == nil { return nil } existenceRowIDs := make([]uint64, len(columnIDs)) - return ef.Import(existenceRowIDs, columnIDs, nil) + // If we don't gratuitously hand-duplicate things in field.Import, + // the fact that fragment.bulkImport rewrites its row and column + // lists can burn us if we don't make a copy before doing the + // existence field write. + columnCopy := make([]uint64, len(columnIDs)) + copy(columnCopy, columnIDs) + options := ImportOptions{} + return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options) +} + +func clearExistenceColumns(tx Tx, index *Index, columnIDs []uint64, shard uint64) error { + ef := index.existenceField() + if ef == nil { + return nil + } + v := ef.view("standard") + if v == nil { + return nil + } + f := v.Fragment(shard) + if f == nil { + return nil + } + _, err := f.ClearRecords(tx, columnIDs) + return err +} + +// ShardDistribution returns an object representing the distribution of shards +// across nodes for each index, distinguishing between primary and replica. +// The structure of this information is [indexName][nodeID][primaryOrReplica][]uint64. +// This function supports a view in the UI. +func (api *API) ShardDistribution(ctx context.Context) map[string]interface{} { + distByIndex := make(map[string]interface{}) + + for idx := range api.holder.indexes { + dist := api.cluster.shardDistributionByIndex(idx) + distByIndex[idx] = dist + } + + return distByIndex } // MaxShards returns the maximum shard number for each index in a map. @@ -1143,6 +2220,19 @@ func (api *API) AvailableShardsByIndex(ctx context.Context) map[string]*roaring. return api.holder.availableShardsByIndex() } +// AvailableShards returns bitmap of available shards for a single index. +func (api *API) AvailableShards(ctx context.Context, indexName string) (*roaring.Bitmap, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.AvailableShards") + defer span.Finish() + + // Find the index. + index := api.holder.Index(indexName) + if index == nil { + return nil, newNotFoundError(ErrIndexNotFound, indexName) + } + return index.AvailableShards(false), nil +} + // StatsWithTags returns an instance of whatever implementation of StatsClient // pilosa is using with the given tags. func (api *API) StatsWithTags(tags []string) stats.StatsClient { @@ -1155,16 +2245,15 @@ func (api *API) StatsWithTags(tags []string) stats.StatsClient { // LongQueryTime returns the configured threshold for logging/statting // long running queries. func (api *API) LongQueryTime() time.Duration { - if api.cluster == nil { - return 0 - } - return api.cluster.longQueryTime + return api.server.longQueryTime } func (api *API) validateShardOwnership(indexName string, shard uint64) error { + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() // Validate that this handler owns the shard. - if !api.cluster.ownsShard(api.Node().ID, indexName, shard) { - api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName) + if !snap.OwnsShard(api.NodeID(), indexName, shard) { + api.server.logger.Errorf("node %s does not own shard %d of index %s", api.NodeID(), shard, indexName) return ErrClusterDoesNotOwnShard } return nil @@ -1176,93 +2265,38 @@ func (api *API) indexField(indexName string, fieldName string, shard uint64) (*I // Find the Index. index := api.holder.Index(indexName) if index == nil { - api.server.logger.Printf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error()) + api.server.logger.Errorf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error()) return nil, nil, newNotFoundError(ErrIndexNotFound, indexName) } // Retrieve field. field := index.Field(fieldName) if field == nil { - api.server.logger.Printf("field error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrFieldNotFound.Error()) + api.server.logger.Errorf("field error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrFieldNotFound.Error()) return nil, nil, newNotFoundError(ErrFieldNotFound, fieldName) } return index, field, nil } -// SetCoordinator makes a new Node the cluster coordinator. -func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { - span, _ := tracing.StartSpanFromContext(ctx, "API.SetCoordinator") - defer span.Finish() - - if err := api.validate(apiSetCoordinator); err != nil { - return nil, nil, errors.Wrap(err, "validating api method") - } - - oldNode = api.cluster.nodeByID(api.cluster.Coordinator) - newNode = api.cluster.nodeByID(id) - if newNode == nil { - return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") - } - - // If the new coordinator is this node, do the SetCoordinator directly. - if newNode.ID == api.Node().ID { - return oldNode, newNode, api.cluster.setCoordinator(newNode) - } - - // Send the set-coordinator message to new node. - err = api.server.SendTo( - newNode, - &SetCoordinatorMessage{ - New: newNode, - }) - if err != nil { - return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) - } - return oldNode, newNode, nil -} - -// RemoveNode puts the cluster into the "RESIZING" state and begins the job of -// removing the given node. -func (api *API) RemoveNode(id string) (*Node, error) { - if err := api.validate(apiRemoveNode); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - removeNode := api.cluster.nodeByID(id) - if removeNode == nil { - if !api.cluster.topologyContainsNode(id) { - return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") - } - removeNode = &Node{ - ID: id, - } - } - - // Start the resize process (similar to NodeJoin) - err := api.cluster.nodeLeave(id) - if err != nil { - return removeNode, errors.Wrap(err, "calling node leave") - } - return removeNode, nil -} - -// ResizeAbort stops the current resize job. -func (api *API) ResizeAbort() error { - if err := api.validate(apiResizeAbort); err != nil { - return errors.Wrap(err, "validating api method") - } - - err := api.cluster.completeCurrentJob(resizeJobStateAborted) - return errors.Wrap(err, "complete current job") -} - // State returns the cluster state which is usually "NORMAL", but could be -// "STARTING", "RESIZING", or potentially others. See cluster.go for more +// "STARTING", or potentially others. See disco.go for more // details. -func (api *API) State() string { +func (api *API) State() (disco.ClusterState, error) { + if err := api.validate(apiState); err != nil { + return "", errors.Wrap(err, "validating api method") + } + return api.cluster.State() } +// ClusterName returns the cluster name. +func (api *API) ClusterName() string { + if api.cluster.Name == "" { + return api.cluster.id + } + return api.cluster.Name +} + // Version returns the Pilosa version. func (api *API) Version() string { return strings.TrimPrefix(Version, "v") @@ -1282,59 +2316,892 @@ func (api *API) Info() serverInfo { CPUMHz: mhz, CPUType: si.CPUModel(), Memory: mem, + StorageBackend: api.holder.txf.TxType(), + ReplicaN: api.cluster.ReplicaN, + ShardHash: api.cluster.Hasher.Name(), + KeyHash: api.cluster.Hasher.Name(), } } // GetTranslateEntryReader provides an entry reader for key translation logs starting at offset. -func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (TranslateEntryReader, error) { +func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader") defer span.Finish() - return api.holder.TranslateEntryReader(ctx, offsets) + + // Ensure all readers are cleaned up if any error. + var a []TranslateEntryReader + defer func() { + if err != nil { + for i := range a { + a[i].Close() // nolint: errcheck + } + } + }() + + // Fetch all index partition readers. + for indexName, indexMap := range offsets { + index := api.holder.Index(indexName) + if index == nil { + return nil, newNotFoundError(ErrIndexNotFound, indexName) + } + + for partitionID, offset := range indexMap.Partitions { + store := index.TranslateStore(partitionID) + if store == nil { + return nil, ErrTranslateStoreNotFound + } + + r, err := store.EntryReader(ctx, uint64(offset)) + if err != nil { + return nil, errors.Wrap(err, "index partition translate reader") + } + a = append(a, r) + } + } + + // Fetch all field readers. + for indexName, indexMap := range offsets { + index := api.holder.Index(indexName) + if index == nil { + return nil, newNotFoundError(ErrIndexNotFound, indexName) + } + + for fieldName, offset := range indexMap.Fields { + field := index.Field(fieldName) + if field == nil { + return nil, newNotFoundError(ErrFieldNotFound, fieldName) + } + store := field.TranslateStore() + if store == nil { + return nil, ErrTranslateStoreNotFound + } + r, err := field.TranslateStore().EntryReader(ctx, uint64(offset)) + if err != nil { + return nil, errors.Wrap(err, "field translate reader") + } + a = append(a, r) + } + } + + return NewMultiTranslateEntryReader(ctx, a), nil +} + +func (api *API) TranslateIndexKey(ctx context.Context, indexName string, key string, writable bool) (uint64, error) { + return api.cluster.translateIndexKey(ctx, indexName, key, writable) +} + +func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) { + return api.cluster.translateIndexIDs(ctx, indexName, ids) } // TranslateKeys handles a TranslateKeyRequest. -func (api *API) TranslateKeys(r io.Reader) ([]byte, error) { +// ErrTranslatingKeyNotFound error will be swallowed here, so the empty response will be returned. +func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) { var req TranslateKeysRequest - if buf, err := ioutil.ReadAll(r); err != nil { + buf, err := ioutil.ReadAll(r) + if err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read translate keys request error")) } else if err := api.Serializer.Unmarshal(buf, &req); err != nil { return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate keys request error")) } // Lookup store for either index or field and translate keys. - store, err := api.holder.TranslateStore(req.Index, req.Field) - if err != nil { - return nil, err + var ids []uint64 + if req.Field == "" { + ids, err = api.cluster.translateIndexKeys(ctx, req.Index, req.Keys, !req.NotWritable) + } else { + field := api.holder.Field(req.Index, req.Field) + if field == nil { + return nil, newNotFoundError(ErrFieldNotFound, req.Field) + } + + if fi := field.ForeignIndex(); fi != "" { + ids, err = api.cluster.translateIndexKeys(ctx, fi, req.Keys, !req.NotWritable) + } else { + ids, err = api.cluster.translateFieldKeys(ctx, field, req.Keys, !req.NotWritable) + } } - ids, err := store.TranslateKeys(req.Keys) - if err != nil { - return nil, err + if err != nil && errors.Cause(err) != ErrTranslatingKeyNotFound { + return nil, errors.WithMessage(err, "translating keys") } // Encode response. - buf, err := api.Serializer.Marshal(&TranslateKeysResponse{IDs: ids}) - if err != nil { + if buf, err = api.Serializer.Marshal(&TranslateKeysResponse{IDs: ids}); err != nil { return nil, errors.Wrap(err, "translate keys response encoding error") } return buf, nil } +// TranslateIDs handles a TranslateIDRequest. +func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error) { + var req TranslateIDsRequest + if buf, err := ioutil.ReadAll(r); err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "read translate ids request error")) + } else if err := api.Serializer.Unmarshal(buf, &req); err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate ids request error")) + } + + // Lookup store for either index or field and translate ids. + var keys []string + if req.Field == "" { + if keys, err = api.cluster.translateIndexIDs(ctx, req.Index, req.IDs); err != nil { + return nil, err + } + } else { + if field := api.holder.Field(req.Index, req.Field); field == nil { + return nil, newNotFoundError(ErrFieldNotFound, req.Field) + } else if fi := field.ForeignIndex(); fi != "" { + keys, err = api.cluster.translateIndexIDs(ctx, fi, req.IDs) + if err != nil { + return nil, err + } + } else if keys, err = api.cluster.translateFieldListIDs(ctx, field, req.IDs); err != nil { + return nil, err + } + } + + // Encode response. + buf, err := api.Serializer.Marshal(&TranslateIDsResponse{Keys: keys}) + if err != nil { + return nil, errors.Wrap(err, "translate ids response encoding error") + } + return buf, nil +} + +// FindIndexKeys looks up column keys in the index, mapping them to IDs. +// If a key does not exist, it will be absent from the resulting map. +func (api *API) FindIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error) { + return api.cluster.findIndexKeys(ctx, index, keys...) +} + +// FindFieldKeys looks up keys in a field, mapping them to IDs. +// If a key does not exist, it will be absent from the resulting map. +func (api *API) FindFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error) { + f := api.holder.Field(index, field) + if f == nil { + return nil, newNotFoundError(ErrFieldNotFound, field) + } + return api.cluster.findFieldKeys(ctx, f, keys...) +} + +// CreateIndexKeys looks up column keys in the index, mapping them to IDs. +// If a key does not exist, it will be created. +func (api *API) CreateIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error) { + return api.cluster.createIndexKeys(ctx, index, keys...) +} + +// CreateFieldKeys looks up keys in a field, mapping them to IDs. +// If a key does not exist, it will be created. +func (api *API) CreateFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error) { + f := api.holder.Field(index, field) + if f == nil { + return nil, newNotFoundError(ErrFieldNotFound, field) + } + return api.cluster.createFieldKeys(ctx, f, keys...) +} + +// MatchField finds the IDs of all field keys matching a filter. +func (api *API) MatchField(ctx context.Context, index, field string, like string) ([]uint64, error) { + f := api.holder.Field(index, field) + if f == nil { + return nil, newNotFoundError(ErrFieldNotFound, field) + } + return api.cluster.matchField(ctx, f, like) +} + // PrimaryReplicaNodeURL returns the URL of the cluster's primary replica. func (api *API) PrimaryReplicaNodeURL() url.URL { - node := api.cluster.PrimaryReplicaNode() + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + + node := snap.PrimaryReplicaNode(api.NodeID()) if node == nil { return url.URL{} } return node.URI.URL() } +func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, remote bool) (*Transaction, error) { + if err := api.validate(apiStartTransaction); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + t, err := api.server.StartTransaction(ctx, id, timeout, exclusive, remote) + + switch err { + case nil: + if exclusive { + api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0) + } else { + api.holder.Stats.Count(MetricTransactionStart, 1, 1.0) + } + case ErrTransactionExclusive: + if exclusive { + api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0) + } else { + api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0) + } + } + if exclusive && t != nil && t.Active { + api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + } + return t, err +} + +func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) { + if err := api.validate(apiFinishTransaction); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + t, err := api.server.FinishTransaction(ctx, id, remote) + if err == nil { + if t.Exclusive { + api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0) + } else { + api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0) + } + } + return t, err +} + +func (api *API) Transactions(ctx context.Context) (map[string]*Transaction, error) { + if err := api.validate(apiTransactions); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + return api.server.Transactions(ctx) +} + +func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) { + if err := api.validate(apiGetTransaction); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + t, err := api.server.GetTransaction(ctx, id, remote) + if err == nil { + if t.Exclusive && t.Active { + api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + } + } + return t, err +} + +func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error) { + if err := api.validate(apiActiveQueries); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + return api.tracker.ActiveQueries(), nil +} + +func (api *API) PastQueries(ctx context.Context, remote bool) ([]PastQueryStatus, error) { + if err := api.validate(apiPastQueries); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + clusterQueries := api.tracker.PastQueries() + + if !remote { + nodes := api.cluster.Nodes() + for _, node := range nodes { + if node.ID == api.server.nodeID { + continue + } + nodeQueries, err := api.server.defaultClient.GetPastQueries(ctx, &node.URI) + if err != nil { + return nil, errors.Wrapf(err, "collecting query history from %s", node.URI) + } + clusterQueries = append(clusterQueries, nodeQueries...) + } + } + + sort.Slice(clusterQueries, func(i, j int) bool { + return clusterQueries[i].Start.After(clusterQueries[j].Start) + }) + + return clusterQueries, nil +} + +func (api *API) ReserveIDs(key IDAllocKey, session [32]byte, offset uint64, count uint64) ([]IDRange, error) { + if err := api.validate(apiIDReserve); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + + if !snap.IsPrimaryFieldTranslationNode(api.NodeID()) { + return nil, errors.New("cannot reserve IDs on a non-primary node") + } + + return api.holder.ida.reserve(key, session, offset, count) +} + +func (api *API) CommitIDs(key IDAllocKey, session [32]byte, count uint64) error { + if err := api.validate(apiIDCommit); err != nil { + return errors.Wrap(err, "validating api method") + } + + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + + if !snap.IsPrimaryFieldTranslationNode(api.NodeID()) { + return errors.New("cannot commit IDs on a non-primary node") + } + + return api.holder.ida.commit(key, session, count) +} + +func (api *API) ResetIDAlloc(index string) error { + if err := api.validate(apiIDReset); err != nil { + return errors.Wrap(err, "validating api method") + } + + // Create a snapshot of the cluster to use for node/partition calculations. + snap := api.cluster.NewSnapshot() + + if !snap.IsPrimaryFieldTranslationNode(api.NodeID()) { + return errors.New("cannot reset IDs on a non-primary node") + } + + return api.holder.ida.reset(index) +} + +func (api *API) WriteIDAllocDataTo(w io.Writer) error { + _, err := api.holder.ida.WriteTo(w) + return err +} +func (api *API) RestoreIDAlloc(r io.Reader) error { + return api.holder.ida.Replace(r) +} + +// TranslateIndexDB is an internal function to load the index keys database +// rd is a boltdb file. +func (api *API) TranslateIndexDB(ctx context.Context, indexName string, partitionID int, rd io.Reader) error { + idx := api.holder.Index(indexName) + if idx == nil { + return fmt.Errorf("index %q not found", indexName) + } + store := idx.TranslateStore(partitionID) + if store == nil { + return fmt.Errorf("index %q has no translate store", indexName) + } + _, err := store.ReadFrom(rd) + return err +} + +// TranslateFieldDB is an internal function to load the field keys database +func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName string, rd io.Reader) error { + idx := api.holder.Index(indexName) + if idx == nil { + return fmt.Errorf("index %q not found", indexName) + } + field := idx.Field(fieldName) + if field == nil { + // Older versions used to accidentally provide an empty translation + // data file for a nonexistent field called "_keys". To make migration + // easier, we politely ignore that. + if fieldName == "_keys" { + return nil + } + return fmt.Errorf("field %q/%q not found", indexName, fieldName) + } + store := field.TranslateStore() + if store == nil { + return fmt.Errorf("field %q/%q has no translate store", indexName, fieldName) + } + _, err := store.ReadFrom(rd) + return err +} + +// RestoreShard +func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64, rd io.Reader) error { + snap := api.cluster.NewSnapshot() + if !snap.OwnsShard(api.server.nodeID, indexName, shard) { + return ErrClusterDoesNotOwnShard // TODO (twg)really just node doesn't own shard but leave for now + } + + idx := api.holder.Index(indexName) + //need to get a dbShard + dbs, err := idx.Txf().dbPerShard.GetDBShard(indexName, shard, idx) + if err != nil { + return err + } + db := dbs.W + finalPath := db.Path() + "/data" + tempPath := finalPath + ".tmp" + o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer o.Close() + + bw := bufio.NewWriter(o) + if _, err = io.Copy(bw, rd); err != nil { + return err + } else if err := bw.Flush(); err != nil { + return err + } else if err := o.Sync(); err != nil { + return err + } else if err := o.Close(); err != nil { + return err + } + + if err != nil { + _ = os.Remove(tempPath) + return err + } + err = db.CloseDB() + if err != nil { + return err + } + err = os.Rename(tempPath, finalPath) + if err != nil { + _ = os.Remove(tempPath) + return err + } + err = db.OpenDB() + if err != nil { + return err + } + tx, err := db.NewTx(false, idx.name, Txo{}) + if err != nil { + return err + } + defer tx.Rollback() + //arguments idx,shard do not matter for rbf they + //are ignored + flvs, err := tx.GetSortedFieldViewList(idx, shard) + if err != nil { + return nil + } + + for _, flv := range flvs { + fld := idx.field(flv.Field) + view := fld.view(flv.View) + if view == nil { + view, err = fld.createViewIfNotExists(flv.View) + if err != nil { + return err + } + } + frag, err := view.CreateFragmentIfNotExists(shard) + if err != nil { + return err + } + err = frag.RebuildRankCache(ctx) + if err != nil { + return err + } + bd, err := view.bitDepth([]uint64{shard}) + if err != nil { + return err + } + err = fld.cacheBitDepth(bd) + if err != nil { + return err + } + } + + return nil +} + +func (api *API) mutexCheckThisNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { + index := api.holder.Index(indexName) + if index == nil { + return nil, newNotFoundError(ErrIndexNotFound, indexName) + } + field := index.Field(fieldName) + if field == nil { + return nil, newNotFoundError(ErrFieldNotFound, fieldName) + } + results, err := field.MutexCheck(ctx, qcx, details, limit) + if err != nil { + return nil, err + } + if limit != 0 && len(results) > limit { + toDel := len(results) - limit + // yes, Go allows you to delete keys you've already seen while + // iterating a map. The spec says that if a value not-yet-reached + // is deleted during iteration, it may or may not appear; this + // carries the implication that deleting things during map iteration + // is safe. + for k := range results { + delete(results, k) + toDel-- + if toDel == 0 { + break + } + } + } + return results, err +} + +// mergeIDLists merges a list of numeric IDs into another list, removing +// duplicates. +func mergeIDLists(dst []uint64, src []uint64) []uint64 { + dst = append(dst, src...) + sort.Slice(dst, func(i, j int) bool { + return dst[i] < dst[j] + }) + // dedup. + n := 1 + prev := dst[0] + for i := 1; i < len(dst); i++ { + if dst[i] != prev { + dst[n] = dst[i] + n++ + } + prev = dst[i] + } + return dst[:n] +} + +// mergeKeyLists merges a list of string IDs into another list, removing +// duplicates. +func mergeKeyLists(dst []string, src []string) []string { + dst = append(dst, src...) + sort.Slice(dst, func(i, j int) bool { + return dst[i] < dst[j] + }) + // dedup. + n := 1 + prev := dst[0] + for i := 1; i < len(dst); i++ { + if dst[i] != prev { + dst[n] = dst[i] + n++ + } + prev = dst[i] + } + return dst[:n] +} + +// MutexCheckNode checks for collisions in a given mutex field. The response is +// a map[shard]map[column]values, not translated. +func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) { + if err := api.validate(apiMutexCheck); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + return api.mutexCheckThisNode(ctx, qcx, indexName, fieldName, details, limit) +} + +// MutexCheck checks a named field for mutex violations, returning a +// map of record IDs to values for records that have multiple values in the +// field. The return will be one of: +// details true: +// map[uint64][]uint64 // unkeyed index, unkeyed field +// map[uint64][]string // unkeyed index, keyed field +// map[string][]uint64 // keyed index, unkeyed field +// map[string][]string // keyed index, keyed field +// details false: +// []uint64 // unkeyed index +// []string // keyed index +func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (result interface{}, err error) { + if err = api.validate(apiMutexCheck); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + index, err := api.Index(ctx, indexName) + if err != nil { + return nil, err + } + field, err := api.Field(ctx, indexName, fieldName) + if err != nil { + return nil, err + } + if field.Type() != FieldTypeMutex { + return nil, errors.New("can only check mutex state for mutex fields") + } + // request data from other nodes as well + snap := api.cluster.NewSnapshot() + eg, _ := errgroup.WithContext(ctx) + myID := api.NodeID() + results := make([]map[uint64]map[uint64][]uint64, len(snap.Nodes)) + for i, node := range snap.Nodes { + i := i // loop variable shadowing is a war crime + if node.ID != myID { + node := node // loop variable shadowing again + eg.Go(func() (err error) { + results[i], err = api.server.defaultClient.MutexCheck(ctx, &node.URI, indexName, fieldName, details, limit) + return err + }) + } else { + eg.Go(func() (err error) { + results[i], err = api.mutexCheckThisNode(ctx, qcx, indexName, fieldName, details, limit) + return err + }) + } + } + err = eg.Wait() + if err != nil { + return nil, err + } + // Set this arbitrarily large so we don't have to be hand-checking for 0 + // throughout. + if limit == 0 { + limit = math.MaxInt32 + } + // We now have a series of maps from shards to maps of record IDs to + // values. But wait! Either the field, or the index, might be using keys, + // and want those translated. So we have to translate those. We'll create + // some tables. + useIndexKeys := index.Keys() + // If we're not doing details, we won't translate field keys even if we could. + useFieldKeys := field.Keys() && details + var indexKeys = map[uint64]string{} + var fieldKeys = map[uint64]string{} + var indexIDs []uint64 + var fieldIDs []uint64 + // We'll use the string "untranslated" as our default value and overwrite + // it with translations. We do check for missing translation values in + // our returns, but just in case, you know? + untranslated := "untranslated" + // We don't know which of four map types we want to be working with, + // but what we can do is make a function which works with that map type + // given the raw integer values, and is a closure with an already-created + // map which has already been stashed in `result`. Because maps are + // reference-y, this should actually work. This function returns true if + // it's hit the limit for length of results. + var process func(uint64, []uint64) bool + if useIndexKeys || useFieldKeys { + for _, nodeResults := range results { + for _, shardResults := range nodeResults { + for record, values := range shardResults { + if useIndexKeys { + if _, ok := indexKeys[record]; !ok { + indexKeys[record] = untranslated + indexIDs = append(indexIDs, record) + } + } + if useFieldKeys { + for _, value := range values { + if _, ok := fieldKeys[value]; !ok { + fieldKeys[value] = untranslated + fieldIDs = append(fieldIDs, value) + } + } + } + } + } + } + // if context is done, return early. + if err := ctx.Err(); err != nil { + return nil, err + } + untranslatedKeys := 0 + // Obtain translation tables for the keys. + if useIndexKeys { + indexKeyList, err := api.cluster.translateIndexIDs(ctx, indexName, indexIDs) + if err != nil { + return nil, errors.Wrap(err, "translating index keys") + } + if len(indexKeyList) != len(indexIDs) { + return nil, fmt.Errorf("translating %d record IDs, got %d keys", len(indexIDs), len(indexKeyList)) + } + for i := range indexIDs { + if indexKeyList[i] != "" { + indexKeys[indexIDs[i]] = indexKeyList[i] + } else { + untranslatedKeys++ + } + } + } + // if context is done, return early. + if err := ctx.Err(); err != nil { + return nil, err + } + if useFieldKeys { + fieldKeyList, err := api.cluster.translateFieldListIDs(ctx, field, fieldIDs) + if err != nil { + return nil, errors.Wrap(err, "translating index keys") + } + if len(fieldKeyList) != len(fieldIDs) { + return nil, fmt.Errorf("translating %d IDs, got %d keys", len(indexIDs), len(fieldKeyList)) + } + for i := range fieldIDs { + if fieldKeyList[i] != "" { + fieldKeys[fieldIDs[i]] = fieldKeyList[i] + } else { + untranslatedKeys++ + } + } + } + if untranslatedKeys > 0 { + api.server.logger.Warnf("translating mutex check results: %d key(s) untranslated", untranslatedKeys) + } + } + + // if context is done, return early. + if err := ctx.Err(); err != nil { + return nil, err + } + + // define the process functions. separated from above code just to make + // it easier to follow/compare them. + if useIndexKeys { + if !details { + outMap := make(map[uint64]struct{}) + outStrings := []string{} + // unlike a map, the slice won't get updated-in-place, so we have + // to assign to result after we're done + defer func() { + if err == nil { + result = outStrings + } + }() + process = func(recordID uint64, valueIDs []uint64) bool { + if _, ok := outMap[recordID]; ok { + return len(outMap) >= limit + } + outMap[recordID] = struct{}{} + outStrings = append(outStrings, indexKeys[recordID]) + return len(outMap) >= limit + } + } else if useFieldKeys { + outMap := make(map[string][]string) + var valueKeys []string + result = outMap + process = func(recordID uint64, valueIDs []uint64) bool { + valueKeys = valueKeys[:0] + for _, id := range valueIDs { + valueKeys = append(valueKeys, fieldKeys[id]) + } + record := indexKeys[recordID] + if existing, ok := outMap[record]; ok { + outMap[record] = mergeKeyLists(existing, valueKeys) + } else if len(outMap) < limit { + // The append is so we can reuse this buffer safely, + // which matters if there's replication, because many + // cases won't need to copy the buffer, they'll just + // copy individual things from it. + outMap[record] = append([]string{}, valueKeys...) + } + return len(outMap) >= limit + } + } else { + outMap := make(map[string][]uint64) + result = outMap + process = func(recordID uint64, values []uint64) bool { + record := indexKeys[recordID] + if existing, ok := outMap[record]; ok { + outMap[record] = mergeIDLists(existing, values) + } else if len(outMap) < limit { + outMap[record] = values + } + return len(outMap) >= limit + } + } + } else { + if !details { + outMap := make(map[uint64]struct{}) + outIDs := []uint64{} + // unlike a map, the slice won't get updated-in-place, so we have + // to assign to result after we're done + defer func() { + if err == nil { + result = outIDs + } + }() + process = func(recordID uint64, valueIDs []uint64) bool { + if _, ok := outMap[recordID]; ok { + return len(outMap) >= limit + } + outMap[recordID] = struct{}{} + outIDs = append(outIDs, recordID) + return len(outMap) >= limit + } + } else if useFieldKeys { + outMap := make(map[uint64][]string) + var valueKeys []string + result = outMap + process = func(record uint64, valueIDs []uint64) bool { + valueKeys = valueKeys[:0] + for _, id := range valueIDs { + valueKeys = append(valueKeys, fieldKeys[id]) + } + if existing, ok := outMap[record]; ok { + outMap[record] = mergeKeyLists(existing, valueKeys) + } else { + // The append is so we can reuse this buffer safely, + // which matters if there's replication, because many + // cases won't need to copy the buffer, they'll just + // copy individual things from it. + outMap[record] = append([]string{}, valueKeys...) + } + return len(outMap) >= limit + } + } else { + outMap := make(map[uint64][]uint64) + result = outMap + process = func(record uint64, values []uint64) bool { + if existing, ok := outMap[record]; ok { + outMap[record] = mergeIDLists(existing, values) + } else { + outMap[record] = values + } + return len(outMap) >= limit + } + } + } + + // if you specify a limit, and you have *different* errors on different + // nodes, we will not check all of the nodes. otherwise there's no practical + // way to get the primary benefit of specifying a limit. +processing: + for _, nodeResults := range results { + if len(nodeResults) == 0 { + continue + } + // if context is done, return early. + if err := ctx.Err(); err != nil { + return nil, err + } + for _, v := range nodeResults { + if len(v) == 0 { + continue + } + counter := 0 + for record, values := range v { + counter++ + if process(record, values) { + break processing + } + // every 65k items or so, check the context for done-ness + if counter%(1<<16) == 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + } + } + } + } + return result, ctx.Err() +} +func (api *API) Plan(ctx context.Context, q string) (*Stmt, error) { + return api.server.PlanSQL(ctx, q) +} + +func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo { + infos := make(map[string]*rbf.DebugInfo) + + for key, dbShard := range api.holder.Txf().dbPerShard.Flatmap { + wrapper, ok := dbShard.W.(*RbfDBWrapper) + if !ok { + continue + } + + skey := fmt.Sprintf("%s/%d", key.index, key.shard) + infos[skey] = wrapper.db.DebugInfo() + } + return infos +} + type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` + ReplicaN int `json:"replicaN"` + ShardHash string `json:"shardHash"` + KeyHash string `json:"keyHash"` Memory uint64 `json:"memory"` CPUType string `json:"cpuType"` CPUPhysicalCores int `json:"cpuPhysicalCores"` CPULogicalCores int `json:"cpuLogicalCores"` CPUMHz int `json:"cpuMHz"` + StorageBackend string `json:"storageBackend"` } type apiMethod int @@ -1352,39 +3219,63 @@ const ( apiFragmentBlockData apiFragmentBlocks apiFragmentData + apiTranslateData + apiFieldTranslateData apiField - apiFieldAttrDiff //apiHosts // not implemented apiImport apiImportValue apiIndex - apiIndexAttrDiff //apiLocalID // not implemented //apiLongQueryTime // not implemented //apiMaxShards // not implemented apiQuery apiRecalculateCaches - apiRemoveNode - apiResizeAbort - //apiSchema // not implemented - apiSetCoordinator + apiSchema apiShardNodes - //apiState // not implemented + apiState //apiStatsWithTags // not implemented //apiVersion // not implemented apiViews apiApplySchema + apiStartTransaction + apiFinishTransaction + apiTransactions + apiGetTransaction + apiActiveQueries + apiPastQueries + apiIDReserve + apiIDCommit + apiIDReset + apiPartitionNodes + apiIngestOperations + apiIngestNodeOperations + apiMutexCheck ) var methodsCommon = map[apiMethod]struct{}{ apiClusterMessage: {}, - apiSetCoordinator: {}, + apiState: {}, } -var methodsResizing = map[apiMethod]struct{}{ - apiFragmentData: {}, - apiResizeAbort: {}, -} +// var methodsDegraded = map[apiMethod]struct{}{ +// apiExportCSV: {}, +// apiFragmentBlockData: {}, +// apiFragmentBlocks: {}, +// apiField: {}, +// apiIndex: {}, +// apiQuery: {}, +// apiRecalculateCaches: {}, +// apiRemoveNode: {}, +// apiShardNodes: {}, +// apiSchema: {}, +// apiViews: {}, +// apiStartTransaction: {}, +// apiFinishTransaction: {}, +// apiTransactions: {}, +// apiGetTransaction: {}, +// apiActiveQueries: {}, +// } var methodsNormal = map[apiMethod]struct{}{ apiCreateField: {}, @@ -1397,15 +3288,28 @@ var methodsNormal = map[apiMethod]struct{}{ apiFragmentBlockData: {}, apiFragmentBlocks: {}, apiField: {}, - apiFieldAttrDiff: {}, + apiFieldTranslateData: {}, apiImport: {}, apiImportValue: {}, apiIndex: {}, - apiIndexAttrDiff: {}, apiQuery: {}, apiRecalculateCaches: {}, - apiRemoveNode: {}, apiShardNodes: {}, + apiSchema: {}, apiViews: {}, apiApplySchema: {}, + apiStartTransaction: {}, + apiFinishTransaction: {}, + apiTransactions: {}, + apiTranslateData: {}, + apiGetTransaction: {}, + apiActiveQueries: {}, + apiPastQueries: {}, + apiIDReserve: {}, + apiIDCommit: {}, + apiIDReset: {}, + apiPartitionNodes: {}, + apiIngestOperations: {}, + apiIngestNodeOperations: {}, + apiMutexCheck: {}, } diff --git a/api/client/grpc.go b/api/client/grpc.go new file mode 100644 index 000000000..636b60d40 --- /dev/null +++ b/api/client/grpc.go @@ -0,0 +1,203 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package client + +import ( + "context" + "crypto/tls" + "sync" + + "github.com/molecula/featurebase/v3/logger" + pb "github.com/molecula/featurebase/v3/proto" + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" +) + +const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody! + +// GRPCClient is a client for working with the gRPC server. +type GRPCClient struct { + dialTargets []string + tlsConfig *tls.Config + logger logger.Logger + + mu sync.RWMutex + conn *grpc.ClientConn + targetIndex int +} + +// NewGRPCClient returns a new instance of GRPCClient. +func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config, logger logger.Logger) (*GRPCClient, error) { + c := &GRPCClient{ + dialTargets: dialTargets, + tlsConfig: tlsConfig, + logger: logger, + } + // resetConn sets GRPCClient.conn when it doesn't + // exist yet. + if err := c.resetConn(); err != nil { + return nil, errors.Wrap(err, "setting connection") + } + + return c, nil +} + +// resetConn resets the gRPC client connection. This method +// can also be used to initially set the client connection +// because it only tries to first close the connection if +// the connection already exists. +func (c *GRPCClient) resetConn() error { + c.mu.Lock() + defer c.mu.Unlock() + + // If an existing connection exists, close it first. + if c.conn != nil { + if err := c.conn.Close(); err != nil { + return errors.Wrap(err, "closing existing connection") + } + } + + var opts []grpc.DialOption + if c.tlsConfig != nil { + creds := credentials.NewTLS(c.tlsConfig) + opts = append(opts, grpc.WithTransportCredentials(creds)) + } else { + opts = append(opts, grpc.WithInsecure()) + } + + opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize))) + + var err error + if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil { + return errors.Wrap(err, "creating new grpc client") + } + + return nil +} + +// getTargetIndex gets the current target index, then increments it for +// next time. Unprotected. +func (c *GRPCClient) getTargetIndex() int { + if len(c.dialTargets) == 0 { + return 0 + } + ret := c.targetIndex + c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets + return ret +} + +// Close closes any connections the client has opened. +func (c *GRPCClient) Close() error { + c.mu.RLock() + defer c.mu.RUnlock() + + if c.conn != nil { + return c.conn.Close() + } + return nil +} + +// Conn returns the gRPC client connection. If the connection +// has gone into state `TransientFailure`, this method tries +// to reset the connection and return that new connection. +func (c *GRPCClient) Conn() *grpc.ClientConn { + c.mu.RLock() + if c.conn == nil { + c.mu.RUnlock() + return nil + } else if c.conn.GetState() != connectivity.TransientFailure { + defer c.mu.RUnlock() + return c.conn + } + c.mu.RUnlock() + + if err := c.resetConn(); err != nil { + c.logger.Errorf("error resetting connection: %s", err) + } + + c.mu.RLock() + defer c.mu.RUnlock() + return c.conn +} + +// Query returns a stream of RowResponse for the given index and PQL string. +func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) { + conn := c.Conn() + + if conn == nil { + return nil, errors.New("client has not established a grpc connection") + } + + grpcClient := pb.NewPilosaClient(conn) + + stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{ + Index: index, + Pql: pql, + }) + if err != nil { + return nil, errors.Wrap(err, "getting stream") + } else if stream == nil { + return nil, errors.New("could not create stream") + } + + return stream, err +} + +// QueryUnary returns a TableResponse for the given index and PQL string. +func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) { + conn := c.Conn() + + if conn == nil { + return nil, errors.New("client has not established a grpc connection") + } + + grpcClient := pb.NewPilosaClient(conn) + + return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{ + Index: index, + Pql: pql, + }) +} + +// Inspect returns a stream of RowResponse for the given index, columns, and filters. +// It is intended to mimic something like "select [fields] from table where recordID IN (...)". +func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) { + conn := c.Conn() + + if conn == nil { + return nil, errors.New("client has not established a grpc connection") + } + + if len(columnIDs) > 0 && len(columnKeys) > 0 { + return nil, errors.New("only provide column ids or keys, not both") + } + + // Convert columns to proto type IdsOrKeys. + idsOrKeys := &pb.IdsOrKeys{} + if len(columnKeys) > 0 { + idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}} + } else { + idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}} + } + + grpcClient := pb.NewPilosaClient(conn) + + stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{ + Index: index, + Columns: idsOrKeys, + FilterFields: fieldFilters, + Limit: limit, + Offset: offset, + Query: query, + }) + + if err != nil { + return nil, errors.Wrap(err, "getting stream") + } else if stream == nil { + return nil, errors.New("could not create stream") + } + + return stream, err +} diff --git a/api_test.go b/api_test.go index 85cb92601..a80cd5661 100644 --- a/api_test.go +++ b/api_test.go @@ -1,112 +1,149 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 package pilosa_test import ( + "bytes" "context" + "encoding/hex" + "encoding/json" + "errors" "fmt" + "io" "math" + "math/rand" + "net/http" + "net/http/httptest" + "os" + "path/filepath" "reflect" + "sort" "strings" "testing" "time" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/boltdb" - "github.com/pilosa/pilosa/v2/http" - "github.com/pilosa/pilosa/v2/server" - "github.com/pilosa/pilosa/v2/test" + "github.com/golang-jwt/jwt" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/authn" + "github.com/molecula/featurebase/v3/boltdb" + "github.com/molecula/featurebase/v3/roaring" + "github.com/molecula/featurebase/v3/server" + "github.com/molecula/featurebase/v3/shardwidth" + "github.com/molecula/featurebase/v3/test" + . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck + + "golang.org/x/sync/errgroup" ) func TestAPI_Import(t *testing.T) { - c := test.MustRunCluster(t, 2, + c := test.MustRunCluster(t, 3, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node2"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() - m0 := c[0] - m1 := c[1] + m0 := c.GetNode(0) + m1 := c.GetNode(1) - t.Run("RowIDColumnKey", func(t *testing.T) { - ctx := context.Background() - index := "rick" - field := "f" + indexNames := map[bool]string{false: "i", true: "ki"} + fieldNames := map[bool]string{false: "f", true: "kf"} - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) + ctx := context.Background() + for ik, indexName := range indexNames { + _, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: ik, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) - if err != nil { - t.Fatalf("creating field: %v", err) + for fk, fieldName := range fieldNames { + if fk { + _, err = m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys()) + } else { + _, err = m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + + } + if err != nil { + t.Fatalf("creating field: %v", err) + } } + } - rowID := uint64(1) - timestamp := int64(0) + N := 10 - // Generate some keyed records. - rowIDs := []uint64{} - colKeys := []string{} - timestamps := []int64{} - for i := 1; i <= 10; i++ { - rowIDs = append(rowIDs, rowID) - timestamps = append(timestamps, timestamp) - colKeys = append(colKeys, fmt.Sprintf("col%d", i)) - } + // Keys are sharded so ordering is not guaranteed. + colKeys := make([]string, N) + rowKeys := make([]string, N) + rowIDs := make([]uint64, N) + colIDs := make([]uint64, N) + for i := range colKeys { + colKeys[i] = fmt.Sprintf("col%d", i) + rowKeys[i] = fmt.Sprintf("row%d", i) + colIDs[i] = (uint64(i) + 1) * 3 + rowIDs[i] = 1 + } + sort.Strings(colKeys) + sort.Strings(rowKeys) - // Import data with keys to the coordinator (node0) and verify that it gets + t.Run("RowIDColumnKey", func(t *testing.T) { + // Import data with keys to the primary and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ - Index: index, - Field: field, - Shard: 0, + Index: indexNames[true], + Field: fieldNames[false], + Shard: 0, // inaccurate, but keys override it RowIDs: rowIDs, ColumnKeys: colKeys, - Timestamps: timestamps, - } - if err := m0.API.Import(ctx, req); err != nil { - t.Fatal(err) } - pql := fmt.Sprintf("Row(%s=%d)", field, rowID) + qcx := m0.API.Txf().NewQcx() + + if err := m0.API.Import(ctx, qcx, req); err != nil { + t.Fatal(err) + } + PanicOn(qcx.Finish()) + + pql := fmt.Sprintf("Row(%s=%d)", fieldNames[false], rowIDs[0]) // Query node0. - if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + var keys []string + if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexNames[true], Query: pql}); err != nil { t.Fatal(err) - } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { - t.Fatalf("unexpected column keys: %#v", keys) + } else { + keys = res.Results[0].(*pilosa.Row).Keys + } + sort.Strings(keys) + if !reflect.DeepEqual(keys, colKeys) { + t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys) } // Query node1. if err := test.RetryUntil(5*time.Second, func() error { - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexNames[true], Query: pql}); err != nil { return err - } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { + } else { + keys = res.Results[0].(*pilosa.Row).Keys + + } + sort.Strings(keys) + if !reflect.DeepEqual(keys, colKeys) { return fmt.Errorf("unexpected column keys: %#v", keys) } return nil @@ -114,11 +151,67 @@ func TestAPI_Import(t *testing.T) { t.Fatal(err) } }) + t.Run("ExpectedErrors", func(t *testing.T) { + t.Skip() // skipping due to change partitioning strategy + ctx := context.Background() + for ik, indexName := range indexNames { + for fk, fieldName := range fieldNames { + req := pilosa.ImportRequest{ + Index: indexName, + Field: fieldName, + Shard: 0, + } + for rik := range indexNames { + if rik { + req.ColumnKeys = colKeys + req.ColumnIDs = nil + } else { + req.ColumnKeys = nil + req.ColumnIDs = colIDs + } + for rfk := range fieldNames { + if rfk { + req.RowKeys = rowKeys + req.RowIDs = nil + } else { + req.RowKeys = nil + req.RowIDs = rowIDs + } + err := func() error { + qcx := m0.API.Txf().NewQcx() + defer qcx.Abort() + err := m0.API.Import(ctx, qcx, req.Clone()) + e2 := qcx.Finish() + if e2 != nil { + t.Fatalf("unexpected error committing: %v", e2) + } + return err + }() + if err != nil { + if rfk == fk && rik == ik { + t.Errorf("unexpected error: schema keys %t/%t, req keys %t/%t: %v", + ik, fk, rik, rfk, err) + } + } else { + if rfk != fk || rik != ik { + t.Errorf("unexpected no error: schema keys %t/%t, req keys %t/%t, req %#v", + ik, fk, rik, rfk, req) + } + } + } + } + } + } + }) // Relies on the previous test creating an index with TrackExistence and // adding some data. t.Run("SchemaHasNoExists", func(t *testing.T) { - schema := m1.API.Schema(context.Background()) + schema, err := m1.API.Schema(context.Background(), false) + if err != nil { + t.Fatal(err) + } + for _, f := range schema[0].Fields { if f.Name == "_exists" { t.Fatalf("found _exists field in schema") @@ -127,115 +220,75 @@ func TestAPI_Import(t *testing.T) { t.Fatalf("found internal field '%s' in schema output", f.Name) } } - - }) - - t.Run("RowKeyColumnID", func(t *testing.T) { - ctx := context.Background() - index := "rkci" - field := "f" - - _, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - _, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys()) - if err != nil { - t.Fatalf("creating field: %v", err) - } - - rowKey := "rowkey" - - // Generate some keyed records. - rowKeys := []string{rowKey, rowKey, rowKey} - colIDs := []uint64{1, 2, pilosa.ShardWidth + 1} - timestamps := []int64{0, 0, 0} - - // Import data with keys to the coordinator (node0) and verify that it gets - // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) - req := &pilosa.ImportRequest{ - Index: index, - Field: field, - Shard: 0, - RowKeys: rowKeys, - ColumnIDs: colIDs, - Timestamps: timestamps, - } - if err := m0.API.Import(ctx, req); err != nil { - t.Fatal(err) - } - - pql := fmt.Sprintf("Row(%s=%s)", field, rowKey) - - // Query node0. - if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { - t.Fatalf("unexpected column ids: %+v", columns) - } - - // Query node1. - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { - t.Fatalf("unexpected column ids: %+v", columns) - } }) } func TestAPI_ImportValue(t *testing.T) { - c := test.MustRunCluster(t, 2, + c := test.MustRunCluster(t, 3, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node2"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), )}, ) defer c.Close() - m0 := c[0] - m1 := c[1] + coord := c.GetPrimary() + m0 := c.GetNode(0) + m1 := c.GetNode(1) + m2 := c.GetNode(2) t.Run("ValColumnKey", func(t *testing.T) { ctx := context.Background() index := "valck" field := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}) + _, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) + _, err = coord.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatalf("creating field: %v", err) } // Generate some keyed records. values := []int64{} - colKeys := []string{} for i := 1; i <= 10; i++ { values = append(values, int64(i)) - colKeys = append(colKeys, fmt.Sprintf("col%d", i)) } - // Import data with keys to the coordinator (node0) and verify that it gets + // Column keys are sharded so their order is not guaranteed. + colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + + // Import data with keys to the primary and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportValueRequest{ Index: index, Field: field, ColumnKeys: colKeys, Values: values, + Shard: 0, // inaccurate but keys override it } - if err := m0.API.ImportValue(ctx, req); err != nil { + + qcx := coord.API.Txf().NewQcx() + if err := coord.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } + PanicOn(qcx.Finish()) pql := fmt.Sprintf("Row(%s>0)", field) @@ -258,6 +311,491 @@ func TestAPI_ImportValue(t *testing.T) { t.Fatal(err) } }) + + t.Run("ValIntEmpty", func(t *testing.T) { + ctx := context.Background() + index := "valintempty" + field := "fld" + createIndexForTest(index, coord, t) + createFieldForTest(index, field, coord, t) + + // Column keys are sharded so their order is not guaranteed. + colKeys := []string{"col2", "col1", "col3"} + values := []int64{1, 2, 3, 4} + + // Import without data, verify that it succeeds + req := &pilosa.ImportValueRequest{ + Index: index, + Field: field, + } + qcx1 := coord.API.Txf().NewQcx() + defer qcx1.Abort() + + // Import with empty request, should succeed + if err := coord.API.ImportValue(ctx, qcx1, req); err != nil { + t.Fatal(err) + } + PanicOn(qcx1.Finish()) + + // Import without data but with columnkeys, verify that it errors + req.ColumnKeys = colKeys + qcx2 := coord.API.Txf().NewQcx() + defer qcx2.Abort() + if err := coord.API.ImportValue(ctx, qcx2, req); err == nil { + t.Fatal("expected error but succeeded") + } + PanicOn(qcx2.Finish()) + + // Import with mismatch column and value lengths + req.Values = values + qcx3 := coord.API.Txf().NewQcx() + defer qcx3.Abort() + if err := coord.API.ImportValue(ctx, qcx3, req); err == nil { + t.Fatal("expected error but succeeded") + } + PanicOn(qcx3.Finish()) + + // Import with data but no columns + req.ColumnKeys = make([]string, 0) + qcx4 := coord.API.Txf().NewQcx() + defer qcx4.Abort() + if err := coord.API.ImportValue(ctx, qcx4, req); err == nil { + t.Fatal("expected error but succeeded") + } + PanicOn(qcx4.Finish()) + + }) + + t.Run("ValDecimalField", func(t *testing.T) { + t.Skip() // skipping due to change partitioning strategy + ctx := context.Background() + index := "valdec" + field := "fdec" + _, err := m2.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = m2.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + // Generate some records. + values := []float64{} + colIDs := []uint64{} + for i := 0; i < 10; i++ { + values = append(values, float64(i)+0.1) + colIDs = append(colIDs, uint64(i)) + } + // Import data with keys to node1 and verify that it gets translated and + // forwarded to the owner of shard 0 (node0; because of offsetModHasher) + req := &pilosa.ImportValueRequest{ + Index: index, + Field: field, + ColumnIDs: colIDs, + FloatValues: values, + } + qcx := m0.API.Txf().NewQcx() + if err := m0.API.ImportValue(ctx, qcx, req); err != nil { + t.Fatal(err) + } + PanicOn(qcx.Finish()) + query := fmt.Sprintf("Row(%s>6)", field) + // Query node0. + if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil { + t.Fatal(err) + } else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) { + t.Fatalf("unexpected column keys: observerd %+v; expected '%+v'", ids, colIDs[6:]) + } + }) + + t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) { + ctx := context.Background() + index := "valdecneg" + field := "fdecneg" + + _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(-1)) + if err == nil { + t.Fatal("expected error creating field") + } + }) + + t.Run("ValTimestampField", func(t *testing.T) { + t.Skip() // skipping due to change partitioning strategy + ctx := context.Background() + index := "valts" + field := "fts" + + _, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + // Generate some records. + values := []time.Time{} + colIDs := []uint64{} + for i := 0; i < 10; i++ { + values = append(values, pilosa.MinTimestamp.Add(time.Duration(i)*time.Second)) + colIDs = append(colIDs, uint64(i)) + } + + // Import data with keys to node1 and verify that it gets translated and + // forwarded to the owner of shard 0 (node0; because of offsetModHasher) + req := &pilosa.ImportValueRequest{ + Index: index, + Field: field, + ColumnIDs: colIDs, + TimestampValues: values, + } + + qcx := m2.API.Txf().NewQcx() + if err := m2.API.ImportValue(ctx, qcx, req); err != nil { + t.Fatal(err) + } + PanicOn(qcx.Finish()) + + query := fmt.Sprintf("Row(%s>='1833-11-24T17:31:50Z')", field) // 6s after MinTimestamp + + // Query node0. + if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil { + t.Fatal(err) + } else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, colIDs[6:]) { + t.Fatalf("unexpected column keys: observerd %+v; expected '%+v'", ids, colIDs[6:]) + } + }) + + t.Run("ValStringField", func(t *testing.T) { + t.Skip() // skipping due to change partitioning strategy + ctx := context.Background() + index := "valstr" + field := "fstr" + + fgnIndex := "fgnvalstr" + + _, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + + _, err = coord.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatalf("creating foreign index: %v", err) + } + _, err = coord.API.CreateField(ctx, index, field, + pilosa.OptFieldTypeInt(0, math.MaxInt64), + pilosa.OptFieldForeignIndex(fgnIndex), + ) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + // Generate some keyed records. + values := []string{} + colIDs := []uint64{} + for i := 0; i < 10; i++ { + value := fmt.Sprintf("strval-%d", (i)*100+10) + values = append(values, value) + colIDs = append(colIDs, uint64(i)) + } + + // Import data with keys to the node0 and verify that it gets translated + // and forwarded to the owner of shard 0 (node1; because of + // offsetModHasher) + req := &pilosa.ImportValueRequest{ + Index: index, + Field: field, + ColumnIDs: colIDs, + StringValues: values, + } + qcx := m0.API.Txf().NewQcx() + if err := m0.API.ImportValue(ctx, qcx, req); err != nil { + t.Fatal(err) + } + PanicOn(qcx.Finish()) + + pql := fmt.Sprintf(`Row(%s=="strval-110")`, field) + + // Query node1. + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + t.Fatal(err) + } else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected columns: observerd %+v; expected '%+v'", ids, []uint64{1}) + } + }) +} + +func TestAPI_Ingest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := test.MustRunCluster(t, 1, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node0"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), + )}, + ) + defer c.Close() + coord := c.GetPrimary() + index := "ingest" + setField := "set" + timeField := "tq" + intField := "int" + + _, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = coord.API.CreateField(ctx, index, setField, pilosa.OptFieldTypeSet("none", 0)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + _, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD", "0")) + if err != nil { + t.Fatalf("creating field: %v", err) + } + _, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 100000)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + t.Run("IngestAPI", func(t *testing.T) { + sampleJson := []byte(` + [ + { + "action": "set", + "records": { + "2": { + "set": [2], + "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [6] } + }, + "5": { "set": [3] }, + "8": { "set": [3] }, + "1": { + "set": [2], + "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } + }, + "4": { "set": [3, 7] } + } + }, + { + "action": "clear", + "record_ids": [ 5, 6, 7 ], + "fields": [ "tq", "set" ] + }, + { + "action": "write", + "records": { + "8": { "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } }, + "9": { "set": [7, 3] } + } + }, + { + "action": "delete", + "record_ids": [ 9 ] + } + ] + `) + // just for set row 3: + // first operation should set it for 4, 5, and 8. + // clear operation should clear it for 5, 6, and 7, leaving it still set for 4 and 8. + // the write operation should clear set for record 8, even though record 8 doesn't + // contain that field in that op, because set is present in record 9, which also + // gets row 3 set. but then we delete 9. + // so after all that we expect Row(set=3) to be 4... + sampleBuf := bytes.NewBuffer(sampleJson) + qcx := coord.API.Txf().NewQcx() + defer func() { + if err := qcx.Finish(); err != nil { + t.Fatalf("finishing qcx: %v", err) + } + }() + err = coord.API.IngestOperations(ctx, qcx, index, sampleBuf) + if err != nil { + t.Fatalf("importing data: %v", err) + } + query := "Row(set=3)" + res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) + if err != nil { + t.Errorf("query: %v", err) + } + r := res.Results[0].(*pilosa.Row).Columns() + if len(r) != 1 || r[0] != 4 { + t.Fatalf("expected row with 4 set, got %d", r) + } + }) + + t.Run("ImportRoaringShard", func(t *testing.T) { + setBuf := &bytes.Buffer{} + setBits := roaring.NewBitmap(7, pilosa.ShardWidth+7) + _, _ = setBits.WriteTo(setBuf) // bytes.Buffer never errors + intBuf := &bytes.Buffer{} + intBits := roaring.NewBitmap(7, pilosa.ShardWidth*2+7) + _, _ = intBits.WriteTo(intBuf) // bytes.Buffer never errors + request := &pilosa.ImportRoaringShardRequest{ + Remote: true, + Views: []pilosa.RoaringUpdate{ + { + Field: setField, + View: "standard", + Set: setBuf.Bytes(), + }, + { + Field: intField, + View: "bsig_" + intField, + Set: intBuf.Bytes(), + }, + }, + } + if err := coord.API.ImportRoaringShard(context.Background(), "ingest", 8, request); err != nil { + t.Fatalf("ingesting: %v", err) + } + + mustQuery := func(t *testing.T, index, query string) pilosa.QueryResponse { + res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) + if err != nil { + t.Fatalf("querying: %v", err) + } + return res + } + + res := mustQuery(t, "ingest", "Row(set=0)") + r := res.Results[0].(*pilosa.Row).Columns() + if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 { + t.Fatalf("expected row with pilosa.ShardWidth*8+7 set, got %d", r) + } + + res = mustQuery(t, "ingest", "Row(set=1)") + r = res.Results[0].(*pilosa.Row).Columns() + if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 { + t.Fatalf("expected row with pilosa.ShardWidth*8+7 set, got %d", r) + } + + res = mustQuery(t, "ingest", "Row(int==1)") + r = res.Results[0].(*pilosa.Row).Columns() + if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 { + t.Fatalf("expected row with, pilosa.ShardWidth*8+7 set, got %d", r) + } + + request = &pilosa.ImportRoaringShardRequest{ + Remote: true, + Views: []pilosa.RoaringUpdate{ + { + Field: setField, + View: "standard", + Clear: setBuf.Bytes(), + }, + { + Field: intField, + View: "bsig_" + intField, + Clear: intBuf.Bytes(), + }, + }, + } + if err := coord.API.ImportRoaringShard(context.Background(), "ingest", 8, request); err != nil { + t.Fatalf("ingesting: %v", err) + } + + res = mustQuery(t, "ingest", "Row(set=0)") + r = res.Results[0].(*pilosa.Row).Columns() + if len(r) != 0 { + t.Fatalf("expected no values after clearing, got: %v", r) + } + + res = mustQuery(t, "ingest", "Row(set=1)") + r = res.Results[0].(*pilosa.Row).Columns() + if len(r) != 0 { + t.Fatalf("expected no values after clearing, got: %v", r) + } + + res = mustQuery(t, "ingest", "Row(int==1)") + r = res.Results[0].(*pilosa.Row).Columns() + if len(r) != 0 { + t.Fatalf("expected no values after clearing, got: %v", r) + } + + }) +} + +// ingestBenchmarkHelper makes it easier to exclude this from benchmark computations +// and profiles. +func ingestBenchmarkHelper() []byte { + buf := &bytes.Buffer{} + buf.WriteString(`[{"action": "write", "records": {`) + comma := "" + now := time.Now().Add(-3840000 * time.Second) + for i := 0; i < 1000000; i++ { + then := now.Add(time.Duration(rand.Int63n(1234567)) * time.Second) + fmt.Fprintf(buf, `%s"%d": { "set": [%d, %d], "int": %d, "tq": { "time": "%s", "values": %d } }`, comma, i, i%2, (i%4)+2, rand.Int63n(163840), + then.Format(time.RFC3339), rand.Int63n(25)) + comma = ", " + } + buf.WriteString(`}}]`) + data := buf.Bytes() + return data +} + +func BenchmarkIngest(b *testing.B) { + b.StopTimer() + data := ingestBenchmarkHelper() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := test.MustRunCluster(b, 1, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node0"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), + )}, + ) + defer c.Close() + + coord := c.GetPrimary() + m0 := c.GetNode(0) + // m1 := c.GetNode(1) + // m2 := c.GetNode(2) + + index := "ingest" + setField := "set" + intField := "int" + tqField := "tq" + _, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false}) + if err != nil { + b.Fatalf("creating index: %v", err) + } + _, err = coord.API.CreateField(ctx, index, setField, pilosa.OptFieldTypeSet("none", 0)) + if err != nil { + b.Fatalf("creating field: %v", err) + } + _, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 163840)) + if err != nil { + b.Fatalf("creating field: %v", err) + } + _, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0")) + if err != nil { + b.Fatalf("creating field: %v", err) + } + b.ReportAllocs() + b.StartTimer() + for i := 0; i < b.N; i++ { + qcx := m0.API.Txf().NewQcx() + defer qcx.Abort() + err = coord.API.IngestOperations(ctx, qcx, index, bytes.NewBuffer(data)) + if err != nil { + b.Fatalf("ingest: %v", err) + } + err = qcx.Finish() + if err != nil { + b.Fatalf("finish: %v", err) + } + } } // offsetModHasher represents a simple, mod-based hashing offset by 1. @@ -266,3 +804,1003 @@ type offsetModHasher struct{} func (*offsetModHasher) Hash(key uint64, n int) int { return int(key+1) % n } + +func (*offsetModHasher) Name() string { return "mod" } + +func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { + c := test.MustRunCluster(t, 1, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node0"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)), + )}, + ) + defer c.Close() + + // plan: + // 1. set a bit + // 2. clear with Import() using the ImportRequest.Clear flag + // 3. verifiy the clear is done. + // repeat for ImportValueRequest and ImportValues() + + m0 := c.GetNode(0) + m0api := m0.API + + ctx := context.Background() + index := "i" + fieldAcct0 := "acct0" + + opts := pilosa.OptFieldTypeInt(-1000, 1000) + + _, err := m0api.CreateIndex(ctx, index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = m0api.CreateField(ctx, index, fieldAcct0, opts) + if err != nil { + t.Fatalf("creating fieldAcct0: %v", err) + } + + iraField := "ira" // set field. + iraRowID := uint64(3) + _, err = m0api.CreateField(ctx, index, iraField) + if err != nil { + t.Fatalf("creating fieldIRA: %v", err) + } + + acctOwnerID := uint64(78) // ColumnID + shard := acctOwnerID / ShardWidth + acct0bal := int64(500) + + ivr0 := &pilosa.ImportValueRequest{ + Index: index, + Field: fieldAcct0, + Shard: shard, + ColumnIDs: []uint64{acctOwnerID}, + Values: []int64{acct0bal}, + } + ir0 := &pilosa.ImportRequest{ + Index: index, + Field: iraField, + Shard: shard, + ColumnIDs: []uint64{acctOwnerID}, + RowIDs: []uint64{iraRowID}, + } + + qcx := m0api.Txf().NewQcx() + if err := m0api.Import(ctx, qcx, ir0.Clone()); err != nil { + t.Fatal(err) + } + if err := m0api.ImportValue(ctx, qcx, ivr0.Clone()); err != nil { + t.Fatal(err) + } + PanicOn(qcx.Finish()) + + bitIsSet := func() bool { + query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID) + res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) + PanicOn(err) + cols := res.Results[0].(*pilosa.Row).Columns() + for i := range cols { + if cols[i] == acctOwnerID { + return true + } + } + return false + } + + if !bitIsSet() { + PanicOn("IRA bit should have been set") + } + + queryAcct := func(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) { + query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID) + res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) + PanicOn(err) + + if len(res.Results) == 0 { + return 0 + } + valCount := res.Results[0].(pilosa.ValCount) + return valCount.Val + } + + bal := queryAcct(m0api, acctOwnerID, fieldAcct0, index) + + if bal != acct0bal { + PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal)) + } + + // clear the bit + qcx = m0api.Txf().NewQcx() + ir0.Clear = true + if err := m0api.Import(ctx, qcx, ir0); err != nil { + t.Fatal(err) + } + PanicOn(qcx.Finish()) + + if bitIsSet() { + PanicOn("IRA bit should have been cleared") + } + + // clear the BSI + qcx = m0api.Txf().NewQcx() + ivr0.Clear = true + if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil { + t.Fatal(err) + } + PanicOn(qcx.Finish()) + + bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index) + if bal != 0 { + PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0)) + } +} + +func TestAPI_IDAlloc(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + + primary := c.GetPrimary().API + + t.Run("Normal", func(t *testing.T) { + key := pilosa.IDAllocKey{ + Index: "normal", + Key: "key", + } + var session [32]byte + _, err := rand.Read(session[:]) + if err != nil { + t.Fatalf("obtaining random bytes: %v", err) + } + + const toReserve = 2 + + ids, err := primary.ReserveIDs(key, session, ^uint64(0), toReserve) + if err != nil { + t.Fatalf("reserving IDs: %v", err) + } + + var numIds uint64 + for _, idr := range ids { + numIds += (idr.Last - idr.First) + 1 + } + if numIds != toReserve { + t.Errorf("expected %d ids but got %d: %v", toReserve, numIds, ids) + } + + err = primary.CommitIDs(key, session, numIds) + if err != nil { + t.Fatalf("committing IDs: %v", err) + } + + err = primary.ResetIDAlloc(key.Index) + if err != nil { + t.Fatalf("resetting ID alloc: %v", err) + } + }) + t.Run("Offset", func(t *testing.T) { + key := pilosa.IDAllocKey{ + Index: "offset", + Key: "key", + } + var session [32]byte + _, err := rand.Read(session[:]) + if err != nil { + t.Fatalf("obtaining random bytes: %v", err) + } + + ids, err := primary.ReserveIDs(key, session, 0, 2) + if err != nil { + t.Fatalf("reserving IDs: %v", err) + } + + { + var numIds uint64 + for _, idr := range ids { + numIds += (idr.Last - idr.First) + 1 + } + if numIds != 2 { + t.Errorf("expected %d ids but got %d: %v", 2, numIds, ids) + } + } + + _, err = rand.Read(session[:]) + if err != nil { + t.Fatalf("obtaining random bytes: %v", err) + } + ids2, err := primary.ReserveIDs(key, session, 1, 2) + if err != nil { + t.Fatalf("reserving IDs with partially increased offset: %v", err) + } + + var numIds uint64 + for _, idr := range ids2 { + numIds += (idr.Last - idr.First) + 1 + } + if numIds != 2 { + t.Errorf("expected %d ids but got %d: %v", 2, numIds, ids2) + } + + if prevEnd, newStart := ids[len(ids)-1].Last, ids2[0].First; prevEnd != newStart { + t.Errorf("expected reuse of last ID (%d), but started with %d", prevEnd, newStart) + } + + err = primary.CommitIDs(key, session, numIds) + if err != nil { + t.Errorf("committing IDs: %v", err) + } + + _, err = rand.Read(session[:]) + if err != nil { + t.Fatalf("obtaining random bytes: %v", err) + } + ids3, err := primary.ReserveIDs(key, session, 0, 2) + var esync pilosa.ErrIDOffsetDesync + if errors.As(err, &esync) { + if esync.Requested != 0 { + t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested) + } + if esync.Base != 3 { + t.Errorf("incorrect base offset: expected %d but got %d", 3, esync.Base) + } + } else if err == nil { + t.Errorf("successfully re-reserved at a committed offset: %v", ids3) + } else { + t.Fatalf("unexpected error when reserving committed IDs: %v", err) + } + + err = primary.ResetIDAlloc(key.Index) + if err != nil { + t.Fatalf("resetting ID alloc: %v", err) + } + }) +} + +type mutexCheckIndex struct { + index *pilosa.Index + indexName string + createdAt int64 + fields map[bool]mutexCheckField +} + +type mutexCheckField struct { + fieldName string + field *pilosa.Field + createdAt int64 +} + +func TestAPI_MutexCheck(t *testing.T) { + c := test.MustNewCluster(t, 3) + for _, c := range c.Nodes { + c.Config.Cluster.ReplicaN = 2 + } + if err := c.Start(); err != nil { + t.Fatalf("starting cluster: %v", err) + } + defer c.Close() + + m0 := c.GetNode(0) + nodesByID := make(map[string]*test.Command, 3) + qcxsByID := make(map[string]*pilosa.Qcx, 3) + for i := 0; i < 3; i++ { + node := c.GetNode(i) + id := node.API.NodeID() + nodesByID[id] = node + } + + indexes := make(map[bool]mutexCheckIndex) + + ctx := context.Background() + for _, keyedIndex := range []bool{false, true} { + indexName := fmt.Sprintf("i%t", keyedIndex) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: keyedIndex, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + if index.CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + indexData := mutexCheckIndex{indexName: indexName, index: index, fields: make(map[bool]mutexCheckField), createdAt: index.CreatedAt()} + for _, keyedField := range []bool{false, true} { + fieldName := fmt.Sprintf("f%t", keyedField) + var field *pilosa.Field + if keyedField { + field, err = m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0), pilosa.OptFieldKeys()) + } else { + field, err = m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0)) + } + if err != nil { + t.Fatalf("creating field: %v", err) + } + if field.CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } + indexData.fields[keyedField] = mutexCheckField{fieldName: fieldName, field: field, createdAt: field.CreatedAt()} + } + indexes[keyedIndex] = indexData + } + + rowIDs := []uint64{0, 1, 2, 3} + colIDs := []uint64{0, 1, 2, 3} + rowKeysBase := []string{"v0", "v1", "v2", "v3"} + colKeysBase := []string{"c0", "c1", "c2", "c3"} + + const nShards = 9 + + // now, try the same thing for each combination of keyed/unkeyed. we + // share code between keyed/unkeyed fields, but for indexes, the logic + // is fundamentally different because we can't know shards in advance. + indexData := indexes[false] + for keyedField, fieldData := range indexData.fields { + t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) { + for id, node := range nodesByID { + qcxsByID[id] = node.API.Txf().NewQcx() + } + for shard := uint64(0); shard < nShards; shard++ { + // restore row/col ID values which can get altered by imports + for i := range rowIDs { + rowIDs[i] = uint64(i) + colIDs[i] = (shard << shardwidth.Exponent) + uint64(i) + (shard % 4) + } + req := &pilosa.ImportRequest{ + Index: indexData.indexName, + IndexCreatedAt: indexData.createdAt, + Field: fieldData.fieldName, + FieldCreatedAt: fieldData.createdAt, + Shard: shard, + ColumnIDs: colIDs, + } + if keyedField { + req.RowKeys = rowKeysBase + } else { + req.RowIDs = rowIDs + } + nodesForShard, err := m0.API.ShardNodes(ctx, indexData.indexName, shard) + if err != nil { + t.Fatalf("obtaining shard list: %v", err) + } + if len(nodesForShard) < 1 { + t.Fatalf("no nodes for shard %d", shard) + } + node := nodesByID[nodesForShard[0].ID] + if err := node.API.Import(ctx, qcxsByID[nodesForShard[0].ID], req); err != nil { + t.Fatalf("importing data: %v", err) + } + } + // and then we break the mutex and close the Qcxs + for id, node := range nodesByID { + field, err := node.API.Field(ctx, indexData.indexName, fieldData.fieldName) + if err != nil { + t.Fatalf("requesting field %s from node %s: %v", fieldData.fieldName, id, err) + } + pilosa.CorruptAMutex(t, field, qcxsByID[id]) + err = qcxsByID[id].Finish() + if err != nil { + t.Fatalf("closing out transaction on node %s: %v", id, err) + } + } + qcx := m0.API.Txf().NewQcx() + defer qcx.Abort() + + // first two shards of each group of 4 should have a collision in + // position 1 + expected := map[uint64]bool{ + (0 << shardwidth.Exponent) + 1: true, + (1 << shardwidth.Exponent) + 1: true, + (4 << shardwidth.Exponent) + 1: true, + (5 << shardwidth.Exponent) + 1: true, + (8 << shardwidth.Exponent) + 1: true, + // So, nShards used to be 10. If you run a complete test, + // with go test -race, and you have the sample input for the + // unrelated TestImportMutexSampleData configured to use 64K + // bit density and 2K rows, everything is fine. If you run a + // partial test, everything is fine. If you run a complete + // test with -race, but you skip TestImportMutexSampleData, + // or reduce either the bit density or the row count, you get + // a very strange panic where the go panic handler panics + // trying to report what happened so we don't get a valid + // stack dump. On Macs. This is as much as I could debug it + // after about 6 hours. Since there's no special reason to + // think we need all 10 shards, and 9 still tests the + // behavior, we're leaving this one a mystery. + // (9 << shardwidth.Exponent) + 1: true, + } + + results, err := m0.API.MutexCheck(ctx, qcx, indexData.indexName, fieldData.fieldName, true, 0) + if err != nil { + t.Fatalf("checking mutexes: %v", err) + } + + if keyedField { + mapped, ok := results.(map[uint64][]string) + if !ok { + t.Fatalf("expected map[uint64][]string, got %T", results) + } + seen := 0 + for k, v := range mapped { + seen++ + if !expected[k] { + t.Fatalf("expected all collisions to be 1 shards (s %% 4 in [0,1]), got %d", k) + } + if len(v) != 2 { + t.Fatalf("expected exactly two collisions") + } + } + if seen != len(expected) { + t.Fatalf("expected exactly %d records to have collisions", len(expected)) + } + } else { + mapped, ok := results.(map[uint64][]uint64) + if !ok { + t.Fatalf("expected map[uint64][]uint64, got %T", results) + } + seen := 0 + for k, v := range mapped { + seen++ + if !expected[k] { + t.Fatalf("expected all collisions to be 1 shards (s %% 4 in [0,1]), got %d", k) + } + if len(v) != 2 { + t.Fatalf("expected exactly two collisions") + } + } + if seen != len(expected) { + t.Fatalf("expected exactly %d records to have collisions, got %d", len(expected), seen) + } + } + + // and let's try with no details and a limit of 3... + results, err = m0.API.MutexCheck(ctx, qcx, indexData.indexName, fieldData.fieldName, false, 3) + if err != nil { + t.Fatalf("checking mutexes: %v", err) + } + mapped, ok := results.([]uint64) + if !ok { + t.Fatalf("expected []uint64, got %T", results) + } + seen := 0 + for _, k := range mapped { + seen++ + if !expected[k] { + t.Fatalf("expected all collisions to be position 1 in shards (s %% 4 in [0,1]), got %d", k) + } + } + if seen != 3 { + t.Fatalf("expected results limited to 3, got %d", seen) + } + }) + } + indexData = indexes[true] + for keyedField, fieldData := range indexData.fields { + t.Run(fmt.Sprintf("%s-%s", indexData.indexName, fieldData.fieldName), func(t *testing.T) { + for id, node := range nodesByID { + qcxsByID[id] = node.API.Txf().NewQcx() + } + req := &pilosa.ImportRequest{ + Index: indexData.indexName, + IndexCreatedAt: indexData.createdAt, + Field: fieldData.fieldName, + FieldCreatedAt: fieldData.createdAt, + Shard: 0, // ignored when using keys + } + rowKeys := make([]string, 0, len(rowKeysBase)*nShards) + colKeys := make([]string, 0, len(rowKeysBase)*nShards) + rowIDs = rowIDs[:0] + for shard := uint64(0); shard < nShards; shard++ { + for i := range rowKeysBase { + colKeys = append(colKeys, fmt.Sprintf("s%d-%s", shard, colKeysBase[i])) + if keyedField { + rowKeys = append(rowKeys, rowKeysBase[i]) + } else { + rowIDs = append(rowIDs, uint64(i)) + } + } + } + req.ColumnKeys = colKeys + if keyedField { + req.RowKeys = rowKeys + } else { + req.RowIDs = rowIDs + } + var id string + var node *test.Command + for id, node = range nodesByID { + break + } + if err := node.API.Import(ctx, qcxsByID[id], req); err != nil { + t.Fatalf("importing data: %v", err) + } + expected, err := node.API.FindIndexKeys(ctx, indexData.indexName, colKeys...) + if err != nil { + t.Fatalf("looking up index keys: %v", err) + } + for key, id := range expected { + // CorruptAMutex should only corrupt things in position 1 of their + // shards... + if id%(1<= apiMethod(len(_apiMethod_index)-1) { diff --git a/attr.go b/attr.go deleted file mode 100644 index 3502688d1..000000000 --- a/attr.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "sort" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/v2/internal" -) - -// Attribute data type enum. -const ( - attrTypeString = 1 - attrTypeInt = 2 - attrTypeBool = 3 - attrTypeFloat = 4 -) - -// AttrStore represents an interface for handling row/column attributes. -type AttrStore interface { - Path() string - Open() error - Close() error - Attrs(id uint64) (m map[string]interface{}, err error) - SetAttrs(id uint64, m map[string]interface{}) error - SetBulkAttrs(m map[uint64]map[string]interface{}) error - Blocks() ([]AttrBlock, error) - BlockData(i uint64) (map[uint64]map[string]interface{}, error) -} - -// nopStore represents an AttrStore that doesn't do anything. -var nopStore AttrStore = nopAttrStore{} - -// newNopAttrStore returns an attr store which does nothing. It returns a global -// object to avoid unnecessary allocations. -func newNopAttrStore(string) AttrStore { return nopStore } - -// nopAttrStore represents a no-op implementation of the AttrStore interface. -type nopAttrStore struct{} - -// Path is a no-op implementation of AttrStore Path method. -func (s nopAttrStore) Path() string { return "" } - -// Open is a no-op implementation of AttrStore Open method. -func (s nopAttrStore) Open() error { return nil } - -// Close is a no-op implementation of AttrStore Close method. -func (s nopAttrStore) Close() error { return nil } - -// Attrs is a no-op implementation of AttrStore Attrs method. -func (s nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return nil, nil } - -// SetAttrs is a no-op implementation of AttrStore SetAttrs method. -func (s nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil } - -// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method. -func (s nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { return nil } - -// Blocks is a no-op implementation of AttrStore Blocks method. -func (s nopAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } - -// BlockData is a no-op implementation of AttrStore BlockData method. -func (s nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } - -// AttrBlock represents a checksummed block of the attribute store. -type AttrBlock struct { - ID uint64 `json:"id"` - Checksum []byte `json:"checksum"` -} - -// attrBlocks represents a list of blocks. -type attrBlocks []AttrBlock - -// Diff returns a list of block ids that are different or are new in other. -// Block lists must be in sorted order. -func (a attrBlocks) Diff(other []AttrBlock) []uint64 { - var ids []uint64 - for { - // Read next block from each list. - var blk0, blk1 *AttrBlock - if len(a) > 0 { - blk0 = &a[0] - } - if len(other) > 0 { - blk1 = &other[0] - } - - // Exit if "a" contains no more blocks. - if blk0 == nil { - return ids - } - - // Add block ID if it's different or if it's only in "a". - if blk1 == nil || blk0.ID < blk1.ID { - ids = append(ids, blk0.ID) - a = a[1:] - } else if blk1.ID < blk0.ID { - other = other[1:] - } else { - if !bytes.Equal(blk0.Checksum, blk1.Checksum) { - ids = append(ids, blk0.ID) - } - a, other = a[1:], other[1:] - } - } -} - -func encodeAttrs(m map[string]interface{}) []*internal.Attr { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - - a := make([]*internal.Attr, len(keys)) - for i := range keys { - a[i] = encodeAttr(keys[i], m[keys[i]]) - } - return a -} - -func decodeAttrs(pb []*internal.Attr) map[string]interface{} { - m := make(map[string]interface{}, len(pb)) - for i := range pb { - key, value := decodeAttr(pb[i]) - m[key] = value - } - return m -} - -// encodeAttr converts a key/value pair into an Attr internal representation. -func encodeAttr(key string, value interface{}) *internal.Attr { - pb := &internal.Attr{Key: key} - switch value := value.(type) { - case string: - pb.Type = attrTypeString - pb.StringValue = value - case float64: - pb.Type = attrTypeFloat - pb.FloatValue = value - case uint64: - pb.Type = attrTypeInt - pb.IntValue = int64(value) - case int64: - pb.Type = attrTypeInt - pb.IntValue = value - case bool: - pb.Type = attrTypeBool - pb.BoolValue = value - } - return pb -} - -// decodeAttr converts from an Attr internal representation to a key/value pair. -func decodeAttr(attr *internal.Attr) (key string, value interface{}) { - switch attr.Type { - case attrTypeString: - return attr.Key, attr.StringValue - case attrTypeInt: - return attr.Key, attr.IntValue - case attrTypeBool: - return attr.Key, attr.BoolValue - case attrTypeFloat: - return attr.Key, attr.FloatValue - default: - return attr.Key, nil - } -} - -// cloneAttrs returns a shallow clone of m. -func cloneAttrs(m map[string]interface{}) map[string]interface{} { - other := make(map[string]interface{}, len(m)) - for k, v := range m { - other[k] = v - } - return other -} - -// EncodeAttrs encodes an attribute map into a byte slice. -func EncodeAttrs(attr map[string]interface{}) ([]byte, error) { - return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)}) -} - -// DecodeAttrs decodes a byte slice into an attribute map. -func DecodeAttrs(v []byte) (map[string]interface{}, error) { - var pb internal.AttrMap - if err := proto.Unmarshal(v, &pb); err != nil { - return nil, err - } - return decodeAttrs(pb.GetAttrs()), nil -} diff --git a/attr_test.go b/attr_test.go deleted file mode 100644 index 38af27c5a..000000000 --- a/attr_test.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "io/ioutil" - "os" - "reflect" - "runtime" - "sync" - "testing" - - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/boltdb" -) - -// Ensure database can set and retrieve column attributes. -func TestAttrStore_Attrs(t *testing.T) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - if err := s.SetAttrs(1, map[string]interface{}{"A": 100, "C": -27}); err != nil { - t.Fatal(err) - } else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil { - t.Fatal(err) - } else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil { - t.Fatal(err) - } - - // Retrieve attributes for column #1. - if m, err := s.Attrs(1); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) { - t.Fatalf("unexpected attrs(1): %#v", m) - } - - // Retrieve attributes for column #2. - if m, err := s.Attrs(2); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) { - t.Fatalf("unexpected attrs(2): %#v", m) - } -} - -// Ensure database returns a non-nil empty map if unset. -func TestAttrStore_Attrs_Empty(t *testing.T) { - s := MustOpenAttrStore() - defer s.Close() - - if m, err := s.Attrs(100); err != nil { - t.Fatal(err) - } else if m == nil || len(m) > 0 { - t.Fatalf("unexpected attrs: %#v", m) - } -} - -// Ensure database can unset attributes if explicitly set to nil. -func TestAttrStore_Attrs_Unset(t *testing.T) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - if err := s.SetAttrs(1, map[string]interface{}{"A": "X", "B": "Y"}); err != nil { - t.Fatal(err) - } else if err := s.SetAttrs(1, map[string]interface{}{"B": nil}); err != nil { - t.Fatal(err) - } - - // Verify attributes. - if m, err := s.Attrs(1); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": "X"}) { - t.Fatalf("unexpected attrs: %#v", m) - } -} - -// Ensure attribute block checksums can be returned. -func TestAttrStore_Blocks(t *testing.T) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil { - t.Fatal(err) - } else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil { - t.Fatal(err) - } else if err := s.SetAttrs(100, map[string]interface{}{"B": "VALUE"}); err != nil { - t.Fatal(err) - } else if err := s.SetAttrs(350, map[string]interface{}{"C": "FOO"}); err != nil { - t.Fatal(err) - } - - // Retrieve blocks. - blks0, err := s.Blocks() - if err != nil { - t.Fatal(err) - } else if len(blks0) != 3 || blks0[0].ID != 0 || blks0[1].ID != 1 || blks0[2].ID != 3 { - t.Fatalf("unexpected blocks: %#v", blks0) - } - - // Change second block. - if err := s.SetAttrs(100, map[string]interface{}{"X": 12}); err != nil { - t.Fatal(err) - } - - // Ensure second block changed. - blks1, err := s.Blocks() - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(blks0[0], blks1[0]) { - t.Fatalf("block 0 mismatch: %#v != %#v", blks0[0], blks1[0]) - } else if reflect.DeepEqual(blks0[1], blks1[1]) { - t.Fatalf("block 1 match: %#v ", blks0[0]) - } else if !reflect.DeepEqual(blks0[2], blks1[2]) { - t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2]) - } -} - -// AttrStore represents a test wrapper for pilosa.AttrStore. -type AttrStore struct { - pilosa.AttrStore -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(string) pilosa.AttrStore { - f, err := ioutil.TempFile("", "pilosa-attr-") - if err != nil { - panic(err) - } - f.Close() - os.Remove(f.Name()) - - return &AttrStore{boltdb.NewAttrStore(f.Name())} -} - -func BenchmarkAttrStore_Duplicate(b *testing.B) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - const n = 5 - for i := 0; i < n; i++ { - if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { - b.Fatal(err) - } - } - - b.ReportAllocs() - b.ResetTimer() - - // Update attributes with an existing subset. - cpuN := runtime.GOMAXPROCS(0) - var wg sync.WaitGroup - errchan := make(chan error) - for i := 0; i < cpuN; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < b.N/cpuN; j++ { - if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { - errchan <- err - } - } - }() - } - go func() { - wg.Wait() - close(errchan) - }() - if err := <-errchan; err != nil { - b.Fatal(err) - } -} - -// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() pilosa.AttrStore { - s := NewAttrStore("") - if err := s.Open(); err != nil { - panic(err) - } - return s -} - -// Close closes the database and removes the underlying data. -func (s *AttrStore) Close() error { - defer os.RemoveAll(s.Path()) - return s.AttrStore.Close() -} diff --git a/audit.go b/audit.go new file mode 100644 index 000000000..bd6cab0f7 --- /dev/null +++ b/audit.go @@ -0,0 +1,13 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa + +import ( + "github.com/molecula/featurebase/v3/testhook" +) + +var NewAuditor func() testhook.Auditor = NewNopAuditor + +func NewNopAuditor() testhook.Auditor { + return testhook.NewNopAuditor() +} diff --git a/audit_internal_test.go b/audit_internal_test.go new file mode 100644 index 000000000..b2dc240d5 --- /dev/null +++ b/audit_internal_test.go @@ -0,0 +1,40 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa + +import ( + "fmt" + "reflect" + + "github.com/molecula/featurebase/v3/testhook" +) + +// These audit hooks are desireable during testing, but not in +// production. +type auditorViewHooks struct{} +type auditorFragmentHooks struct{} + +// static type checks +var _ testhook.RegistryHookLive = &auditorViewHooks{} +var _ testhook.RegistryHookLive = &auditorFragmentHooks{} + +func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("view %s still open", o.(*view).name) + } + return nil +} + +func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("fragment %s still open", o.(*fragment).path()) + } + return nil +} + +func GetInternalTestHooks() testhook.RegistryHooks { + return map[reflect.Type]testhook.RegistryHook{ + reflect.TypeOf((*view)(nil)): &auditorViewHooks{}, + reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{}, + } +} diff --git a/audit_test.go b/audit_test.go new file mode 100644 index 000000000..70e828f3b --- /dev/null +++ b/audit_test.go @@ -0,0 +1,95 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa_test + +import ( + "fmt" + "os" + "reflect" + + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/testhook" +) + +// AuditLeaksOn is a global switch to turn on resource +// leak checking at the end of a test run. +var AuditLeaksOn = true + +// for tests, we use a single shared auditor used by all of the holders. +var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks) + +// These audit hooks are desireable during testing, but not in +// production. +type auditorIndexHooks struct{} +type auditorFieldHooks struct{} +type auditorHolderHooks struct{} + +// static type checking +var _ testhook.RegistryHookLive = &auditorIndexHooks{} +var _ testhook.RegistryHookLive = &auditorFieldHooks{} +var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{} +var _ testhook.RegistryHookLive = &auditorHolderHooks{} + +var testHooks = map[reflect.Type]testhook.RegistryHook{ + reflect.TypeOf((*pilosa.Index)(nil)): &auditorIndexHooks{}, + reflect.TypeOf((*pilosa.Field)(nil)): &auditorFieldHooks{}, + reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{}, +} + +func init() { + if !AuditLeaksOn { + return + } + for k, v := range pilosa.GetInternalTestHooks() { + testHooks[k] = v + } + testhook.RegisterPreTestHook(func() error { + pilosa.NewAuditor = NewTestAuditor + return nil + }) + testhook.RegisterPostTestHook(func() error { + err, errs := globalTestAuditor.FinalCheck() + if err != nil { + for i, e := range errs { + fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e) + } + } + return err + }) +} + +func NewTestAuditor() testhook.Auditor { + return globalTestAuditor +} + +func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name()) + } + return nil +} + +func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name()) + } + return nil +} + +func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error { + path := o.(*pilosa.Holder).Path() + if path == "" { + fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n", + ent.Stack) + } else { + os.RemoveAll(o.(*pilosa.Holder).Path()) + } + return err +} + +func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path()) + } + return nil +} diff --git a/authn/authenticate.go b/authn/authenticate.go new file mode 100644 index 000000000..da7f2bb85 --- /dev/null +++ b/authn/authenticate.go @@ -0,0 +1,443 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 + +// Package authn handles authentication +package authn + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/golang-jwt/jwt" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + + "github.com/molecula/featurebase/v3/logger" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +const ( + // AccessCookieName is the name of the cookie that holds the access token. + AccessCookieName = "molecula-chip" + + // RefreshCookieName is the name of the cookie that holds the refresh token. + RefreshCookieName = "refresh-molecula-chip" + + // RefreshHeaderName is the name of the header that holds the refresh token. + RefreshHeaderName = "X-Molecula-Refresh-Token" + + // ContextValueAccessToken is the key used to set AccessTokens in a ctx. + ContextValueAccessToken = "Access" + + // ContextValueRefreshToken is the key used to set RefreshTokens in a ctx. + ContextValueRefreshToken = "Refresh" +) + +// cachedGroups is used to hold groups and when they were last cached +type cachedGroups struct { + cacheTime time.Time + groups []Group +} + +// UserInfo holds the information about the user from the token +type UserInfo struct { + UserID string `json:"userid"` + UserName string `json:"username"` + Groups []Group `json:"groups"` + Expiry time.Time `json:"expiry"` + Token string `json:"token"` + RefreshToken string `json:"refreshtoken"` +} + +// Group holds group information for an authenticated user +type Group struct { + GroupID string `json:"id"` + GroupName string `json:"displayName"` +} + +// Groups holds a slice of Group for marshalling from JSON +type Groups struct { + NextLink string `json:"@odata.nextLink"` + Groups []Group `json:"value"` +} + +// Auth holds state, configuration, and utilities needed for authentication. +type Auth struct { + logger logger.Logger + accessCookieName string + refreshCookieName string + secretKey []byte + groupEndpoint string + logoutEndpoint string + fbURL string // fbURL is the domain featurebase is hosted on, used for post logout redirection + oAuthConfig *oauth2.Config + cacheTTL time.Duration // cacheTTL is used to determine if a cached item should be refreshed or not + groupsCache map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships + lastCacheClean time.Time // last cache clean is the time that the cache was last cleaned + allowedNetworks []net.IPNet // list of allowed networks for ingest +} + +// NewAuth instantiates and returns a new Auth struct +func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string, configuredIPs []string) (auth *Auth, err error) { + auth = &Auth{ + logger: logger, + accessCookieName: AccessCookieName, + refreshCookieName: RefreshCookieName, + groupEndpoint: groupEndpoint, + logoutEndpoint: logout, + fbURL: url, + oAuthConfig: &oauth2.Config{ + RedirectURL: fmt.Sprintf("%s/redirect", url), + ClientID: clientID, + ClientSecret: clientSecret, + Scopes: scopes, + Endpoint: oauth2.Endpoint{ + AuthURL: authURL, + TokenURL: tokenURL, + }, + }, + groupsCache: map[string]cachedGroups{}, + cacheTTL: 10 * time.Minute, + lastCacheClean: time.Now(), + } + + if auth.secretKey, err = decodeHex(secretKey); err != nil { + return nil, errors.Wrap(err, "decoding secret key") + } + + // convert IPs and add them to allowed networks + err = auth.convertIP(configuredIPs) + if err != nil { + return nil, err + } + + return auth, nil +} + +// CleanOAuthConfig returns a's oauthConfig without the client secret +func (a Auth) CleanOAuthConfig() oauth2.Config { + b := *a.oAuthConfig + b.ClientSecret = "" + return b +} + +// SecretKey is a convenient function to get the SecretKey from an Auth struct +func (a Auth) SecretKey() []byte { + return a.secretKey +} + +// refreshToken refreshes a given access/refresh token pair +func (a *Auth) refreshToken(access, refresh string) (string, string, error) { + resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL, + url.Values{ + "grant_type": {"refresh_token"}, + "refresh_token": {refresh}, + "client_id": {a.oAuthConfig.ClientID}, + "client_secret": {a.oAuthConfig.ClientSecret}, + }, + ) + + if err != nil { + return "", "", errors.Wrap(err, "refreshing token") + } + + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("refreshing token: %s", resp.Status) + } + + defer resp.Body.Close() + + var t oauth2.Token + if err := json.NewDecoder(resp.Body).Decode(&t); err != nil { + return "", "", errors.Wrap(err, "decoding refreshed token") + } + + // remove the old groups from the groups cache + delete(a.groupsCache, access) + + return t.AccessToken, t.RefreshToken, nil +} + +// Authenticate takes in a auth token `access` and returns UserInfo from that token +// it is caller's responsibility to inform the user that the access token has been refreshed +func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) { + // clean up the cache every 30 minutes or so + if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute { + a.cleanCache() + } + + if len(access) == 0 { + return nil, fmt.Errorf("auth token is empty") + } + + // NOTE: we are using ParseUnverified here because the IDP validates the + // token's signature when we get the user's groups, we just need to make + // sure it's not expired and is well-formed + token, _, err := new(jwt.Parser).ParseUnverified(access, &jwt.MapClaims{}) + // well-formed-ness check + if token == nil || token.Claims == nil || err != nil { + return nil, fmt.Errorf("parsing auth token: %v", err) + } + + claims := *token.Claims.(*jwt.MapClaims) + + // expiry check + if exp, ok := claims["exp"]; ok { + var expiry int64 + switch v := exp.(type) { + case string: + expiry, err = strconv.ParseInt(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("parsing exp string: %v", err) + } + case float64: + expiry = int64(v) + case int64: + expiry = v + } + + if expiry < time.Now().UTC().Unix() { + access, refresh, err = a.refreshToken(access, refresh) + if err != nil { + return nil, fmt.Errorf("token is expired: %w", err) + } + } + } + + userInfo := UserInfo{ + Token: access, + RefreshToken: refresh, + Groups: []Group{}, + } + + if uid, ok := claims["oid"].(string); ok { + userInfo.UserID = uid + } + if name, ok := claims["name"].(string); ok { + userInfo.UserName = name + } + + if userInfo.Groups, err = a.getGroups(access); err != nil { + return nil, errors.Wrap(err, "getting groups") + } + + return &userInfo, nil +} + +// cleanCache removes old items from our cache +func (a *Auth) cleanCache() { + for access, tkn := range a.groupsCache { + // if it's been more than 24 hours since the groups were cached + if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour { + // remove it from our cache + delete(a.groupsCache, access) + } + } + a.lastCacheClean = time.Now() +} + +// Login redirects a user to login to their configured oAuth authorize endpoint +func (a *Auth) Login(w http.ResponseWriter, r *http.Request) { + authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL) + http.Redirect(w, r, authURL, http.StatusTemporaryRedirect) +} + +// Logout clears out the user's cookie, removes the token from our cache, and +// redirects user to IdP's logout endpoint +func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) { + // remove the access token from a.groupsCache + if access, err := r.Cookie(a.accessCookieName); err == nil { + delete(a.groupsCache, access.Value) + } + // clear cookie + http.SetCookie(w, &http.Cookie{ + Name: a.accessCookieName, + Value: "", + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + Expires: time.Unix(0, 0), + }) + http.SetCookie(w, &http.Cookie{ + Name: a.refreshCookieName, + Value: "", + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + Expires: time.Unix(0, 0), + }) + + http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect) +} + +// Redirect handles the oAuth /redirect endpoint. It gets an access token and +// returns it to the user in the form of a cookie +func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) { + token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline) + if err != nil { + a.logger.Warnf("getting token from IdP: %+v", err) + http.Error(w, "Bad Request", http.StatusBadRequest) + return + } + + a.SetCookie(w, token.AccessToken, token.RefreshToken, token.Expiry) + http.Redirect(w, r, "/", http.StatusTemporaryRedirect) +} + +// getGroups gets the group membership for a given token from configured IdP +func (a *Auth) getGroups(token string) ([]Group, error) { + var groups Groups + + gc, ok := a.groupsCache[token] + if ok && (time.Now().Sub(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 { + return gc.groups, nil + } + + nextLink := a.groupEndpoint + for nextLink != "" { + req, err := http.NewRequest("GET", nextLink, nil) + if err != nil { + return nil, errors.Wrap(err, "creating new request to group endpoint") + } + + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + response, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.Wrap(err, "getting group membership info") + } + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("getting group membership info: %s", response.Status) + } + + var g Groups + if err = json.NewDecoder(response.Body).Decode(&g); err != nil { + return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response") + } + + response.Body.Close() + groups.Groups = append(groups.Groups, g.Groups...) + nextLink = g.NextLink + } + + if len(groups.Groups) == 0 { + return nil, fmt.Errorf("no groups found") + } + + a.groupsCache[token] = cachedGroups{ + cacheTime: time.Now(), + groups: groups.Groups, + } + return groups.Groups, nil +} + +func (a *Auth) SetCookie(w http.ResponseWriter, access, refresh string, expiry time.Time) error { + http.SetCookie(w, &http.Cookie{ + Name: a.refreshCookieName, + Value: refresh, + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + Expires: expiry, + }) + + http.SetCookie(w, &http.Cookie{ + Name: a.accessCookieName, + Value: access, + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + Expires: expiry, + }) + return nil +} + +func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) error { + mCookies := map[string]string{} + if c, ok := md["cookie"]; ok { + for _, cookie := range c { + name, val := parseCookie(cookie) + mCookies[name] = val + } + } + + mCookies[a.accessCookieName] = access + mCookies[a.refreshCookieName] = refresh + + cookies := []string{} + for name, val := range mCookies { + cookies = append(cookies, name+"="+val) + } + + md["cookie"] = cookies + return grpc.SetHeader(ctx, md) +} + +func decodeHex(hexstr string) ([]byte, error) { + data, err := hex.DecodeString(hexstr) + if err != nil { + return nil, errors.Wrap(err, "decoding hex string to byte slice") + } + if len(data) != 32 { + return nil, fmt.Errorf("invalid key length") + } + return data, nil +} + +func (a *Auth) convertIP(configuredIPs []string) error { + sz := len(configuredIPs) + nets := make([]net.IPNet, sz) + for i, ip := range configuredIPs { + // skip empty strings + if ip == "" { + sz-- + continue + } + // for IPs passed without a subnet, append /32 to only allow 1 IP + // this step is needed because ParseCIDR method assumes a CIDR address + if !strings.Contains(ip, "/") { + ip = ip + "/32" + } + _, subnet, err := net.ParseCIDR(ip) + if err != nil { + return errors.Wrapf(err, "parsing CIDR for %v", ip) + } + nets[i] = *subnet + } + a.allowedNetworks = nets[:sz] + return nil +} + +// if IP is in allowed networks, then return true to grant admin permissions +func (a *Auth) CheckAllowedNetworks(clientIP string) bool { + clientIP = strings.Split(clientIP, ":")[0] + convertedIP := net.ParseIP(clientIP) + for _, network := range a.allowedNetworks { + if network.Contains(convertedIP) { + return true + } + } + return false +} + +func parseCookie(cookie string) (name, data string) { + vals := strings.Split(cookie, "=") + if len(vals) == 0 { + vals = []string{"", ""} + } else if len(vals) < 2 { + vals = append(vals, "") + } + return vals[0], vals[1] +} diff --git a/authn/authenticate_internal_test.go b/authn/authenticate_internal_test.go new file mode 100644 index 000000000..664754891 --- /dev/null +++ b/authn/authenticate_internal_test.go @@ -0,0 +1,766 @@ +package authn + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt" + "github.com/molecula/featurebase/v3/logger" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +func NewTestAuth(t *testing.T) *Auth { + t.Helper() + var ( + ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71" + ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" + AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize" + TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token" + GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true" + LogoutURL = "https://login.microsoftonline.com/common/oauth2/v2.0/logout" + Scopes = []string{"https://graph.microsoft.com/.default", "offline_access"} + Key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" + configuredIPs = []string{} + ) + + a, err := NewAuth( + logger.NewStandardLogger(os.Stdout), + "http://localhost:10101/", + Scopes, + AuthorizeURL, + TokenURL, + GroupEndpointURL, + LogoutURL, + ClientID, + ClientSecret, + Key, + configuredIPs, + ) + if err != nil { + t.Fatalf("building auth object%s", err) + } + return a +} + +func TestSetGRPCMetadata(t *testing.T) { + a := NewTestAuth(t) + for name, md := range map[string]metadata.MD{ + "empty": {}, + "something": {"cookie": []string{a.accessCookieName + "=something"}}, + "somethingElse": {"cookie": []string{ + a.accessCookieName + "=something", + a.refreshCookieName + "=something", + }}, + "otherCookies": {"cookie": []string{a.accessCookieName + "=something", "blah=blah"}}, + } { + t.Run(name, func(t *testing.T) { + ogCookies, _ := md["cookie"] + ctx := grpc.NewContextWithServerTransportStream( + metadata.NewIncomingContext(context.TODO(), + md, + ), + NewServerTransportStream(), + ) + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + t.Fatalf("expected ok, got: %v", ok) + } + err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!") + if err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + if err := grpc.SendHeader(ctx, md); err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + md, ok = metadata.FromIncomingContext(ctx) + if !ok { + t.Fatalf("expected ok, got: %v", ok) + } + c, ok := md["cookie"] + if !ok { + t.Fatalf("expected ok, got: %v", ok) + } + var accessCookie, refreshCookie string + for _, cookie := range c { + if strings.HasPrefix(cookie, a.accessCookieName) { + accessCookie = cookie + } else if strings.HasPrefix(cookie, a.refreshCookieName) { + refreshCookie = cookie + } + if refreshCookie != "" && accessCookie != "" { + break + } + } + + exp := a.accessCookieName + "=accesstoken!" + if accessCookie != exp { + t.Fatalf("expected '%v', got '%v'", exp, accessCookie) + } + exp = a.refreshCookieName + "=refreshtoken!" + if refreshCookie != exp { + t.Fatalf("expected '%v', got '%v'", exp, refreshCookie) + } + + for _, cookie := range c { + if strings.HasPrefix(cookie, a.accessCookieName) || strings.HasPrefix(cookie, a.refreshCookieName) { + continue + } + found := false + + for _, ogCookie := range ogCookies { + if cookie == ogCookie { + found = true + break + } + } + if !found { + t.Fatal("SetGRPCMetadata did not maintain the previous cookie list") + } + } + }) + } +} + +func TestAuth(t *testing.T) { + a := NewTestAuth(t) + t.Run("SetCookie", func(t *testing.T) { + w := httptest.NewRecorder() + err := a.SetCookie(w, "access", "refresh", time.Now().Add(time.Hour)) + if err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + + if w.Result().Cookies()[0].Value == "" { + t.Errorf("expected something, got empty string") + } + + if got, want := w.Result().Cookies()[0].Path, "/"; got != want { + t.Fatalf("path=%s, want %s", got, want) + } + }) + + t.Run("KeyLength", func(t *testing.T) { + _, err := NewAuth( + logger.NewStandardLogger(os.Stdout), + "http://localhost:10101/", + []string{"https://graph.microsoft.com/.default", "offline_access"}, + "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize", + "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token", + "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true", + "https://login.microsoftonline.com/common/oauth2/v2.0/logout", + "e9088663-eb08-41d7-8f65-efb5f54bbb71", + "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", + "DEADBEEFD", + []string{}, + ) + if err == nil || !strings.Contains(err.Error(), "decoding secret key") { + t.Fatalf("expected error decoding secret key got: %v", err) + } + }) + t.Run("GetSecretKey", func(t *testing.T) { + want, _ := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + if got := a.SecretKey(); !bytes.Equal(got, want) { + t.Fatalf("expected %v, got %v", got, want) + } + }) +} + +func TestAuthenticate(t *testing.T) { + cases := []struct { + name string + uid string + uname string + exp int64 + refresh bool + refreshToken string + malformed bool + empty bool + groups []Group + err error + }{ + { + name: "GoodToken", + uid: "42", + uname: "A. Token", + groups: []Group{ + { + GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + GroupName: "adminGroup", + }, + }, + }, + { + name: "Malformed", + malformed: true, + err: fmt.Errorf("parsing auth token: token contains an invalid number of segments"), + }, + { + name: "Empty", + empty: true, + err: fmt.Errorf("auth token is empty"), + }, + { + name: "ExpiredTokenNoRefresh", + uid: "42", + uname: "A. Token", + groups: []Group{ + { + GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + GroupName: "adminGroup", + }, + }, + exp: -17764800, + err: fmt.Errorf("token is expired: refreshing token: 400 Bad Request"), + }, + { + name: "ExpiredTokenYesRefresh", + uid: "42", + uname: "A. Token", + groups: []Group{ + { + GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + GroupName: "adminGroup", + }, + }, + refresh: true, + refreshToken: "refreshToken", + exp: -17764800, + }, + { + name: "ExpiredTokenYesRefreshButError", + uid: "42", + uname: "A. Token", + groups: []Group{ + { + GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + GroupName: "adminGroup", + }, + }, + refresh: true, + refreshToken: "blah!!", + exp: -17764800, + err: fmt.Errorf("token is expired: refreshing token: 403 Forbidden"), + }, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + // setup the test + a := NewTestAuth(t) + token := "" + var err error + if !test.malformed && !test.empty { + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = test.uid + claims["name"] = test.uname + if test.exp != 0 { + claims["exp"] = float64(test.exp) + } + token, err = tkn.SignedString(a.SecretKey()) + if err != nil { + t.Fatalf("unexpected error when signing token %v", err) + } + } else if !test.empty { + token = "asdfasdfasdfasdF" + } + if len(test.groups) > 0 { + a.groupsCache[token] = cachedGroups{time.Now(), test.groups} + } + if test.refresh { + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + refresh := r.Form.Get("refresh_token") + if refresh != test.refreshToken { + t.Fatalf("refresh token not passed properly, expected %v, got %v", test.refreshToken, refresh) + return + } + if refresh != "refreshToken" { + http.Error(w, "bad token", http.StatusForbidden) + } + + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = test.uid + claims["name"] = test.uname + expiry := float64(time.Now().Add(2 * time.Hour).Unix()) + claims["exp"] = expiry + fresh, err := tkn.SignedString(a.SecretKey()) + if err != nil { + t.Fatalf("unexpected error when signing token %v", err) + } + + a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups} + fmt.Fprintf(w, `{"access_token": "`+fresh+`", "refresh_token": "blah", "token_type": "bearer", "expires": `+strconv.FormatFloat(expiry, 'f', 0, 64)+` }`) + })) + defer srv.Close() + a.oAuthConfig.Endpoint.TokenURL = srv.URL + } + + // do the actual testing + uinfo, err := a.Authenticate(token, test.refreshToken) + // okay this part kind of sucks bc we need to check errors and i + // dont want to write a whole new test for things that should have + // errors just to avoid this mess. errors.Is doesn't work either + if (test.err == nil && err != nil) || (test.err != nil && err == nil) { + t.Fatalf("expected %v, but got %v", test.err, err) + } else if test.err != nil && err != nil { + if test.err.Error() != err.Error() { + t.Fatalf("expected %v, but got %v", test.err, err) + } else { + return + } + } + + if !reflect.DeepEqual(uinfo.Groups, test.groups) { + t.Fatalf("expected %v, got %v", test.groups, uinfo.Groups) + } + if !reflect.DeepEqual(uinfo.UserID, test.uid) { + t.Fatalf("expected %v, got %v", test.uid, uinfo.UserID) + } + if !reflect.DeepEqual(uinfo.UserName, test.uname) { + t.Fatalf("expected %v, got %v", test.uname, uinfo.UserName) + } + }) + } +} + +func TestAuthenticate_CleanCache(t *testing.T) { + // this deserves its own test bc it has gross setup required + t.Run("should clean", func(t *testing.T) { + a := NewTestAuth(t) + now := time.Now() + a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}} + a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}} + a.lastCacheClean = now.Add(-45 * time.Minute) + + _, _ = a.Authenticate("this doesn't matter", "this doesn't matter?") + if a.lastCacheClean.Sub(now) <= time.Nanosecond { + t.Fatalf("cache should have been cleaned") + } + if _, ok := a.groupsCache["oldy"]; ok { + t.Errorf("oldy should have been deleted") + } + if _, ok := a.groupsCache["goldy"]; !ok { + t.Errorf("goldy should not have been deleted") + } + }) + t.Run("shouldn't clean", func(t *testing.T) { + a := NewTestAuth(t) + now := time.Now() + a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}} + a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}} + a.lastCacheClean = now + + _, _ = a.Authenticate("this doesn't matter", "this doesn't matter?") + if a.lastCacheClean.Sub(now) >= time.Nanosecond { + t.Fatalf("cache should not have been cleaned") + } + if _, ok := a.groupsCache["oldy"]; !ok { + t.Errorf("oldy should not have been deleted") + } + if _, ok := a.groupsCache["goldy"]; !ok { + t.Errorf("goldy should not have been deleted") + } + }) + +} + +func TestGetGroups(t *testing.T) { + a := NewTestAuth(t) + a.groupsCache = map[string]cachedGroups{ + "the world is changed": { + cacheTime: time.Now(), + groups: []Group{ + { + GroupID: "a han noston ned wilith", + GroupName: "I smell it in the air", + }, + }, + }, + } + srvNext := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := json.Marshal( + Groups{ + Groups: []Group{ + { + GroupID: "han mathon ne chae", + GroupName: "I feel it in the earth", + }, + }, + }, + ) + if err != nil { + t.Fatalf("unexpected error marshalling groups response: %v", err) + } + fmt.Fprintf(w, "%s", body) + })) + defer srvNext.Close() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := json.Marshal( + Groups{ + NextLink: srvNext.URL, + Groups: []Group{ + { + GroupID: "han mathon ne nen", + GroupName: "i feel it in the water", + }, + }, + }, + ) + if err != nil { + t.Fatalf("unexpected error marshalling groups response: %v", err) + } + fmt.Fprintf(w, "%s", body) + })) + defer srv.Close() + a.groupEndpoint = srv.URL + + for name, test := range map[string]struct { + token string + groups []Group + }{ + "InCache": { + token: "the world is changed", + groups: []Group{ + { + GroupID: "a han noston ned wilith", + GroupName: "I smell it in the air", + }, + }, + }, + "NotInCache": { + token: "i smell it in the air", + groups: []Group{ + { + GroupID: "han mathon ne nen", + GroupName: "i feel it in the water", + }, + { + GroupID: "han mathon ne chae", + GroupName: "I feel it in the earth", + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) { + t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err) + } + }) + } + +} + +func TestDecodeHex(t *testing.T) { + t.Run("cantDecode", func(t *testing.T) { + _, err := decodeHex("gggg") + if err == nil { + t.Fatalf("expected err cannot decode slice, got nil") + } + }) + t.Run("tooSmall", func(t *testing.T) { + _, err := decodeHex("DEADBEEF") + if err == nil { + t.Fatalf("expected err wrong length, got nil") + } + }) + t.Run("tooBig", func(t *testing.T) { + _, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + if err == nil { + t.Fatalf("expected err wrong length, got nil") + } + }) + t.Run("justRight", func(t *testing.T) { + _, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + if err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) +} + +func TestHandlers(t *testing.T) { + a := NewTestAuth(t) + t.Run("login", func(t *testing.T) { + req := httptest.NewRequest("GET", "/login", nil) + w := httptest.NewRecorder() + a.Login(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusTemporaryRedirect { + t.Fatalf("expected redirect, got %v", resp.StatusCode) + } + redirect := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL) + if got, err := resp.Location(); err != nil || got.String() != redirect { + t.Fatalf("expected %v, got %v", redirect, got.Path) + } + }) + t.Run("logout", func(t *testing.T) { + req := httptest.NewRequest("GET", "/logout", nil) + w := httptest.NewRecorder() + req.AddCookie( + &http.Cookie{ + Name: a.accessCookieName, + Value: "test", + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + Expires: time.Unix(3000000, 0), + }, + ) + + req.AddCookie( + &http.Cookie{ + Name: a.refreshCookieName, + Value: "test", + Path: "/", + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + Expires: time.Unix(3000000, 0), + }, + ) + + a.groupsCache["test"] = cachedGroups{} + a.Logout(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusTemporaryRedirect { + t.Fatalf("expected redirect, got %v", resp.StatusCode) + } + redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL) + if got, err := resp.Location(); err != nil || got.String() != redirect { + t.Fatalf("expected %v, got %v", redirect, got.Path) + } + for _, c := range resp.Cookies() { + if c.Name == a.accessCookieName || c.Name == a.refreshCookieName { + if c.Value != "" { + t.Fatalf("cookie not set to empty value!") + } + want := time.Unix(0, 0).Unix() + got := c.Expires.Unix() + if want != got { + t.Fatalf("expected %v, got %v", want, got) + } + } + } + if _, ok := a.groupsCache["test"]; ok { + t.Fatalf("groups not deleted!") + } + }) + t.Run("redirectGood", func(t *testing.T) { + req := httptest.NewRequest("GET", "/redirect", nil) + w := httptest.NewRecorder() + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = "user id" + claims["name"] = "user name" + expiresIn := 2 * time.Hour + exp := time.Now().Add(expiresIn) + expiry := float64(exp.Unix()) + claims["exp"] = expiry + fresh, err := tkn.SignedString(a.SecretKey()) + if err != nil { + t.Fatalf("unexpected error when signing token %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}` + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + w.Write([]byte(body)) + })) + a.oAuthConfig.Endpoint.TokenURL = srv.URL + a.Redirect(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusTemporaryRedirect { + t.Fatalf("expected redirect, got %v", resp.StatusCode) + } + if got, err := resp.Location(); err != nil || got.String() != "/" { + t.Fatalf("expected %v, got %v", "/", got.Path) + } + cookies := resp.Cookies() + for _, c := range cookies { + if c.Name == a.accessCookieName && c.Value != fresh { + t.Fatalf("expected %v, got %v", exp, c.Value) + } else if c.Name == a.refreshCookieName && c.Value != "blah" { + t.Fatalf("expected %v, got %v", "blah", c.Value) + } + } + }) + + t.Run("redirectBad", func(t *testing.T) { + req := httptest.NewRequest("GET", "/redirect", nil) + w := httptest.NewRecorder() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Server Error", http.StatusInternalServerError) + })) + a.oAuthConfig.Endpoint.TokenURL = srv.URL + a.Redirect(w, req) + resp := w.Result() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected BadRequest, got %v", resp.StatusCode) + } + }) + +} + +// This type is used for mocking ServerTransportStreams in tests +type ServerTransportStream struct { + md metadata.MD + method string +} + +func NewServerTransportStream() *ServerTransportStream { + return &ServerTransportStream{ + md: metadata.MD{}, + method: "test", + } +} + +func (s *ServerTransportStream) Method() string { + return s.method +} + +func (s *ServerTransportStream) SetHeader(md metadata.MD) error { + s.md = md + return nil +} + +func (s *ServerTransportStream) SendHeader(md metadata.MD) error { + _ = md + return nil +} + +func (s *ServerTransportStream) SetTrailer(md metadata.MD) error { + _ = md + return nil +} + +func TestCleanOAuthConfig(t *testing.T) { + a := NewTestAuth(t) + res := a.CleanOAuthConfig() + assertEqual("", res.ClientSecret, t) + assertEqual(a.oAuthConfig.ClientID, res.ClientID, t) + assertEqual(a.oAuthConfig.RedirectURL, res.RedirectURL, t) + assertEqual(a.oAuthConfig.Scopes, res.Scopes, t) + assertEqual(a.oAuthConfig.Endpoint, res.Endpoint, t) +} + +func assertEqual(exp, got interface{}, t *testing.T) { + if !reflect.DeepEqual(exp, got) { + t.Fatalf("expected %v, got %v", exp, got) + } +} + +func TestCheckAllowedNetworks(t *testing.T) { + + tests := []struct { + requestIP string + configuredIPs []string + isAdmin bool + }{ + { + requestIP: "10.0.0.1", + configuredIPs: []string{"10.0.0.1"}, + isAdmin: true, + }, + { + requestIP: "10.0.0.3", + configuredIPs: []string{"10.0.0.1", "10.0.0.2"}, + isAdmin: false, + }, + { + requestIP: "10.0.0.2", + configuredIPs: []string{"10.0.0.1/30"}, + isAdmin: true, + }, + // it is possible for the client IP to have a port + { + requestIP: "10.0.0.2:22", + configuredIPs: []string{"10.0.0.1/30"}, + isAdmin: true, + }, + { + requestIP: "10.1.0.3", + configuredIPs: []string{"10.0.0.1/32"}, + isAdmin: false, + }, + { + requestIP: "10.0.0.254", + configuredIPs: []string{"10.0.0.1/24"}, + isAdmin: true, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) { + a := NewTestAuth(t) + if err := a.convertIP(test.configuredIPs); err != nil { + t.Fatalf("failed to convert IPs from strings to net.IP: %v", err) + } + got := a.CheckAllowedNetworks(test.requestIP) + if got != test.isAdmin { + t.Fatalf("expected %v, got %v", test.isAdmin, got) + } + }) + } +} + +func TestConvertIP(t *testing.T) { + + tests := []struct { + configuredIPs []string + convertedIPs []net.IPNet + }{ + { + configuredIPs: []string{"10.0.0.1"}, + convertedIPs: []net.IPNet{ + {IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(32, 32)}, + }, + }, + { + configuredIPs: []string{"10.0.0.1/30"}, + convertedIPs: []net.IPNet{ + {IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(30, 32)}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) { + a := NewTestAuth(t) + if err := a.convertIP(test.configuredIPs); err != nil { + t.Fatalf("failed to convert IPs from strings to net.IP: %v", err) + } + + if len(a.allowedNetworks) != len(test.convertedIPs) { + t.Fatalf("expected len of %v networks, got %v", len(test.convertedIPs), len(a.allowedNetworks)) + } + + for i := range a.allowedNetworks { + expected, got := test.convertedIPs[i], a.allowedNetworks[i] + if got.IP.String() != expected.IP.String() { + t.Fatalf("for IP, expected %v, got %v", expected.IP, got.IP) + } + if got.Mask.String() != expected.Mask.String() { + t.Fatalf("for mask, expected %v, got %v", expected.Mask.String(), got.Mask.String()) + } + } + }) + } +} diff --git a/authz/authorization.go b/authz/authorization.go new file mode 100644 index 000000000..e8deda26f --- /dev/null +++ b/authz/authorization.go @@ -0,0 +1,131 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 + +package authz + +import ( + "fmt" + "io" + "io/ioutil" + + "github.com/molecula/featurebase/v3/authn" + + "gopkg.in/yaml.v2" +) + +type GroupPermissions struct { + Permissions map[string]map[string]Permission `yaml:"user-groups"` + Admin string `yaml:"admin"` +} + +type Permission string + +const ( + None Permission = "" + Read Permission = "read" + Write Permission = "write" + Admin Permission = "admin" +) + +// Satisfies returns whether `p` satisfies the permissions required by `b` +func (p Permission) Satisfies(b Permission) bool { + switch p { + case "": + return b == "" + case "read": + return b == "" || b == "read" + case "write": + return b == "" || b == "read" || b == "write" + case "admin": + return b == "" || b == "read" || b == "write" || b == "admin" + } + return false +} + +func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) { + permsData, err := ioutil.ReadAll(permsFile) + + if err != nil { + return fmt.Errorf("reading permissions failed with error: %s", err) + } + + err = yaml.UnmarshalStrict(permsData, &p) + if err != nil { + return fmt.Errorf("unmarshalling permissions failed with error: %s", err) + } + + return +} + +func (p *GroupPermissions) GetPermissions(user *authn.UserInfo, index string) (permission Permission, errors error) { + groups := user.Groups + if admin := p.IsAdmin(groups); admin { + return Admin, nil + } + + allPermissions := map[Permission]bool{ + Write: false, + Read: false, + } + + if len(groups) == 0 { + return None, fmt.Errorf("user is not part of any groups in identity provider") + } + + var groupsDenied []string + for _, group := range groups { + if _, ok := p.Permissions[group.GroupID]; ok { + if perm, ok := p.Permissions[group.GroupID][index]; ok { + allPermissions[perm] = true + } else { + return None, fmt.Errorf("user %s does not have permission to index %s", user.UserID, index) + } + } else { + groupsDenied = append(groupsDenied, group.GroupID) + } + } + + if len(groupsDenied) == len(groups) { + return None, fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied) + } + + if allPermissions[Write] { + return Write, nil + } else if allPermissions[Read] { + return Read, nil + } else { + return None, fmt.Errorf("no permissions found") + } +} + +func (p *GroupPermissions) IsAdmin(groups []authn.Group) bool { + for _, group := range groups { + if p.Admin == group.GroupID { + return true + } + } + return false +} + +func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission Permission) (indexList []string) { + // if user is admin, find all indexes in permissions file and return them + if p.IsAdmin(groups) { + for groupId := range p.Permissions { + for index := range p.Permissions[groupId] { + indexList = append(indexList, index) + } + } + return indexList + } + + for _, group := range groups { + if _, ok := p.Permissions[group.GroupID]; ok { + for index, permission := range p.Permissions[group.GroupID] { + if permission.Satisfies(desiredPermission) { + indexList = append(indexList, index) + } + } + } + } + return indexList +} diff --git a/authz/authorization_test.go b/authz/authorization_test.go new file mode 100644 index 000000000..203940ef8 --- /dev/null +++ b/authz/authorization_test.go @@ -0,0 +1,305 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package authz_test + +import ( + "fmt" + "reflect" + "sort" + "strings" + "testing" + + "github.com/molecula/featurebase/v3/authn" + "github.com/molecula/featurebase/v3/authz" +) + +func TestAuth_ReadPermissionsFile(t *testing.T) { + + singleInput := `user-groups: + "dca35310-ecda-4f23-86cd-876aee55906b": + "test": "read" +admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` + + multiInput := `user-groups: + "dca35310-ecda-4f23-86cd-876aee55906b": + "test": "read" + "test2": "write" + "dca35310-ecda-4f23-86cd-876aee559900": + "test": "write" +admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` + + singlePermission := authz.GroupPermissions{ + Permissions: map[string]map[string]authz.Permission{ + "dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read}, + }, + Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + } + + multiPermission := authz.GroupPermissions{ + Permissions: map[string]map[string]authz.Permission{ + "dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read, "test2": authz.Write}, + "dca35310-ecda-4f23-86cd-876aee559900": {"test": authz.Write}}, + Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + } + + tests := []struct { + input string + output authz.GroupPermissions + }{ + {singleInput, singlePermission}, + {multiInput, multiPermission}, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + permFile := strings.NewReader(test.input) + + var p authz.GroupPermissions + err := p.ReadPermissionsFile(permFile) + if err != nil { + t.Fatalf("readPermissionsFile error: %s", err) + } + + if !reflect.DeepEqual(p, test.output) { + t.Fatalf("expected output %s, but got %s", test.output, p) + } + }, + ) + } +} + +func TestAuth_GetPermissions(t *testing.T) { + + // initializes different example of permissions file in yaml + permissions1 := `"user-groups": + "dca35310-ecda-4f23-86cd-876aee55906b": + "test": "read" +admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` + + permissions2 := `"user-groups": + "dca35310-ecda-4f23-86cd-876aee559900": + "test": "write" +admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` + + permissions3 := `"user-groups": + "dca35310-ecda-4f23-86cd-876aee55906b": + "test": "write" + "test2": "read" + "dca35310-ecda-4f23-86cd-876aee559900": + "test": "read" +admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` + + permissions4 := `"user-groups": + "dca35310-ecda-4f23-86cd-876aee559900": + "test": "" +admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` + + // initializes groups that are returned from identity provider + groupName := "name" + groupsList1 := []authn.Group{} + groupsList2 := []authn.Group{{ + GroupID: "fake-group", + GroupName: groupName}} + groupsList3 := []authn.Group{ + {GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: groupName}, + {GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: groupName}, + } + groupsList4 := []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: groupName}} + + tests := []struct { + yamlData string + groups []authn.Group + index string + userAccess authz.Permission + err string + }{ + { + permissions1, + groupsList1, + "test", + authz.None, + "user is not part of any groups in identity provider", + }, + { + permissions1, + groupsList3, + "test1", + authz.None, + "does not have permission to index", + }, + { + permissions2, + groupsList2, + "test", + authz.None, + "does not have permission to FeatureBase", + }, + { + permissions1, + groupsList3, + "test", + authz.Read, + "", + }, + { + permissions2, + groupsList3, + "test", + authz.Write, + "", + }, + { + permissions3, + groupsList4, + "test", + authz.Admin, + "", + }, + { + permissions4, + groupsList3, + "test", + authz.None, + "no permissions found", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + + permFile := strings.NewReader(test.yamlData) + + var p authz.GroupPermissions + if err := p.ReadPermissionsFile(permFile); err != nil { + t.Errorf("Error: %s", err) + } + + p1, err := p.GetPermissions(&authn.UserInfo{Groups: test.groups}, test.index) + + if p1 != test.userAccess { + t.Errorf("expected permission to be %s, but got %s", test.userAccess, p1) + } + + if err != nil { + if !strings.Contains(err.Error(), test.err) { + t.Errorf("expected error to contain %s, but got %s", test.err, err.Error()) + } + } + + }) + } +} + +func TestAuth_IsAdmin(t *testing.T) { + + group1 := []authn.Group{ + {GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"}, + } + + group2 := []authn.Group{ + {GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"}, + } + + groupPermissions := authz.GroupPermissions{ + Permissions: map[string]map[string]authz.Permission{ + "dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Write}, + }, + Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + } + + tests := []struct { + groups []authn.Group + groupPermissions authz.GroupPermissions + output bool + }{ + { + group1, groupPermissions, true, + }, + { + group2, groupPermissions, false, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + p := test.groupPermissions + resp := p.IsAdmin(test.groups) + if resp != test.output { + t.Errorf("expected %t, but got %t", test.output, resp) + } + }) + } +} + +func TestAuth_GetAuthorizedIndexList(t *testing.T) { + + group1 := []authn.Group{ + {GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"}, + } + + group2 := []authn.Group{ + {GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"}, + } + + group3 := []authn.Group{ + {GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: "group-name"}, + } + + p := authz.GroupPermissions{ + Permissions: map[string]map[string]authz.Permission{ + "dca35310-ecda-4f23-86cd-876aee55906b": { + "test1": authz.Read, + "test2": authz.Write, + }, + "dca35310-ecda-4f23-86cd-876aee559900": { + "test3": authz.Read, + }, + }, + Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", + } + + tests := []struct { + groups []authn.Group + permission authz.Permission + output []string + }{ + { + group1, + authz.Read, + []string{"test1", "test2"}, + }, + { + group1, + authz.Write, + []string{"test2"}, + }, + { + group3, + authz.Write, + nil, + }, + { + group2, + authz.Read, + []string{"test1", "test2", "test3"}, + }, + { + group2, + authz.Write, + []string{"test1", "test2", "test3"}, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + + indexList := p.GetAuthorizedIndexList(test.groups, test.permission) + sort.Strings(indexList) + + if !reflect.DeepEqual(indexList, test.output) { + t.Errorf("expected %s, but got %s", test.output, indexList) + } + }) + } + +} diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go deleted file mode 100644 index f29cec183..000000000 --- a/boltdb/attrstore.go +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package boltdb - -import ( - "bytes" - - "encoding/binary" - "fmt" - "sort" - "sync" - "time" - - "github.com/cespare/xxhash" - - "github.com/boltdb/bolt" - "github.com/pilosa/pilosa/v2" - "github.com/pkg/errors" -) - -// attrBlockSize is the size of attribute blocks for anti-entropy. -const attrBlockSize = 100 - -// attrCache represents a cache for attributes. -type attrCache struct { - mu sync.RWMutex - attrs map[uint64]map[string]interface{} -} - -// Get returns the cached attributes for a given id. -func (c *attrCache) Get(id uint64) map[string]interface{} { - c.mu.RLock() - defer c.mu.RUnlock() - attrs := c.attrs[id] - if attrs == nil { - return nil - } - - // Make a copy for safety - ret := make(map[string]interface{}) - for k, v := range attrs { - ret[k] = v - } - return ret -} - -// Set updates the cached attributes for a given id. -func (c *attrCache) Set(id uint64, attrs map[string]interface{}) { - c.mu.Lock() - defer c.mu.Unlock() - c.attrs[id] = attrs -} - -// attrStore represents a storage layer for attributes. -type attrStore struct { - mu sync.RWMutex - path string - db *bolt.DB - attrCache *attrCache -} - -// newAttrCache returns a new instance of AttrCache. -func newAttrCache() *attrCache { - return &attrCache{ - attrs: make(map[uint64]map[string]interface{}), - } -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(path string) pilosa.AttrStore { - return &attrStore{ - path: path, - attrCache: newAttrCache(), - } -} - -// Path returns path to the store's data file. -func (s *attrStore) Path() string { return s.path } - -// Open opens and initializes the store. -func (s *attrStore) Open() error { - // Open storage. - db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return errors.Wrap(err, "opening storage") - } - s.db = db - - // Initialize database. - if err := s.db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte("attrs")) - return err - }); err != nil { - return errors.Wrap(err, "initializing") - } - - return nil -} - -// Close closes the store. -func (s *attrStore) Close() error { - if s.db != nil { - s.db.Close() - } - return nil -} - -// Attrs returns a set of attributes by ID. -func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) { - s.mu.RLock() - defer s.mu.RUnlock() - - // Check cache for map. - if m = s.attrCache.Get(id); m != nil { - return m, nil - } - - // Find attributes from storage. - if err = s.db.View(func(tx *bolt.Tx) error { - m, err = txAttrs(tx, id) - return err - }); err != nil { - return nil, errors.Wrap(err, "finding attributes") - } - - // Add to cache. - s.attrCache.Set(id, m) - - return m, nil -} - -// SetAttrs sets attribute values for a given ID. -func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error { - // Ignore empty maps. - if len(m) == 0 { - return nil - } - - // Check if the attributes already exist under a read-only lock. - if attr, err := s.Attrs(id); err != nil { - return errors.Wrap(err, "checking attrs") - } else if attr != nil && mapContains(attr, m) { - return nil - } - - // Obtain write lock. - s.mu.Lock() - defer s.mu.Unlock() - - var attr map[string]interface{} - if err := s.db.Update(func(tx *bolt.Tx) error { - tmp, err := txUpdateAttrs(tx, id, m) - if err != nil { - return err - } - attr = tmp - - return nil - }); err != nil { - return errors.Wrap(err, "updating store") - } - - // Swap attributes map in cache. - s.attrCache.Set(id, attr) - - return nil -} - -// SetBulkAttrs sets attribute values for a set of ids. -func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { - s.mu.Lock() - defer s.mu.Unlock() - - attrs := make(map[uint64]map[string]interface{}) - if err := s.db.Update(func(tx *bolt.Tx) error { - // Collect and sort keys. - ids := make([]uint64, 0, len(m)) - for id := range m { - ids = append(ids, id) - } - sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) - - // Update attributes for each id. - for _, id := range ids { - attr, err := txUpdateAttrs(tx, id, m[id]) - if err != nil { - return err - } - attrs[id] = attr - } - - return nil - }); err != nil { - return err - } - - // Swap attributes map in cache. - for id, attr := range attrs { - s.attrCache.Set(id, attr) - } - - return nil -} - -// Blocks returns a list of all blocks in the store. -func (s *attrStore) Blocks() (blocks []pilosa.AttrBlock, err error) { - err = s.db.View(func(tx *bolt.Tx) error { - // Wrap cursor to segment by block. - cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize) - - // Iterate over each block. - for cur.nextBlock() { - block := pilosa.AttrBlock{ID: cur.blockID()} - - // Compute checksum of every key/value in block. - h := xxhash.New() - for k, v := cur.next(); k != nil; k, v = cur.next() { - // hash function writes don't usually need to be checked - _, _ = h.Write(k) - _, _ = h.Write(v) - } - block.Checksum = h.Sum(nil) - - // Append block. - blocks = append(blocks, block) - } - return nil - }) - if err != nil { - return nil, errors.Wrap(err, "getting blocks") - } - return blocks, nil -} - -// BlockData returns all data for a single block. -func (s *attrStore) BlockData(i uint64) (m map[uint64]map[string]interface{}, err error) { - m = make(map[uint64]map[string]interface{}) - - // Start read-only transaction. - err = s.db.View(func(tx *bolt.Tx) error { - // Move to the start of the block. - min := u64tob(i * attrBlockSize) - max := u64tob((i + 1) * attrBlockSize) - cur := tx.Bucket([]byte("attrs")).Cursor() - for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { - // Exit if we're past the end of the block. - if bytes.Compare(k, max) != -1 { - break - } - - // Decode attribute map and associate with id. - attrs, err := pilosa.DecodeAttrs(v) - if err != nil { - return errors.Wrap(err, "decoding attrs") - } - m[btou64(k)] = attrs - - } - return nil - }) - if err != nil { - return nil, errors.Wrap(err, "getting block data") - } - return m, nil -} - -// txAttrs returns a map of attributes for an id. -func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) { - v := tx.Bucket([]byte("attrs")).Get(u64tob(id)) - if v == nil { - return emptyMap, nil - } - return pilosa.DecodeAttrs(v) -} - -// txUpdateAttrs updates the attributes for an id. -// Returns the new combined set of attributes for the id. -func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) { - attr, err := txAttrs(tx, id) - if err != nil { - return nil, err - } - - // Create a new map if it is empty so we don't update emptyMap. - if len(attr) == 0 { - attr = make(map[string]interface{}, len(m)) - } - - // Merge attributes with original values. - // Nil values should delete keys. - for k, v := range m { - if v == nil { - delete(attr, k) - continue - } - - switch v := v.(type) { - case int: - attr[k] = int64(v) - case uint: - attr[k] = int64(v) - case uint64: - attr[k] = int64(v) - case string, int64, bool, float64: - attr[k] = v - default: - return nil, fmt.Errorf("invalid attr type: %T", v) - } - } - - // Marshal and save new values. - buf, err := pilosa.EncodeAttrs(attr) - if err != nil { - return nil, errors.Wrap(err, "encoding attrs") - } - if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil { - return nil, errors.Wrap(err, "saving attrs") - } - return attr, nil -} - -// u64tob encodes v to big endian encoding. -func u64tob(v uint64) []byte { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, v) - return b -} - -// btou64 decodes b from big endian encoding. -func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } - -// emptyMap is a reusable map that contains no keys. -var emptyMap = make(map[string]interface{}) - -// mapContains returns true if all keys & values of subset are in m. -func mapContains(m, subset map[string]interface{}) bool { - for k, v := range subset { - value, ok := m[k] - if !ok || value != v { - return false - } - } - return true -} - -// blockCursor represents a cursor for iterating over blocks of a bolt bucket. -type blockCursor struct { - cur *bolt.Cursor - base uint64 - n uint64 - - buf struct { - key []byte - value []byte - filled bool - } -} - -// newBlockCursor returns a new block cursor that wraps cur using n sized blocks. -func newBlockCursor(c *bolt.Cursor, n int) blockCursor { // nolint: unparam - cur := blockCursor{ - cur: c, - n: uint64(n), - } - cur.buf.key, cur.buf.value = c.First() - cur.buf.filled = true - return cur -} - -// blockID returns the current block ID. Only valid after call to nextBlock(). -func (cur *blockCursor) blockID() uint64 { return cur.base } - -// nextBlock moves the cursor to the next block. -// Returns true if another block exists, otherwise returns false. -func (cur *blockCursor) nextBlock() bool { - if cur.buf.key == nil { - return false - } - - cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n - return true -} - -// next returns the next key/value within the block. -// Returns nils at the end of the block. -func (cur *blockCursor) next() (key, value []byte) { - // Use buffered value, if set. - if cur.buf.filled { - key, value = cur.buf.key, cur.buf.value - cur.buf.filled = false - return key, value - } - - // Read next key. - key, value = cur.cur.Next() - - // Fill buffer for EOF. - if key == nil { - cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false - return nil, nil - } - - // Parse key and buffer if outside of block. - id := binary.BigEndian.Uint64(key) - if id/cur.n > cur.base { - cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true - return nil, nil - } - - return key, value -} diff --git a/boltdb/translate.go b/boltdb/translate.go index bca1e5f92..775151240 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -1,39 +1,53 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 package boltdb import ( + "bytes" "context" + "encoding/binary" + "fmt" + "io" "os" "path/filepath" "sync" "time" - "github.com/boltdb/bolt" - "github.com/pilosa/pilosa/v2" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/roaring" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" + + "runtime/pprof" ) +var _ = pprof.StartCPUProfile + var ( // ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader // and the underlying store is closed. ErrTranslateStoreClosed = errors.New("boltdb: translate store closing") + + // ErrTranslateKeyNotFound is returned when translating key + // and the underlying store returns an empty set + ErrTranslateKeyNotFound = errors.New("boltdb: translating key returned empty set") + + bucketKeys = []byte("keys") + bucketIDs = []byte("ids") + bucketFree = []byte("free") + freeKey = []byte("free") +) + +const ( + // snapshotExt is the file extension used for an in-process snapshot. + snapshotExt = ".snapshotting" + + errFmtTranslateBucketNotFound = "boltdb: translate bucket '%s' not found" ) // OpenTranslateStore opens and initializes a boltdb translation store. -func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) { - s := NewTranslateStore(index, field) +func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) { + s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled) s.Path = path if err := s.Open(); err != nil { return nil, err @@ -45,46 +59,71 @@ func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error var _ pilosa.TranslateStore = &TranslateStore{} // TranslateStore is an on-disk storage engine for translating string-to-uint64 values. +// An empty string will be converted into the sentinel byte slice: +// var emptyKey = []byte{ +// 0x00, 0x00, 0x00, +// 0x4d, 0x54, 0x4d, 0x54, // MTMT +// 0x00, +// 0xc2, 0xa0, // NO-BREAK SPACE +// 0x00, +// } type TranslateStore struct { mu sync.RWMutex db *bolt.DB - index string - field string + index string + field string + partitionID int + partitionN int once sync.Once closing chan struct{} - readOnly bool - writeNotify chan struct{} + readOnly bool + fsyncEnabled bool + writeNotify chan struct{} // File path to database file. Path string } // NewTranslateStore returns a new instance of TranslateStore. -func NewTranslateStore(index, field string) *TranslateStore { +func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore { return &TranslateStore{ - index: index, - field: field, - closing: make(chan struct{}), - writeNotify: make(chan struct{}), + index: index, + field: field, + partitionID: partitionID, + partitionN: partitionN, + closing: make(chan struct{}), + writeNotify: make(chan struct{}), + fsyncEnabled: fsyncEnabled, } } // Open opens the translate file. func (s *TranslateStore) Open() (err error) { - if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { + + // add the path to the problem database if we panic handling it. + defer func() { + r := recover() + if r != nil { + panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r)) + } + }() + + if err := os.MkdirAll(filepath.Dir(s.Path), 0750); err != nil { return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path)) - } else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil { + } else if s.db, err = bolt.Open(s.Path, 0600, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil { return errors.Wrapf(err, "open file: %s", err) } // Initialize buckets. if err := s.db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucketIfNotExists([]byte("keys")); err != nil { + if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil { return err - } else if _, err := tx.CreateBucketIfNotExists([]byte("ids")); err != nil { + } else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil { + return err + } else if _, err := tx.CreateBucketIfNotExists(bucketFree); err != nil { return err } return nil @@ -108,6 +147,11 @@ func (s *TranslateStore) Close() (err error) { return nil } +// PartitionID returns the partition id the store was initialized with. +func (s *TranslateStore) PartitionID() int { + return s.partitionID +} + // ReadOnly returns true if the store is in read-only mode. func (s *TranslateStore) ReadOnly() bool { s.mu.RLock() @@ -135,109 +179,139 @@ func (s *TranslateStore) Size() int64 { return tx.Size() } -// TranslateKeys converts a string key to an integer ID. -// If key does not have an associated id then one is created. -func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) { - // Find id by key under read lock. - if err := s.db.View(func(tx *bolt.Tx) error { - id = findIDByKey(tx.Bucket([]byte("keys")), key) - return nil - }); err != nil { - return 0, err - } else if id != 0 { - return id, nil - } - - if s.ReadOnly() { - return 0, pilosa.ErrTranslateStoreReadOnly - } - - // Find or create id under write lock. - var written bool - if err := s.db.Update(func(tx *bolt.Tx) (err error) { - bkt := tx.Bucket([]byte("keys")) - if id = findIDByKey(bkt, key); id != 0 { - return nil - } else if id, err = bkt.NextSequence(); err != nil { - return err - } else if err := bkt.Put([]byte(key), u64tob(id)); err != nil { - return err - } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil { - return err +// FindKeys looks up the ID for each key. +// Keys are not created if they do not exist. +// Missing keys are not considered errors, so the length of the result may be less than that of the input. +func (s *TranslateStore) FindKeys(keys ...string) (map[string]uint64, error) { + result := make(map[string]uint64, len(keys)) + err := s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketKeys) + if bkt == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys) + } + for _, key := range keys { + id, _ := findIDByKey(bkt, key) + if id == 0 { + // The key does not exist. + continue + } + + result[key] = id } - written = true return nil - }); err != nil { - return 0, err + }) + if err != nil { + return nil, err } - if written { - s.notifyWrite() - } - - return id, nil + return result, nil } -// TranslateKeys converts a string key to an integer ID. -// If key does not have an associated id then one is created. -func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) { - if len(keys) == 0 { - return nil, nil - } - - // Allocate slice for ID mapping. - ids = make([]uint64, len(keys)) - - // Find ids by key under read lock. - var found int - if err := s.db.View(func(tx *bolt.Tx) error { - bkt := tx.Bucket([]byte("keys")) - for i, key := range keys { - if id := findIDByKey(bkt, key); id != 0 { - ids[i] = id - found++ - } - } - return nil - }); err != nil { - return nil, err - } else if found == len(keys) { - return ids, nil - } +// translateTransactionSize governs the number of writes to a single +// boltDB bucket we will make in a single db.Update(), before starting +// a new Update. We do this because Put() is quadratic, but Commit is +// expensive enough that we want to do a fair number of updates before +// paying for it. +const translateTransactionSize = 16384 +// CreateKeys maps all keys to IDs, creating the IDs if they do not exist. +// If the translator is read-only, this will return an error. +func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) { if s.ReadOnly() { - return ids, pilosa.ErrTranslateStoreReadOnly + return nil, pilosa.ErrTranslateStoreReadOnly } - // Find or create ids under write lock if any keys were not found. - var written bool - if err := s.db.Update(func(tx *bolt.Tx) (err error) { - bkt := tx.Bucket([]byte("keys")) - for i, key := range keys { - if ids[i] != 0 { - continue + written := false + result := make(map[string]uint64, len(keys)) + idScratch := make([]byte, translateTransactionSize*8) + for len(keys) > 0 { + // boltdb performs badly if you write really large numbers of + // keys all at once... + err := s.db.Update(func(tx *bolt.Tx) error { + keyBucket := tx.Bucket(bucketKeys) + if keyBucket == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys) } + idBucket := tx.Bucket(bucketIDs) + if idBucket == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs) + } + freeBucket := tx.Bucket(bucketFree) + if freeBucket == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketFree) + } + puts := 0 - if ids[i] = findIDByKey(bkt, key); ids[i] != 0 { - continue - } else if ids[i], err = bkt.NextSequence(); err != nil { - return err - } else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil { - return err - } else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil { - return err + // we create a freeIDGetter to reduce marshalling + getter := newFreeIDGetter(freeBucket) + defer getter.Close() + + for idx, key := range keys { + id, boltKey := findIDByKey(keyBucket, key) + if id != 0 { + result[key] = id + continue + } + // see if we can re-use any IDs first + if id = getter.GetFreeID(); id == 0 { + id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + } + idBytes := idScratch[puts*8 : puts*8+8] + binary.BigEndian.PutUint64(idBytes, id) + puts++ + if err := keyBucket.Put(boltKey, idBytes); err != nil { + return err + } else if err := idBucket.Put(idBytes, boltKey); err != nil { + return err + } + result[key] = id + written = true + if puts == translateTransactionSize { + keys = keys[idx+1:] + return nil + } } - written = true + keys = keys[len(keys):] + return nil + }) + if err != nil { + return nil, err } - return nil - }); err != nil { - return nil, err } - if written { s.notifyWrite() } - return ids, nil + + return result, nil +} + +// Match finds the IDs of all keys matching a filter. +func (s *TranslateStore) Match(filter func([]byte) bool) ([]uint64, error) { + var matches []uint64 + err := s.db.View(func(tx *bolt.Tx) error { + // This uses the id bucket instead of the key bucket so that matches are produced in sorted order. + idBucket := tx.Bucket(bucketIDs) + if idBucket == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs) + } + + return idBucket.ForEach(func(id, key []byte) error { + if bytes.Equal(key, emptyKey) { + key = nil + } + + if filter(key) { + matches = append(matches, btou64(id)) + } + + return nil + }) + }) + if err != nil { + return nil, err + } + + return matches, nil } // TranslateID converts an integer ID to a string key. @@ -248,7 +322,7 @@ func (s *TranslateStore) TranslateID(id uint64) (string, error) { return "", err } defer func() { _ = tx.Rollback() }() - return findKeyByID(tx.Bucket([]byte("ids")), id), nil + return findKeyByID(tx.Bucket(bucketIDs), id), nil } // TranslateIDs converts a list of integer IDs to a list of string keys. @@ -263,9 +337,11 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) { } defer func() { _ = tx.Rollback() }() + bucket := tx.Bucket(bucketIDs) + keys := make([]string, len(ids)) for i, id := range ids { - keys[i] = findKeyByID(tx.Bucket([]byte("ids")), id) + keys[i] = findKeyByID(bucket, id) } return keys, nil } @@ -273,9 +349,9 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) { // ForceSet writes the id/key pair to the store even if read only. Used by replication. func (s *TranslateStore) ForceSet(id uint64, key string) error { if err := s.db.Update(func(tx *bolt.Tx) (err error) { - if err := tx.Bucket([]byte("keys")).Put([]byte(key), u64tob(id)); err != nil { + if err := tx.Bucket(bucketKeys).Put([]byte(key), u64tob(id)); err != nil { return err - } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil { + } else if err := tx.Bucket(bucketIDs).Put(u64tob(id), []byte(key)); err != nil { return err } return nil @@ -286,7 +362,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error { return nil } -// Reader returns a reader that streams the underlying data file. +// EntryReader returns a reader that streams the underlying data file. func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) { ctx, cancel := context.WithCancel(ctx) return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil @@ -311,9 +387,7 @@ func (s *TranslateStore) notifyWrite() { // MaxID returns the highest id in the store. func (s *TranslateStore) MaxID() (max uint64, err error) { if err := s.db.View(func(tx *bolt.Tx) error { - if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil { - max = btou64(key) - } + max = maxID(tx) return nil }); err != nil { return 0, err @@ -321,6 +395,61 @@ func (s *TranslateStore) MaxID() (max uint64, err error) { return max, nil } +// WriteTo writes the contents of the store to the writer. +func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) { + tx, err := s.db.Begin(false) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + return tx.WriteTo(w) +} + +// ReadFrom reads the content and overwrites the existing store. +func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) { + // Close store. + if err := s.Close(); err != nil { + return 0, errors.Wrap(err, "closing store") + } + + // Create a temporary file to snapshot to. + snapshotPath := s.Path + snapshotExt + file, err := os.Create(snapshotPath) + if err != nil { + return n, errors.Wrap(err, "creating snapshot file") + } + + // Write payload to snapshot. + if n, err = io.Copy(file, r); err != nil { + file.Close() + return n, errors.Wrap(err, "snapshot write to") + } + + // we close the file here so we don't still have it open when trying + // to open it in a moment. + file.Close() + + // Move snapshot to data file location. + if err := os.Rename(snapshotPath, s.Path); err != nil { + return n, errors.Wrap(err, "renaming snapshot") + } + + // Re-open the store. + if err := s.Open(); err != nil { + return n, errors.Wrap(err, "re-opening store") + } + + return n, nil +} + +// MaxID returns the highest id in the store. +func maxID(tx *bolt.Tx) uint64 { + if key, _ := tx.Bucket(bucketIDs).Cursor().Last(); key != nil { + return btou64(key) + } + return 0 +} + type TranslateEntryReader struct { ctx context.Context store *TranslateStore @@ -353,7 +482,7 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { var found bool if err := r.store.db.View(func(tx *bolt.Tx) error { // Find ID/key lookup at offset or later. - cur := tx.Bucket([]byte("ids")).Cursor() + cur := tx.Bucket(bucketIDs).Cursor() key, value := cur.Seek(u64tob(r.offset)) if key == nil { return nil @@ -387,13 +516,203 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { } } -func findIDByKey(bkt *bolt.Bucket, key string) uint64 { - if value := bkt.Get([]byte(key)); value != nil { - return btou64(value) +type boltWrapper struct { + tx *bolt.Tx + db *bolt.DB +} + +func (w *boltWrapper) Commit() error { + if w.tx != nil { + return w.tx.Commit() } - return 0 + return nil +} + +func (w *boltWrapper) Rollback() { + if w.tx != nil { + w.tx.Rollback() + } +} +func (s *TranslateStore) FreeIDs() (*roaring.Bitmap, error) { + result := roaring.NewBitmap() + err := s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketFree) + if bkt == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys) + } + b := bkt.Get(freeKey) + err := result.UnmarshalBinary(b) + if err != nil { + return err + } + return nil + }) + return result, err +} +func (s *TranslateStore) MergeFree(tx *bolt.Tx, newIDs *roaring.Bitmap) error { + bkt := tx.Bucket(bucketFree) + b := bkt.Get(freeKey) + buf := new(bytes.Buffer) + if b != nil { //if existing combine with newIDs + before := roaring.NewBitmap() + err := before.UnmarshalBinary(b) + if err != nil { + return err + } + final := newIDs.Union(before) + _, err = final.WriteTo(buf) + if err != nil { + return err + } + } else { + newIDs.WriteTo(buf) + } + return bkt.Put(freeKey, buf.Bytes()) +} + +// Delete removes the lookeup pairs in order to make avialble for reuse but doesn't commit the +// transaction for that is tied to the associated rbf transaction being successful +func (s *TranslateStore) Delete(records *roaring.Bitmap) (pilosa.Commitor, error) { + tx, err := s.db.Begin(true) + if err != nil { + return nil, err + } + keyBucket := tx.Bucket(bucketKeys) + idBucket := tx.Bucket(bucketIDs) + ids := records.Slice() + for i := range ids { + id := u64tob(ids[i]) + boltKey := idBucket.Get(id) + err = keyBucket.Delete(boltKey) + if err != nil { + tx.Rollback() + return &boltWrapper{}, err + } + err = idBucket.Delete(id) + if err != nil { + tx.Rollback() + return &boltWrapper{}, err + } + + } + return &boltWrapper{tx: tx}, s.MergeFree(tx, records) +} + +// emptyKey is a sentinel byte slice which stands for "" as a key. +var emptyKey = []byte{ + 0x00, 0x00, 0x00, + 0x4d, 0x54, 0x4d, 0x54, // MTMT + 0x00, + 0xc2, 0xa0, // NO-BREAK SPACE + 0x00, +} + +func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) { + var boltKey []byte + if key == "" { + boltKey = emptyKey + } else { + boltKey = []byte(key) + } + + if value := bkt.Get(boltKey); value != nil { + return btou64(value), boltKey + } + return 0, boltKey +} + +// freeIDGetter reduces the amount of marshaling required to get multiple ids +type freeIDGetter struct { + freeBucket *bolt.Bucket + b *roaring.Bitmap + changed bool +} + +// newFreeIDGetter initializes a new freeIDGetter. If at any point there is a +// failure, it returns an error. +// +// NOTE: For changes to be persisted to the bucket, you must call +// (*freeIDGetter).Close() +func newFreeIDGetter(freeBucket *bolt.Bucket) *freeIDGetter { + g := &freeIDGetter{ + freeBucket: freeBucket, + } + // we ignore this value because it's okay if we dont have a bitmap just yet + _ = g.getBitmap() + return g +} + +func (g *freeIDGetter) getBitmap() bool { + if g.b == nil { + // get the bitmap from freeBucket + value := g.freeBucket.Get(freeKey) + if value == nil { + return false + } + // turn the value into a bitmap + b := roaring.NewBitmap() + if err := b.UnmarshalBinary(value); err != nil { + return false + } + g.b = b + } + return true +} + +// GetFreeID tries to get a free ID from the free id bucket. If at any point it +// fails to do so, it returns a 0. Otherwise, it returns the first free ID in the +// bucket +func (g *freeIDGetter) GetFreeID() (id uint64) { + if !g.getBitmap() { + return 0 + } + // get the first free id + id, ok := g.b.Min() + if !ok { + return 0 + } + // remove that id from the free id bitmap + if changed, err := g.b.RemoveN(id); changed == 0 || err != nil { + return 0 + } else { + g.changed = true + } + return id +} + +// Close persists any changes to the bitmap back to the bucket and then nils the +// references for safety. +func (g *freeIDGetter) Close() error { + if g.changed { + // convert bitmap to binary + buf, err := g.b.MarshalBinary() + if err != nil { + return errors.Wrap(err, "closing free ID Getter") + } + // put updated bitmap back into the freeBucket + if err := g.freeBucket.Put(freeKey, buf); err != nil { + return errors.Wrap(err, "closing free ID Getter") + } + } + g.b = nil + g.freeBucket = nil + return nil } func findKeyByID(bkt *bolt.Bucket, id uint64) string { - return string(bkt.Get(u64tob(id))) + boltKey := bkt.Get(u64tob(id)) + if bytes.Equal(boltKey, emptyKey) { + return "" + } + return string(boltKey) } + +// u64tob encodes v to big endian encoding. +func u64tob(v uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, v) + return b +} + +// btou64 decodes b from big endian encoding. +func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } diff --git a/boltdb/translate_internal_test.go b/boltdb/translate_internal_test.go new file mode 100644 index 000000000..29d5c6fbb --- /dev/null +++ b/boltdb/translate_internal_test.go @@ -0,0 +1,107 @@ +package boltdb + +import ( + "path/filepath" + "testing" + + "github.com/molecula/featurebase/v3/roaring" + bolt "go.etcd.io/bbolt" +) + +func TestGetFreeID(t *testing.T) { + boltDir := t.TempDir() + db, err := bolt.Open(filepath.Join(boltDir, "testDB"), 0600, nil) + if err != nil { + t.Fatalf("unexpected error opening test boltdb: %v", err) + } + defer db.Close() + + makeTestBucket := func(tx *bolt.Tx, b *roaring.Bitmap) *bolt.Bucket { + if b == nil { + t.Fatalf("unexpected nil bitmap") + } + free, err := tx.CreateBucketIfNotExists(bucketFree) + if err != nil { + t.Fatalf("unexpected error making freeBucket: %v", err) + } + buf, err := b.MarshalBinary() + if err != nil { + t.Fatalf("unexpected error marshaling bitmap (%v) to binary: %v", b, err) + } + if err := free.Put(freeKey, buf); err != nil { + t.Fatalf("unexpected error adding data (%v) to freeBucket: %v", b, err) + } + return free + } + + for name, test := range map[string]struct { + bits *roaring.Bitmap + want uint64 + }{ + "bucket is there, but nobody's home": { + bits: roaring.NewBitmap(), + want: 0, + }, + "good bucket": { + bits: roaring.NewBitmap(1, 2, 34, 55, 9000), + want: 1, + }, + } { + t.Run(name, func(t *testing.T) { + tx, err := db.Begin(true) + if err != nil { + t.Fatalf("unexpected error starting bolt transaction: %v", err) + } + defer tx.Rollback() + freeBucket := makeTestBucket(tx, test.bits) + + getter := newFreeIDGetter(freeBucket) + defer getter.Close() + if got := getter.GetFreeID(); got != test.want { + t.Fatalf("expected %v got %v", test.want, got) + } + }) + } + + t.Run("CorrectOrdering", func(t *testing.T) { + tx, err := db.Begin(true) + if err != nil { + t.Fatalf("unexpected error starting bolt transaction: %v", err) + } + defer tx.Rollback() + + bucket := makeTestBucket(tx, roaring.NewBitmap(1, 34, 2, 55, 9000)) + + getter := newFreeIDGetter(bucket) + defer getter.Close() + for _, want := range []uint64{1, 2, 34, 55, 9000} { + if got := getter.GetFreeID(); got != want { + t.Fatalf("expected %v got %v", want, got) + } + } + if got := getter.GetFreeID(); got != 0 { + t.Fatalf("expected 0 got %v", got) + } + }) + + t.Run("NotABitmap", func(t *testing.T) { + tx, err := db.Begin(true) + if err != nil { + t.Fatalf("unexpected error starting bolt transaction: %v", err) + } + defer tx.Rollback() + + free, err := tx.CreateBucketIfNotExists(bucketFree) + if err != nil { + t.Fatalf("unexpected error making freeBucket: %v", err) + } + if err := free.Put(freeKey, []byte("this isn't right!")); err != nil { + t.Fatalf("unexpected error adding data to freeBucket: %v", err) + } + getter := newFreeIDGetter(free) + defer getter.Close() + if got := getter.GetFreeID(); got != 0 { + t.Fatalf("expected 0 got %v", got) + } + }) +} diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 7c3d0e3d5..6093c04e5 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -1,125 +1,99 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 package boltdb_test import ( + "bytes" "context" - "io/ioutil" - "os" + "fmt" + "reflect" + "strconv" "testing" "time" - "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/boltdb" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/boltdb" + "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/roaring" + "github.com/molecula/featurebase/v3/testhook" ) -func TestTranslateStore_TranslateKey(t *testing.T) { - s := MustOpenNewTranslateStore() +//var vv = pilosa.VV + +func TestTranslateStore_CreateKeys(t *testing.T) { + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) - // Ensure initial key translates to ID 1. - if id, err := s.TranslateKey("foo"); err != nil { + ids, err := s.CreateKeys("abc", "abc") + if err != nil { t.Fatal(err) - } else if got, want := id, uint64(1); got != want { - t.Fatalf("TranslateKey()=%d, want %d", got, want) + } else if _, ok := ids["abc"]; !ok { + t.Fatalf(`missing "abc"; got %v`, ids) + } else if len(ids) > 1 { + t.Fatalf("expected one key, got %d in %v", len(ids), ids) } - // Ensure next key autoincrements. - if id, err := s.TranslateKey("bar"); err != nil { + // Ensure different keys translate to different IDs. + ids1, err := s.CreateKeys("foo", "bar") + if err != nil { t.Fatal(err) - } else if got, want := id, uint64(2); got != want { - t.Fatalf("TranslateKey()=%d, want %d", got, want) - } - - // Ensure retranslating existing key returns original ID. - if id, err := s.TranslateKey("foo"); err != nil { - t.Fatal(err) - } else if got, want := id, uint64(1); got != want { - t.Fatalf("TranslateKey()=%d, want %d", got, want) - } -} - -func TestTranslateStore_TranslateKeys(t *testing.T) { - s := MustOpenNewTranslateStore() - defer MustCloseTranslateStore(s) - - // Ensure initial keys translate to incrementing IDs. - if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { - t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { - t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(2); got != want { - t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) + } else if foo, bar := ids1["foo"], ids1["bar"]; foo == bar { + t.Fatalf(`"foo" and "bar" map back to the same ID %d`, foo) } // Ensure retranslation returns original IDs. - if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { + if ids, err := s.CreateKeys("bar", "foo"); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { - t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(2); got != want { - t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) + } else if !reflect.DeepEqual(ids, ids1) { + t.Fatalf("retranslation produced result %v which is different from original translation %v", ids, ids1) } // Ensure retranslating with existing and non-existing keys returns correctly. - if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil { + if ids, err := s.CreateKeys("foo", "baz", "bar"); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { - t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(3); got != want { - t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) - } else if got, want := ids[2], uint64(2); got != want { - t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want) + } else if got, want := ids["foo"], ids1["foo"]; got != want { + t.Fatalf(`mismatched ID %d for "foo" (previously %d)`, got, want) + } else if _, ok := ids["baz"]; !ok { + t.Fatalf(`missing translation for "baz"; got %v`, ids) + } else if got, want := ids["bar"], ids1["bar"]; got != want { + t.Fatalf(`mismatched ID %d for "bar" (previously %d)`, got, want) } } func TestTranslateStore_TranslateID(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Setup initial keys. - if _, err := s.TranslateKey("foo"); err != nil { - t.Fatal(err) - } else if _, err := s.TranslateKey("bar"); err != nil { + ids, err := s.CreateKeys("foo", "bar", "") + if err != nil { t.Fatal(err) } // Ensure IDs can be translated back to keys. - if key, err := s.TranslateID(1); err != nil { - t.Fatal(err) - } else if got, want := key, "foo"; got != want { - t.Fatalf("TranslateID()=%s, want %s", got, want) - } - - if key, err := s.TranslateID(2); err != nil { - t.Fatal(err) - } else if got, want := key, "bar"; got != want { - t.Fatalf("TranslateID()=%s, want %s", got, want) + for key, id := range ids { + k, err := s.TranslateID(id) + if err != nil { + t.Fatal(err) + } + if k != key { + t.Fatalf("TranslateID()=%s, want %s", k, key) + } } } func TestTranslateStore_TranslateIDs(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Setup initial keys. - if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { + ids, err := s.CreateKeys("foo", "bar") + if err != nil { t.Fatal(err) } // Ensure IDs can be translated back to keys. - if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil { + if keys, err := s.TranslateIDs([]uint64{ids["foo"], ids["bar"], 1}); err != nil { t.Fatal(err) } else if got, want := keys[0], "foo"; got != want { t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want) @@ -130,13 +104,114 @@ func TestTranslateStore_TranslateIDs(t *testing.T) { } } +func TestTranslateStore_FindKeys(t *testing.T) { + cases := []struct { + name string + data []string + lookup []string + }{ + { + name: "All", + data: []string{"plugh", "xyzzy", "h"}, + lookup: []string{"plugh", "xyzzy", "h"}, + }, + { + name: "Extra", + data: []string{"plugh", "xyzzy", "h"}, + lookup: []string{"plugh", "xyzzy", "h", "65"}, + }, + { + name: "None", + data: []string{"a", "b", "c"}, + lookup: []string{"d", "e"}, + }, + { + name: "Empty", + lookup: []string{"h"}, + }, + { + name: "LookupNothing", + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + s := MustOpenNewTranslateStore(t) + defer MustCloseTranslateStore(s) + + var naiveMap map[string]uint64 + if c.data != nil { + // Load in key data. + keys := c.data + ids, err := s.CreateKeys(keys...) + if err != nil { + t.Errorf("failed to import keys: %v", err) + return + } + if len(ids) != len(keys) { + t.Errorf("mapped %d keys to %d ids", len(keys), len(ids)) + return + } + naiveMap = ids + } + + // Compute expected lookup result. + result := map[string]uint64{} + for _, key := range c.lookup { + id, ok := naiveMap[key] + if !ok { + // The key is expected to be missing. + continue + } + + result[key] = id + } + + // Find the keys. + found, err := s.FindKeys(c.lookup...) + if err != nil { + t.Errorf("failed to find keys: %v", err) + } else if !reflect.DeepEqual(result, found) { + t.Errorf("expected %v but found %v", result, found) + } + }) + } +} + +func TestTranslateStore_MaxID(t *testing.T) { + s := MustOpenNewTranslateStore(t) + defer MustCloseTranslateStore(s) + + // Generate a bunch of keys. + var lastk uint64 + for i := 0; i < 1026; i++ { + key := strconv.Itoa(i) + ids, err := s.CreateKeys(key) + if err != nil { + t.Fatalf("translating %d: %v", i, err) + } + lastk = ids[key] + } + + // Verify the max ID. + max, err := s.MaxID() + if err != nil { + t.Fatalf("checking max ID: %v", err) + } + if max != lastk { + t.Fatalf("last key is %d but max is %d", lastk, max) + } +} + func TestTranslateStore_EntryReader(t *testing.T) { t.Run("OK", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Create multiple new keys. - if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { + ids1, err := s.CreateKeys("foo", "bar") + if err != nil { t.Fatal(err) } @@ -151,7 +226,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read first entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(1); got != want { + } else if got, want := entry.ID, ids1["foo"]; got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "foo"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -160,21 +235,22 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read next entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(2); got != want { + } else if got, want := entry.ID, ids1["bar"]; got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "bar"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) } // Insert next key while reader is open. - if _, err := s.TranslateKey("baz"); err != nil { + ids2, err := s.CreateKeys("baz") + if err != nil { t.Fatal(err) } // Read newly created entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(3); got != want { + } else if got, want := entry.ID, ids2["baz"]; got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "baz"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -188,7 +264,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Ensure reader will read as soon as a new write comes in using WriteNotify(). t.Run("WriteNotify", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Start reader from initial position. @@ -198,20 +274,25 @@ func TestTranslateStore_EntryReader(t *testing.T) { } defer r.Close() + // cache holds the translated key id so we can check it later + cache := make(chan uint64) + // Insert key in separate goroutine. // Sleep momentarily to reader hangs. translateErr := make(chan error) go func() { time.Sleep(100 * time.Millisecond) - if _, err := s.TranslateKey("foo"); err != nil { + ids, err := s.CreateKeys("foo") + if err != nil { translateErr <- err } + cache <- ids["foo"] }() var entry pilosa.TranslateEntry if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(1); got != want { + } else if got, want := entry.ID, <-cache; got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "foo"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -226,7 +307,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Ensure exits read on close. t.Run("Close", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Start reader from initial position. @@ -260,7 +341,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Ensure exits read on store close. t.Run("StoreClose", func(t *testing.T) { - s := MustOpenNewTranslateStore() + s := MustOpenNewTranslateStore(t) defer MustCloseTranslateStore(s) // Start reader from initial position. @@ -294,24 +375,138 @@ func TestTranslateStore_EntryReader(t *testing.T) { } // MustNewTranslateStore returns a new TranslateStore with a temporary path. -func MustNewTranslateStore() *boltdb.TranslateStore { - f, err := ioutil.TempFile("", "") +func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { + f, err := testhook.TempFile(tb, "translate-store") if err != nil { panic(err) } else if err := f.Close(); err != nil { panic(err) } - s := boltdb.NewTranslateStore("I", "F") + s := boltdb.NewTranslateStore("I", "F", 0, disco.DefaultPartitionN, false) s.Path = f.Name() return s } +func TestTranslateStore_Delete(t *testing.T) { + s := MustOpenNewTranslateStore(t) + defer MustCloseTranslateStore(s) + + // Setup initial keys. + ids, err := s.CreateKeys("foo", "bar", "deleteme") + if err != nil { + t.Fatal(err) + } + + records := roaring.NewBitmap(ids["deleteme"]) + c, err := s.Delete(records) + if err != nil { + t.Fatal(err) + } + if err = c.Commit(); err != nil { + t.Fatal(err) + } + r, e := s.FreeIDs() + if e != nil { + t.Fatal(err) + } + freeids := r.Slice() + if len(freeids) == 0 { + t.Fatalf("expected to have free id") + } + if freeids[0] != ids["deleteme"] { + t.Fatalf("expected [%v] and got %v", ids["deleteme"], freeids[0]) + } + + records2 := roaring.NewBitmap(ids["foo"]) + c, err = s.Delete(records2) + if err != nil { + t.Fatal(err) + } + if err = c.Commit(); err != nil { + t.Fatal(err) + } + r, e = s.FreeIDs() + if e != nil { + t.Fatal(err) + } + freeids = r.Slice() + if len(freeids) != 2 { + t.Fatalf("expected to have 2 free ids") + } +} +func TestTranslateStore_ReadWrite(t *testing.T) { + t.Run("WriteTo_ReadFrom", func(t *testing.T) { + s := MustOpenNewTranslateStore(t) + defer MustCloseTranslateStore(s) + + batch0 := []string{} + for i := 0; i < 100; i++ { + batch0 = append(batch0, fmt.Sprintf("key%d", i)) + } + batch1 := []string{} + for i := 100; i < 200; i++ { + batch1 = append(batch1, fmt.Sprintf("key%d", i)) + } + + // Populate the store with the keys in batch0. + batch0IDs, err := s.CreateKeys(batch0...) + if err != nil { + t.Fatal(err) + } + + // Put the contents of the store into a buffer. + buf := bytes.NewBuffer(nil) + expN := s.Size() + + // After this, the buffer should contain batch0. + if n, err := s.WriteTo(buf); err != nil { + t.Fatalf("writing to buffer: %s", err) + } else if n != expN { + t.Fatalf("expected buffer size: %d, but got: %d", expN, n) + } + + // Populate the store with the keys in batch1. + batch1IDs, err := s.CreateKeys(batch1...) + if err != nil { + t.Fatal(err) + } + + expIDs := map[string]uint64{ + "key50": batch0IDs["key50"], + "key150": batch1IDs["key150"], + } + + // Check the IDs for a key from each batch. + if ids, err := s.FindKeys("key50", "key150"); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expIDs, ids) { + t.Fatalf("first expected ids: %v, but got: %v", expIDs, ids) + } + + // Reset the contents of the store with the data in the buffer. + if n, err := s.ReadFrom(buf); err != nil { + t.Fatalf("reading from buffer: %s", err) + } else if n != expN { + t.Fatalf("expected buffer size: %d, but got: %d", expN, n) + } + + // This time, we expect the second key to be different because + // we overwrote the store, and then just set that key. + if ids, err := s.CreateKeys("key50", "key150"); err != nil { + t.Fatal(err) + } else if ids["key50"] != expIDs["key50"] { + t.Fatalf("last expected ids[key50]: %d, but got: %d", expIDs["key50"], ids["key50"]) + } else if ids["key150"] == expIDs["key150"] { + t.Fatalf("last expected different ids[key150]: %d, but got: %d", expIDs["key150"], ids["key150"]) + } + }) +} // MustOpenNewTranslateStore returns a new, opened TranslateStore. -func MustOpenNewTranslateStore() *boltdb.TranslateStore { - s := MustNewTranslateStore() +func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { + s := MustNewTranslateStore(tb) if err := s.Open(); err != nil { - panic(err) + tb.Fatalf("opening s: %v", err) } return s } @@ -320,7 +515,5 @@ func MustOpenNewTranslateStore() *boltdb.TranslateStore { func MustCloseTranslateStore(s *boltdb.TranslateStore) { if err := s.Close(); err != nil { panic(err) - } else if err := os.Remove(s.Path); err != nil { - panic(err) } } diff --git a/broadcast.go b/broadcast.go index 6f2245992..5836efb62 100644 --- a/broadcast.go +++ b/broadcast.go @@ -1,22 +1,11 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 package pilosa import ( "fmt" + "github.com/molecula/featurebase/v3/disco" "github.com/pkg/errors" ) @@ -26,11 +15,22 @@ type Serializer interface { Unmarshal([]byte, Message) error } +// NopSerializer represents a Serializer that doesn't do anything. +var NopSerializer Serializer = &nopSerializer{} + +type nopSerializer struct{} + +// Marshal is a no-op implementation of Serializer Marshal method. +func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil } + +// Unmarshal is a no-op implementation of Serializer Unmarshal method. +func (*nopSerializer) Unmarshal([]byte, Message) error { return nil } + // broadcaster is an interface for broadcasting messages. type broadcaster interface { SendSync(Message) error SendAsync(Message) error - SendTo(*Node, Message) error + SendTo(*disco.Node, Message) error } // Message is the interface implemented by all core pilosa types which can be serialized to messages. @@ -49,7 +49,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil } func (nopBroadcaster) SendAsync(Message) error { return nil } // SendTo is a no-op implementation of Broadcaster SendTo method. -func (nopBroadcaster) SendTo(*Node, Message) error { return nil } +func (nopBroadcaster) SendTo(*disco.Node, Message) error { return nil } // Broadcast message types. const ( @@ -61,14 +61,17 @@ const ( messageTypeCreateView messageTypeDeleteView messageTypeClusterStatus - messageTypeResizeInstruction - messageTypeResizeInstructionComplete - messageTypeSetCoordinator - messageTypeUpdateCoordinator + messageTypeUNUSED0 // used to be ResizeInstruction + messageTypeUNUSED1 // used to be ResizeInstructionComplete messageTypeNodeState messageTypeRecalculateCaches + messageTypeLoadSchemaMessage messageTypeNodeEvent messageTypeNodeStatus + messageTypeTransaction + messageTypeUNUSED2 // used to be ResizeNodeMessage + messageTypeUNUSED3 // used to be ResizeAbortMessage + messageTypeUpdateField ) // MarshalInternalMessage serializes the pilosa message and adds pilosa internal @@ -100,22 +103,20 @@ func getMessage(typ byte) Message { return &DeleteViewMessage{} case messageTypeClusterStatus: return &ClusterStatus{} - case messageTypeResizeInstruction: - return &ResizeInstruction{} - case messageTypeResizeInstructionComplete: - return &ResizeInstructionComplete{} - case messageTypeSetCoordinator: - return &SetCoordinatorMessage{} - case messageTypeUpdateCoordinator: - return &UpdateCoordinatorMessage{} case messageTypeNodeState: return &NodeStateMessage{} case messageTypeRecalculateCaches: return &RecalculateCaches{} + case messageTypeLoadSchemaMessage: + return &LoadSchemaMessage{} case messageTypeNodeEvent: return &NodeEvent{} case messageTypeNodeStatus: return &NodeStatus{} + case messageTypeTransaction: + return &TransactionMessage{} + case messageTypeUpdateField: + return &UpdateFieldMessage{} default: panic(fmt.Sprintf("unknown message type %d", typ)) } @@ -139,22 +140,20 @@ func getMessageType(m Message) byte { return messageTypeDeleteView case *ClusterStatus: return messageTypeClusterStatus - case *ResizeInstruction: - return messageTypeResizeInstruction - case *ResizeInstructionComplete: - return messageTypeResizeInstructionComplete - case *SetCoordinatorMessage: - return messageTypeSetCoordinator - case *UpdateCoordinatorMessage: - return messageTypeUpdateCoordinator case *NodeStateMessage: return messageTypeNodeState case *RecalculateCaches: return messageTypeRecalculateCaches + case *LoadSchemaMessage: + return messageTypeLoadSchemaMessage case *NodeEvent: return messageTypeNodeEvent case *NodeStatus: return messageTypeNodeStatus + case *TransactionMessage: + return messageTypeTransaction + case *UpdateFieldMessage: + return messageTypeUpdateField default: panic(fmt.Sprintf("don't have type for message %#v", m)) } diff --git a/bsi.go b/bsi.go new file mode 100644 index 000000000..2f7b427ef --- /dev/null +++ b/bsi.go @@ -0,0 +1,284 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa + +import ( + "math/bits" + + "github.com/molecula/featurebase/v3/roaring" +) + +// bsiData contains BSI-structured data. +type bsiData []*Row + +// pivotDescending loops over nonzero BSI values in descending order. +// For each value, the provided function is called with the value and a slice of the associated columns. +// If limit or offset are not-nil, they will be applied. +// Applying a limit or offset may modify the pointed-to value. +func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) { + // This "pivot" algorithm works by treating the BSI data as a tree. + // Each branch of this tree corresponds to a power-of-2-sized range of BSI values. + // Each range is subdivided into 2 ranges of half size, which form lower branches. + // Eventually, a range of width 1 cannot be subdivided and forms a leaf. + // At each branch and leaf, there is a bitmap of all columns within the corresponding range. + // The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range. + // This function uses a depth-first search over this virtual tree. + + switch { + case !filter.Any(): + // There are no remaining data. + + case offset != nil && *offset >= filter.Count(): + // Skip this entire branch. + *offset -= filter.Count() + + case limit != nil && *limit == 0: + // The limit has been reached. + // No more data is necessary. + + case len(bsi) == 0: + // This is a leaf node. + cols := filter.Columns() + if offset != nil { + cols = cols[*offset:] + *offset = 0 + } + if limit != nil { + if *limit < uint64(len(cols)) { + cols = cols[:*limit] + } + *limit -= uint64(len(cols)) + } + fn(branch, cols...) + + default: + // Pivot over the highest bit. + upperBranch, lowerBranch := branch|(1< 0: + c = roaring.NewContainerArrayCopy(b.array) + b.array = b.array[:0] + default: + return + } + + if b.bm == nil { + b.bm = roaring.NewBitmap() + } + if old := b.bm.Containers.Get(b.key); old != nil { + c = roaring.Union(c, old) + } + b.bm.Containers.Put(b.key, c) +} + +// Add a value to the bitmap. +// Values must be added sequentially. +func (b *rowBuilder) Add(v uint64) { + vkey := v / (1 << 16) + if b.key != vkey { + // This is a new key, so flush the old one. + b.flushKey() + b.key = vkey + } + + if b.mask != nil { + // Add to the mask. + b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64))) + b.mask[uint16(v)/64] |= 1 << (v % 64) + return + } + + // Add to an array. + b.array = append(b.array, uint16(v)) + if len(b.array) >= roaring.ArrayMaxSize { + // The array is too big. + // Convert it to a bitmask. + m := [1024]uint64{} + for _, v := range b.array { + m[v/64] |= 1 << (v % 64) + } + b.n = int32(len(b.array)) + b.array = b.array[:0] + b.mask = &m + } +} + +// Build a Row from stored data. +// This resets the builder. +func (b *rowBuilder) Build() *Row { + // Flush the active key to the bitmap. + b.flushKey() + + // Remove the bitmap and convert it to a Row. + bm := b.bm + b.bm = nil + if bm == nil { + return NewRow() + } + return NewRowFromBitmap(bm) +} + +// bsiBuilder assembles BSI data. +// It is optimized for the case in which values are generated sequentially. +type bsiBuilder []rowBuilder + +// Insert a value into the BSI data. +// Columns must be inserted sequentially, and duplicates are not allowed. +func (b *bsiBuilder) Insert(col, val uint64) { + for val != 0 { + i := bits.TrailingZeros64(val) + val &^= 1 << i + for len(*b) <= i { + *b = append(*b, rowBuilder{}) + } + (*b)[i].Add(col) + } +} + +// Build BSI data. +// This resets the builder. +func (b *bsiBuilder) Build() bsiData { + builders := *b + *b = builders[:0] + rows := make(bsiData, len(builders)) + for i := range builders { + rows[i] = builders[i].Build() + } + return rows +} diff --git a/bsi_test.go b/bsi_test.go new file mode 100644 index 000000000..e11e316f7 --- /dev/null +++ b/bsi_test.go @@ -0,0 +1,160 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa + +import ( + "fmt" + "math/rand" + "sort" + "testing" +) + +// TestBSIAdd does a number of iterations. For each iteration, it +// generates a random number of ids, and two random values for each id +// to add together. +func TestBSIAdd(t *testing.T) { + // TODO wouldn't it be cool if our test suite had a randomized + // burn-in mode where you could run any test which supported it + // with a random seed and way more iterations? + rnd := rand.New(rand.NewSource(99)) + //numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1) + idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth) + var builderA, builderB bsiBuilder + // a and b are generated slices of numbers to add together + var a, b []uint64 + // idToIndex maps record ids to indexes in a and b + idToIndex := make(map[int]int) + // indexToID has the record id for each value in a and b + indexToID := []uint64{} + + min := 999999999 + max := 0 + + for iteration := 0; iteration < 1; iteration++ { + t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) { + // reset generated data + a, b = a[:0], b[:0] + indexToID = indexToID[:0] + for k := range idToIndex { + delete(idToIndex, k) + } + + // z generates the values, they can be fairly large, but are usually small + z := rand.NewZipf(rnd, 1.3, 7, 1<<44) + id := -1 + for i := 0; true; i++ { + // get the next id, skipping a random amount + id = id + int(idZipf.Uint64()+1) + if id >= ShardWidth { + if i < min { + min = i + } + if max < i { + max = i + } + break + } + idToIndex[id] = int(i) + indexToID = append(indexToID, uint64(id)) + + // append a random value to each data slice + a = append(a, z.Uint64()) + b = append(b, z.Uint64()) + } + + // build the BSIs based on the data slices and generated IDs + for index, id := range indexToID { + va, vb := a[index], b[index] + builderA.Insert(uint64(id), va) + builderB.Insert(uint64(id), vb) + } + dataA, dataB := builderA.Build(), builderB.Build() + dataC := addBSI(dataA, dataB) + + // build results from added bsiData; results[i] should hold a[i]+b[i] + results := make([]uint64, len(a)) + dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) { + for _, id := range ids { + results[idToIndex[int(id)]] = count + } + }) + + for i, res := range results { + if res != a[i]+b[i] { + t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results) + } + } + }) + } +} + +type bsiAddCase struct { + positions []uint64 + a []uint64 + b []uint64 +} + +func (b bsiAddCase) Len() int { + return len(b.positions) +} + +// Less reports whether the element with +// index i should sort before the element with index j. +func (b bsiAddCase) Less(i, j int) bool { + return b.positions[i] < b.positions[j] +} + +// Swap swaps the elements with indexes i and j. +func (b bsiAddCase) Swap(i, j int) { + b.positions[i], b.positions[j] = b.positions[j], b.positions[i] + b.a[i], b.a[j] = b.a[j], b.a[i] + b.b[i], b.b[j] = b.b[j], b.b[i] +} + +// TestBSIAddCases tests specific cases of bsiAdd (would generally be +// pulled from randomly generated ones from TestBSIAdd upon failure). +func TestBSIAddCases(t *testing.T) { + tests := []bsiAddCase{ + { + positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274}, + a: []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4}, + b: []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7}, + }, + { + positions: []uint64{17570, 54607}, + a: []uint64{1010, 71}, + b: []uint64{25, 150}, + }, + } + + var builderA, builderB bsiBuilder + for i, tst := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) { + t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b)) + } + sort.Sort(tst) + + for i := 0; i < len(tst.a); i++ { + builderA.Insert(tst.positions[i], tst.a[i]) + builderB.Insert(tst.positions[i], tst.b[i]) + } + + dataA, dataB := builderA.Build(), builderB.Build() + dataC := addBSI(dataA, dataB) + // maps id to count + results := make(map[uint64]uint64) + dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) { + for _, id := range ids { + results[id] = count + } + }) + + for i, id := range tst.positions { + if results[id] != tst.a[i]+tst.b[i] { + t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i]) + } + } + }) + } +} diff --git a/cache.go b/cache.go index de836b8c8..907a551d7 100644 --- a/cache.go +++ b/cache.go @@ -1,29 +1,20 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 package pilosa import ( "bytes" + "encoding/json" "fmt" "io" "sort" "sync" "time" - "github.com/pilosa/pilosa/v2/lru" - "github.com/pilosa/pilosa/v2/stats" + "github.com/molecula/featurebase/v3/lru" + pb "github.com/molecula/featurebase/v3/proto" + "github.com/molecula/featurebase/v3/stats" + "github.com/pkg/errors" ) const ( @@ -52,6 +43,9 @@ type cache interface { // SetStats defines the stats client used in the cache. SetStats(s stats.StatsClient) + + // Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused. + Clear() } // lruCache represents a least recently used Cache implementation. @@ -59,14 +53,17 @@ type lruCache struct { cache *lru.Cache counts map[uint64]uint64 stats stats.StatsClient + // maxEntries is saved to support Clear which recreates the cache. + maxEntries uint32 } // newLRUCache returns a new instance of LRUCache. func newLRUCache(maxEntries uint32) *lruCache { c := &lruCache{ - cache: lru.New(int(maxEntries)), - counts: make(map[uint64]uint64), - stats: stats.NopStatsClient, + cache: lru.New(int(maxEntries)), + counts: make(map[uint64]uint64), + stats: stats.NopStatsClient, + maxEntries: maxEntries, } c.cache.OnEvicted = c.onEvicted return c @@ -118,7 +115,8 @@ func (c *lruCache) Top() []bitmapPair { Count: n, }) } - sort.Sort(bitmapPairs(a)) + pairs := bitmapPairs(a) + sort.Sort(&pairs) return a } @@ -127,6 +125,13 @@ func (c *lruCache) SetStats(s stats.StatsClient) { c.stats = s } +func (c *lruCache) Clear() { + for k := range c.counts { + delete(c.counts, k) + } + c.cache = lru.New(int(c.maxEntries)) +} + func (c *lruCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) } // Ensure LRUCache implements Cache. @@ -134,9 +139,12 @@ var _ cache = &lruCache{} // rankCache represents a cache with sorted entries. type rankCache struct { - mu sync.Mutex - entries map[uint64]uint64 - rankings []bitmapPair // cached, ordered list + // TODO why does this have a lock and lruCache doesn't? + mu sync.Mutex + entries map[uint64]uint64 + rankings bitmapPairs // cached, ordered list + rankingsRead bool + dirty bool updateN int updateTime time.Time @@ -164,14 +172,35 @@ func NewRankCache(maxEntries uint32) *rankCache { } } +func (c *rankCache) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + for k := range c.entries { + delete(c.entries, k) + } + c.rankings = c.rankings[:0] + c.rankingsRead = false + c.dirty = false + + c.updateN = 0 + c.updateTime = time.Time{} + c.thresholdValue = 0 +} + // Add adds a count to the cache. func (c *rankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() + + // Flag the cache as dirty. + // This forces recalculation if top is called before the cache is recalculated. + c.dirty = true + // Ignore if the column count is below the threshold, // unless the count is 0, which is effectively used // to clear the cache value. if n < c.thresholdValue && n > 0 { + delete(c.entries, id) return } @@ -184,11 +213,25 @@ func (c *rankCache) Add(id uint64, n uint64) { func (c *rankCache) BulkAdd(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() + + // Flag the cache as dirty. + // This forces recalculation if top is called before the cache is recalculated. + c.dirty = true + if n < c.thresholdValue { + delete(c.entries, id) return } c.entries[id] = n + + // FB-1206: Periodically invalidate the cache when we are bulk loading + // as this can take up an upbounded amount of memory. This is especially + // true when restoring shards as all rows will be added. + if len(c.entries) > int(2*c.maxEntries) { + c.stats.Count(MetricRecalculateCache, 1, 1.0) + c.recalculate() + } } // Get returns a count for a given id. @@ -209,12 +252,15 @@ func (c *rankCache) Len() int { func (c *rankCache) IDs() []uint64 { c.mu.Lock() defer c.mu.Unlock() - a := make([]uint64, 0, len(c.entries)) - for id := range c.entries { - a = append(a, id) + if len(c.entries) == 0 { + return nil } - sort.Sort(uint64Slice(a)) - return a + ids := make([]uint64, 0, len(c.entries)) + for id := range c.entries { + ids = append(ids, id) + } + sort.Sort(uint64Slice(ids)) + return ids } // Invalidate recalculates the entries by rank. @@ -228,7 +274,7 @@ func (c *rankCache) Invalidate() { func (c *rankCache) Recalculate() { c.mu.Lock() defer c.mu.Unlock() - c.stats.Count("cache.recalculate", 1, 1.0) + c.stats.Count(MetricRecalculateCache, 1, 1.0) c.recalculate() } @@ -236,27 +282,42 @@ func (c *rankCache) invalidate() { // Don't invalidate more than once every X seconds. // TODO: consider making this configurable. if time.Since(c.updateTime).Seconds() < 10 { + // Skipping recalculation means that the ranked cache's growth is unbounded. + // This is somewhat necessary for now since recalculation is not cheap. + // The cache will remain flagged as dirty and will be recalculated if Top is called. + // This may cause unexpected memory growth, so record it in metrics for debugging purposes. + c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0) + // Ensure that we're marked as dirty even if we weren't otherwise. + c.dirty = true return } - c.stats.Count("cache.invalidate", 1, 1.0) + c.stats.Count(MetricInvalidateCache, 1, 1.0) c.recalculate() } func (c *rankCache) recalculate() { + if c.rankingsRead { + c.rankings = nil + c.rankingsRead = false + } + // Convert cache to a sorted list. - rankings := make([]bitmapPair, 0, len(c.entries)) + rankings := c.rankings[:0] + if cap(rankings) < len(c.entries) { + rankings = make([]bitmapPair, 0, len(c.entries)) + } for id, cnt := range c.entries { rankings = append(rankings, bitmapPair{ ID: id, Count: cnt, }) } - sort.Sort(bitmapPairs(rankings)) + c.rankings = rankings + sort.Sort(&c.rankings) // Store the count of the item at the threshold index. - c.rankings = rankings length := len(c.rankings) - c.stats.Gauge("RankCache", float64(length), 1.0) + c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0) var removeItems []bitmapPair // cached, ordered list if length > int(c.maxEntries) { @@ -272,11 +333,14 @@ func (c *rankCache) recalculate() { // If size is larger than the threshold then trim it. if len(c.entries) > c.thresholdBuffer { - c.stats.Count("cache.threshold", 1, 1.0) + c.stats.Count(MetricCacheThresholdReached, 1, 1.0) for _, pair := range removeItems { delete(c.entries, pair.ID) } } + + // The cache is no longer dirty. + c.dirty = false } // SetStats defines the stats client used in the cache. @@ -285,7 +349,19 @@ func (c *rankCache) SetStats(s stats.StatsClient) { } // Top returns an ordered list of pairs. -func (c *rankCache) Top() []bitmapPair { return c.rankings } +func (c *rankCache) Top() []bitmapPair { + c.mu.Lock() + defer c.mu.Unlock() + + if c.dirty { + // The cache is dirty, so we need to recalculate it to get a consistent view. + c.stats.Count(MetricReadDirtyCache, 1, 1.0) + c.recalculate() + } + + c.rankingsRead = true + return c.rankings +} // WriteTo writes the cache to w. func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) { @@ -309,17 +385,68 @@ type bitmapPair struct { // bitmapPairs is a sortable list of BitmapPair objects. type bitmapPairs []bitmapPair -func (p bitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p bitmapPairs) Len() int { return len(p) } -func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } +func (p *bitmapPairs) Swap(i, j int) { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] } +func (p *bitmapPairs) Len() int { return len(*p) } +func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count } // Pair holds an id/count pair. type Pair struct { ID uint64 `json:"id"` - Key string `json:"key,omitempty"` + Key string `json:"key"` Count uint64 `json:"count"` } +// PairField is a Pair with its associated field. +type PairField struct { + Pair Pair + Field string +} + +func (p PairField) Clone() (r PairField) { + return PairField{ + Pair: p.Pair, + Field: p.Field, + } +} + +// ToTable implements the ToTabler interface. +func (p PairField) ToTable() (*pb.TableResponse, error) { + return pb.RowsToTable(p, 1) +} + +// ToRows implements the ToRowser interface. +func (p PairField) ToRows(callback func(*pb.RowResponse) error) error { + if p.Pair.Key != "" { + return callback(&pb.RowResponse{ + Headers: []*pb.ColumnInfo{ + {Name: p.Field, Datatype: "string"}, + {Name: "count", Datatype: "uint64"}, + }, + Columns: []*pb.ColumnResponse{ + {ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}}, + }, + }) + } else { + return callback(&pb.RowResponse{ + Headers: []*pb.ColumnInfo{ + {Name: p.Field, Datatype: "uint64"}, + {Name: "count", Datatype: "uint64"}, + }, + Columns: []*pb.ColumnResponse{ + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}}, + }, + }) + } +} + +// MarshalJSON marshals PairField into a JSON-encoded byte slice, +// excluding `Field`. +func (p PairField) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Pair) +} + // Pairs is a sortable slice of Pair objects. type Pairs []Pair @@ -395,6 +522,82 @@ func (p Pairs) String() string { return buf.String() } +// PairsField is a Pairs object with its associated field. +type PairsField struct { + Pairs []Pair + Field string +} + +func (p *PairsField) Clone() (r *PairsField) { + r = &PairsField{ + Pairs: make([]Pair, len(p.Pairs)), + Field: p.Field, + } + copy(r.Pairs, p.Pairs) + return +} + +// ToTable implements the ToTabler interface. +func (p *PairsField) ToTable() (*pb.TableResponse, error) { + return pb.RowsToTable(p, len(p.Pairs)) +} + +// ToRows implements the ToRowser interface. +func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error { + // Determine if the ID has string keys. + var stringKeys bool + if len(p.Pairs) > 0 { + if p.Pairs[0].Key != "" { + stringKeys = true + } + } + + dtype := "uint64" + if stringKeys { + dtype = "string" + } + ci := []*pb.ColumnInfo{ + {Name: p.Field, Datatype: dtype}, + {Name: "count", Datatype: "uint64"}, + } + for _, pair := range p.Pairs { + if stringKeys { + if err := callback(&pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + {ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}}, + }}); err != nil { + return errors.Wrap(err, "calling callback") + } + } else { + if err := callback(&pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}}, + }}); err != nil { + return errors.Wrap(err, "calling callback") + } + } + ci = nil //only send on the first + } + return nil +} + +// MarshalJSON marshals PairsField into a JSON-encoded byte slice, +// excluding `Field`. +func (p PairsField) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Pairs) +} + +// int64Slice represents a sortable slice of int64 numbers. +type int64Slice []int64 + +func (p int64Slice) Len() int { return len(p) } +func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 @@ -402,66 +605,6 @@ func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p uint64Slice) Len() int { return len(p) } func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } -// merge combines p and other to a unique sorted set of values. -// p and other must both have unique sets and be sorted. -func (p uint64Slice) merge(other []uint64) []uint64 { - ret := make([]uint64, 0, len(p)) - - i, j := 0, 0 - for i < len(p) && j < len(other) { - a, b := p[i], other[j] - if a == b { - ret = append(ret, a) - i, j = i+1, j+1 - } else if a < b { - ret = append(ret, a) - i++ - } else { - ret = append(ret, b) - j++ - } - } - - if i < len(p) { - ret = append(ret, p[i:]...) - } else if j < len(other) { - ret = append(ret, other[j:]...) - } - - return ret -} - -// bitmapCache provides an interface for caching full bitmaps. -type bitmapCache interface { - Fetch(id uint64) (*Row, bool) - Add(id uint64, b *Row) -} - -// simpleCache implements BitmapCache -// it is meant to be a short-lived cache for cases where writes are continuing to access -// the same row within a short time frame (i.e. good for write-heavy loads) -// A read-heavy use case would cause the cache to get bigger, potentially causing the -// node to run out of memory. -type simpleCache struct { - cache map[uint64]*Row -} - -// Fetch retrieves the bitmap at the id in the cache. -func (s *simpleCache) Fetch(id uint64) (*Row, bool) { - m, ok := s.cache[id] - return m, ok -} - -// Add adds the bitmap to the cache, keyed on the id. A nil row means -// deleting the row from the cache. -func (s *simpleCache) Add(id uint64, b *Row) { - if b != nil { - s.cache[id] = b - } else { - delete(s.cache, id) - } -} - // nopCache represents a no-op Cache implementation. type nopCache struct { stats stats.StatsClient @@ -481,6 +624,7 @@ func (c nopCache) Invalidate() {} func (c nopCache) Len() int { return 0 } func (c nopCache) Recalculate() {} func (c nopCache) SetStats(stats.StatsClient) {} +func (c nopCache) Clear() {} func (c nopCache) Top() []bitmapPair { return []bitmapPair{} diff --git a/cache_test.go b/cache_test.go index de1bd3a0b..29b150061 100644 --- a/cache_test.go +++ b/cache_test.go @@ -1,27 +1,16 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 package pilosa_test import ( + "reflect" "testing" - "github.com/pilosa/pilosa/v2" + pilosa "github.com/molecula/featurebase/v3" ) -// Ensure a bitmap query can be executed. -func TestCache_Rank(t *testing.T) { +// Ensure cache stays constrained to its configured size. +func TestCache_Rank_Size(t *testing.T) { cacheSize := uint32(3) cache := pilosa.NewRankCache(cacheSize) for i := 1; i < int(2*cacheSize); i++ { @@ -31,5 +20,66 @@ func TestCache_Rank(t *testing.T) { if cache.Len() != int(cacheSize) { t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize) } - +} + +// Ensure cache entries set below threshold are handled appropriately. +func TestCache_Rank_Threshold(t *testing.T) { + cacheSize := uint32(5) + cache := pilosa.NewRankCache(cacheSize) + for i := 1; i < int(2*cacheSize); i++ { + cache.Add(uint64(i), 3) + } + + // Set the cache value for rows 4 and 5 to a number below the threshold + // value (which is 3), and ensure that they gets zeroed out. + cache.Add(4, 1) + cache.BulkAdd(5, 1) + cache.Recalculate() + + if cache.Get(4) != 0 { + t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0) + } + if cache.Get(5) != 0 { + t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0) + } +} + +// Test that consecutive writes show up in Top. +// On later writes, the cache skips recalculation to save CPU time. +// This used to mean that the later writes would not show up in Top. +// Now, the cache is flagged as dirty and recalculated during the call to Top. +func TestCache_Rank_Dirty(t *testing.T) { + cacheSize := uint32(5) + cache := pilosa.NewRankCache(cacheSize) + + type pair struct{ ID, Count uint64 } + expect := []pair{ + {5, 2}, + {4, 1}, + } + + for _, v := range expect { + cache.Add(v.ID, v.Count) + } + + var got []pair + for _, p := range cache.Top() { + got = append(got, pair(p)) + } + + if !reflect.DeepEqual(expect, got) { + t.Fatalf("wrote %v but got %v", expect, got) + } +} + +func TestCache_Rank_BulkAdd(t *testing.T) { + const cacheSize = 10 + cache := pilosa.NewRankCache(uint32(cacheSize)) + + for i := uint64(0); i < 1000; i++ { + cache.BulkAdd(i, i) + if n := cache.Len(); n > cacheSize*2 { + t.Fatalf("entry count exceed 2x cache size: %d", n) + } + } } diff --git a/catcher.go b/catcher.go new file mode 100644 index 000000000..19caf53de --- /dev/null +++ b/catcher.go @@ -0,0 +1,248 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package pilosa + +import ( + "github.com/molecula/featurebase/v3/roaring" + txkey "github.com/molecula/featurebase/v3/short_txkey" + "github.com/molecula/featurebase/v3/vprint" +) + +// catcher is useful to report error locations with a +// Stack dump before the complexity +// of the executor_test swallows up +// the location of a PanicOn. +type catcherTx struct { + b Tx +} + +func newCatcherTx(b Tx) *catcherTx { + return &catcherTx{b: b} +} + +func init() { + // keep golangci-lint happy + _ = newCatcherTx +} + +var _ Tx = (*catcherTx)(nil) + +func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { + return c.b.NewTxIterator(index, field, view, shard) +} + +func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) +} + +func (c *catcherTx) Rollback() { + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + c.b.Rollback() +} + +func (c *catcherTx) Commit() error { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Commit() +} + +func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.RoaringBitmap(index, field, view, shard) +} + +func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Container(index, field, view, shard, key) +} + +func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.PutContainer(index, field, view, shard, key, rc) +} + +func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.RemoveContainer(index, field, view, shard, key) +} + +func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Add(index, field, view, shard, a...) +} + +func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Remove(index, field, view, shard, a...) +} + +func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Contains(index, field, view, shard, key) +} + +func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) +} + +func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.ForEach(index, field, view, shard, fn) +} + +func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.ForEachRange(index, field, view, shard, start, end, fn) +} + +func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Count(index, field, view, shard) +} + +func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Max(index, field, view, shard) +} + +func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.Min(index, field, view, shard) +} + +func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.CountRange(index, field, view, shard, start, end) +} + +func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) { + + defer func() { + if r := recover(); r != nil { + vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack()) + vprint.PanicOn(r) + } + }() + return c.b.OffsetRange(index, field, view, shard, offset, start, end) +} + +func (c *catcherTx) Type() string { + return c.b.Type() +} + +func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + return GenericApplyFilter(c, index, field, view, shard, ckey, filter) +} + +func (c *catcherTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) { + return c.b.ApplyRewriter(index, field, view, shard, ckey, filter) +} + +func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { + return c.b.GetSortedFieldViewList(idx, shard) +} + +func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) { + return 0, nil +} diff --git a/client.go b/client.go deleted file mode 100644 index 3b815a6bd..000000000 --- a/client.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "context" - "io" -) - -// Bit represents the intersection of a row and a column. It can be specified by -// integer ids or string keys. -type Bit struct { - RowID uint64 - ColumnID uint64 - RowKey string - ColumnKey string - Timestamp int64 -} - -// FieldValue represents the value for a column within a -// range-encoded field. -type FieldValue struct { - ColumnID uint64 - ColumnKey string - Value int64 -} - -// InternalClient should be implemented by any struct that enables any transport between nodes -// TODO: Refactor -// Note from Travis: Typically an interface containing more than two or three methods is an indication that -// something hasn't been architected correctly. -// While I understand that putting the entire Client behind an interface might require this many methods, -// I don't want to let it go unquestioned. -type InternalClient interface { - MaxShardByIndex(ctx context.Context) (map[string]uint64, error) - Schema(ctx context.Context) ([]*IndexInfo, error) - PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error - CreateIndex(ctx context.Context, index string, opt IndexOptions) error - FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) - Nodes(ctx context.Context) ([]*Node, error) - Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) - Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error - ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error - EnsureIndex(ctx context.Context, name string, options IndexOptions) error - EnsureField(ctx context.Context, indexName string, fieldName string) error - EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error - ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error - ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error - ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error - CreateField(ctx context.Context, index, field string) error - CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error - FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) - ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, uri *URI, msg []byte) error - RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error -} - -//=============== - -// InternalQueryClient is the internal interface for querying a node. -type InternalQueryClient interface { - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) -} - -type nopInternalQueryClient struct{} - -func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { - return nil, nil -} - -func newNopInternalQueryClient() *nopInternalQueryClient { - return &nopInternalQueryClient{} -} - -var _ InternalQueryClient = newNopInternalQueryClient() - -//=============== - -type nopInternalClient struct{} - -func newNopInternalClient() nopInternalClient { - return nopInternalClient{} -} - -var _ InternalClient = newNopInternalClient() - -func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) { - return nil, nil -} -func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } -func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error { - return nil -} - -func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { - return nil -} -func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { - return nil, nil -} -func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) { - return nil, nil -} -func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { - return nil, nil -} -func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { - return nil, nil -} -func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error { - return nil -} -func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error { - return nil -} -func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { - return nil -} -func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { - return nil -} -func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { - return nil -} -func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { - return nil -} -func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error { - return nil -} -func (n nopInternalClient) ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error { - return nil -} -func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { - return nil -} -func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } -func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { - return nil -} -func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { - return nil, nil -} -func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { - return nil, nil, nil -} -func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - return nil, nil -} -func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - return nil, nil -} -func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { - return nil -} -func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) { - return nil, nil -} diff --git a/client/README.md b/client/README.md new file mode 100644 index 000000000..6b1efebc7 --- /dev/null +++ b/client/README.md @@ -0,0 +1,85 @@ +# Go Client for Pilosa + +Go client for Pilosa high performance distributed index. + +## Usage + +If you have the pilosa repo in your `GOPATH`, +you can import the library in your code using: + +```go +import "github.com/pilosa/pilosa/v2/client" +``` + + +### Quick overview + +Assuming [Pilosa](https://github.com/pilosa/pilosa) server is running at `localhost:10101` (the default): + +```go +package main + +import ( + "fmt" + + "github.com/pilosa/pilosa/v2/client" +) + +func main() { + // Create the default client + cli := client.DefaultClient() + + // Retrieve the schema + schema, err := cli.Schema() + + // Create an Index object + myindex := schema.Index("myindex") + + // Create a Field object + myfield := myindex.Field("myfield") + + // make sure the index and the field exists on the server + err := cli.SyncSchema(schema) + + // Send a Set query. If err is non-nil, response will be nil. + response, err := cli.Query(myfield.Set(5, 42)) + + // Send a Row query. If err is non-nil, response will be nil. + response, err = cli.Query(myfield.Row(5)) + + // Get the result + result := response.Result() + // Act on the result + if result != nil { + columns := result.Row().Columns + fmt.Println("Got columns: ", columns) + } + + // You can batch queries to improve throughput + response, err = cli.Query(myindex.BatchQuery( + myfield.Row(5), + myfield.Row(10))) + if err != nil { + fmt.Println(err) + } + + for _, result := range response.Results() { + // Act on the result + fmt.Println(result.Row().Columns) + } +} +``` + +## Documentation + +### Data Model and Queries + +See: [Data Model and Queries](docs/data-model-queries.md) + +### Executing Queries + +See: [Server Interaction](docs/server-interaction.md) + +### Other Documentation + +* [Tracing](docs/tracing.md) diff --git a/client/batch.go b/client/batch.go new file mode 100644 index 000000000..b75301fbb --- /dev/null +++ b/client/batch.go @@ -0,0 +1,1778 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package client + +import ( + "bytes" + "math/bits" + "sort" + "sync" + "time" + + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/client/egpool" + "github.com/molecula/featurebase/v3/logger" + "github.com/molecula/featurebase/v3/roaring" + "github.com/pkg/errors" +) + +// Batch defaults. +const ( + DefaultKeyTranslateBatchSize = 100000 +) + +// TODO if using column translation, column ids might get way out of +// order. Could be worth sorting everything after translation (as an +// option?). Instead of sorting all simultaneously, it might be faster +// (more cache friendly) to sort ids and save the swap ops to apply to +// everything else that needs to be sorted. Note: we're already doing +// some sorting in importValueData and importMutexData, so if we +// implement it at the top level, remember to remove it there. + +// TODO support clearing values? nil values in records are ignored, +// but perhaps we could have a special type indicating that a bit or +// value should explicitly be cleared? + +// RecordBatch is a Pilosa ingest interface designed to allow for +// maximum throughput on common workloads. Users should call Add() +// with a Row object until it returns ErrBatchNowFull, at which time +// they should call Import(), and then repeat. +// +// Add will not modify or otherwise retain the Row once it returns, so +// it is recommended that callers reuse the same Row with repeated +// calls to Add, just modifying its values appropriately in between +// calls. This avoids allocating a new slice of Values for each +// inserted Row. +// +// The supported types of the values in Row.Values are implementation +// defined. Similarly, the supported types for Row.ID are +// implementation defined. +type RecordBatch interface { + Add(Row) error + + // Import does translation, creates the fragment files, and then, + // if we're not using split batch mode, imports everything to + // Pilosa. It then resets internal data structures for the next + // batch. If we are using split batch mode, it saves the fragment + // data to the batch, resets all other internal structures, and + // continues. + // Split batch mode DOES NOT CURRENTLY SUPPORT MUTEX OR INT FIELDS! + Import() error + + // Len reports the number of records which have been added to the + // batch since the last call to Import (or since it was created). + Len() int + + // Flush is only applicable in split batch mode where it actually + // imports the stored data to Pilosa. Otherwise it simply returns + // nil. + Flush() error +} + +// agedTranslation combines a translation with a recording of when it was last used. +type agedTranslation struct { + id uint64 + lastUsed uint64 +} + +// Batch implements RecordBatch. +// +// It supports Values of type string, uint64, int64, or nil. The +// following table describes what Pilosa field each type of value must +// map to. Fields are set up when calling "NewBatch". +// +// | type | pilosa field type | options | +// |--------+-------------------+-----------| +// | string | set | keys=true | +// | uint64 | set | any | +// | int64 | int | any | +// | float64| decimal | scale | +// | nil | any | | +// +// nil values are ignored. +type Batch struct { + client *Client + index *Index + header []*Field + headerMap map[string]*Field + + // prevDuration records the time that each doImport() takes. This + // is used to set the timeout for transactions to a reasonable + // value based on the last import. It starts with a conservative + // default set in NewBatch. + prevDuration time.Duration + + // ids is a slice of length batchSize of record IDs + ids []uint64 + + // rowIDs is a map of field index (in the header) to slices of + // length batchSize which contain row IDs. + rowIDs map[int][]uint64 + // clearRowIDs is a map[fieldIndex][idsIndex]rowID we don't expect + // clears to happen very often, so we store the idIndex/value + // mapping in a map rather than a slice as we do for rowIDs. This + // is a potentially temporary workaround to allow packed boolean + // fields to clear "false" values. Packed fields may be more + // completely supported by Pilosa in future. + clearRowIDs map[int]map[int]uint64 + + // rowIDSets is a map from field name to a batchSize slice of + // slices of row IDs. When a given record can have more than one + // value for a field, rowIDSets stores that information. + rowIDSets map[string][][]uint64 + + // values holds the values for each record of an int field + values map[string][]int64 + + // times holds a time for each record. (if any of the fields are time fields) + times []QuantizedTime + + // nullIndices holds a slice of indices into b.ids for each + // integer field which has nil values. + nullIndices map[string][]uint64 + + // TODO support bool fields. + + // for each field, keep a map of key to which record indexes that key mapped to + toTranslate map[int]map[string][]int + toTranslateClear map[int]map[string][]int + + // toTranslateSets is a map from field name to a map of string + // keys that need to be translated to sets of record indexes which + // those keys map to. + toTranslateSets map[string]map[string][]int + + // toTranslateID maps each string key to a record index - this + // will get translated into Batch.rowIDs + toTranslateID []string + + colTranslations map[string]agedTranslation + rowTranslations map[string]map[string]agedTranslation + cycle uint64 + maxAge uint64 + + // staleTime tracks the time the first record of the batch was inserted + // plus the maxStaleness, in order to raise ErrBatchNowStale if the + // maxStaleness has elapsed + staleTime time.Time + maxStaleness time.Duration + + // Maximum number of keys to translate at one time. + keyTranslateBatchSize int + + log logger.Logger + + // experimental — only used by FlushToFragments which is an + // alternative to Import which just builds the bitmap data for a + // batch without actually importing it. + splitBatchMode bool + frags fragments + clearFrags fragments + + useShardTransactionalEndpoint bool +} + +func (b *Batch) Len() int { return len(b.ids) } + +// BatchOption is a functional option for Batch objects. +type BatchOption func(b *Batch) error + +func OptLogger(l logger.Logger) BatchOption { + return func(b *Batch) error { + b.log = l + return nil + } +} + +func OptSplitBatchMode(on bool) BatchOption { + return func(b *Batch) error { + b.splitBatchMode = on + return nil + } +} + +func OptCacheMaxAge(age uint64) BatchOption { + return func(b *Batch) error { + b.maxAge = age + return nil + } +} + +func OptMaxStaleness(t time.Duration) BatchOption { + return func(b *Batch) error { + b.maxStaleness = t + return nil + } +} + +func OptKeyTranslateBatchSize(v int) BatchOption { + return func(b *Batch) error { + b.keyTranslateBatchSize = v + return nil + } +} + +// OptUseShardTransactionalEndpoint tells the batch to import using +// the newer shard-transactional endpoint. +func OptUseShardTransactionalEndpoint(use bool) BatchOption { + return func(b *Batch) error { + b.useShardTransactionalEndpoint = use + return nil + } +} + +// NewBatch initializes a new Batch object which will use the given +// Pilosa client, index, set of fields, and will take "size" records +// before returning ErrBatchNowFull. The positions of the Fields in +// 'fields' correspond to the positions of values in the Row's Values +// passed to Batch.Add(). +func NewBatch(client *Client, size int, index *Index, fields []*Field, opts ...BatchOption) (*Batch, error) { + if len(fields) == 0 || size == 0 { + return nil, errors.New("can't batch with no fields or batch size") + } + headerMap := make(map[string]*Field, len(fields)) + rowIDs := make(map[int][]uint64, len(fields)) + values := make(map[string][]int64) + tt := make(map[int]map[string][]int, len(fields)) + ttSets := make(map[string]map[string][]int) + hasTime := false + for i, field := range fields { + headerMap[field.Name()] = field + opts := field.Opts() + switch typ := opts.Type(); typ { + case FieldTypeDefault, FieldTypeSet, FieldTypeTime: + if opts.Keys() { + tt[i] = make(map[string][]int) + ttSets[field.Name()] = make(map[string][]int) + } + hasTime = typ == FieldTypeTime || hasTime + case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: + // tt line only needed if int field is string foreign key + tt[i] = make(map[string][]int) + values[field.Name()] = make([]int64, 0, size) + case FieldTypeMutex: + // similar to set/time fields, but no need to support sets + // of values (hence no ttSets) + if opts.Keys() { + tt[i] = make(map[string][]int) + } + rowIDs[i] = make([]uint64, 0, size) + default: + return nil, errors.Errorf("field type '%s' is not currently supported through Batch", typ) + } + } + + b := &Batch{ + client: client, + header: fields, + headerMap: headerMap, + prevDuration: time.Minute * 11, + index: index, + ids: make([]uint64, 0, size), + rowIDs: rowIDs, + clearRowIDs: make(map[int]map[int]uint64), + rowIDSets: make(map[string][][]uint64), + values: values, + nullIndices: make(map[string][]uint64), + toTranslate: tt, + toTranslateClear: make(map[int]map[string][]int), + toTranslateSets: ttSets, + colTranslations: make(map[string]agedTranslation), + rowTranslations: make(map[string]map[string]agedTranslation), + maxAge: 64, + maxStaleness: time.Duration(0), + keyTranslateBatchSize: DefaultKeyTranslateBatchSize, + + log: logger.NopLogger, + + frags: make(fragments), + clearFrags: make(fragments), + } + if hasTime { + b.times = make([]QuantizedTime, 0, size) + } + for _, opt := range opts { + err := opt(b) + if err != nil { + return nil, errors.Wrap(err, "applying options") + } + } + return b, nil +} + +// Row represents a single record which can be added to a Batch. +type Row struct { + ID interface{} + // Values map to the slice of fields in Batch.header + Values []interface{} + // Clears' int key is an index into Batch.header + Clears map[int]interface{} + // Time applies to all time fields + Time QuantizedTime +} + +// QuantizedTime represents a moment in time down to some granularity +// (year, month, day, or hour). +type QuantizedTime struct { + ymdh [10]byte +} + +// Set sets the Quantized time to the given timestamp (down to hour +// granularity). +func (qt *QuantizedTime) Set(t time.Time) { + copy(qt.ymdh[:], t.Format("2006010215")) +} + +// SetYear sets the quantized time's year, but leaves month, day, and +// hour untouched. +func (qt *QuantizedTime) SetYear(year string) { + copy(qt.ymdh[:4], year) +} + +// SetMonth sets the QuantizedTime's month, but leaves year, day, and +// hour untouched. +func (qt *QuantizedTime) SetMonth(month string) { + copy(qt.ymdh[4:6], month) +} + +// SetDay sets the QuantizedTime's day, but leaves year, month, and +// hour untouched. +func (qt *QuantizedTime) SetDay(day string) { + copy(qt.ymdh[6:8], day) +} + +// SetHour sets the QuantizedTime's hour, but leaves year, month, and +// day untouched. +func (qt *QuantizedTime) SetHour(hour string) { + copy(qt.ymdh[8:10], hour) +} + +func (qt *QuantizedTime) Time() (time.Time, error) { + return time.Parse("2006010215", string(qt.ymdh[:])) +} + +// Reset sets the time to the zero value which generates no time views. +func (qt *QuantizedTime) Reset() { + for i := range qt.ymdh { + qt.ymdh[i] = 0 + } +} + +// views builds the list of Pilosa views for this particular time, +// given a quantum. +func (qt *QuantizedTime) views(q TimeQuantum) ([]string, error) { + zero := QuantizedTime{} + if *qt == zero { + return nil, nil + } + views := make([]string, 0, len(q)) + for _, unit := range q { + switch unit { + case 'Y': + if qt.ymdh[0] == 0 { + return nil, errors.New("no data set for year") + } + views = append(views, string(qt.ymdh[:4])) + case 'M': + if qt.ymdh[4] == 0 { + return nil, errors.New("no data set for month") + } + views = append(views, string(qt.ymdh[:6])) + case 'D': + if qt.ymdh[6] == 0 { + return nil, errors.New("no data set for day") + } + views = append(views, string(qt.ymdh[:8])) + case 'H': + if qt.ymdh[8] == 0 { + return nil, errors.New("no data set for hour") + } + views = append(views, string(qt.ymdh[:10])) + } + } + return views, nil +} + +func (b *Batch) getColTranslation(key string) (uint64, bool) { + trans, ok := b.colTranslations[key] + if ok { + trans.lastUsed = b.cycle + b.colTranslations[key] = trans + } + return trans.id, ok +} + +func (b *Batch) getRowTranslation(field, key string) (uint64, bool) { + trans, ok := b.rowTranslations[field][key] + if ok { + trans.lastUsed = b.cycle + b.rowTranslations[field][key] = trans + } + return trans.id, ok +} + +// Add adds a record to the batch. Performance will be best if record +// IDs are shard-sorted. That is, all records which belong to the same +// Pilosa shard are added adjacent to each other. If the records are +// also in-order within a shard this will likely help as well. Add +// clears rec.Clears when it returns normally (either a nil error or +// BatchNowFull). +func (b *Batch) Add(rec Row) error { + // Clear recValues and rec.Clears upon return. + defer func() { + for i := range rec.Values { + rec.Values[i] = nil + } + for k := range rec.Clears { + delete(rec.Clears, k) + } + }() + + if len(b.ids) == cap(b.ids) { + return ErrBatchAlreadyFull + } + if len(rec.Values) != len(b.header) { + return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.header), len(rec.Values)) + } + + handleStringID := func(rid string) error { + if rid == "" { + return errors.Errorf("record identifier cannot be an empty string") + } + if colID, ok := b.getColTranslation(rid); ok { + b.ids = append(b.ids, colID) + } else { + if b.toTranslateID == nil { + b.toTranslateID = make([]string, cap(b.ids)) + } + b.toTranslateID[len(b.ids)] = rid + b.ids = append(b.ids, 0) + } + return nil + } + var err error + + switch rid := rec.ID.(type) { + case uint64: + b.ids = append(b.ids, rid) + case string: + err := handleStringID(rid) + if err != nil { + return err + } + case []byte: + err = handleStringID(string(rid)) + if err != nil { + return err + } + default: // TODO support nil ID as being auto-allocated. + return errors.Errorf("unsupported id type %T value %v", rid, rid) + } + + // curPos is the current position in b.ids, rowIDs[*], etc. + curPos := len(b.ids) - 1 + + if b.times != nil { + b.times = append(b.times, rec.Time) + } + + for i := 0; i < len(rec.Values); i++ { + field := b.header[i] + switch val := rec.Values[i].(type) { + case string: + if field.Opts().Type() != FieldTypeInt { + // nil-extend + for len(b.rowIDs[i]) < curPos { + b.rowIDs[i] = append(b.rowIDs[i], nilSentinel) + } + rowIDs := b.rowIDs[i] + // empty string is not a valid value at this point (Pilosa refuses to translate it) + if val == "" { // + b.rowIDs[i] = append(rowIDs, nilSentinel) + + } else if rowID, ok := b.getRowTranslation(field.Name(), val); ok { + b.rowIDs[i] = append(rowIDs, rowID) + } else { + ints, ok := b.toTranslate[i][val] + if !ok { + ints = make([]int, 0) + } + ints = append(ints, curPos) + b.toTranslate[i][val] = ints + b.rowIDs[i] = append(rowIDs, 0) + } + } else if field.Opts().Type() == FieldTypeInt { + if val == "" { + // copied from the `case nil:` section for ints and decimals + b.values[field.Name()] = append(b.values[field.Name()], 0) + nullIndices, ok := b.nullIndices[field.Name()] + if !ok { + nullIndices = make([]uint64, 0) + } + nullIndices = append(nullIndices, uint64(curPos)) + b.nullIndices[field.Name()] = nullIndices + } else if intVal, ok := b.getRowTranslation(field.Name(), val); ok { + b.values[field.Name()] = append(b.values[field.Name()], int64(intVal)) + } else { + ints, ok := b.toTranslate[i][val] + if !ok { + ints = make([]int, 0) + } + ints = append(ints, curPos) + b.toTranslate[i][val] = ints + b.values[field.Name()] = append(b.values[field.Name()], 0) + } + } + case uint64: + // nil-extend + for len(b.rowIDs[i]) < curPos { + b.rowIDs[i] = append(b.rowIDs[i], nilSentinel) + } + b.rowIDs[i] = append(b.rowIDs[i], val) + case int64: + b.values[field.Name()] = append(b.values[field.Name()], val) + case []string: + if len(val) == 0 { + continue + } + rowIDSets, ok := b.rowIDSets[field.Name()] + if !ok { + rowIDSets = make([][]uint64, len(b.ids)-1, cap(b.ids)) + b.rowIDSets[field.Name()] = rowIDSets + } + for len(rowIDSets) < len(b.ids)-1 { + rowIDSets = append(rowIDSets, nil) // nil extend + } + + rowIDs := make([]uint64, 0, len(val)) + for _, k := range val { + if k == "" { + continue + } + if rowID, ok := b.getRowTranslation(field.Name(), k); ok { + rowIDs = append(rowIDs, rowID) + } else { + ttsets, ok := b.toTranslateSets[field.Name()] + if !ok { + ttsets = make(map[string][]int) + b.toTranslateSets[field.Name()] = make(map[string][]int) + } + ints, ok := ttsets[k] + if !ok { + ints = make([]int, 0, 1) + } + ints = append(ints, curPos) + b.toTranslateSets[field.Name()][k] = ints + } + } + b.rowIDSets[field.Name()] = append(rowIDSets, rowIDs) + case []uint64: + if len(val) == 0 { + continue + } + rowIDSets, ok := b.rowIDSets[field.Name()] + if !ok { + rowIDSets = make([][]uint64, len(b.ids)-1, cap(b.ids)) + } + for len(rowIDSets) < len(b.ids)-1 { + rowIDSets = append(rowIDSets, nil) // nil extend + } + b.rowIDSets[field.Name()] = append(rowIDSets, val) + case nil: + t := field.Opts().Type() + if t == FieldTypeInt || t == FieldTypeDecimal || t == FieldTypeTimestamp { + b.values[field.Name()] = append(b.values[field.Name()], 0) + nullIndices, ok := b.nullIndices[field.Name()] + if !ok { + nullIndices = make([]uint64, 0) + } + nullIndices = append(nullIndices, uint64(curPos)) + b.nullIndices[field.Name()] = nullIndices + + } else { + // only append nil to rowIDs if this field already has + // rowIDs. Otherwise, this could be a []string or + // []uint64 field where we've only seen nil values so + // far. when we see a uint64 or string value, we'll + // "nil-extend" rowIDs to make sure it's the right + // length. + if rowIDs, ok := b.rowIDs[i]; ok { + b.rowIDs[i] = append(rowIDs, nilSentinel) + } + } + default: + return errors.Errorf("Val %v Type %[1]T is not currently supported. Use string, uint64 (row id), or int64 (integer value)", val) + } + } + + for i, uval := range rec.Clears { + field := b.header[i] + if _, ok := b.clearRowIDs[i]; !ok { + b.clearRowIDs[i] = make(map[int]uint64) + } + switch val := uval.(type) { + case string: + clearRows := b.clearRowIDs[i] + // translate val and add to clearRows + if rowID, ok := b.getRowTranslation(field.Name(), val); ok { + clearRows[curPos] = rowID + } else { + _, ok := b.toTranslateClear[i] + if !ok { + b.toTranslateClear[i] = make(map[string][]int) + } + ints, ok := b.toTranslateClear[i][val] + if !ok { + ints = make([]int, 0) + } + ints = append(ints, curPos) + b.toTranslateClear[i][val] = ints + } + case uint64: + b.clearRowIDs[i][curPos] = val + case nil: + if field.Opts().Type() == FieldTypeMutex { + for len(b.rowIDs[i]) <= curPos { + b.rowIDs[i] = append(b.rowIDs[i], nilSentinel) + } + b.rowIDs[i][len(b.rowIDs[i])-1] = clearSentinel + } + + default: + return errors.Errorf("Clearing a value '%v' Type %[1]T is not currently supported (field '%s')", val, field.Name()) + } + // nil extend b.rowIDs so we don't run into a horrible bug + // where we skip doing clears because b.rowIDs doesn't have a + // value for this field + for len(b.rowIDs[i]) <= curPos { + b.rowIDs[i] = append(b.rowIDs[i], nilSentinel) + } + + } + + if len(b.ids) == cap(b.ids) { + return ErrBatchNowFull + } + if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking + if len(b.ids) == 1 { + b.staleTime = time.Now().Add(b.maxStaleness) + } else if time.Now().After(b.staleTime) { + return ErrBatchNowStale + } + } + return nil +} + +// ErrBatchNowFull, similar to io.EOF, is a marker error to notify the +// user of a batch that it is time to call Import. +var ErrBatchNowFull = errors.New("batch is now full - you cannot add any more records (though the one you just added was accepted)") + +// ErrBatchAlreadyFull is a real error saying that Batch.Add did not +// complete because the batch was full. +var ErrBatchAlreadyFull = errors.New("batch was already full, record was rejected") + +// ErrBatchNowStale indicates that the oldest record in the batch is older than +// the maxStaleness value of the batch. Like ErrBatchNowFull, the error does +// not mean the record was rejected. +var ErrBatchNowStale = errors.New("batch is stale and needs to be imported (however, record was accepted)") + +// Import does translation, creates the fragment files, and then, +// if we're not using split batch mode, imports everything to +// Pilosa. It then resets internal data structures for the next +// batch. If we are using split batch mode, it saves the fragment +// data to the batch, resets all other internal structures, and +// continues. split batch mode DOES NOT CURRENTLY SUPPORT MUTEX +// OR INT FIELDS! +func (b *Batch) Import() error { + start := time.Now() + trns, err := b.client.StartTransaction("", b.prevDuration*10, false, time.Hour) + if err != nil { + return errors.Wrap(err, "starting transaction") + } + defer func() { + trnsl, err := b.client.FinishTransaction(trns.ID) + if err != nil { + b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl) + } + b.client.Stats.Timing(MetricBatchImportDurationSeconds, time.Since(start), 1.0) + }() + + size := len(b.ids) + transStart := time.Now() + // first we need to translate the toTranslate, then fill out the missing row IDs + err = b.doTranslation() + if err != nil { + return errors.Wrap(err, "doing Translation") + } + transTime := time.Now() + b.log.Printf("translating batch of %d took: %v", size, transTime.Sub(transStart)) + + frags, clearFrags, err := b.makeFragments(b.frags, b.clearFrags) + if err != nil { + return errors.Wrap(err, "making fragments (flush)") + } + if b.useShardTransactionalEndpoint { + // TODO handle bool? + frags, clearFrags, err = b.makeSingleValFragments(frags, clearFrags) + if err != nil { + return errors.Wrap(err, "making single val fragments") + } + } + + makeTime := time.Now() + b.log.Printf("making fragments for batch of %d took %v", size, makeTime.Sub(transTime)) + + if b.splitBatchMode { + b.frags = frags + b.clearFrags = clearFrags + } else { + b.frags = make(fragments) + b.clearFrags = make(fragments) + // create bitmaps out of each field in b.rowIDs and import. Also + // import int data. + if !b.useShardTransactionalEndpoint { + err = b.doImport(frags, clearFrags) + if err != nil { + return errors.Wrap(err, "doing import") + } + b.log.Printf("importing fragments took %v", time.Since(makeTime)) + } else { + err = b.doImportShardTransactional(frags, clearFrags) + if err != nil { + return errors.Wrap(err, "doing shard transactional import") + } + } + } + + b.reset() + return nil +} + +// Flush is only applicable in split batch mode where it actually +// imports the stored data to Pilosa. Otherwise it simply returns +// nil. +func (b *Batch) Flush() error { + if !b.splitBatchMode { + return nil + } + start := time.Now() + + trns, err := b.client.StartTransaction("", b.prevDuration*10, false, time.Hour) + if err != nil { + return errors.Wrap(err, "starting transaction") + } + defer func() { + trnsl, err := b.client.FinishTransaction(trns.ID) + if err != nil { + b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl) + } + b.client.Stats.Timing(MetricBatchFlushDurationSeconds, time.Since(start), 1.0) + }() + + importStart := time.Now() + err = b.doImport(b.frags, b.clearFrags) + if err != nil { + return errors.Wrap(err, "doing import (ImportFragments)") + } + + b.log.Debugf("superbatch import took %v", time.Since(importStart)) + + b.reset() + b.frags = make(fragments) + b.clearFrags = make(fragments) + return nil +} + +func (b *Batch) doTranslation() error { + eg := egpool.Group{PoolSize: 20} + + // Translate the column keys. + eg.Go(func() error { + // Deduplicate keys to translate. + dedup := make(map[string]struct{}) + var keys []string + for _, key := range b.toTranslateID { + if key == "" { + continue + } + + if _, ok := dedup[key]; ok { + continue + } + dedup[key] = struct{}{} + + keys = append(keys, key) + } + if len(keys) == 0 { + // There are no column keys to translate. + return nil + } + + // Create the keys. + start := time.Now() + trans, err := b.createIndexKeys(b.index, keys...) + if err != nil { + return errors.Wrap(err, "translating col keys") + } + if len(trans) != len(keys) { + return errors.Errorf("requested IDs for %d column keys but got %d back", len(keys), len(trans)) + } + b.log.Debugf("translating %d column keys took %v", len(keys), time.Since(start)) + + // Apply keys to translation cache. + for key, id := range trans { + b.colTranslations[key] = agedTranslation{ + id: id, + lastUsed: b.cycle, + } + } + + // Translate remaining keys in batch. + for index, ttkey := range b.toTranslateID { + if ttkey == "" { + continue + } + + b.ids[index] = trans[ttkey] + } + + return nil + }) + + // creating a lock up here for the rowCache(s) which we get + // below. Usually this isn't needed, but sometimes I think the + // same rowCache gets used repeatedly because the same field is in + // there multiple times, and that can lead to race + // conditions. Need to understand this better, but gonna see if + // this avoids the races. + rowCacheLock := &sync.Mutex{} + + // Translate the row keys. + for i, tt := range b.toTranslate { + // Skip this if there are no keys to translate. + ttc := b.toTranslateClear[i] + if len(tt) == 0 && len(ttc) == 0 { + continue + } + + // Look up the associated field. + field := b.header[i] + fieldName := field.Name() + + // Fetch the translation cache. + rowCache := b.rowTranslations[fieldName] + if rowCache == nil { + rowCache = make(map[string]agedTranslation) + b.rowTranslations[fieldName] = rowCache + } + + i, tt := i, tt + eg.Go(func() error { + // Collect the keys to translate. + keys := make([]string, 0, len(tt)+len(ttc)) + for k := range tt { + keys = append(keys, k) + } + for k := range ttc { + keys = append(keys, k) + } + + // Create the keys. + start := time.Now() + trans, err := b.createFieldKeys(field, keys...) + if err != nil { + return errors.Wrap(err, "translating field keys") + } + b.log.Debugf("translating %d field keys for %s took %v", len(trans), fieldName, time.Since(start)) + + // Apply keys to translation cache. + rowCacheLock.Lock() + for key, id := range trans { + rowCache[key] = agedTranslation{ + id: id, + lastUsed: b.cycle, + } + } + rowCacheLock.Unlock() + + switch ftype := field.Opts().Type(); ftype { + case FieldTypeSet, FieldTypeMutex, FieldTypeTime: + // Fill out missing IDs in local batch records with translated IDs. + rows := b.rowIDs[i] + for key, idxs := range tt { + id, ok := trans[key] + if !ok { + return errors.Errorf("key translation missing: %q in field %q", key, fieldName) + } + + for _, i := range idxs { + rows[i] = id + } + } + + // Fill out missing IDs in clear lists. + clearRows := b.clearRowIDs[i] + for key, idxs := range ttc { + id, ok := trans[key] + if !ok { + return errors.Errorf("key translation missing: %q in field %q", key, fieldName) + } + + for _, i := range idxs { + clearRows[i] = id + } + } + + case FieldTypeInt: + // Handle foreign key int fields — fill out b.values instead of b.rows. + vals := b.values[fieldName] + for key, idxs := range tt { + id, ok := trans[key] + if !ok { + return errors.Errorf("key translation missing: %q in field %q", key, fieldName) + } + + for _, i := range idxs { + vals[i] = int64(id) + } + } + + default: + return errors.Errorf("unexpected field type for translation: %q", ftype) + } + + return nil + }) + } + + for fieldName, tt := range b.toTranslateSets { + // Skip this if there are no keys to translate. + if len(tt) == 0 { + continue + } + + // Look up the associated field. + field := b.headerMap[fieldName] + + // Fetch the translation cache. + rowCache := b.rowTranslations[fieldName] + if rowCache == nil { + rowCache = make(map[string]agedTranslation) + b.rowTranslations[fieldName] = rowCache + } + + fieldName, tt := fieldName, tt + eg.Go(func() error { + // Collect the keys to translate. + keys := make([]string, 0, len(tt)) + for k := range tt { + keys = append(keys, k) + } + + // Create the keys. + start := time.Now() + trans, err := b.createFieldKeys(field, keys...) + if err != nil { + return errors.Wrap(err, "translating field keys") + } + b.log.Debugf("translating %d field keys for %s took %v", len(trans), fieldName, time.Since(start)) + + // Apply keys to translation cache. + rowCacheLock.Lock() + for key, id := range trans { + rowCache[key] = agedTranslation{ + id: id, + lastUsed: b.cycle, + } + } + rowCacheLock.Unlock() + + // Fill out missing IDs in local batch records with translated IDs. + rowIDSets := b.rowIDSets[fieldName] + for key, idxs := range tt { + id, ok := trans[key] + if !ok { + return errors.Errorf("key translation missing: %q in field %q", key, fieldName) + } + + for _, i := range idxs { + rowIDSets[i] = append(rowIDSets[i], id) + } + } + + return nil + }) + } + + return eg.Wait() +} + +func (b *Batch) createIndexKeys(index *Index, keys ...string) (map[string]uint64, error) { + batchSize := b.keyTranslateBatchSize + if batchSize <= 0 || len(keys) <= batchSize { + return b.client.CreateIndexKeys(index, keys...) + } + + results := make(map[string]uint64, len(keys)) + for len(keys) > 0 { + keySlice := keys + if len(keySlice) > batchSize { + keySlice = keySlice[:batchSize] + } + + trans, err := b.client.CreateIndexKeys(index, keySlice...) + if err != nil { + return nil, err + } else if len(trans) != len(keySlice) { + return nil, errors.Errorf("requested IDs for %d column keys but got %d back", len(keySlice), len(trans)) + } + for key, id := range trans { + results[key] = id + } + + keys = keys[len(keySlice):] + } + + return results, nil +} + +func (b *Batch) createFieldKeys(field *Field, keys ...string) (map[string]uint64, error) { + batchSize := b.keyTranslateBatchSize + if batchSize <= 0 || len(keys) <= batchSize { + return b.client.CreateFieldKeys(field, keys...) + } + + results := make(map[string]uint64, len(keys)) + for len(keys) > 0 { + keySlice := keys + if len(keySlice) > batchSize { + keySlice = keySlice[:batchSize] + } + + trans, err := b.client.CreateFieldKeys(field, keySlice...) + if err != nil { + return nil, err + } else if len(trans) != len(keySlice) { + return nil, errors.Errorf("requested IDs for %d row keys but got %d back", len(keySlice), len(trans)) + } + for key, id := range trans { + results[key] = id + } + + keys = keys[len(keySlice):] + } + + return results, nil +} + +func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error { + start := time.Now() + requests := make(map[uint64]*featurebase.ImportRoaringShardRequest) + getOrCreate := func(requests map[uint64]*featurebase.ImportRoaringShardRequest, shard uint64) *featurebase.ImportRoaringShardRequest { + request, ok := requests[shard] + if !ok { + request = &featurebase.ImportRoaringShardRequest{ + Remote: true, // the client will send to all replicas TODO probably rename before merge + Views: make([]featurebase.RoaringUpdate, 0, 1), + } + requests[shard] = request + } + return request + } + + for fragKey, viewMap := range frags { + request := getOrCreate(requests, fragKey.shard) + + for view, bitmap := range viewMap { + buf := &bytes.Buffer{} + _, err := bitmap.WriteTo(buf) + if err != nil { + return errors.Wrap(err, "serializing bitmap") + } + request.Views = append(request.Views, featurebase.RoaringUpdate{Field: fragKey.field, View: view, Set: buf.Bytes()}) + + // handle clear bitmap now if it exists so we don't have to go searching later + if clearVM := clearFrags.GetViewMap(fragKey.shard, fragKey.field); clearVM != nil { + if clearBitmap, ok := clearVM[view]; ok { + clearBuf := &bytes.Buffer{} + _, err := clearBitmap.WriteTo(clearBuf) + if err != nil { + return errors.Wrap(err, "serializing clear bitmap") + } + request.Views[len(request.Views)-1].Clear = clearBuf.Bytes() + // delete from clearFrags so any remaining we know for sure must be added new + clearFrags.DeleteView(fragKey.shard, fragKey.field, view) + } + } + } + } + + for fragKey, viewMap := range clearFrags { + request := getOrCreate(requests, fragKey.shard) + + for view, bitmap := range viewMap { + buf := &bytes.Buffer{} + _, err := bitmap.WriteTo(buf) + if err != nil { + return errors.Wrap(err, "serializing bitmap") + } + request.Views = append(request.Views, featurebase.RoaringUpdate{Field: fragKey.field, View: view, Clear: buf.Bytes()}) + } + } + + b.client.Stats.Timing(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0) + start = time.Now() + eg := egpool.Group{PoolSize: 20} + for shard, request := range requests { + shard := shard + request := request + eg.Go(func() error { + return b.client.ImportRoaringShard(b.index.Name(), shard, request) + }) + } + err := eg.Wait() + dur := time.Since(start) + b.client.Stats.Timing(MetricBatchShardImportDurationSeconds, dur, 1.0) + b.log.Printf("import shard took: %v\n", dur) + return errors.Wrap(err, "doing shard-transactional imports") +} + +func (b *Batch) doImport(frags, clearFrags fragments) error { + + start := time.Now() + eg := egpool.Group{PoolSize: 20} + // TODO, currently this relies on upstream behavior of + // makeFragments to guarantee that any shard/field combination in + // clearFrags also has a shard/field in frags. We're only + // iterating over frags and then checking to see if clearFrags has + // the same keys. If we optimized makeFragments to skip adding + // things to frags which had no set bits (e.g. if we were only + // clearing things), then this code would need to be updated to + // ensure that it looked at the things in clearFrags which were + // *not* in frags. + for fragmentKey, viewMap := range frags { + field := fragmentKey.field + shard := fragmentKey.shard + viewMap := viewMap + + eg.Go(func() error { + clearViewMap := clearFrags.GetViewMap(shard, field) + if len(clearViewMap) > 0 { + startx := time.Now() + err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, clearViewMap, true) + if err != nil { + return errors.Wrapf(err, "import clearing clearing data for %s", field) + } + b.log.Debugf("imp-roar-clr %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(startx)) + } + + starty := time.Now() + err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, viewMap, false) + b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(starty)) + return errors.Wrapf(err, "importing data for %s", field) + }) + } + eg.Go(func() error { return b.importValueData() }) + eg.Go(func() error { return b.importMutexData() }) + + err := eg.Wait() + if err != nil { + if pferr := anyCause(ErrPreconditionFailed, eg.Errors()...); pferr != nil { + return pferr + } + return err + } + b.prevDuration = time.Since(start) + return nil +} + +func anyCause(cause error, errs ...error) error { + if cause == nil { + return nil + } + + for _, err := range errs { + if errors.Cause(err) == cause { + return err + } + } + return nil +} + +func (b *Batch) shardWidth() uint64 { + shardWidth := b.index.ShardWidth() + if shardWidth == 0 { + shardWidth = DefaultShardWidth + } + return shardWidth +} + +// this is kind of bad as it means we can never import column id +// ^uint64(0) which is a valid column ID. I think it's unlikely to +// matter much in practice (we could maybe special case it somewhere +// if needed though). +var nilSentinel = ^uint64(0) + +// clearSentinel indicates that we're trying to clear all values for +// this field of this record +var clearSentinel = nilSentinel - 1 + +func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments, error) { + shardWidth := b.shardWidth() + emptyClearRows := make(map[int]uint64) + + // create _exists fragments if needed + if b.index.Opts().TrackExistence() { + var curBM *roaring.Bitmap + curShard := ^uint64(0) // impossible sentinel value for shard. + for _, col := range b.ids { + if col/shardWidth != curShard { + curShard = col / shardWidth + curBM = frags.GetOrCreate(curShard, "_exists", "") + } + curBM.DirectAdd(col % shardWidth) + } + } + + for i, rowIDs := range b.rowIDs { + if len(rowIDs) == 0 { + continue // this can happen when the values that came in for this field were string slices + } + clearRows := b.clearRowIDs[i] + if clearRows == nil { + clearRows = emptyClearRows + } + field := b.header[i] + opts := field.Opts() + if opts.Type() == FieldTypeMutex { + continue // we handle mutex fields separately — they can't use importRoaring + } + curShard := ^uint64(0) // impossible sentinel value for shard. + var curBM *roaring.Bitmap + var clearBM *roaring.Bitmap + for j := range b.ids { + col := b.ids[j] + row := nilSentinel + if len(rowIDs) > j { + // this is to protect against what i believe is a bug in the idk.DeleteSentinel logic in handling nil entries + // this will prevent a crash by assuming missing entries are nil entries which i think is ok + // TODO (twg) find where the nil entry was not added on the idk side ~ingest.go batchFromSchema method + row = rowIDs[j] + } + + if col/shardWidth != curShard { + curShard = col / shardWidth + curBM = frags.GetOrCreate(curShard, field.Name(), "") + clearBM = clearFrags.GetOrCreate(curShard, field.Name(), "") + } + if row != nilSentinel { + // TODO this is super ugly, but we want to avoid setting + // bits on the standard view in the specific case when + // there isn't one. Should probably refactor this whole + // loop to be more general w.r.t. views. Also... tests for + // the NoStandardView case would be great. + if !(opts.Type() == FieldTypeTime && opts.NoStandardView()) { + curBM.DirectAdd(row*shardWidth + (col % shardWidth)) + } + if opts.Type() == FieldTypeTime { + views, err := b.times[j].views(opts.TimeQuantum()) + if err != nil { + return nil, nil, errors.Wrap(err, "calculating views") + } + for _, view := range views { + tbm := frags.GetOrCreate(curShard, field.Name(), view) + tbm.DirectAdd(row*shardWidth + (col % shardWidth)) + } + } + } + + clearRow, ok := clearRows[j] + if ok { + clearBM.DirectAddN(clearRow*shardWidth + (col % shardWidth)) + // we're going to execute the clear before the set, so + // we want to make sure that at this point, the "set" + // fragments don't contain the bit that we're clearing + curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth)) + } + } + } + + for fname, rowIDSets := range b.rowIDSets { + if len(rowIDSets) == 0 { + continue + } else if len(rowIDSets) < len(b.ids) { + // rowIDSets is guaranteed to have capacity == to b.ids, + // but if the last record had a nil for this field, it + // might not have the same length, so we re-slice it to + // ensure the lengths are the same. + rowIDSets = rowIDSets[:len(b.ids)] + } + field := b.headerMap[fname] + opts := field.Opts() + curShard := ^uint64(0) // impossible sentinel value for shard. + var curBM *roaring.Bitmap + for j := range b.ids { + col, rowIDs := b.ids[j], rowIDSets[j] + if len(rowIDs) == 0 { + continue + } + if col/shardWidth != curShard { + curShard = col / shardWidth + curBM = frags.GetOrCreate(curShard, fname, "") + } + // TODO this is super ugly, but we want to avoid setting + // bits on the standard view in the specific case when + // there isn't one. Should probably refactor this whole + // loop to be more general w.r.t. views. Also... tests for + // the NoStandardView case would be great. + if !(opts.Type() == FieldTypeTime && opts.NoStandardView()) { + for _, row := range rowIDs { + curBM.DirectAdd(row*shardWidth + (col % shardWidth)) + } + } + if opts.Type() == FieldTypeTime { + views, err := b.times[j].views(opts.TimeQuantum()) + if err != nil { + return nil, nil, errors.Wrap(err, "calculating views") + } + for _, view := range views { + tbm := frags.GetOrCreate(curShard, fname, view) + for _, row := range rowIDs { + tbm.DirectAdd(row*shardWidth + (col % shardWidth)) + } + } + } + } + } + return frags, clearFrags, nil +} + +func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments, fragments, error) { + shardWidth := b.shardWidth() + ids := make([]uint64, len(b.ids)) + + // ------------------------- + // int-like fields + // ------------------------- + for fieldName, bvalues := range b.values { + ids = ids[:len(b.ids)] + + // trim out null values from ids and values. + nullIndices := b.nullIndices[fieldName] + + i, n := uint64(0), 0 + for _, nullIndex := range nullIndices { + copy(ids[n:], b.ids[i:nullIndex]) + n += copy(bvalues[n:], bvalues[i:nullIndex]) + i = nullIndex + 1 + } + + copy(ids[n:], b.ids[i:]) + n += copy(bvalues[n:], bvalues[i:]) + ids, bvalues = ids[:n], bvalues[:n] + + if len(ids) == 0 { + continue + } + + sc := &valsByIDsSortable{ids: ids, vals: bvalues, width: shardWidth} + if !sort.IsSorted(sc) { + sort.Stable(sc) + } + field := b.headerMap[fieldName] + base := field.Options().base + if field.Options().Type() == FieldTypeTimestamp { + base = 0 + } + + shard := ids[0] / shardWidth + bitmap := frags.GetOrCreate(shard, fieldName, "bsig_"+fieldName) // TODO... grab bsig_ prefix from elsewhere + for i, id := range ids { + if i+1 < len(ids) { + // we only want the last value set for each id + if ids[i+1] == id { + continue + } + } + if shard != id/shardWidth { + shard = id / shardWidth + bitmap = frags.GetOrCreate(shard, fieldName, "bsig_"+fieldName) + } + fragmentColumn := id % shardWidth + bitmap.Add(fragmentColumn) // existence bit + svalue := bvalues[i] - base + negative := svalue < 0 + var value uint64 + if negative { + bitmap.Add(shardWidth + fragmentColumn) // set sign bit + value = uint64(svalue * -1) + } else { + value = uint64(svalue) + } + lz := bits.LeadingZeros64(value) + row := uint64(2) + for mask := uint64(0x1); mask <= 1<<(64-lz) && mask != 0; mask = mask << 1 { + if value&mask > 0 { + bitmap.Add(row*shardWidth + fragmentColumn) + } + row++ + } + } + } + + // ------------------------- + // mutex fields + // ------------------------- + for findex, rowIDs := range b.rowIDs { + field := b.header[findex] + if field.Opts().Type() != FieldTypeMutex { + continue + } + ids = ids[:0] + + // get slice of column ids for non-nil rowIDs and cut nil row + // IDs out of rowIDs. + idsIndex := 0 + for i, id := range b.ids { + rowID := rowIDs[i] + if rowID == nilSentinel { + continue + } + rowIDs[idsIndex] = rowID + ids = append(ids, id) + idsIndex++ + } + rowIDs = rowIDs[:idsIndex] + + if len(ids) == 0 { + continue + } + + sc := &rowsByIDsSortable{ids: ids, rows: rowIDs, width: shardWidth} + if !sort.IsSorted(sc) { + sort.Stable(sc) + } + + shard := ids[0] / shardWidth + bitmap := frags.GetOrCreate(shard, field.Name(), "standard") + clearBM := clearFrags.GetOrCreate(shard, field.Name(), "standard") + for i, id := range ids { + if i+1 < len(ids) { + // we only want the last value set for each id + if ids[i+1] == id { + continue + } + } + row := rowIDs[i] + if shard != id/shardWidth { + shard = id / shardWidth + bitmap = frags.GetOrCreate(shard, field.Name(), "standard") + clearBM = clearFrags.GetOrCreate(shard, field.Name(), "standard") + } + fragmentColumn := id % shardWidth + clearBM.Add(fragmentColumn) // Will use this to clear columns. + if row != clearSentinel { + // clearSentinel is used for deletion + // so this value should only be added if its not clearSentinel + bitmap.Add(row*shardWidth + fragmentColumn) + } + } + } + + return frags, clearFrags, nil +} + +type valsByIDsSortable struct { + ids []uint64 + vals []int64 + // shard width so we can compare by shard instead of ID + width uint64 +} + +func (v *valsByIDsSortable) Len() int { return len(v.ids) } + +func (v *valsByIDsSortable) Less(i, j int) bool { return v.ids[i] < v.ids[j] } +func (v *valsByIDsSortable) Swap(i, j int) { + v.ids[i], v.ids[j] = v.ids[j], v.ids[i] + v.vals[i], v.vals[j] = v.vals[j], v.vals[i] +} + +// importValueData imports data for int fields. +func (b *Batch) importValueData() error { + shardWidth := b.index.ShardWidth() + if shardWidth == 0 { + shardWidth = DefaultShardWidth + } + eg := egpool.Group{PoolSize: 20} + + ids := make([]uint64, len(b.ids)) + for fieldName, bvalues := range b.values { + ids = ids[:len(b.ids)] + + // trim out null values from ids and values. + nullIndices := b.nullIndices[fieldName] + + i, n := uint64(0), 0 + for _, nullIndex := range nullIndices { + copy(ids[n:], b.ids[i:nullIndex]) + n += copy(bvalues[n:], bvalues[i:nullIndex]) + i = nullIndex + 1 + } + + copy(ids[n:], b.ids[i:]) + n += copy(bvalues[n:], bvalues[i:]) + ids, bvalues = ids[:n], bvalues[:n] + + // now do imports by shard + if len(ids) == 0 { + continue // TODO test this "all nil" case + } + + sc := &valsByIDsSortable{ids: ids, vals: bvalues, width: shardWidth} + if !sort.IsSorted(sc) { + sort.Stable(sc) // TODO(jaffee) this was sort.Sort which I think is a bug. If we get multiple of the same record w/in a batch with different int values, the last one needs to win. We need a test for this. + } + + curShard := ids[0] / shardWidth + startIdx := 0 + for i := 1; i <= len(ids); i++ { + var recordID uint64 + if i < len(ids) { + recordID = ids[i] + } else { + recordID = (curShard + 2) * shardWidth + } + + if recordID/shardWidth != curShard { + endIdx := i + shard := curShard + field := b.headerMap[fieldName] + path, data, err := b.client.EncodeImportValues(field, shard, bvalues[startIdx:endIdx], ids[startIdx:endIdx], false) + if err != nil { + return errors.Wrap(err, "encoding import values") + } + eg.Go(func() error { + start := time.Now() + err := b.client.DoImportValues(b.index.Name(), shard, path, data) + b.log.Debugf("imp-vals %s,shard:%d,data:%d %v", field, shard, len(data), time.Since(start)) + return errors.Wrapf(err, "importing values for field = %s", field) + }) + startIdx = i + curShard = recordID / shardWidth + } + } + } + err := eg.Wait() + if err != nil { + if pferr := anyCause(ErrPreconditionFailed, eg.Errors()...); pferr != nil { + return pferr + } + return err + } + return errors.Wrap(err, "importing value data") +} + +type rowsByIDsSortable struct { + ids []uint64 + rows []uint64 + // shard width so we can compare by shard instead of ID + width uint64 +} + +func (v *rowsByIDsSortable) Len() int { return len(v.ids) } + +func (v *rowsByIDsSortable) Less(i, j int) bool { return v.ids[i] < v.ids[j] } +func (v *rowsByIDsSortable) Swap(i, j int) { + v.ids[i], v.ids[j] = v.ids[j], v.ids[i] + v.rows[i], v.rows[j] = v.rows[j], v.rows[i] +} + +// TODO this should work for bools as well - just need to support them +// at batch creation time and when calling Add, I think. +func (b *Batch) importMutexData() error { + shardWidth := b.index.ShardWidth() + if shardWidth == 0 { + shardWidth = DefaultShardWidth + } + + eg := egpool.Group{PoolSize: 20} + ids := make([]uint64, 0, len(b.ids)) + for findex, rowIDs := range b.rowIDs { + field := b.header[findex] + if field.Opts().Type() != FieldTypeMutex { + continue + } + ids = ids[:0] + + // get slice of column ids for non-nil rowIDs and cut nil row + // IDs out of rowIDs. + idsIndex := 0 + for i, id := range b.ids { + rowID := rowIDs[i] + if rowID == nilSentinel { + continue + } + rowIDs[idsIndex] = rowID + ids = append(ids, id) + idsIndex++ + } + rowIDs = rowIDs[:idsIndex] + + if len(ids) == 0 { + continue + } + + sc := &rowsByIDsSortable{ids: ids, rows: rowIDs, width: shardWidth} + if !sort.IsSorted(sc) { + sort.Stable(sc) + } + + curShard := ids[0] / shardWidth + startIdx := 0 + for i := 1; i <= len(ids); i++ { + var recordID uint64 + if i < len(ids) { + recordID = ids[i] + } else { + recordID = (curShard + 2) * shardWidth + } + + if recordID/shardWidth != curShard { + endIdx := i + shard := curShard + field := field + path, data, err := b.client.EncodeImport(field, shard, rowIDs[startIdx:endIdx], ids[startIdx:endIdx], false) + if err != nil { + return errors.Wrap(err, "encoding mutex import") + } + eg.Go(func() error { + start := time.Now() + err := b.client.DoImport(b.index.Name(), shard, path, data) + b.log.Debugf("imp-mux %s,shard:%d,data:%d %v", field.Name(), shard, len(data), time.Since(start)) + return errors.Wrapf(err, "importing values for field = %s", field) + }) + startIdx = i + curShard = recordID / shardWidth + } + } + } + err := eg.Wait() + if err != nil { + if pferr := anyCause(ErrPreconditionFailed, eg.Errors()...); pferr != nil { + return pferr + } + return err + } + return errors.Wrap(err, "importing mutex data") +} + +// reset is called at the end of importing to ready the batch for the +// next round. Where possible it does not re-allocate memory. +func (b *Batch) reset() { + b.ids = b.ids[:0] + b.times = b.times[:0] + for i, rowIDs := range b.rowIDs { + b.rowIDs[i] = rowIDs[:0] + } + for _, tt := range b.toTranslate { + for k := range tt { + delete(tt, k) // TODO pool these slices + } + } + for _, tts := range b.toTranslateSets { + for k := range tts { + delete(tts, k) + } + } + for field, rowIDSet := range b.rowIDSets { + for i := range rowIDSet { + rowIDSet[i] = nil + } + b.rowIDSets[field] = rowIDSet[:0] + } + for _, rowIDs := range b.clearRowIDs { + for k := range rowIDs { + delete(rowIDs, k) + } + } + for _, clearMap := range b.toTranslateClear { + for k := range clearMap { + delete(clearMap, k) + } + } + for i := range b.toTranslateID { + b.toTranslateID[i] = "" + } + for k := range b.values { + delete(b.values, k) // TODO pool these slices + } + for k := range b.nullIndices { + delete(b.nullIndices, k) // TODO pool these slices + } + b.cycle++ + for k, trans := range b.colTranslations { + if trans.lastUsed-b.cycle > b.maxAge { + delete(b.colTranslations, k) + } + } + for field, rowTranslations := range b.rowTranslations { + for k, trans := range rowTranslations { + if trans.lastUsed-b.cycle > b.maxAge { + delete(rowTranslations, k) + } + } + + if len(rowTranslations) == 0 { + delete(b.rowTranslations, field) + } + } +} + +// map[shard][field][view]fragmentData +type fragments map[fragmentKey]map[string]*roaring.Bitmap + +type fragmentKey struct { + shard uint64 + field string +} + +func (f fragments) GetOrCreate(shard uint64, field, view string) *roaring.Bitmap { + key := fragmentKey{shard, field} + viewMap, ok := f[key] + if !ok { + viewMap = make(map[string]*roaring.Bitmap) + f[key] = viewMap + } + bm, ok := viewMap[view] + if !ok { + bm = roaring.NewBTreeBitmap() + viewMap[view] = bm + } + return bm +} + +func (f fragments) GetViewMap(shard uint64, field string) map[string]*roaring.Bitmap { + key := fragmentKey{shard, field} + viewMap, ok := f[key] + if !ok { + return nil + } + // Remove any views which have an empty bitmap. + // TODO: Ideally we would prevent allocating the empty bitmap to begin with, + // but the logic is a bit tricky, and since we don't want to spend too much + // time on it right now, we're leaving that for a future exercise. + for k, v := range viewMap { + if v.Count() == 0 { + delete(viewMap, k) + } + } + return viewMap +} + +func (f fragments) DeleteView(shard uint64, field, view string) { + vm := f.GetViewMap(shard, field) + if vm == nil { + return + } + delete(vm, view) +} diff --git a/client/batch_test.go b/client/batch_test.go new file mode 100644 index 000000000..b7a968848 --- /dev/null +++ b/client/batch_test.go @@ -0,0 +1,1841 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "fmt" + "math/rand" + "reflect" + "sort" + "strconv" + "testing" + "time" + + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/test" + + "github.com/pkg/errors" +) + +func NewTestClient(t *testing.T, c *test.Cluster) *Client { + client, err := NewClient(c.Nodes[0].URL()) + if err != nil { + t.Fatal(err) + } + return client +} + +func TestAgainstCluster(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + client := NewTestClient(t, c) + t.Run("string-slice-combos", func(t *testing.T) { testStringSliceCombos(t, c, client) }) + t.Run("import-batch-ints", func(t *testing.T) { testImportBatchInts(t, c, client) }) + t.Run("import-batch-sorting", func(t *testing.T) { testImportBatchSorting(t, c, client) }) + t.Run("test-trim-null", func(t *testing.T) { testTrimNull(t, c, client) }) + t.Run("test-string-slice-empty-and-nil", func(t *testing.T) { testStringSliceEmptyAndNil(t, c, client) }) + t.Run("test-string-slice", func(t *testing.T) { testStringSlice(t, c, client) }) + t.Run("test-single-clear-batch-regression", func(t *testing.T) { testSingleClearBatchRegression(t, c, client) }) + t.Run("test-batches", func(t *testing.T) { testBatches(t, c, client) }) + t.Run("batches-strings-ids", func(t *testing.T) { testBatchesStringIDs(t, c, client) }) + t.Run("test-batch-staleness", func(t *testing.T) { testBatchStaleness(t, c, client) }) + t.Run("test-import-batch-multiple-ints", func(t *testing.T) { testImportBatchMultipleInts(t, c, client) }) + t.Run("test-import-batch-multiple-timestamps", func(t *testing.T) { testImportBatchMultipleTimestamps(t, c, client) }) + t.Run("test-import-batch-sets-clears", func(t *testing.T) { testImportBatchSetsAndClears(t, c, client) }) + t.Run("test-topn-cache-regression", func(t *testing.T) { testTopNCacheRegression(t, c, client) }) + t.Run("test-multiple-int-same-batch", func(t *testing.T) { testMultipleIntSameBatch(t, c, client) }) + t.Run("test-mutex-clearing-regression", func(t *testing.T) { mutexClearRegression(t, c, client) }) + t.Run("test-mutex-nil-clear-id", func(t *testing.T) { mutexNilClearID(t, c, client) }) + t.Run("test-mutex-nil-clear-key", func(t *testing.T) { mutexNilClearKey(t, c, client) }) +} + +func testStringSliceCombos(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-string-slice-combos") + fields := make([]*Field, 1) + fields[0] = idx.Field("a1", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 5, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + + records := []Row{ + {ID: uint64(0), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(1), Values: []interface{}{[]string{"z"}}}, + {ID: uint64(2), Values: []interface{}{[]string{}}}, + {ID: uint64(3), Values: []interface{}{[]string{"q", "r", "s", "t", "c"}}}, + {ID: uint64(4), Values: []interface{}{nil}}, + {ID: uint64(5), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(6), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(7), Values: []interface{}{[]string{"z"}}}, + {ID: uint64(8), Values: []interface{}{[]string{}}}, + {ID: uint64(9), Values: []interface{}{[]string{"q", "r", "s", "t"}}}, + {ID: uint64(10), Values: []interface{}{nil}}, + {ID: uint64(11), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(12), Values: []interface{}{[]string{}}}, + {ID: uint64(13), Values: []interface{}{[]string{}}}, + } + + err = ingestRecords(records, b) + if err != nil { + t.Fatalf("importing: %v", err) + } + + a1 := fields[0] + + result := tq(t, client, a1.TopN(10)) + rez := sortableCRI(result.CountItems()) + sort.Sort(rez) + exp := sortableCRI{ + {Key: "a", Count: 4}, + {Key: "b", Count: 4}, + {Key: "c", Count: 5}, + {Key: "q", Count: 2}, + {Key: "r", Count: 2}, + {Key: "s", Count: 2}, + {Key: "t", Count: 2}, + {Key: "z", Count: 2}, + } + sort.Sort(exp) + errorIfNotEqual(t, exp, rez) + + result = tq(t, client, a1.Row("a")) + errorIfNotEqual(t, result.Row().Columns, []uint64{0, 5, 6, 11}) + result = tq(t, client, a1.Row("b")) + errorIfNotEqual(t, result.Row().Columns, []uint64{0, 5, 6, 11}) + result = tq(t, client, a1.Row("c")) + errorIfNotEqual(t, result.Row().Columns, []uint64{0, 3, 5, 6, 11}) + result = tq(t, client, a1.Row("z")) + errorIfNotEqual(t, result.Row().Columns, []uint64{1, 7}) + result = tq(t, client, a1.Row("q")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + result = tq(t, client, a1.Row("r")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + result = tq(t, client, a1.Row("s")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + result = tq(t, client, a1.Row("t")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + + result = tq(t, client, idx.RawQuery("Count(All())")) + errorIfNotEqual(t, result.Count(), int64(14)) +} + +func errorIfNotEqual(t *testing.T, exp, got interface{}) { + t.Helper() + if !reflect.DeepEqual(exp, got) { + t.Errorf("unequal exp/got:\n%v\n%v", exp, got) + } +} + +type sortableCRI []CountResultItem + +func (s sortableCRI) Len() int { return len(s) } +func (s sortableCRI) Less(i, j int) bool { + if s[i].Count != s[j].Count { + return s[i].Count > s[j].Count + } + if s[i].ID != s[j].ID { + return s[i].ID < s[j].ID + } + if s[i].Key != s[j].Key { + return s[i].Key < s[j].Key + } + return true +} +func (s sortableCRI) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func tq(t *testing.T, client *Client, query PQLQuery) QueryResult { + resp, err := client.Query(query) + if err != nil { + t.Fatalf("querying: %v", err) + } + return resp.Results()[0] +} + +func ingestRecords(records []Row, batch *Batch) error { + for _, rec := range records { + err := batch.Add(rec) + if err == ErrBatchNowFull { + err = batch.Import() + if err != nil { + return errors.Wrap(err, "importing batch") + } + } else if err != nil { + return errors.Wrap(err, "while adding record") + } + } + if batch.Len() > 0 { + err := batch.Import() + if err != nil { + return errors.Wrap(err, "importing batch") + } + } + return nil +} + +func testImportBatchInts(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-import-batch-ints") + field := idx.Field("anint", OptFieldTypeInt()) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 3, idx, []*Field{field}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + r := Row{Values: make([]interface{}, 1)} + + for i := uint64(0); i < 3; i++ { + r.ID = i + r.Values[0] = int64(i) + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + r.ID = uint64(0) + r.Values[0] = nil + err = b.Add(r) + if err != nil { + t.Fatalf("adding after import: %v", err) + } + r.ID = uint64(1) + r.Values[0] = int64(7) + err = b.Add(r) + if err != nil { + t.Fatalf("adding second after import: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("second import: %v", err) + } + + resp, err := client.Query(idx.BatchQuery(field.Equals(0), field.Equals(7), field.Equals(2))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + for i, result := range resp.Results() { + if !reflect.DeepEqual(result.Row().Columns, []uint64{uint64(i)}) { + t.Errorf("expected %v for %d, but got %v", []uint64{uint64(i)}, i, result.Row().Columns) + } + } +} + +func testImportBatchSorting(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-import-batch-sorting") + field := idx.Field("anint", OptFieldTypeInt()) + field2 := idx.Field("amutex", OptFieldTypeMutex(CacheTypeNone, 0)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 100, idx, []*Field{field, field2}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + r := Row{Values: make([]interface{}, 2)} + + rnd := rand.New(rand.NewSource(7)) + + // generate 100 records randomly spread/ordered across multiple + // shards to test sorting on int/mutex fields + for i := 0; i < 100; i++ { + id := rnd.Intn(10_000_000) + r.ID = uint64(id) + r.Values[0] = int64(id) + r.Values[1] = uint64(id) + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("second import: %v", err) + } + + resp, err := client.Query(idx.RawQuery("Count(All())")) + if err != nil { + t.Fatalf("querying: %v", err) + } + if res := resp.Results()[0]; res.Count() != 100 { + t.Fatalf("unexpected result: %+v", res) + } +} + +func testTrimNull(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-trim-null") + field := idx.Field("empty", OptFieldTypeInt()) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + b, err := NewBatch(client, 3, idx, []*Field{field}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + b.nullIndices = make(map[string][]uint64, 1) + b.nullIndices[field.Name()] = []uint64{0, 1, 2} + r := Row{Values: make([]interface{}, 1)} + for i := 0; i < 3; i++ { + r.ID = uint64(i) + r.Values[0] = int64(i) + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + resp, err := client.Query(idx.BatchQuery(field.Equals(0), field.Equals(1), field.Equals(2))) + if err != nil { + t.Fatalf("querying: %v", err) + } + for i, result := range resp.Results() { + if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) { + t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns) + } + } + + b, err = NewBatch(client, 4, idx, []*Field{field}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + r = Row{Values: make([]interface{}, 1)} + for i := 10; i < 40; i += 10 { + r.ID = uint64(i) + r.Values[0] = int64(i) + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + + r.ID = uint64(40) + r.Values[0] = nil + err = b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + resp, err = client.Query(idx.BatchQuery(field.Equals(10), field.Equals(40), field.Equals(20), field.Equals(30))) + if err != nil { + t.Fatalf("querying: %v", err) + } + for i, result := range resp.Results() { + if i == 1 { + if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) { + t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns) + } + } else { + if !reflect.DeepEqual(result.Row().Columns, []uint64{result.Row().Columns[0]}) { + t.Errorf("expected %#v for %d, but got %#v", []uint64{result.Row().Columns[0]}, i, result.Row().Columns) + } + } + } + +} + +func testStringSliceEmptyAndNil(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-string-slice-nil") + fields := make([]*Field, 1) + fields[0] = idx.Field("strslice", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + // first create a batch and test adding a single value with empty + // string - this failed with a translation error at one point, and + // how we catch it and treat it like a nil. + b, err := NewBatch(client, 2, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + r := Row{Values: make([]interface{}, len(fields))} + r.ID = uint64(1) + r.Values[0] = "" + err = b.Add(r) + if err != nil { + t.Fatalf("adding: %v", err) + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + // now create a batch and add a mixture of string slice values + b, err = NewBatch(client, 6, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + r = Row{Values: make([]interface{}, len(fields))} + r.ID = uint64(0) + r.Values[0] = []string{"a"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding to batch: %v", err) + } + + r.ID = uint64(1) + r.Values[0] = nil + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with nil stringslice to r: %v", err) + } + + r.ID = uint64(2) + r.Values[0] = []string{"a", "b", "z"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with idslice to r: %v", err) + } + + r.ID = uint64(3) + r.Values[0] = []string{"b", "c"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with stringslice to r: %v", err) + } + + r.ID = uint64(4) + r.Values[0] = []string{} + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with stringslice to r: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + rows := []interface{}{"a", "b", "c", "z"} + resp, err := client.Query(idx.BatchQuery(fields[0].Row(rows[0]), fields[0].Row(rows[1]), fields[0].Row(rows[2]), fields[0].Row(rows[3]))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + // TODO test is flaky because we can't guarantee what a,b,c map to + expectations := [][]uint64{{0, 2}, {2, 3}, {3}, {2}} + for i, re := range resp.Results() { + if !reflect.DeepEqual(re.Row().Columns, expectations[i]) { + t.Errorf("expected row %v to have columns %v, but got %v", rows[i], expectations[i], re.Row().Columns) + } + } + +} + +func testStringSlice(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-string-slice") + fields := make([]*Field, 1) + fields[0] = idx.Field("strslice", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 3, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + + rowmap := map[string]uint64{ + "c": 9, + "d": 10, + "f": 13, + } + b.rowTranslations["strslice"] = make(map[string]agedTranslation) + for k, id := range rowmap { + b.rowTranslations["strslice"][k] = agedTranslation{ + id: id, + } + } + + r := Row{Values: make([]interface{}, len(fields))} + r.ID = uint64(0) + r.Values[0] = []string{"a"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding to batch: %v", err) + } + if got := b.toTranslateSets["strslice"]["a"]; !reflect.DeepEqual(got, []int{0}) { + t.Fatalf("expected []int{0}, got: %v", got) + } + + r.ID = uint64(1) + r.Values[0] = []string{"a", "b", "c"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding to batch: %v", err) + } + if got := b.toTranslateSets["strslice"]["a"]; !reflect.DeepEqual(got, []int{0, 1}) { + t.Fatalf("expected []int{0,1}, got: %v", got) + } + if got := b.toTranslateSets["strslice"]["b"]; !reflect.DeepEqual(got, []int{1}) { + t.Fatalf("expected []int{1}, got: %v", got) + } + if got, ok := b.toTranslateSets["strslice"]["c"]; ok { + t.Fatalf("should be nothing at c, got: %v", got) + } + if got := b.rowIDSets["strslice"][1]; !reflect.DeepEqual(got, []uint64{9}) { + t.Fatalf("expected c to map to rowID 9 but got %v", got) + } + + r.ID = uint64(2) + r.Values[0] = []string{"d", "e", "f"} + err = b.Add(r) + if err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + if got, ok := b.toTranslateSets["strslice"]["d"]; ok { + t.Fatalf("should be nothing at d, got: %v", got) + } + if got, ok := b.toTranslateSets["strslice"]["f"]; ok { + t.Fatalf("should be nothing at f, got: %v", got) + } + if got := b.toTranslateSets["strslice"]["e"]; !reflect.DeepEqual(got, []int{2}) { + t.Fatalf("expected []int{2}, got: %v", got) + } + if got := b.rowIDSets["strslice"][2]; !reflect.DeepEqual(got, []uint64{10, 13}) { + t.Fatalf("expected c to map to rowID 9 but got %v", got) + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("translating: %v", err) + } + + if got0 := b.rowIDSets["strslice"][0]; len(got0) != 1 { + t.Errorf("after translation, rec 0, wrong len: %v", got0) + } else if got1 := b.rowIDSets["strslice"][1]; len(got1) != 3 || got1[0] != 9 || (got1[1] != got0[0] && got1[2] != got0[0]) { + t.Errorf("after translation, rec 1: %v, rec 0: %v", got1, got0) + } else if got2 := b.rowIDSets["strslice"][2]; len(got2) != 3 || got2[0] != 10 || got2[1] != 13 || got2[2] == got1[2] || got2[2] == got0[0] { + t.Errorf("after translation, rec 2: %v", got2) + } + + frags, clearFrags, err := b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + result := resp.Result() + if !reflect.DeepEqual(result.Row().Columns, []uint64{0, 1}) { + t.Fatalf("expected a to be [0,1], got %v", result.Row().Columns) + } +} + +func testSingleClearBatchRegression(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-single-clear-batch-regression") + numFields := 1 + fields := make([]*Field, numFields) + fields[0] = idx.Field("zero", OptFieldKeys(true)) + + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + _, err = client.Query(fields[0].Set("row1", 1)) + if err != nil { + t.Fatalf("setting bit: %v", err) + } + + b, err := NewBatch(client, 1, idx, fields) + if err != nil { + t.Fatalf("getting new batch: %v", err) + } + r := Row{ID: uint64(1), Values: make([]interface{}, numFields), Clears: make(map[int]interface{})} + r.Values[0] = nil + r.Clears[0] = "row1" + err = b.Add(r) + if err != ErrBatchNowFull { + t.Fatalf("wrong error from batch add: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("error importing: %v", err) + } + + resp, err := client.Query(fields[0].Row("row1")) + if err != nil { + t.Fatalf("error querying: %v", err) + } + result := resp.Results()[0].Row().Columns + if len(result) != 0 { + t.Fatalf("unexpected values in row: result %+v", result) + } + +} + +func testBatches(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-batches") + numFields := 5 + fields := make([]*Field, numFields) + fields[0] = idx.Field("zero", OptFieldKeys(true)) + fields[1] = idx.Field("one", OptFieldKeys(true)) + fields[2] = idx.Field("two", OptFieldKeys(true)) + fields[3] = idx.Field("three", OptFieldTypeInt()) + fields[4] = idx.Field("four", OptFieldTypeTime(TimeQuantumYearMonthDay)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + b, err := NewBatch(client, 10, idx, fields) + if err != nil { + t.Fatalf("getting new batch: %v", err) + } + r := Row{Values: make([]interface{}, numFields), Clears: make(map[int]interface{})} + r.Time.Set(time.Date(2019, time.January, 2, 15, 45, 0, 0, time.UTC)) + + for i := 0; i < 9; i++ { + r.ID = uint64(i) + if i%2 == 0 { + r.Values[0] = "a" + r.Values[1] = "b" + r.Values[2] = "c" + r.Values[3] = int64(99) + r.Values[4] = uint64(1) + r.Time.SetMonth("01") + } else { + r.Values[0] = "x" + r.Values[1] = "y" + r.Values[2] = "z" + r.Values[3] = int64(-10) + r.Values[4] = uint64(1) + r.Time.SetMonth("02") + } + if i == 8 { + r.Values[0] = nil + r.Clears[1] = uint64(97) + r.Clears[2] = "c" + r.Values[3] = nil + r.Values[4] = nil + } + err := b.Add(r) + if err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + + } + + if len(b.toTranslate[0]) != 2 { + t.Fatalf("wrong number of keys in toTranslate[0]") + } + for k, ints := range b.toTranslate[0] { + if k == "a" { + if !reflect.DeepEqual(ints, []int{0, 2, 4, 6}) { + t.Fatalf("wrong ints for key a in field zero: %v", ints) + } + } else if k == "x" { + if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) { + t.Fatalf("wrong ints for key x in field zero: %v", ints) + } + + } else { + t.Fatalf("unexpected key %s", k) + } + } + if !reflect.DeepEqual(b.toTranslateClear, map[int]map[string][]int{2: {"c": {8}}}) { + t.Errorf("unexpected toTranslateClear: %+v", b.toTranslateClear) + } + if !reflect.DeepEqual(b.clearRowIDs, map[int]map[int]uint64{1: {8: 97}, 2: {}}) { + t.Errorf("unexpected clearRowIDs: %+v", b.clearRowIDs) + } + + if !reflect.DeepEqual(b.values["three"], []int64{99, -10, 99, -10, 99, -10, 99, -10, 0}) { + t.Fatalf("unexpected values: %v", b.values["three"]) + } + if !reflect.DeepEqual(b.nullIndices["three"], []uint64{8}) { + t.Fatalf("unexpected nullIndices: %v", b.nullIndices["three"]) + } + + if len(b.toTranslate[1]) != 2 { + t.Fatalf("wrong number of keys in toTranslate[1]") + } + for k, ints := range b.toTranslate[1] { + if k == "b" { + if !reflect.DeepEqual(ints, []int{0, 2, 4, 6, 8}) { + t.Fatalf("wrong ints for key b in field one: %v", ints) + } + } else if k == "y" { + if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) { + t.Fatalf("wrong ints for key y in field one: %v", ints) + } + + } else { + t.Fatalf("unexpected key %s", k) + } + } + + if len(b.toTranslate[2]) != 2 { + t.Fatalf("wrong number of keys in toTranslate[2]") + } + for k, ints := range b.toTranslate[2] { + if k == "c" { + if !reflect.DeepEqual(ints, []int{0, 2, 4, 6, 8}) { + t.Fatalf("wrong ints for key c in field two: %v", ints) + } + } else if k == "z" { + if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) { + t.Fatalf("wrong ints for key z in field two: %v", ints) + } + + } else { + t.Fatalf("unexpected key %s", k) + } + } + + err = b.Add(r) + if err != ErrBatchNowFull { + t.Fatalf("should have gotten full batch error, but got %v", err) + } + + err = b.Add(r) + if err != ErrBatchAlreadyFull { + t.Fatalf("should have gotten already full batch error, but got %v", err) + } + + if !reflect.DeepEqual(b.values["three"], []int64{99, -10, 99, -10, 99, -10, 99, -10, 0, 0}) { + t.Fatalf("unexpected values: %v", b.values["three"]) + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("doing translation: %v", err) + } + + for fidx, rowIDs := range b.rowIDs { + // we don't know which key will get translated first, but we do know the pattern + if fidx == 0 { + if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, nilSentinel, nilSentinel}) && + !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, nilSentinel, nilSentinel}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + + } else if fidx == 4 { + if !reflect.DeepEqual(rowIDs, []uint64{1, 1, 1, 1, 1, 1, 1, 1, nilSentinel, nilSentinel}) { + t.Fatalf("unexpected rowids for time field") + } + } else if fidx == 3 { + if len(rowIDs) != 0 { + t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs) + } + } else { + if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, 1, nilSentinel}) && !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, 2, nilSentinel}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + } + } + + if !reflect.DeepEqual(b.clearRowIDs[1], map[int]uint64{8: 97}) { + t.Errorf("unexpected clearRowIDs after translation: %+v", b.clearRowIDs[1]) + } + if !reflect.DeepEqual(b.clearRowIDs[2], map[int]uint64{8: 2}) && !reflect.DeepEqual(b.clearRowIDs[2], map[int]uint64{8: 1}) { + t.Errorf("unexpected clearRowIDs: after translation%+v", b.clearRowIDs[2]) + } + + frags, clearFrags, err := b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + b.reset() + + for i := 9; i < 19; i++ { + r.ID = uint64(i) + if i%2 == 0 { + r.Values[0] = "a" + r.Values[1] = "b" + r.Values[2] = "c" + r.Values[3] = int64(99) + r.Values[4] = uint64(1) + } else { + r.Values[0] = "x" + r.Values[1] = "y" + r.Values[2] = "z" + r.Values[3] = int64(-10) + r.Values[4] = uint64(2) + } + err := b.Add(r) + if i != 18 && err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + if i == 18 && err != ErrBatchNowFull { + t.Fatalf("unexpected err: %v", err) + } + } + + // should do nothing + err = b.doTranslation() + if err != nil { + t.Fatalf("doing translation: %v", err) + } + + frags, clearFrags, err = b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + for fidx, rowIDs := range b.rowIDs { + if fidx == 3 { + if len(rowIDs) != 0 { + t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs) + } + continue + } + // we don't know which key will get translated first, but we do know the pattern + if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, 1, 2}) && !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, 2, 1}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + } + + b.reset() + + for i := 19; i < 29; i++ { + r.ID = uint64(i) + if i%2 == 0 { + r.Values[0] = "d" + r.Values[1] = "e" + r.Values[2] = "f" + r.Values[3] = int64(100) + r.Values[4] = uint64(3) + } else { + r.Values[0] = "u" + r.Values[1] = "v" + r.Values[2] = "w" + r.Values[3] = int64(0) + r.Values[4] = uint64(4) + } + err := b.Add(r) + if i != 28 && err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + if i == 28 && err != ErrBatchNowFull { + t.Fatalf("unexpected err: %v", err) + } + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("doing translation: %v", err) + } + + frags, clearFrags, err = b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + for fidx, rowIDs := range b.rowIDs { + // we don't know which key will get translated first, but we do know the pattern + if fidx == 3 { + if len(rowIDs) != 0 { + t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs) + } + continue + } + if !reflect.DeepEqual(rowIDs, []uint64{3, 4, 3, 4, 3, 4, 3, 4, 3, 4}) && !reflect.DeepEqual(rowIDs, []uint64{4, 3, 4, 3, 4, 3, 4, 3, 4, 3}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + } + + frags, _, err = b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Fatalf("making fragments: %v", err) + } + + var n int + for key := range frags { + if key.shard == 0 { + n++ + } + } + if n != 5 { // zero, one, two, four (three is an int field so not in fragments) + _exists + t.Fatalf("there should be 5 views, but have %d", n) + } + + resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"), + fields[1].Row("b"), + fields[2].Row("c"), + fields[3].Equals(99))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results := resp.Results() + for _, j := range []int{0, 2, 3} { + cols := results[j].Row().Columns + if !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 10, 12, 14, 16, 18}) { + t.Fatalf("unexpected columns for a: %v", cols) + } + } + res := results[1] + + if cols := res.Row().Columns; !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 8, 10, 12, 14, 16, 18}) { + t.Fatalf("unexpected columns for field 1 row b: %v", cols) + } + + resp, err = client.Query(idx.BatchQuery(fields[0].Row("d"), + fields[1].Row("e"), + fields[2].Row("f"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results = resp.Results() + for _, res := range results { + cols := res.Row().Columns + if !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) { + t.Fatalf("unexpected columns: %v", cols) + } + } + + resp, err = client.Query(idx.BatchQuery(fields[3].GT(-11), + fields[3].Equals(0), + fields[3].Equals(100), + fields[4].Range(1, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC), time.Date(2019, time.January, 29, 0, 0, 0, 0, time.UTC)), + fields[4].Range(1, time.Date(2019, time.February, 1, 0, 0, 0, 0, time.UTC), time.Date(2019, time.February, 29, 0, 0, 0, 0, time.UTC)))) + if err != nil { + t.Fatalf("querying: %v", err) + } + results = resp.Results() + + if cols := results[0].Row().Columns; !reflect.DeepEqual(cols, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}) { + t.Fatalf("all columns (but 8) should be greater than -11, but got: %v", cols) + } + + if cols := results[1].Row().Columns; !reflect.DeepEqual(cols, []uint64{19, 21, 23, 25, 27}) { + t.Fatalf("wrong cols for ==0: %v", cols) + } + + if cols := results[2].Row().Columns; !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) { + t.Fatalf("wrong cols for ==100: %v", cols) + } + + cols := results[3].Row().Columns + exp := []uint64{0, 2, 4, 6, 10, 12, 14, 16, 18} + if !reflect.DeepEqual(cols, exp) { + t.Fatalf("wrong cols for January: got/want\n%v\n%v", cols, exp) + } + + cols = results[4].Row().Columns + exp = []uint64{1, 3, 5, 7} + if !reflect.DeepEqual(cols, exp) { + t.Fatalf("wrong cols for January: got/want\n%v\n%v", cols, exp) + } + + b.reset() + r.ID = uint64(0) + r.Values[0] = "x" + r.Values[1] = "b" + r.Clears[0] = "a" + r.Clears[1] = "b" // b should get cleared + err = b.Add(r) + if err != nil { + t.Fatalf("adding with clears: %v", err) + } + err = b.Import() + if err != nil { + t.Fatalf("importing w/clears: %v", err) + } + resp, err = client.Query(idx.BatchQuery( + fields[0].Row("a"), + fields[0].Row("x"), + fields[1].Row("b"), + )) + if err != nil { + t.Fatalf("querying after clears: %v", err) + } + if arow := resp.Results()[0].Row().Columns; arow[0] == 0 { + t.Errorf("shouldn't have id 0 in row a after clearing! %v", arow) + } + if xrow := resp.Results()[1].Row().Columns; xrow[0] != 0 { + t.Errorf("should have id 0 in row x after setting %v", xrow) + } + if brow := resp.Results()[2].Row().Columns; brow[0] == 0 { + t.Errorf("shouldn't have id 0 in row b after clearing! %v", brow) + } + + // TODO test importing across multiple shards +} + +func testBatchesStringIDs(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("batches-strings-ids", OptIndexKeys(true)) + fields := make([]*Field, 3) + fields[0] = idx.Field("zero", OptFieldKeys(true)) + fields[1] = idx.Field("one", OptFieldTypeMutex(CacheTypeNone, 0), OptFieldKeys(true)) + fields[2] = idx.Field("two", OptFieldTypeTime("YMDH"), OptFieldKeys(true)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 3, idx, fields) + if err != nil { + t.Fatalf("getting new batch: %v", err) + } + + r := Row{Values: make([]interface{}, 3)} + r.Time.Set(time.Date(2019, time.January, 2, 15, 45, 0, 0, time.UTC)) + + for i := 0; i < 3; i++ { + r.ID = strconv.Itoa(i) + if i%2 == 0 { + r.Values[0] = "a" + r.Values[1] = "b" + r.Values[2] = "c" + r.Time.SetMonth("01") + } else { + r.Values[0] = "x" + r.Values[1] = "y" + r.Values[2] = "z" + r.Time.SetMonth("02") + } + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("unexpected err adding record: %v", err) + } + } + + if len(b.toTranslateID) != 3 { + t.Fatalf("id translation table unexpected size: %v", b.toTranslateID) + } + for i, k := range b.toTranslateID { + if ik, err := strconv.Atoi(k); err != nil || ik != i { + t.Errorf("unexpected toTranslateID key %s at index %d", k, i) + } + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("translating: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("x"), fields[1].Row("b"), fields[1].Row("y"), fields[2].Row("c"), fields[2].Row("z"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results := resp.Results() + for i, res := range results { + cols := res.Row().Keys + if i%2 == 0 && !reflect.DeepEqual(cols, []string{"0", "2"}) && !reflect.DeepEqual(cols, []string{"2", "0"}) { + t.Fatalf("unexpected columns: %v", cols) + } + if i%2 == 1 && !reflect.DeepEqual(cols, []string{"1"}) { + t.Fatalf("unexpected columns: %v", cols) + } + } + + b.reset() + + r.ID = "1" + r.Values[0] = "a" + err = b.Add(r) + if err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + + r.ID = "3" + r.Values[0] = "z" + err = b.Add(r) + if err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + resp, err = client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("z"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results = resp.Results() + for i, res := range results { + cols := res.Row().Keys + if err := isPermutationOf(cols, []string{"0", "1", "2"}); i == 0 && err != nil { + t.Fatalf("unexpected columns: %v: %v", cols, err) + } + if i == 1 && !reflect.DeepEqual(cols, []string{"3"}) { + t.Fatalf("unexpected columns: %v", cols) + } + } + +} + +func isPermutationOf(one, two []string) error { + if len(one) != len(two) { + return errors.Errorf("different lengths %d and %d", len(one), len(two)) + } +outer: + for _, vOne := range one { + for j, vTwo := range two { + if vOne == vTwo { + two = append(two[:j], two[j+1:]...) + continue outer + } + } + return errors.Errorf("%s in one but not two", vOne) + } + if len(two) != 0 { + return errors.Errorf("vals in two but not one: %v", two) + } + return nil +} + +func TestQuantizedTime(t *testing.T) { + cases := []struct { + name string + time time.Time + year string + month string + day string + hour string + quantum TimeQuantum + reset bool + exp []string + expErr string + }{ + { + name: "no time quantum", + expErr: "", + }, + { + name: "no time quantum with data", + year: "2017", + exp: []string{}, + expErr: "", + }, + { + name: "no data", + quantum: TimeQuantumYear, + exp: nil, + expErr: "", + }, + { + name: "timestamp", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "YMDH", + exp: []string{"2013", "201310", "20131016", "2013101617"}, + }, + { + name: "timestamp-less-granular", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "YM", + exp: []string{"2013", "201310"}, + }, + { + name: "timestamp-mid-granular", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "MD", + exp: []string{"201310", "20131016"}, + }, + { + name: "justyear", + year: "2013", + quantum: "Y", + exp: []string{"2013"}, + }, + { + name: "justyear-wantmonth", + year: "2013", + quantum: "YM", + expErr: "no data set for month", + }, + { + name: "timestamp-changeyear", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + year: "2019", + quantum: "YMDH", + exp: []string{"2019", "201910", "20191016", "2019101617"}, + }, + { + name: "yearmonthdayhour", + year: "2013", + month: "10", + day: "16", + hour: "17", + quantum: "YMDH", + exp: []string{"2013", "201310", "20131016", "2013101617"}, + }, + { + name: "timestamp-changehour", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + hour: "05", + quantum: "MDH", + exp: []string{"201310", "20131016", "2013101605"}, + }, + { + name: "timestamp", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "YMDH", + reset: true, + exp: nil, + }, + } + + for i, test := range cases { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + tq := QuantizedTime{} + var zt time.Time + if zt != test.time { + tq.Set(test.time) + } + if test.year != "" { + tq.SetYear(test.year) + } + if test.month != "" { + tq.SetMonth(test.month) + } + if test.day != "" { + tq.SetDay(test.day) + } + if test.hour != "" { + tq.SetHour(test.hour) + } + if test.reset { + tq.Reset() + } + + views, err := tq.views(test.quantum) + if !reflect.DeepEqual(views, test.exp) { + t.Errorf("unexpected views, got/want:\n%v\n%v\n", views, test.exp) + } + if (err != nil && err.Error() != test.expErr) || (err == nil && test.expErr != "") { + t.Errorf("unexpected error, got/want:\n%v\n%s\n", err, test.expErr) + } + }) + } + +} + +func testBatchStaleness(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-batch-staleness") + field := idx.Field("anint", OptFieldTypeInt()) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 3, idx, []*Field{field}, OptMaxStaleness(time.Millisecond)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + r := Row{ID: uint64(0), Values: []interface{}{int64(0)}} + err = b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + + // sleep so batch becomes stale + time.Sleep(time.Millisecond) + + r = Row{ID: uint64(1), Values: []interface{}{int64(0)}} + err = b.Add(r) + if err != ErrBatchNowStale { + t.Fatal("batch expected to be stale") + } +} + +func testImportBatchMultipleInts(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-import-batch-multi-int") + field := idx.Field("anint", OptFieldTypeInt()) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 6, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + r := Row{Values: make([]interface{}, 1)} + + vals := []int64{16, 8, 32, 1, 2, 4} + for i := uint64(0); i < 6; i++ { + r.ID = uint64(1) + r.Values[0] = vals[i] + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + if resp, err := client.Query(field.Equals(4)); err != nil { + t.Fatalf("querying: %v", err) + } else if res := resp.Results()[0].Row().Columns; len(res) != 1 || res[0] != 1 { + t.Fatalf("unepxected result: %v", res) + } + +} + +// testImportBatchMultipleTimestamps tests if nils are handles correctly for TS in batch imports +func testImportBatchMultipleTimestamps(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-import-batch-multi-timestamp") + field := idx.Field("ts2", OptFieldTypeTimestamp(time.Unix(0, 0), "s")) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b1, err := NewBatch(client, 6, idx, []*Field{field}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + b2, err := NewBatch(client, 6, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + batches := []*Batch{b1, b2} + + for j := 0; j < 2; j++ { + t.Run(fmt.Sprintf("batch %d", j), func(t *testing.T) { + b := batches[j] + r := Row{Values: make([]interface{}, 1)} + + rawVals := []interface{}{int64(16), int64(8), int64(32), nil, int64(2), int64(4)} + chkVals := []int64{16, 8, 32, 0, 2, 4} + chkImport := []interface{}{time.Unix(16, 0), time.Unix(8, 0), time.Unix(32, 0), nil, time.Unix(2, 0), time.Unix(4, 0)} + cols := []uint64{0, 1, 2, 3, 4, 5} + for i := range cols { + r.ID = cols[i] + r.Values[0] = rawVals[i] + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + + if b.nullIndices[field.name][0] != 3 { + t.Fatalf("unexpected nulls, got/want: %v/%v", b.nullIndices[field.name], []uint64{3}) + } + for i, val := range chkVals { + if b.values[field.name][i] != val { + t.Fatalf("unexpected value, got/want: %v/%v", b.values[field.name][i], val) + } + if b.ids[i] != cols[i] { + t.Fatalf("unexpected id, got/want: %v/%v", b.ids[i], cols[i]) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + qr := c.Query(t, idx.name, `Extract(All(), Rows(ts2))`) + results := qr.Results[0].(featurebase.ExtractedTable) + for k, res := range results.Columns { + if chkImport[k] != nil { + if res.Rows[0] != chkImport[k].(time.Time).UTC() { + t.Fatalf("unexpected result, got/want: %v/%v", res.Rows[0], chkImport[k].(time.Time).UTC()) + } + } else { + if res.Rows[0] != nil { + t.Fatalf("unexpected result, got/want: %v/%v", res.Rows[0], nil) + } + } + } + }) + } +} + +func testImportBatchSetsAndClears(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-import-batch-set-and-clear") + field := idx.Field("aset", OptFieldTypeSet(featurebase.DefaultCacheType, featurebase.DefaultCacheSize)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 6, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + r := Row{ + Values: make([]interface{}, 1), + Clears: make(map[int]interface{}), + } + + vals := []uint64{1, 2, 3, 1, 5, 6} + clears := []interface{}{nil, uint64(1), uint64(3), nil, uint64(2), uint64(4)} + for i := uint64(0); i < 6; i++ { + r.ID = i%3 + 1 + r.Values[0] = vals[i] + if clears[i] != nil { + r.Clears[0] = clears[i] + } + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + if resp, err := client.Query(field.TopN(6)); err != nil { + t.Fatalf("querying topn: %v", err) + } else if res := resp.Result().CountItems(); len(res) != 3 { + t.Fatalf("unexpected topn: %+v", res) + } + + exp := [][]uint64{ + {}, + {1}, + {}, + {}, + {}, + {2}, + {3}, + } + for row := 0; row < 7; row++ { + resp, err := client.Query(field.Row(row)) + if err != nil { + t.Fatalf("querying: %v", err) + } + res := resp.Results()[0].Row().Columns + if !reflect.DeepEqual(exp[row], res) && !(len(exp[row]) == 0 && len(res) == 0) { + t.Errorf("row: %d, exp: %v, got %v", row, exp[row], res) + } + } + +} + +// testTopNCacheRegression recreates an issue we saw in an IDK test +// where if a value is completely removed (all bits unset from a row), +// it didn't get removed from the cache beacuse a full recalculation +// had no way to clear the cache, it would just reset existing +// values. We added Clear on the cache interface to fix this. +func testTopNCacheRegression(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-topn-cache-regression") + field := idx.Field("aset", OptFieldTypeSet(featurebase.DefaultCacheType, featurebase.DefaultCacheSize)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 3, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + records := []struct { + ID uint64 + Set interface{} + Clear interface{} + }{ + {0, 1, nil}, + {featurebase.ShardWidth, 1, nil}, + {featurebase.ShardWidth * 2, nil, 1}, + {featurebase.ShardWidth * 2, nil, 1}, + {0, nil, 1}, + {featurebase.ShardWidth, nil, 1}, + {featurebase.ShardWidth, 1, nil}, + {featurebase.ShardWidth, nil, nil}, + } + + for _, rec := range records { + if rec.Set != nil { + rec.Set = uint64(rec.Set.(int)) + } + row := Row{ + ID: rec.ID, + Values: []interface{}{rec.Set}, + } + if rec.Clear != nil { + row.Clears = map[int]interface{}{0: uint64(rec.Clear.(int))} + } + + err := b.Add(row) + if err == ErrBatchNowFull { + if err := b.Import(); err != nil { + t.Fatalf("importing: %v", err) + } + } + } + if err := b.Import(); err != nil { + t.Fatalf("importing: %v", err) + } + + if resp, err := client.Query(field.TopN(6)); err != nil { + t.Fatalf("querying topn: %v", err) + } else if res := resp.Result().CountItems(); len(res) != 1 { + t.Fatalf("unexpected topn: %+v", res) + } else if res[0].ID != 1 || res[0].Count != 1 { + t.Fatalf("unexpected topn result: %v", res) + } +} + +// testMultipleIntSameBatch checks that if the same ID is added multiple times with different values that only the last value is set and the bits aren't mixed together. It adds a different ID in between the two same ones which triggered a bug because we were sorting by shard rather than ID. +func testMultipleIntSameBatch(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-multiple-int-same-batch") + field := idx.Field("age", OptFieldTypeInt(0, 10000)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 4, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + if err := b.Add(Row{ + ID: uint64(1), + Values: []interface{}{int64(1)}, + }); err != nil { + t.Fatalf("adding to batch: %v", err) + } + if err := b.Add(Row{ + ID: uint64(2), + Values: []interface{}{int64(0)}, + }); err != nil { + t.Fatalf("adding to batch: %v", err) + } + if err := b.Add(Row{ + ID: uint64(1), + Values: []interface{}{int64(2)}, + }); err != nil { + t.Fatalf("adding to batch: %v", err) + } + + if err := b.Import(); err != nil { + t.Fatalf("importing: %v", err) + } + + if resp, err := client.Query(field.Sum(nil)); err != nil { + t.Fatalf("querying sum: %v", err) + } else if res := resp.Result().Value(); res != 2 { + t.Errorf("unexpected sum: %+v", res) + } +} + +// mutexClearRegression checks for a bug where shards beyond the first +// one in a batch did not get any bits set in their clear bitmap, and +// in fact, all the bits were set in the clear bitmap for the first +// shard. +func mutexClearRegression(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-multiple-mut-same-batch") + field := idx.Field("mut", OptFieldTypeMutex(CacheTypeNone, 0)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 11, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + col := uint64(0) + row := uint64(1) + for i := uint64(0); i <= 21; i++ { + col = (i%2+1)*featurebase.ShardWidth + i%5 + row = i % 3 + if err := b.Add(Row{ + ID: col, + Values: []interface{}{row}, + }); err == ErrBatchNowFull { + if err := b.Import(); err != nil { + t.Fatalf("importing: %v", err) + } + resp, err := client.Query(idx.GroupBy(field.Rows(), field.Rows())) + if err != nil { + t.Fatalf("querying groupby: %v", err) + } + groupCounts := resp.Result().GroupCounts() + for j, gc := range groupCounts { + if gc.Groups[0].RowID != gc.Groups[1].RowID { + t.Errorf("zmismatched group at after %d batch: %d, %v", j, i, gc) + } + } + } else if err != nil { + t.Fatalf("adding to batch: %v", err) + } + + } + if err := b.Import(); err != nil { + t.Fatalf("importing: %v", err) + } + + resp, err := client.Query(idx.GroupBy(field.Rows(), field.Rows())) + if err != nil { + t.Fatalf("querying groupby: %v", err) + } + groupCounts := resp.Result().GroupCounts() + for i, gc := range groupCounts { + if gc.Groups[0].RowID != gc.Groups[1].RowID { + t.Fatalf("bmismatched group at %d, %v", i, gc) + } + } +} + +// test clearing record with explict nil +func mutexNilClearID(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-mut-nil-clear-id") + field := idx.Field("mut", OptFieldTypeMutex(CacheTypeNone, 0)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 11, idx, []*Field{field}, OptUseShardTransactionalEndpoint(true)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + col := uint64(0) + row := uint64(1) + // populate mutex with some data + for i := uint64(0); i < 11; i++ { + col = (i%2+1)*featurebase.ShardWidth + i%5 + row = i % 3 + if err := b.Add(Row{ + ID: col, + Values: []interface{}{row}, + }); err == ErrBatchNowFull { + if err := b.Import(); err != nil { + t.Fatalf("importing: %v", err) + } + } else if err != nil { + t.Fatalf("adding to batch: %v", err) + } + + } + // example data just copyied from test above + // confirm expected data + resp, err := client.Query(idx.RawQuery("Row(mut=0)")) + if err != nil { + t.Fatalf("Fetching data: %v", err) + } + items := resp.Result().Row().Columns + // delete item 0 + b.Add( + Row{ + ID: items[0], + Values: []interface{}{nil}, + Clears: map[int]interface{}{0: nil}, + }, + ) + b.Import() + items = items[1:] + // confirm record removed + resp, err = client.Query(idx.RawQuery("Row(mut=0)")) + if err != nil { + t.Fatalf("Fetching data: %v", err) + } + errorIfNotEqual(t, resp.Result().Row().Columns, items) + +} + +// similar test to above but with string keys +func mutexNilClearKey(t *testing.T, c *test.Cluster, client *Client) { + schema := NewSchema() + idx := schema.Index("test-mut-nil-clear-key", OptIndexKeys(true)) + fields := make([]*Field, 1) + fields[0] = idx.Field("mut", OptFieldTypeMutex(CacheTypeNone, 0), OptFieldKeys(true)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 3, idx, fields) + if err != nil { + t.Fatalf("getting new batch: %v", err) + } + + r := Row{Values: make([]interface{}, 1)} + + for i := 0; i < 3; i++ { + r.ID = strconv.Itoa(i) + if i%2 == 0 { + r.Values[0] = "a" + } else { + r.Values[0] = "x" + } + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("unexpected err adding record: %v", err) + } + } + + if len(b.toTranslateID) != 3 { + t.Fatalf("id translation table unexpected size: %v", b.toTranslateID) + } + for i, k := range b.toTranslateID { + if ik, err := strconv.Atoi(k); err != nil || ik != i { + t.Errorf("unexpected toTranslateID key %s at index %d", k, i) + } + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("translating: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + resp, err := client.Query(idx.RawQuery(`Row(mut="a")`)) + errorIfNotEqual(t, resp.Result().Row().Keys, []string{"0", "2"}) + + r.ID = "2" + r.Values[0] = nil + r.Clears = map[int]interface{}{0: nil} + err = b.Add(r) + if err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + resp, err = client.Query(idx.RawQuery(`Row(mut="a")`)) + if err != nil { + t.Fatalf("importing: %v", err) + } + errorIfNotEqual(t, resp.Result().Row().Keys, []string{"0"}) +} diff --git a/client/client.go b/client/client.go new file mode 100644 index 000000000..47a115528 --- /dev/null +++ b/client/client.go @@ -0,0 +1,1781 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. +package client + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/golang/protobuf/proto" //nolint:staticcheck + pilosa "github.com/molecula/featurebase/v3" + fbproto "github.com/molecula/featurebase/v3/encoding/proto" // TODO use this everywhere and get rid of proto import + "github.com/molecula/featurebase/v3/logger" + pnet "github.com/molecula/featurebase/v3/net" + "github.com/molecula/featurebase/v3/pb" + "github.com/molecula/featurebase/v3/pql" + "github.com/molecula/featurebase/v3/roaring" + "github.com/molecula/featurebase/v3/stats" + "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +// PQLVersion is the version of PQL expected by the client +const PQLVersion = "1.0" + +// DefaultShardWidth is used if an index doesn't have it defined. +const DefaultShardWidth = pilosa.ShardWidth + +const maxHosts = 10 + +// Client is the HTTP client for Pilosa server. +type Client struct { + cluster *Cluster + client *http.Client + logger logger.Logger + primaryURI *pnet.URI + primaryLock *sync.RWMutex + manualFragmentNode *fragmentNode + manualServerURI *pnet.URI + tracer opentracing.Tracer + Stats stats.StatsClient + // An exponential backoff algorithm retries requests exponentially (if an HTTP request fails), + // increasing the waiting time between retries up to a maximum backoff time. + maxBackoff time.Duration + maxRetries int + + nat map[pnet.URI]pnet.URI + + shardNodes shardNodes + tick *time.Ticker + done chan struct{} + + AuthToken string +} + +func (c *Client) getURIsForShard(index string, shard uint64) ([]*pnet.URI, error) { + uris, ok := c.shardNodes.Get(index, shard) + if ok { + return uris, nil + } + fragmentNodes, err := c.fetchFragmentNodes(index, shard) + if err != nil { + return nil, errors.Wrap(err, "trying to look up nodes for shard") + } + uris = make([]*pnet.URI, 0, len(fragmentNodes)) + for _, fn := range fragmentNodes { + uris = append(uris, fn.URI()) + } + c.shardNodes.Put(index, shard, uris) + return uris, nil +} + +func (c *Client) runChangeDetection() { + for { + select { + case <-c.tick.C: + c.detectClusterChanges() + case <-c.done: + return + } + } +} + +func (c *Client) Close() error { + c.tick.Stop() + close(c.done) + return nil +} + +// detectClusterChanges chooses a random index and shard from the +// shardNodes cache and deletes it. It then looks it up from Pilosa to +// see if it still matches, and if not it drops the whole cache. +func (c *Client) detectClusterChanges() { + c.shardNodes.mu.Lock() + needsUnlock := true + // we rely on Go's random map iteration order to get a random + // element. If it doesn't end up being random, it shouldn't + // actually matter. + for index, shardMap := range c.shardNodes.data { + for shard, uris := range shardMap { + delete(shardMap, shard) + c.shardNodes.data[index] = shardMap + c.shardNodes.mu.Unlock() + needsUnlock = false + newURIs, err := c.getURIsForShard(index, shard) // refetch URIs from server. + if err != nil { + c.logger.Errorf("problem invalidating shard node cache: %v", err) + return + } + if len(uris) != len(newURIs) { + c.logger.Printf("invalidating shard node cache old: %v, new: %v", uris, newURIs) + c.shardNodes.Invalidate() + return + } + for i := range uris { + u1, u2 := uris[i], newURIs[i] + if *u1 != *u2 { + c.logger.Printf("invalidating shard node cache, uri mismatch at %d old: %v, new: %v", i, uris, newURIs) + c.shardNodes.Invalidate() + return + } + } + break + } + break + } + if needsUnlock { + c.shardNodes.mu.Unlock() + } +} + +// DefaultClient creates a client with the default address and options. +func DefaultClient() *Client { + return newClientWithCluster(NewClusterWithHost(pnet.DefaultURI()), nil) +} + +func newClientFromAddresses(addresses []string, options *ClientOptions) (*Client, error) { + uris := make([]*pnet.URI, len(addresses)) + for i, address := range addresses { + uri, err := pnet.NewURIFromAddress(address) + if err != nil { + return nil, err + } + uris[i] = uri + } + cluster := NewClusterWithHost(uris...) + client := newClientWithCluster(cluster, options) + return client, nil +} + +func newClientWithCluster(cluster *Cluster, options *ClientOptions) *Client { + client := newClientWithOptions(options) + client.cluster = cluster + return client +} + +func newClientWithURI(uri *pnet.URI, options *ClientOptions) *Client { + client := newClientWithOptions(options) + if options.manualServerAddress { + fNode := newFragmentNodeFromURI(uri) + client.manualFragmentNode = &fNode + client.manualServerURI = uri + client.cluster = NewClusterWithHost() + } + client.cluster = NewClusterWithHost(uri) + return client +} + +func newClientWithOptions(options *ClientOptions) *Client { + if options == nil { + options = &ClientOptions{} + } + options = options.withDefaults() + + c := &Client{ + client: newHTTPClient(options.withDefaults()), + logger: logger.NewStandardLogger(os.Stderr), + primaryLock: &sync.RWMutex{}, + + shardNodes: newShardNodes(), + tick: time.NewTicker(time.Minute), + done: make(chan struct{}), + + nat: options.nat, + } + + if options.tracer == nil { + c.tracer = NoopTracer{} + } else { + c.tracer = options.tracer + } + if options.stats == nil { + c.Stats = stats.NopStatsClient + } else { + c.Stats = options.stats + } + + c.maxRetries = *options.retries + c.maxBackoff = 2 * time.Minute + go c.runChangeDetection() + return c + +} + +// NewClient creates a client with the given address, URI, or cluster and options. +func NewClient(addrURIOrCluster interface{}, options ...ClientOption) (*Client, error) { + var cluster *Cluster + clientOptions := &ClientOptions{ + nat: make(map[pnet.URI]pnet.URI), + } + err := clientOptions.addOptions(options...) + if err != nil { + return nil, err + } + + switch u := addrURIOrCluster.(type) { + case string: + uri, err := pnet.NewURIFromAddress(u) + if err != nil { + return nil, err + } + return newClientWithURI(uri, clientOptions), nil + case []string: + if len(u) == 1 { + uri, err := pnet.NewURIFromAddress(u[0]) + if err != nil { + return nil, err + } + return newClientWithURI(uri, clientOptions), nil + } else if clientOptions.manualServerAddress { + return nil, ErrSingleServerAddressRequired + } + return newClientFromAddresses(u, clientOptions) + case *pnet.URI: + uriCopy := *u + return newClientWithURI(&uriCopy, clientOptions), nil + case []*pnet.URI: + if len(u) == 1 { + uriCopy := *u[0] + return newClientWithURI(&uriCopy, clientOptions), nil + } else if clientOptions.manualServerAddress { + return nil, ErrSingleServerAddressRequired + } + cluster = NewClusterWithHost(u...) + case *Cluster: + cluster = u + case nil: + cluster = NewClusterWithHost() + default: + return nil, ErrAddrURIClusterExpected + } + + return newClientWithCluster(cluster, clientOptions), nil +} + +// Query runs the given query against the server with the given options. +// Pass nil for default options. +func (c *Client) Query(query PQLQuery, options ...interface{}) (*QueryResponse, error) { + span := c.tracer.StartSpan("Client.Query") + defer span.Finish() + + if err := query.Error(); err != nil { + return nil, err + } + queryOptions := &QueryOptions{} + err := queryOptions.addOptions(options...) + if err != nil { + return nil, err + } + serializedQuery := query.Serialize() + reqData, err := makeRequestData(serializedQuery.String(), queryOptions) + if err != nil { + return nil, errors.Wrap(err, "making request data") + } + path := fmt.Sprintf("/index/%s/query", query.Index().name) + _, respData, err := c.HTTPRequest("POST", path, reqData, c.augmentHeaders(defaultProtobufHeaders())) + if err != nil { + return nil, err + } + iqr := &pb.QueryResponse{} + err = proto.Unmarshal(respData, iqr) + if err != nil { + return nil, err + } + queryResponse, err := newQueryResponseFromInternal(iqr) + if err != nil { + return nil, err + } + return queryResponse, nil +} + +// CreateIndex creates an index on the server using the given Index struct. +func (c *Client) CreateIndex(index *Index) error { + span := c.tracer.StartSpan("Client.CreateIndex") + defer span.Finish() + + data := []byte(index.options.String()) + path := fmt.Sprintf("/index/%s", index.name) + status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil)) + if err != nil { + return errors.Wrapf(err, "creating index: %s", index.name) + } + var resp struct { + CreatedAt int64 `json:"createdAt,omitempty"` + } + if err := json.Unmarshal(body, &resp); err == nil && resp.CreatedAt != 0 { + index.createdAt = resp.CreatedAt + } + + if status == http.StatusConflict { + return ErrIndexExists + } + return nil +} + +// CreateField creates a field on the server using the given Field struct. +func (c *Client) CreateField(field *Field) error { + span := c.tracer.StartSpan("Client.CreateField") + defer span.Finish() + + data := []byte(field.options.String()) + path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name) + status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil)) + if err != nil { + return errors.Wrapf(err, "creating field: %s in index: %s", field.name, field.index.name) + } + var resp struct { + CreatedAt int64 `json:"createdAt,omitempty"` + } + if err := json.Unmarshal(body, &resp); err == nil && resp.CreatedAt != 0 { + field.createdAt = resp.CreatedAt + } + + if status == http.StatusConflict { + return ErrFieldExists + } + return nil +} + +// EnsureIndex creates an index on the server if it does not exist. +func (c *Client) EnsureIndex(index *Index) error { + err := c.CreateIndex(index) + if err == ErrIndexExists { + return nil + } + return errors.Wrap(err, "creating index") +} + +func (c *Client) SyncIndex(index *Index) error { + err := c.EnsureIndex(index) + if err != nil { + return errors.Wrapf(err, "ensuring index exists") + } + + for name, field := range index.fields { + if name == "_exists" { + continue + } + err = c.EnsureField(field) + if err != nil { + return errors.Wrapf(err, "ensuring field") + } + } + + return nil +} + +// EnsureField creates a field on the server if it doesn't exists. +func (c *Client) EnsureField(field *Field) error { + err := c.CreateField(field) + if err == ErrFieldExists { + return nil + } + return err +} + +// DeleteIndex deletes an index on the server. +func (c *Client) DeleteIndex(index *Index) error { + if index != nil { + return c.DeleteIndexByName(index.Name()) + } + return nil +} + +// DeleteIndexByName deletes the named index on the server. +func (c *Client) DeleteIndexByName(index string) error { + span := c.tracer.StartSpan("Client.DeleteIndex") + defer span.Finish() + + path := fmt.Sprintf("/index/%s", index) + _, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil)) + return err +} + +// DeleteField deletes a field on the server. +func (c *Client) DeleteField(field *Field) error { + span := c.tracer.StartSpan("Client.DeleteField") + defer span.Finish() + + path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name) + _, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil)) + return err +} + +// SyncSchema updates a schema with the indexes and fields on the server and +// creates the indexes and fields in the schema on the server side. +// This function does not delete indexes and the fields on the server side nor in the schema. +func (c *Client) SyncSchema(schema *Schema) error { + span := c.tracer.StartSpan("Client.SyncSchema") + defer span.Finish() + schema.mu.Lock() + defer schema.mu.Unlock() + serverSchema, err := c.Schema() + if err != nil { + return err + } + serverSchema.mu.RLock() + defer serverSchema.mu.RUnlock() + + return c.syncSchema(schema, serverSchema) +} + +func (c *Client) syncSchema(schema *Schema, serverSchema *Schema) error { + var err error + + // find out local - remote schema + diffSchema := schema.diff(serverSchema) + // create the indexes and fields which doesn't exist on the server side + for indexName, index := range diffSchema.indexes { + if _, ok := serverSchema.indexes[indexName]; !ok { + err = c.EnsureIndex(index) + if err != nil { + return errors.Wrap(err, "ensuring index") + } + } + for name, field := range index.fields { + if name == "_exists" { + continue + } + err = c.EnsureField(field) + if err != nil { + return errors.Wrapf(err, "ensuring field") + } + } + } + + // find out remote - local schema + diffSchema = serverSchema.diff(schema) + for indexName, index := range diffSchema.indexes { + if localIndex, ok := schema.indexes[indexName]; !ok { + schema.indexes[indexName] = index + } else { + for fieldName, field := range index.fields { + localIndex.fields[fieldName] = field + } + } + } + + return nil +} + +// Schema returns the indexes and fields on the server. +func (c *Client) Schema() (*Schema, error) { + span := c.tracer.StartSpan("Client.Schema") + defer span.Finish() + + var indexes []SchemaIndex + indexes, err := c.readSchema() + if err != nil { + return nil, err + } + schema := NewSchema() + for _, indexInfo := range indexes { + index := schema.indexWithOptions(indexInfo.Name, indexInfo.CreatedAt, indexInfo.ShardWidth, indexInfo.Options.asIndexOptions()) + + for _, fieldInfo := range indexInfo.Fields { + index.fieldWithOptions(fieldInfo.Name, fieldInfo.CreatedAt, fieldInfo.Options.asFieldOptions()) + } + } + return schema, nil +} + +// Import imports data for a single shard using the regular import +// endpoint rather than import-roaring. This is good for e.g. mutex or +// bool fields where import-roaring is not supported. +func (c *Client) Import(field *Field, shard uint64, vals, ids []uint64, clear bool) error { + path, data, err := c.EncodeImport(field, shard, vals, ids, clear) + if err != nil { + return errors.Wrap(err, "encoding import request") + } + err = c.DoImport(field.index.Name(), shard, path, data) + return errors.Wrap(err, "doing import") +} + +// EncodeImport computes the HTTP path and payload for an import +// request. It is typically followed by a call to DoImport. +func (c *Client) EncodeImport(field *Field, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error) { + msg := &pb.ImportRequest{ + Index: field.index.Name(), + IndexCreatedAt: field.index.CreatedAt(), + Field: field.Name(), + FieldCreatedAt: field.CreatedAt(), + Shard: shard, + RowIDs: vals, + ColumnIDs: ids, + } + data, err = proto.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.index.Name(), field.Name(), strconv.FormatBool(clear)) + return path, data, nil +} + +// DoImport takes a path and data payload (normally from EncodeImport +// or EncodeImportValues), logs the import, finds all nodes which own +// this shard, and concurrently imports to those nodes. +func (c *Client) DoImport(index string, shard uint64, path string, data []byte) error { + // Unlike ImportRoaring, Pilosa does not forward requests to the + // .../import endpoint to all replicas, so we must do that + // here. Yes this is odd. To make it worse, if a ../import request + // was made with keys that needed to be translated server side, + // Pilosa would handle sending the translated data to all the + // appropriate nodes and replicas. + + uris, err := c.getURIsForShard(index, shard) + if err != nil { + return errors.Wrap(err, "getting uris") + } + + eg := errgroup.Group{} + for _, uri := range uris { + uri := uri + eg.Go(func() error { + return c.importData(uri, path, data) + }) + } + return errors.Wrap(eg.Wait(), "importing to nodes") +} + +// EncodeImportValues computes the HTTP path and payload for an +// import-values request. It is typically followed by a call to +// DoImportValues. +func (c *Client) EncodeImportValues(field *Field, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error) { + msg := &pb.ImportValueRequest{ + Index: field.index.Name(), + IndexCreatedAt: field.index.CreatedAt(), + Field: field.Name(), + FieldCreatedAt: field.CreatedAt(), + Shard: shard, + ColumnIDs: ids, + Values: vals, + } + data, err = proto.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling ImportValue to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.index.Name(), field.Name(), strconv.FormatBool(clear)) + return path, data, nil +} + +// ImportValues takes the given integer values and column ids (which +// must all be in the given shard) and imports them into the given +// index,field,shard on all nodes which should hold that shard. It +// assumes that the ids have been translated from keys if necessary +// and so tells Pilosa to ignore checking if the index uses column +// keys. ImportValues wraps EncodeImportValues and DoImportValues — +// these are broken out and exported so that performance conscious +// users can re-use the same vals and ids byte buffers for local +// encoding, while performing the imports concurrently. +func (c *Client) ImportValues(field *Field, shard uint64, vals []int64, ids []uint64, clear bool) error { + path, data, err := c.EncodeImportValues(field, shard, vals, ids, clear) + if err != nil { + return errors.Wrap(err, "encoding import-values request") + } + err = c.DoImportValues(field.index.Name(), shard, path, data) + return errors.Wrap(err, "doing import values") +} + +// DoImportValues is deprecated. Use DoImport. +func (c *Client) DoImportValues(index string, shard uint64, path string, data []byte) error { + return c.DoImport(index, shard, path, data) +} + +func (c *Client) fetchFragmentNodes(indexName string, shard uint64) ([]fragmentNode, error) { + if c.manualFragmentNode != nil { + return []fragmentNode{*c.manualFragmentNode}, nil + } + path := fmt.Sprintf("/internal/fragment/nodes?shard=%d&index=%s", shard, indexName) + _, body, err := c.HTTPRequest("GET", path, []byte{}, c.augmentHeaders(nil)) + if err != nil { + return nil, err + } + fragmentNodes := []fragmentNode{} + var fragmentNodeURIs []fragmentNodeRoot + err = json.Unmarshal(body, &fragmentNodeURIs) + if err != nil { + return nil, errors.Wrap(err, "unmarshaling fragment node URIs") + } + for _, nodeURI := range fragmentNodeURIs { + fragmentNodes = append(fragmentNodes, nodeURI.URI) + } + return fragmentNodes, nil +} + +func (c *Client) fetchPrimaryNode() (fragmentNode, error) { + if c.manualFragmentNode != nil { + return *c.manualFragmentNode, nil + } + status, err := c.Status() + if err != nil { + return fragmentNode{}, err + } + for _, node := range status.Nodes { + if node.IsPrimary { + nodeURI := node.URI.URI().Translate(c.nat) + return fragmentNode{ //nolint:gosimple + Scheme: nodeURI.Scheme, + Host: nodeURI.Host, + Port: nodeURI.Port, + }, nil + } + } + return fragmentNode{}, errors.New("Primary node not found") +} + +func (c *Client) importData(uri *pnet.URI, path string, data []byte) error { + if status, _, err := c.doRequest(uri, "POST", path, c.augmentHeaders(defaultProtobufHeaders()), data); err != nil { + return errors.Wrapf(err, "import to %s", uri.HostPort()) + } else if status == http.StatusPreconditionFailed { + return ErrPreconditionFailed + } + + return nil +} + +// ImportRoaringShard imports into the shard-transactional endpoint. +func (c *Client) ImportRoaringShard(index string, shard uint64, request *pilosa.ImportRoaringShardRequest) error { + uris, err := c.getURIsForShard(index, shard) + if err != nil { + return errors.Wrap(err, "getting URIs for import") + } + + data, err := fbproto.DefaultSerializer.Marshal(request) + if err != nil { + return errors.Wrap(err, "marshaling") + } + eg := errgroup.Group{} + for _, uri := range uris { + uri := uri + eg.Go(func() error { + return c.importData(uri, fmt.Sprintf("/index/%s/shard/%d/import-roaring", index, shard), data) + }) + } + err = eg.Wait() + return errors.Wrap(err, "importing") +} + +// ImportRoaringBitmap can import pre-made bitmaps for a number of +// different views into the given field/shard. If the view name in the +// map is an empty string, the standard view will be used. +func (c *Client) ImportRoaringBitmap(field *Field, shard uint64, views map[string]*roaring.Bitmap, clear bool) error { + uris, err := c.getURIsForShard(field.index.Name(), shard) + if err != nil { + return errors.Wrap(err, "getting URIs for import") + } + err = c.importRoaringBitmap(uris[0], field, shard, views, &ImportOptions{clear: clear}) + return errors.Wrap(err, "importing bitmap") +} + +func (c *Client) importRoaringBitmap(uri *pnet.URI, field *Field, shard uint64, views viewImports, options *ImportOptions) error { + protoViews := []*pb.ImportRoaringRequestView{} + for name, bmp := range views { + buf := &bytes.Buffer{} + _, err := bmp.WriteTo(buf) + if err != nil { + return errors.Wrap(err, "marshalling bitmap") + } + protoViews = append(protoViews, &pb.ImportRoaringRequestView{ + Name: name, + Data: buf.Bytes(), + }) + } + params := url.Values{} + params.Add("clear", strconv.FormatBool(options.clear)) + path := makeRoaringImportPath(field, shard, params) + req := &pb.ImportRoaringRequest{ + Clear: options.clear, + Views: protoViews, + IndexCreatedAt: field.index.CreatedAt(), + FieldCreatedAt: field.CreatedAt(), + } + data, err := proto.Marshal(req) + if err != nil { + return err + } + + header := c.augmentHeaders(defaultProtobufHeaders()) + status, _, err := c.doRequest(uri, "POST", path, header, data) + if err != nil { + return errors.Wrapf(err, "roaring import to %s, status: %d", uri.HostPort(), status) + } + if status == http.StatusPreconditionFailed { + return ErrPreconditionFailed + } + + return nil +} + +// ExportField exports columns for a field. +func (c *Client) ExportField(field *Field) (io.Reader, error) { + span := c.tracer.StartSpan("Client.ExportField") + defer span.Finish() + + var shardsMax map[string]uint64 + var err error + + status, err := c.Status() + if err != nil { + return nil, err + } + shardsMax, err = c.shardsMax() + if err != nil { + return nil, err + } + status.indexMaxShard = shardsMax + shardURIs, err := c.statusToNodeShardsForIndex(status, field.index.Name()) + if err != nil { + return nil, err + } + + return newExportReader(c, shardURIs, field), nil +} + +// Info returns the server's configuration/host information. +func (c *Client) Info() (Info, error) { + span := c.tracer.StartSpan("Client.Info") + defer span.Finish() + + _, data, err := c.HTTPRequest("GET", "/info", nil, c.augmentHeaders(nil)) + if err != nil { + return Info{}, errors.Wrap(err, "requesting /info") + } + info := Info{} + err = json.Unmarshal(data, &info) + if err != nil { + return Info{}, errors.Wrap(err, "unmarshaling /info data") + } + return info, nil +} + +// Status returns the server's status. +func (c *Client) Status() (Status, error) { + span := c.tracer.StartSpan("Client.Status") + defer span.Finish() + + _, data, err := c.HTTPRequest("GET", "/status", nil, nil) + if err != nil { + return Status{}, errors.Wrap(err, "requesting /status") + } + status := Status{} + err = json.Unmarshal(data, &status) + if err != nil { + return Status{}, errors.Wrap(err, "unmarshaling /status data") + } + return status, nil +} + +func (c *Client) readSchema() ([]SchemaIndex, error) { + _, data, err := c.HTTPRequest("GET", "/schema", nil, c.augmentHeaders(nil)) + if err != nil { + return nil, errors.Wrap(err, "requesting /schema") + } + schemaInfo := SchemaInfo{} + err = json.Unmarshal(data, &schemaInfo) + if err != nil { + return nil, errors.Wrap(err, "unmarshaling /schema data") + } + return schemaInfo.Indexes, nil +} + +func (c *Client) IngestSchema(reqBody map[string]interface{}) (body []byte, err error) { + data, err := json.Marshal(reqBody) + if err != nil { + return data, errors.Wrap(err, "error building Schema body to Ingest") + } + return c.IngestRequest("/internal/schema", data) +} + +func (c *Client) IngestData(index string, reqBody []map[string]interface{}) (body []byte, err error) { + data, err := json.Marshal(reqBody) + if err != nil { + return data, errors.Wrap(err, "error building request body to Ingest") + } + return c.IngestRequest("/internal/ingest/"+index, data) +} + +func (c *Client) IngestRequest(uri string, data []byte) (body []byte, err error) { + var header = make(map[string]string) + header["Content-Type"] = "application/json" + header["Accept"] = "application/json" + header["User-Agent"] = "pilosa/" + pilosa.Version + status, body, err := c.HTTPRequest("POST", uri, data, header) + if err != nil { + return nil, errors.Wrapf(err, "requesting %s status: %d", uri, status) + } + return body, err +} + +func (c *Client) shardsMax() (map[string]uint64, error) { + _, data, err := c.HTTPRequest("GET", "/internal/shards/max", nil, nil) + if err != nil { + return nil, errors.Wrap(err, "requesting /internal/shards/max") + } + m := map[string]map[string]uint64{} + err = json.Unmarshal(data, &m) + if err != nil { + return nil, errors.Wrap(err, "unmarshaling /internal/shards/max data") + } + return m["standard"], nil +} + +// HTTPRequest sends an HTTP request to the Pilosa server (used by idk) +func (c *Client) HTTPRequest(method string, path string, data []byte, headers map[string]string) (status int, body []byte, err error) { + span := c.tracer.StartSpan("Client.HTTPRequest") + + status, body, err = c.httpRequest(method, path, data, headers, false) + span.Finish() + return +} + +// httpRequest makes a request to the cluster - use this when you want the +// client to choose a host, and it doesn't matter if the request goes to a +// specific host +func (c *Client) httpRequest(method string, path string, data []byte, headers map[string]string, usePrimary bool) (int, []byte, error) { + if data == nil { + data = []byte{} + } + + var ( + status int + body []byte + err error + ) + // try request on host, if it fails, try again on primary + for i := 0; i <= 1; i++ { + host, herr := c.host(usePrimary) + if herr != nil { + return status, nil, errors.Wrapf(herr, "getting host, previous err: %v", err) + } + // doRequest implements expotential backoff + status, body, err = c.doRequest(host, method, path, c.augmentHeaders(headers), data) + // conditions when primary should not be tried + if err == nil || usePrimary || path == "/status" { + break + } + + usePrimary = true + } + + if err != nil { + err = errors.Wrap(err, ErrHTTPRequest.Error()) + } + + return status, body, err +} + +// host returns the first URI that applies, in this order: +// - a non-nil manualServerURI +// - primary URI (if usePrimary = true) +// - the next host from the node list (round-robin) +func (c *Client) host(usePrimary bool) (*pnet.URI, error) { + if c.manualServerURI != nil { + return c.manualServerURI, nil + } + var host *pnet.URI + if usePrimary { + c.primaryLock.RLock() + host = c.primaryURI + c.primaryLock.RUnlock() + if host == nil { + c.primaryLock.Lock() + if c.primaryURI == nil { + node, err := c.fetchPrimaryNode() + if err != nil { + c.primaryLock.Unlock() + return nil, errors.Wrap(err, "fetching primary node") + } + if host, err = pnet.NewURIFromAddress(fmt.Sprintf("%s://%s:%d", node.Scheme, node.Host, node.Port)); err != nil { + return nil, errors.Wrap(err, "parsing primary node URL") + } + } else { + host = c.primaryURI + } + c.primaryURI = host + c.primaryLock.Unlock() + } + } else { + // get a host from the cluster + host = c.cluster.Host() + if host == nil { + return nil, ErrEmptyCluster + } + } + return host, nil +} + +// doRequest creates and performs an http request. +func (c *Client) doRequest(host *pnet.URI, method, path string, headers map[string]string, data []byte) (int, []byte, error) { + var ( + req *http.Request + resp *http.Response + err error + sleepTime time.Duration + rand = rand.New(rand.NewSource(time.Now().UnixNano())) + ) + for retry := 0; ; { + if req, err = buildRequest(host, method, path, headers, data); err != nil { + return 0, nil, errors.Wrap(err, "building request") + } + if resp, err = c.client.Do(req); err != nil { + return 0, nil, errors.Wrap(err, "sending request") + } + if warning := resp.Header.Get("warning"); warning != "" { + c.logger.Warnf(warning) + } + + buf := bytes.NewBuffer(make([]byte, 0, 1+resp.ContentLength)) + _, err = buf.ReadFrom(resp.Body) + _ = resp.Body.Close() + if err != nil { + return resp.StatusCode, nil, errors.Wrap(err, "reading response body") + } + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + // [200, 300): OK + return resp.StatusCode, buf.Bytes(), nil + + case resp.StatusCode == 409: + // 409 Conflict + return resp.StatusCode, buf.Bytes(), nil + + case resp.StatusCode == 412: + // 412 Precondition Failed + return resp.StatusCode, buf.Bytes(), nil + + case resp.StatusCode == 429: + // 429 Too Many Requests + // A Retry-After header might be included to this response indicating how long to wait before making a new request. + if ms, _ := strconv.Atoi(resp.Header.Get("Retry-After")); ms > 0 { + sleepTime = time.Duration(ms) * time.Millisecond + } else { + sleepTime = time.Duration(1<= 400 && resp.StatusCode < 500: + // don't retry any 400 level errors + return resp.StatusCode, nil, errors.New(strings.TrimSpace(buf.String())) + + case resp.StatusCode == 503: + // This indicates that Pilosa is not ready to service this request, + // typically during startup. In this case, it's ok to give Pilosa + // some time and try again. + sleepTime = time.Duration(1<= c.maxRetries { + // If the error here is nil, we still want to return an error because + // we've hit the max retries limit. If an error exists, wrap it. + errMsg := fmt.Sprintf("max retries (%d) exceeded", c.maxRetries) + if err == nil { + return resp.StatusCode, nil, errors.New(errMsg) + } + return resp.StatusCode, nil, errors.Wrap(err, errMsg) + } + // The client can continue retrying after it has reached the maxBackoff time. + if sleepTime > c.maxBackoff { + return resp.StatusCode, nil, errors.Wrapf(err, "max backoff (%s) time exceeded", c.maxBackoff) + } + retry++ + c.logger.Errorf("request failed with: '%v' status: %d, retrying %d after %v ", err, resp.StatusCode, retry, sleepTime) + time.Sleep(sleepTime) + } + // Unreachable code +} + +// statusToNodeShardsForIndex finds the hosts which contains shards for the given index +func (c *Client) statusToNodeShardsForIndex(status Status, indexName string) (map[uint64]*pnet.URI, error) { + result := make(map[uint64]*pnet.URI) + if maxShard, ok := status.indexMaxShard[indexName]; ok { + for shard := 0; shard <= int(maxShard); shard++ { + fragmentNodes, err := c.fetchFragmentNodes(indexName, uint64(shard)) + if err != nil { + return nil, err + } + if len(fragmentNodes) == 0 { + return nil, ErrNoFragmentNodes + } + node := fragmentNodes[0] + uri := &pnet.URI{ + Host: node.Host, + Port: node.Port, + Scheme: node.Scheme, + } + + result[uint64(shard)] = uri + } + } else { + return nil, ErrNoShard + } + return result, nil +} + +func (c *Client) augmentHeaders(headers map[string]string) map[string]string { + if headers == nil { + headers = map[string]string{} + } + + // TODO: move the following block to NewClient once cluster-resize support branch is merged. + version := strings.TrimPrefix(Version, "v") + + headers["User-Agent"] = fmt.Sprintf("pilosa/client/%s", version) + if c.AuthToken != "" { + headers["Authorization"] = c.AuthToken + } + return headers +} + +// FindFieldKeys looks up the IDs associated with specified keys in a field. +// If a key does not exist, the result will not include it. +func (c *Client) FindFieldKeys(field *Field, keys ...string) (map[string]uint64, error) { + path := fmt.Sprintf("/internal/translate/field/%s/%s/keys/find", field.index.name, field.name) + + reqData, err := json.Marshal(keys) + if err != nil { + return nil, errors.Wrap(err, "marshalling request") + } + + headers := c.augmentHeaders(map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }) + + status, body, err := c.HTTPRequest(http.MethodPost, path, reqData, headers) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + if status != http.StatusOK { + return nil, errors.Errorf("find field keys request failed (status: %d): %q", status, string(body)) + } + + var result map[string]uint64 + err = json.Unmarshal(body, &result) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling response") + } + + return result, nil +} + +// CreateFieldKeys looks up the IDs associated with specified keys in a field. +// If a key does not exist, it will be created. +func (c *Client) CreateFieldKeys(field *Field, keys ...string) (map[string]uint64, error) { + path := fmt.Sprintf("/internal/translate/field/%s/%s/keys/create", field.index.name, field.name) + + reqData, err := json.Marshal(keys) + if err != nil { + return nil, errors.Wrap(err, "marshalling request") + } + + headers := c.augmentHeaders(map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }) + + status, body, err := c.httpRequest(http.MethodPost, path, reqData, headers, field.options.foreignIndex == "") + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + if status != http.StatusOK { + return nil, errors.Errorf("find field keys request failed (status: %d): %q", status, string(body)) + } + + var result map[string]uint64 + err = json.Unmarshal(body, &result) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling response") + } + + return result, nil +} + +// FindIndexKeys looks up the IDs associated with specified column keys in an index. +// If a key does not exist, the result will not include it. +func (c *Client) FindIndexKeys(idx *Index, keys ...string) (map[string]uint64, error) { + path := fmt.Sprintf("/internal/translate/index/%s/keys/find", idx.name) + + reqData, err := json.Marshal(keys) + if err != nil { + return nil, errors.Wrap(err, "marshalling request") + } + + headers := c.augmentHeaders(map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }) + + status, body, err := c.HTTPRequest(http.MethodPost, path, reqData, headers) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + if status != http.StatusOK { + return nil, errors.Errorf("find field keys request failed (status: %d): %q", status, string(body)) + } + + var result map[string]uint64 + err = json.Unmarshal(body, &result) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling response") + } + + return result, nil +} + +// CreateIndexKeys looks up the IDs associated with specified column keys in an index. +// If a key does not exist, it will be created. +func (c *Client) CreateIndexKeys(idx *Index, keys ...string) (map[string]uint64, error) { + path := fmt.Sprintf("/internal/translate/index/%s/keys/create", idx.name) + + reqData, err := json.Marshal(keys) + if err != nil { + return nil, errors.Wrap(err, "marshalling request") + } + + headers := c.augmentHeaders(map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }) + + status, body, err := c.HTTPRequest(http.MethodPost, path, reqData, headers) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + if status != http.StatusOK { + return nil, errors.Errorf("find field keys request failed (status: %d): %q", status, string(body)) + } + + var result map[string]uint64 + err = json.Unmarshal(body, &result) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling response") + } + + return result, nil +} + +type TransactionResponse struct { + Transaction *pilosa.Transaction `json:"transaction,omitempty"` + Error string `json:"error,omitempty"` +} + +// StartTransaction tries to start a new transaction in Pilosa. It +// will continue trying until at least requestTimeout time has +// passed. If it fails due to an exclusive transaction already +// existing, it will return that transaction along with a non-nil +// error. +func (c *Client) StartTransaction(id string, timeout time.Duration, exclusive bool, requestTimeout time.Duration) (*pilosa.Transaction, error) { + return c.startTransaction(id, timeout, exclusive, time.Now().Add(requestTimeout)) +} + +func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bool, deadline time.Time) (*pilosa.Transaction, error) { + trns := pilosa.Transaction{ + ID: id, + Timeout: timeout, + Exclusive: exclusive, + } + bod, err := json.Marshal(&trns) + if err != nil { + return nil, errors.Wrap(err, "marshalling transaction") + } + + status, data, err := c.httpRequest("POST", "/transaction", bod, c.augmentHeaders(defaultJSONHeaders()), true) + if status == http.StatusConflict && time.Now().Before(deadline) { + // if we're getting StatusConflict after all the usual timeouts/retries, keep retrying until the deadline + time.Sleep(time.Second) + return c.startTransaction(id, timeout, exclusive, deadline) + } + if err != nil { + return nil, err + } + + tr := &TransactionResponse{} + uerr := json.Unmarshal(data, &tr) + if uerr != nil { + if err != nil { + return nil, errors.Wrap(err, "unmarshal failed after") + } + return nil, errors.Wrap(uerr, "couldn't decode body") + } + + if tr.Error != "" { + err = errors.New(tr.Error) + } + + return tr.Transaction, err +} + +func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) { + _, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, c.augmentHeaders(defaultJSONHeaders()), true) + if err != nil && len(data) == 0 { + return nil, err + } + + tr := &TransactionResponse{} + uerr := json.Unmarshal(data, &tr) + if uerr != nil { + if err != nil { + return nil, errors.Wrap(err, "unmarshal failed after") + } + return nil, errors.Wrap(uerr, "couldn't decode body") + } + + if tr.Error != "" { + err = errors.New(tr.Error) + } + + return tr.Transaction, err +} + +func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) { + _, respData, err := c.httpRequest("GET", "/transactions", nil, c.augmentHeaders(defaultJSONHeaders()), true) + if err != nil { + return nil, errors.Wrap(err, "getting transactions") + } + + trnsMap := make(map[string]*pilosa.Transaction) + err = json.Unmarshal(respData, &trnsMap) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling transactions") + } + return trnsMap, nil +} + +func (c *Client) GetTransaction(id string) (*pilosa.Transaction, error) { + _, data, err := c.httpRequest("GET", "/transaction/"+id, nil, c.augmentHeaders(defaultJSONHeaders()), true) + if err != nil { + return nil, err + } + + tr := &TransactionResponse{} + uerr := json.Unmarshal(data, &tr) + if uerr != nil { + if err != nil { + return nil, errors.Wrap(err, "unmarshal failed after") + } + return nil, errors.Wrap(uerr, "couldn't decode body") + } + + if tr.Error != "" { + err = errors.New(tr.Error) + } + + return tr.Transaction, err +} + +func defaultProtobufHeaders() map[string]string { + return map[string]string{ + "Content-Type": "application/x-protobuf", + "Accept": "application/x-protobuf", + "PQL-Version": PQLVersion, + } +} + +func defaultJSONHeaders() map[string]string { + return map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + "PQL-Version": PQLVersion, + } +} + +func buildRequest(host *pnet.URI, method, path string, headers map[string]string, data []byte) (*http.Request, error) { + request, err := http.NewRequest(method, host.Normalize()+path, bytes.NewReader(data)) + if err != nil { + return nil, err + } + + for k, v := range headers { + request.Header.Set(k, v) + } + + return request, nil +} + +func newHTTPClient(options *ClientOptions) *http.Client { + transport := &http.Transport{ + Dial: (&net.Dialer{ + Timeout: options.ConnectTimeout, + }).Dial, + TLSClientConfig: options.TLSConfig, + MaxIdleConnsPerHost: options.PoolSizePerRoute, + MaxIdleConns: options.TotalPoolSize, + } + return &http.Client{ + Transport: transport, + Timeout: options.SocketTimeout, + } +} + +func makeRequestData(query string, options *QueryOptions) ([]byte, error) { + request := &pb.QueryRequest{ + Query: query, + Shards: options.Shards, + } + r, err := proto.Marshal(request) + if err != nil { + return nil, errors.Wrap(err, "marshaling request to protobuf") + } + return r, nil +} + +func makeRoaringImportPath(field *Field, shard uint64, params url.Values) string { + return fmt.Sprintf("/index/%s/field/%s/import-roaring/%d?%s", + field.index.name, field.name, shard, params.Encode()) +} + +type viewImports map[string]*roaring.Bitmap + +// ClientOptions control the properties of client connection to the server. +type ClientOptions struct { + SocketTimeout time.Duration + ConnectTimeout time.Duration + PoolSizePerRoute int + TotalPoolSize int + TLSConfig *tls.Config + manualServerAddress bool + tracer opentracing.Tracer + retries *int + stats stats.StatsClient + nat map[pnet.URI]pnet.URI +} + +func (co *ClientOptions) addOptions(options ...ClientOption) error { + for _, option := range options { + err := option(co) + if err != nil { + return err + } + } + return nil +} + +// ClientOption is used when creating a PilosaClient struct. +type ClientOption func(options *ClientOptions) error + +// OptClientSocketTimeout is the maximum idle socket time in nanoseconds +func OptClientSocketTimeout(timeout time.Duration) ClientOption { + return func(options *ClientOptions) error { + options.SocketTimeout = timeout + return nil + } +} + +// OptClientConnectTimeout is the maximum time to connect in nanoseconds. +func OptClientConnectTimeout(timeout time.Duration) ClientOption { + return func(options *ClientOptions) error { + options.ConnectTimeout = timeout + return nil + } +} + +// OptClientPoolSizePerRoute is the maximum number of active connections in the pool to a host. +func OptClientPoolSizePerRoute(size int) ClientOption { + return func(options *ClientOptions) error { + options.PoolSizePerRoute = size + return nil + } +} + +// OptClientTotalPoolSize is the maximum number of connections in the pool. +func OptClientTotalPoolSize(size int) ClientOption { + return func(options *ClientOptions) error { + options.TotalPoolSize = size + return nil + } +} + +// OptClientTLSConfig contains the TLS configuration. +func OptClientTLSConfig(config *tls.Config) ClientOption { + return func(options *ClientOptions) error { + options.TLSConfig = config + return nil + } +} + +// OptClientManualServerAddress forces the client use only the manual server address +func OptClientManualServerAddress(enabled bool) ClientOption { + return func(options *ClientOptions) error { + options.manualServerAddress = enabled + return nil + } +} + +// OptClientTracer sets the Open Tracing tracer +// See: https://opentracing.io +func OptClientTracer(tracer opentracing.Tracer) ClientOption { + return func(options *ClientOptions) error { + options.tracer = tracer + return nil + } +} + +// OptClientRetries sets the number of retries on HTTP request failures. +func OptClientRetries(retries int) ClientOption { + return func(options *ClientOptions) error { + if retries < 0 { + return errors.New("retries must be non-negative") + } + options.retries = &retries + return nil + } +} + +// OptClientStatsClient sets a stats client, such as Prometheus +func OptClientStatsClient(stats stats.StatsClient) ClientOption { + return func(options *ClientOptions) error { + options.stats = stats + return nil + } +} + +// OptClientNAT sets a NAT map used to translate the advertised URI to something +// else (for example, when accessing pilosa running in docker). +func OptClientNAT(nat map[string]string) ClientOption { + return func(options *ClientOptions) error { + // covert the strings to URIs + m := make(map[pnet.URI]pnet.URI) + for k, v := range nat { + if kuri, err := pnet.NewURIFromAddress(k); err != nil { + return errors.Wrapf(err, "converting string to URI: %s", k) + } else if vuri, err := pnet.NewURIFromAddress(v); err != nil { + return errors.Wrapf(err, "converting string to URI: %s", v) + } else { + m[*kuri] = *vuri + } + } + options.nat = m + return nil + } +} + +func (co *ClientOptions) withDefaults() (updated *ClientOptions) { + // copy options so the original is not updated + updated = &ClientOptions{} + *updated = *co + // impose defaults + if updated.SocketTimeout <= 0 { + updated.SocketTimeout = time.Second * 300 + } + if updated.ConnectTimeout <= 0 { + updated.ConnectTimeout = time.Second * 60 + } + if updated.PoolSizePerRoute <= 0 { + updated.PoolSizePerRoute = 50 + } + if updated.TotalPoolSize <= 0 { + updated.TotalPoolSize = 500 + } + if updated.TLSConfig == nil { + updated.TLSConfig = &tls.Config{} + } + if updated.retries == nil { + retries := 2 + updated.retries = &retries + } + return +} + +// QueryOptions contains options to customize the Query function. +type QueryOptions struct { + // Shards restricts query to a subset of shards. Queries all shards if nil. + Shards []uint64 +} + +func (qo *QueryOptions) addOptions(options ...interface{}) error { + for i, option := range options { + switch o := option.(type) { + case nil: + if i != 0 { + return ErrInvalidQueryOption + } + continue + case *QueryOptions: + if i != 0 { + return ErrInvalidQueryOption + } + *qo = *o + case QueryOption: + err := o(qo) + if err != nil { + return err + } + default: + return ErrInvalidQueryOption + } + } + return nil +} + +// QueryOption is used when using options with a client.Query, +type QueryOption func(options *QueryOptions) error + +// OptQueryShards restricts the set of shards on which a query operates. +func OptQueryShards(shards ...uint64) QueryOption { + return func(options *QueryOptions) error { + options.Shards = append(options.Shards, shards...) + return nil + } +} + +// ImportOptions are the options for controlling the importer +type ImportOptions struct { + threadCount int + batchSize int + wantRoaring *bool + clear bool + skipSort bool +} + +// ImportOption is used when running imports. +type ImportOption func(options *ImportOptions) error + +// OptImportThreadCount is the number of goroutines allocated for import. +func OptImportThreadCount(count int) ImportOption { + return func(options *ImportOptions) error { + options.threadCount = count + return nil + } +} + +// OptImportBatchSize is the number of records read before importing them. +func OptImportBatchSize(batchSize int) ImportOption { + return func(options *ImportOptions) error { + options.batchSize = batchSize + return nil + } +} + +// OptImportClear sets clear import, which clears bits instead of setting them. +func OptImportClear(clear bool) ImportOption { + return func(options *ImportOptions) error { + options.clear = clear + return nil + } +} + +// OptImportRoaring enables importing using roaring bitmaps which is more performant. +func OptImportRoaring(enable bool) ImportOption { + return func(options *ImportOptions) error { + options.wantRoaring = &enable + return nil + } +} + +// OptImportSort tells the importer whether or not to sort batches of records, on +// by default. Sorting imposes some performance cost, especially on data that's +// already sorted, but dramatically improves performance in pathological +// cases. It is enabled by default because the pathological cases are awful, +// and the performance hit is comparatively small, but the performance cost can +// be significant if you know your data is sorted. +func OptImportSort(sorting bool) ImportOption { + return func(options *ImportOptions) error { + // skipSort is expressed negatively because we want to + // keep sorting enabled by default, so the zero value should + // be that default behavior. The client option expresses it + // positively because that's easier for API users. + options.skipSort = !sorting + return nil + } +} + +type fragmentNodeRoot struct { + URI fragmentNode `json:"uri"` +} + +type fragmentNode struct { + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` +} + +func newFragmentNodeFromURI(uri *pnet.URI) fragmentNode { + return fragmentNode{ + Scheme: uri.Scheme, + Host: uri.Host, + Port: uri.Port, + } +} + +func (node fragmentNode) URI() *pnet.URI { + return &pnet.URI{ + Scheme: node.Scheme, + Host: node.Host, + Port: node.Port, + } +} + +// Info contains the configuration/host information from a Pilosa server. +type Info struct { + ShardWidth uint64 `json:"shardWidth"` // width of each shard + Memory uint64 `json:"memory"` // approximate host physical memory + CPUType string `json:"cpuType"` // "brand name string" from cpuid + CPUPhysicalCores int `json:"CPUPhysicalCores"` // physical cores (cpuid) + CPULogicalCores int `json:"CPULogicalCores"` // logical cores cpuid + CPUMHz uint64 `json:"CPUMHz"` // estimated clock speed +} + +// Status contains the status information from a Pilosa server. +type Status struct { + Nodes []StatusNode `json:"nodes"` + State string `json:"state"` + LocalID string `json:"localID"` + indexMaxShard map[string]uint64 +} + +// StatusNode contains information about a node in the cluster. +type StatusNode struct { + ID string `json:"id"` + URI StatusURI `json:"uri"` + IsPrimary bool `json:"isPrimary"` +} + +// StatusURI contains node information. +type StatusURI struct { + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` +} + +// URI returns the StatusURI as a URI. +func (s StatusURI) URI() pnet.URI { + return pnet.URI{ + Scheme: s.Scheme, + Host: s.Host, + Port: s.Port, + } +} + +// SchemaInfo contains the indexes. +type SchemaInfo struct { + Indexes []SchemaIndex `json:"indexes"` +} + +// SchemaIndex contains index information. +type SchemaIndex struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options SchemaOptions `json:"options"` + Fields []SchemaField `json:"fields"` + Shards []uint64 `json:"shards"` + ShardWidth uint64 `json:"shardWidth"` +} + +// SchemaField contains field information. +type SchemaField struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options SchemaOptions `json:"options"` +} + +// SchemaOptions contains options for a field or an index. +type SchemaOptions struct { + FieldType FieldType `json:"type"` + CacheType string `json:"cacheType"` + CacheSize uint `json:"cacheSize"` + TimeQuantum string `json:"timeQuantum"` + TTL time.Duration `json:"ttl"` + Min pql.Decimal `json:"min"` + Max pql.Decimal `json:"max"` + Scale int64 `json:"scale"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView"` + TrackExistence bool `json:"trackExistence"` + TimeUnit string `json:"timeUnit"` + Base int64 `json:"base"` + Epoch time.Time `json:"epoch"` +} + +func (so SchemaOptions) asIndexOptions() *IndexOptions { + return &IndexOptions{ + keys: so.Keys, + keysSet: true, + trackExistence: so.TrackExistence, + trackExistenceSet: true, + } +} + +func (so SchemaOptions) asFieldOptions() *FieldOptions { + return &FieldOptions{ + fieldType: so.FieldType, + cacheSize: int(so.CacheSize), + cacheType: CacheType(so.CacheType), + timeQuantum: TimeQuantum(so.TimeQuantum), + ttl: so.TTL, + min: so.Min, + max: so.Max, + scale: so.Scale, + keys: so.Keys, + noStandardView: so.NoStandardView, + timeUnit: so.TimeUnit, + base: so.Base, + epoch: so.Epoch, + } +} + +type exportReader struct { + client *Client + shardURIs map[uint64]*pnet.URI + field *Field + body []byte + bodyIndex int + currentShard uint64 + shardCount uint64 +} + +func newExportReader(client *Client, shardURIs map[uint64]*pnet.URI, field *Field) *exportReader { + return &exportReader{ + client: client, + shardURIs: shardURIs, + field: field, + shardCount: uint64(len(shardURIs)), + } +} + +// Read updates the passed array with the exported CSV data and returns the number of bytes read +func (r *exportReader) Read(p []byte) (n int, err error) { + if r.currentShard >= r.shardCount { + err = io.EOF + return + } + if r.body == nil { + uri := r.shardURIs[r.currentShard] + headers := map[string]string{ + "Accept": "text/csv", + } + path := fmt.Sprintf("/export?index=%s&field=%s&shard=%d", + r.field.index.Name(), r.field.Name(), r.currentShard) + _, respData, err := r.client.doRequest(uri, "GET", path, headers, nil) + if err != nil { + return 0, errors.Wrap(err, "doing export request") + } + r.body = respData + r.bodyIndex = 0 + } + n = copy(p, r.body[r.bodyIndex:]) + r.bodyIndex += n + if n >= len(r.body) { + r.body = nil + r.currentShard++ + } + return +} diff --git a/client/client_it_test.go b/client/client_it_test.go new file mode 100644 index 000000000..aea626caa --- /dev/null +++ b/client/client_it_test.go @@ -0,0 +1,893 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package client + +import ( + "bytes" + "fmt" + "io/ioutil" + "testing" + "time" + + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/disco" + pnet "github.com/molecula/featurebase/v3/net" + "github.com/molecula/featurebase/v3/roaring" + "github.com/molecula/featurebase/v3/shardwidth" + "github.com/molecula/featurebase/v3/test" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +var ( + testIndex *Index + testIndexWithKeys *Index + testIndexWithKeysNoTrack *Index + testIndexAtomicRecord *Index + testIndexKeyTranslation *Index + + testField *Field + testFieldTimestamp *Field + testFieldInt *Field + testFieldTimeQuantum *Field + testFieldInt0 *Field + testFieldInt1 *Field +) + +func setup(t *testing.T, cli *Client) { + t.Helper() + + testSchema := NewSchema() + testIndex = testSchema.Index("test-index") + testIndexWithKeys = testSchema.Index("test-index-keys", OptIndexKeys(true)) + testIndexWithKeysNoTrack = testSchema.Index("test-index-keys-notrack", + OptIndexKeys(true), + OptIndexTrackExistence(false), + ) + testField = testIndex.Field("test-field") + testFieldTimeQuantum = testIndex.Field("test-field-timequantum", OptFieldTypeTime(TimeQuantumYear)) + testFieldTimestamp = testIndex.Field("test-field-timestamp", OptFieldTypeTimestamp(time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC), "s")) + testFieldInt = testIndex.Field("test-field-int", OptFieldTypeInt(0, 100000)) + testIndexKeyTranslation = testSchema.Index("test-index-key-translation", OptIndexKeys(true)) + + testIndexAtomicRecord = testSchema.Index("test-index-atomic-record") + testFieldInt0 = testIndexAtomicRecord.Field("test-field-int0", OptFieldTypeInt(-1000, 1000)) + testFieldInt1 = testIndexAtomicRecord.Field("test-field-int1", OptFieldTypeInt(-1000, 1000)) + + require.NoErrorf(t, cli.SyncSchema(testSchema), "SyncSchema") +} + +func tearDown(t *testing.T, cli *Client) { + t.Helper() + + for _, i := range []*Index{testIndex, testIndexWithKeys, testIndexWithKeysNoTrack, testIndexAtomicRecord, testIndexKeyTranslation} { + require.NoErrorf(t, cli.DeleteIndex(i), "DeleteIndex(%s)", i.name) + } +} + +func TestClientAgainstCluster(t *testing.T) { + for size, replicaN := 3, 1; replicaN <= 2; replicaN++ { + testName := fmt.Sprintf("%d.%d", size, replicaN) + t.Run(testName, func(t *testing.T) { + + // Start size.replicaN cluster + c := test.MustNewCluster(t, size) + for _, n := range c.Nodes { + n.Config.Cluster.ReplicaN = replicaN + } + err := c.Start() + require.NoError(t, err, "Start cluster "+testName) + + urls := make([]string, len(c.Nodes)) + for i, n := range c.Nodes { + urls[i] = n.URL() + } + defer c.Close() + + // Create a new client for the cluster + cli, err := newClientFromAddresses(urls, &ClientOptions{}) + require.NoErrorf(t, err, "newClientFromAddresses(%v): %v", urls, err) + defer cli.Close() + + t.Run("GetStatus", func(t *testing.T) { + status, err := cli.Status() + require.NoErrorf(t, err, "GET /status") + + require.Equalf(t, disco.ClusterStateNormal, disco.ClusterState(status.State), "GET /status") + }) + + t.Run("QueryRow", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + resp, err := cli.Query(testField.Row(1)) + require.NoErrorf(t, err, "Query Row") + require.NotNil(t, resp, "Response should not be nil") + }) + + t.Run("IntBase", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testIndex.Field("intbase", OptFieldTypeInt(-10, -5)) + testIndex.Field("intbaseplus", OptFieldTypeInt(5, 10)) + err = cli.SyncIndex(testIndex) + require.NoError(t, err) + + schema, err := cli.Schema() + require.NoError(t, err) + + if base := schema.Index("test-index").Field("intbase").Options().base; base != -5 { + t.Fatalf("unexpected base is not -5: %d", base) + } + if base := schema.Index("test-index").Field("intbaseplus").Options().base; base != 5 { + t.Fatalf("unexpected base is not 5: %d", base) + } + }) + + t.Run("QueryWithShards", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + shardWidth := uint64(1 << shardwidth.Exponent) + + _, err := cli.Query(testField.Set(1, 1)) + require.NoErrorf(t, err, "Set(1, %d)", 1) + + _, err = cli.Query(testField.Set(1, shardWidth)) + require.NoErrorf(t, err, "Set(1, %d)", shardWidth) + + _, err = cli.Query(testField.Set(1, shardWidth*3)) + require.NoErrorf(t, err, "Set(1, %d)", shardWidth*3) + + resp, err := cli.Query(testField.Row(1), OptQueryShards(0, 3)) + require.NoErrorf(t, err, "Row(1) OptQueryShards(0, 3)") + + cols := resp.Result().Row().Columns + require.Equalf(t, []uint64{1, shardWidth * 3}, cols, "Unexpected results: %#v", cols) + }) + + t.Run("OrmCount", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldCount := testIndex.Field("test-field-count") + err := cli.EnsureField(testFieldCount) + require.NoError(t, err) + + qry := testIndex.BatchQuery( + testFieldCount.Set(10, 20), + testFieldCount.Set(10, 21), + testFieldCount.Set(15, 25), + ) + _, err = cli.Query(qry) + require.NoErrorf(t, err, "BatchQuery") + + resp, err := cli.Query(testIndex.Count(testFieldCount.Row(10))) + require.NoErrorf(t, err, "Count") + require.Equalf(t, int64(2), resp.Result().Count(), "Count") + }) + + t.Run("DecimalField", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldDec := testIndex.Field("test-field-dec", OptFieldTypeDecimal(3)) + err := cli.EnsureField(testFieldDec) + require.NoError(t, err) + + sch, err := cli.Schema() + require.NoErrorf(t, err, "Schema") + + idx := sch.indexes[testIndex.name] + opts := idx.Field(testFieldDec.name).Options() + require.Equalf(t, int64(3), opts.scale, "%s scale", testFieldDec.name) + }) + + t.Run("IntersectReturns", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldSegments := testIndex.Field("test-field-segments") + err := cli.EnsureField(testFieldSegments) + require.NoError(t, err) + + qry1 := testIndex.BatchQuery( + testFieldSegments.Set(2, 10), + testFieldSegments.Set(2, 15), + testFieldSegments.Set(3, 10), + testFieldSegments.Set(3, 20), + ) + _, err = cli.Query(qry1) + require.NoErrorf(t, err, "BatchQuery") + + qry2 := testIndex.Intersect(testFieldSegments.Row(2), testFieldSegments.Row(3)) + resp, err := cli.Query(qry2) + require.NoErrorf(t, err, "Intersect") + + require.Equalf(t, 1, len(resp.Results()), "Intersect number of results") + require.Equalf(t, []uint64{10}, resp.Result().Row().Columns, "Intersect columns results") + }) + + t.Run("TopNReturns", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldTopN := testIndex.Field("test-field-topn") + err := cli.EnsureField(testFieldTopN) + require.NoError(t, err) + + qry := testIndex.BatchQuery( + testFieldTopN.Set(10, 5), + testFieldTopN.Set(10, 10), + testFieldTopN.Set(10, 15), + testFieldTopN.Set(20, 5), + testFieldTopN.Set(30, 5), + ) + _, err = cli.Query(qry) + require.NoErrorf(t, err, "BatchQuery") + + // XXX: The following is required to make this test pass. See: https://github.com/molecula/featurebase/issues/625 + _, _, err = cli.HTTPRequest("POST", "/recalculate-caches", nil, nil) + require.NoErrorf(t, err, "POST /recalculate-caches") + + resp, err := cli.Query(testFieldTopN.TopN(2)) + require.NoErrorf(t, err, "TopN(2)") + + items := resp.Result().CountItems() + require.Equalf(t, 2, len(items), "TopN result CountItems") + + item := items[0] + require.Equalf(t, uint64(10), item.ID, "TopN result item[0].ID") + require.Equalf(t, uint64(3), item.Count, "TopN result item[0].Count") + }) + + t.Run("MinMaxRow", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldMinMax := testIndex.Field("test-field-minmax") + err := cli.EnsureField(testFieldMinMax) + require.NoError(t, err) + + qry := testIndex.BatchQuery( + testFieldMinMax.Set(10, 5), + testFieldMinMax.Set(10, 10), + testFieldMinMax.Set(10, 15), + testFieldMinMax.Set(20, 5), + testFieldMinMax.Set(30, 5), + ) + _, err = cli.Query(qry) + require.NoErrorf(t, err, "Setting bits") + + resp, err := cli.Query(testFieldMinMax.MinRow()) + require.NoErrorf(t, err, "MinRow") + + min := resp.Result().CountItem().ID + require.Equalf(t, uint64(10), min, "Min") + + resp, err = cli.Query(testFieldMinMax.MaxRow()) + require.NoErrorf(t, err, "MaxRow") + + max := resp.Result().CountItem().ID + require.Equalf(t, uint64(30), max, "Max") + }) + + t.Run("SetMutexField", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldMutex := testIndex.Field("test-field-mutex", OptFieldTypeMutex(CacheTypeDefault, 0)) + err := cli.EnsureField(testFieldMutex) + require.NoError(t, err) + + // can set mutex + _, err = cli.Query(testFieldMutex.Set(1, 100)) + require.NoErrorf(t, err, "Set(1, 100)") + + resp, err := cli.Query(testFieldMutex.Row(1)) + require.NoErrorf(t, err, "Row(1)") + + target := []uint64{100} + require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") + + // setting another row removes the previous + _, err = cli.Query(testFieldMutex.Set(42, 100)) + require.NoErrorf(t, err, "Set(42, 100)") + + resp, err = cli.Query(testIndex.BatchQuery( + testFieldMutex.Row(1), + testFieldMutex.Row(42), + )) + require.NoErrorf(t, err, "BatchQuery") + + target1 := []uint64(nil) + target42 := []uint64{100} + require.Equalf(t, target1, resp.Results()[0].Row().Columns, "Row Results[0] Columns") + require.Equalf(t, target42, resp.Results()[1].Row().Columns, "Row Results[1] Columns") + }) + + t.Run("SetBoolField", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldBool := testIndex.Field("test-field-bool", OptFieldTypeBool()) + err := cli.EnsureField(testFieldBool) + require.NoError(t, err) + + // can set bool + _, err = cli.Query(testFieldBool.Set(true, 100)) + require.NoErrorf(t, err, "Set(true, 100)") + + resp, err := cli.Query(testFieldBool.Row(true)) + require.NoErrorf(t, err, "Row(true)") + + target := []uint64{100} + require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("ClearRowQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldClear := testIndex.Field("test-field-clear") + err := cli.EnsureField(testFieldClear) + require.NoError(t, err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldClear.Set(1, 100), + testFieldClear.Set(1, 200), + )) + require.NoErrorf(t, err, "Set(1, 100) Set(1, 200)") + + resp, err := cli.Query(testFieldClear.Row(1)) + require.NoErrorf(t, err, "Row(1)") + + target := []uint64{100, 200} + require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") + + _, err = cli.Query(testFieldClear.ClearRow(1)) + require.NoErrorf(t, err, "ClearRow(1)") + + resp, err = cli.Query(testFieldClear.Row(1)) + require.NoErrorf(t, err, "Row(1)") + + target = []uint64(nil) + require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("RowsQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldRows := testIndex.Field("test-field-rows") + err := cli.EnsureField(testFieldRows) + require.NoError(t, err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldRows.Set(1, 100), + testFieldRows.Set(1, 200), + testFieldRows.Set(2, 200), + )) + require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)") + + resp, err := cli.Query(testFieldRows.Rows()) + require.NoErrorf(t, err, "Rows") + + target := RowIdentifiersResult{ + IDs: []uint64{1, 2}, + } + require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result") + }) + + t.Run("UnionRowsQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldRows := testIndex.Field("test-field-rows") + err := cli.EnsureField(testFieldRows) + require.NoError(t, err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldRows.Set(1, 100), + testFieldRows.Set(1, 200), + testFieldRows.Set(2, 200), + )) + require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)") + + resp, err := cli.Query(testFieldRows.Rows().Union()) + require.NoErrorf(t, err, "Rows Union") + + target := []uint64{100, 200} + require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("LikeQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldLike := testIndex.Field("test-field-like", OptFieldKeys(true)) + err := cli.EnsureField(testFieldLike) + require.NoError(t, err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldLike.Set("a", 100), + testFieldLike.Set("b", 200), + testFieldLike.Set("bc", 200), + )) + require.NoErrorf(t, err, "Set(a, 100) Set(b, 200) Set(bc, 200)") + + resp, err := cli.Query(testFieldLike.Like("b%")) + require.NoErrorf(t, err, `Like(b%)`) + + target := RowIdentifiersResult{ + Keys: []string{"b", "bc"}, + } + require.Equalf(t, target, resp.Result().RowIdentifiers(), "RowIdentifiers Result") + }) + + t.Run("GroupByQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldGroupBy := testIndex.Field("test-field-group-by") + err := cli.EnsureField(testFieldGroupBy) + require.NoError(t, err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldGroupBy.Set(1, 100), + testFieldGroupBy.Set(1, 200), + testFieldGroupBy.Set(2, 200), + )) + require.NoErrorf(t, err, "Set(1, 100) Set(1, 200) Set(2, 200)") + + resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows())) + require.NoErrorf(t, err, `Like(b%)`) + + target := []GroupCount{ + {Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 1}}, Count: 2}, + {Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 2}}, Count: 1}, + } + + assertGroupBy(t, target, resp.Result().GroupCounts()) + }) + + t.Run("GroupByQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldGroupBy := testIndex.Field("test-field-group-by-int", OptFieldTypeInt(-10, 10)) + err := cli.EnsureField(testFieldGroupBy) + require.NoError(t, err) + + _, err = cli.Query(testIndex.RawQuery(` + Set(0, test-field-group-by-int=1) + Set(1, test-field-group-by-int=2) + + Set(2, test-field-group-by-int=-2) + Set(3, test-field-group-by-int=-1) + + Set(4, test-field-group-by-int=4) + + Set(10, test-field-group-by-int=0) + Set(100, test-field-group-by-int=0) + Set(1000, test-field-group-by-int=0) + Set(10000, test-field-group-by-int=0) + Set(100000, test-field-group-by-int=0) + `)) + require.NoError(t, err, "Set(0..100000)") + + resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows())) + require.NoErrorf(t, err, `GroupBy(Rows)`) + + var a, b, c, d, e, f int64 = -2, -1, 0, 1, 2, 4 + target := []GroupCount{ + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &a}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &b}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &c}}, Count: 5}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &d}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &e}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &f}}, Count: 1}, + } + assertGroupBy(t, target, resp.Result().GroupCounts()) + }) + + t.Run("CreateDeleteIndexField", func(t *testing.T) { + tmpIndex := NewIndex("tmp-index") + tmpField := tmpIndex.Field("tmp-field") + + err := cli.CreateIndex(tmpIndex) + require.NoError(t, err) + + err = cli.CreateField(tmpField) + require.NoError(t, err) + + err = cli.DeleteField(tmpField) + require.NoError(t, err) + + err = cli.DeleteIndex(tmpIndex) + require.NoError(t, err) + }) + + t.Run("ErrorCreatingIndexField", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + require.ErrorIs(t, cli.CreateIndex(testIndex), ErrIndexExists) + require.ErrorIs(t, cli.CreateField(testField), ErrFieldExists) + }) + + t.Run("Failover", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + uri, _ := pnet.NewURIFromAddress("does-not-resolve.foo.bar") + tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0)) + + _, err := tmpcli.Query(testIndex.All()) + require.Error(t, err, ErrHTTPRequest) + }) + + t.Run("InvalidQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + _, _, err := cli.HTTPRequest("INVALID METHOD", "/foo", nil, nil) + require.Error(t, err) + + _, err = cli.Query(testIndex.RawQuery("Invalid query")) + require.Error(t, err) + }) + + t.Run("Sync", func(t *testing.T) { + testIndexRemote := NewIndex("test-index-remote") + err := cli.EnsureIndex(testIndexRemote) + require.NoError(t, err) + + testFieldRemote := testIndexRemote.Field("test-field-remote") + err = cli.EnsureField(testFieldRemote) + require.NoError(t, err) + + schema := NewSchema() + idx1 := schema.Index("index-1") + idx1.Field("field-1-1") + idx1.Field("field-1-2") + + idx2 := schema.Index("index-2") + idx2.Field("field-2-1") + schema.Index(testIndexRemote.Name()) + + err = cli.SyncSchema(schema) + require.NoError(t, err) + + err = cli.DeleteIndex(testIndexRemote) + require.NoError(t, err) + + err = cli.DeleteIndex(idx1) + require.NoError(t, err) + + err = cli.DeleteIndex(idx2) + require.NoError(t, err) + }) + + t.Run("FetchFragmentNodes", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + nodes, err := cli.fetchFragmentNodes(testIndex.Name(), 0) + require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name) + require.Equalf(t, replicaN, len(nodes), "len(nodes)") + + // running the same for coverage + nodes, err = cli.fetchFragmentNodes(testIndex.Name(), 0) + require.NoErrorf(t, err, "fetchFragmentNodes(%s, 0)", testIndex.name) + require.Equalf(t, replicaN, len(nodes), "len(nodes)") + }) + + t.Run("RowRangeQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldRange := testIndex.Field("test-field-range", OptFieldTypeTime(TimeQuantumMonthDayHour)) + err := cli.EnsureField(testFieldRange) + require.NoError(t, err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldRange.SetTimestamp(10, 100, time.Date(2017, time.January, 1, 0, 0, 0, 0, time.UTC)), + testFieldRange.SetTimestamp(10, 100, time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC)), + testFieldRange.SetTimestamp(10, 100, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)), + )) + require.NoErrorf(t, err, "BatchQuery SetTimestamp") + + start := time.Date(2017, time.January, 5, 0, 0, 0, 0, time.UTC) + end := time.Date(2018, time.January, 5, 0, 0, 0, 0, time.UTC) + resp, err := cli.Query(testFieldRange.RowRange(10, start, end)) + require.NoErrorf(t, err, "RowRange(10, %v, %v)", start, end) + + target := []uint64{100} + require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") + }) + t.Run("StoreQuery", func(t *testing.T) { + schema := NewSchema() + testIndexStore := schema.Index("test-index-store") + testFieldFrom := testIndexStore.Field("test-field-from") + testFieldTo := testIndexStore.Field("test-field-to") + err := cli.SyncSchema(schema) + require.NoError(t, err) + + defer func() { + cerr := cli.DeleteIndex(testIndexStore) + require.NoErrorf(t, cerr, "failed to delete index: %v", testIndexStore.name) + }() + + _, err = cli.Query(testIndexStore.BatchQuery( + testFieldFrom.Set(10, 100), + testFieldFrom.Set(10, 200), + testFieldTo.Store(testFieldFrom.Row(10), 1), + )) + require.NoErrorf(t, err, "Set(10, 100) Set(10, 200) Store(Row(10), 1)") + + resp, err := cli.Query(testFieldTo.Row(1)) + require.NoErrorf(t, err, "Row(1)") + + target := []uint64{100, 200} + require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("MultipleClientKeyQuery", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldMultiClient := testIndexWithKeys.Field("test-field-multiclient") + err := cli.EnsureField(testFieldMultiClient) + require.NoError(t, err) + + eg := &errgroup.Group{} + for i := 0; i < 10; i++ { + rowID := uint64(i) + eg.Go(func() error { + _, e := cli.Query(testFieldMultiClient.Set(rowID, "col")) + return e + }) + } + require.NoError(t, eg.Wait()) + }) + + t.Run("ExportRowIDColumnID", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldExport := testIndex.Field("test-field-export") + err := cli.EnsureField(testFieldExport) + require.NoError(t, err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldExport.Set(1, 1), + testFieldExport.Set(1, 10), + testFieldExport.Set(2, 1048577), + ), nil) + require.NoErrorf(t, err, "Set(1, 1) Set(1, 10) Set(2, 1048577)") + + r, err := cli.ExportField(testFieldExport) + require.NoErrorf(t, err, "ExportField") + + b, err := ioutil.ReadAll(r) + require.NoError(t, err) + + target := "1,1\n1,10\n2,1048577\n" + require.Equalf(t, target, string(b), "Export Field Response") + }) + + t.Run("ExportRowIDColumnKey", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldExport := testIndexWithKeys.Field("test-field-export") + err := cli.EnsureField(testFieldExport) + require.NoError(t, err) + + _, err = cli.Query(testIndexWithKeys.BatchQuery( + testFieldExport.Set(1, "one"), + testFieldExport.Set(1, "ten"), + testFieldExport.Set(2, "big-number"), + ), nil) + require.NoErrorf(t, err, "Set(1, one) Set(1, ten) Set(2, big-number)") + + r, err := cli.ExportField(testFieldExport) + require.NoErrorf(t, err, "ExportField") + + b, err := ioutil.ReadAll(r) + require.NoError(t, err) + + target := "1,one\n1,ten\n2,big-number\n" + require.Equalf(t, target, string(b), "Export Field Response") + }) + + t.Run("TranslateRowKeys", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + testFieldTranslate := testIndexKeyTranslation.Field("test-field-translate", OptFieldKeys(true)) + err := cli.EnsureField(testFieldTranslate) + require.NoError(t, err) + + trans, err := cli.CreateFieldKeys(testFieldTranslate, "key1", "key2") + require.NoErrorf(t, err, "CreateFieldKeys") + + target := map[string]uint64{"key1": 1, "key2": 2} + require.Equalf(t, target, trans, "CreateFieldKeys") + + trans, err = cli.FindFieldKeys(testFieldTranslate, "key1", "key2", "key3") + require.NoErrorf(t, err, "FindFieldKeys") + + require.Equalf(t, target, trans, "FindFieldKeys") + }) + + t.Run("TranslateColKeys", func(t *testing.T) { + setup(t, cli) + defer tearDown(t, cli) + + created, err := cli.CreateIndexKeys(testIndexKeyTranslation, "key1", "key2") + require.NoErrorf(t, err, "CreateIndexKeys") + if _, ok := created["key1"]; !ok { + t.Error("key1 missing") + } + if _, ok := created["key2"]; !ok { + t.Error("key2 missing") + } + + found, err := cli.FindIndexKeys(testIndexKeyTranslation, "key1", "key2", "key3") + require.NoErrorf(t, err, "FindIndexKeys") + + require.Equalf(t, created, found, "IndexKeys") + }) + + t.Run("Transactions", func(t *testing.T) { + trns, err := cli.StartTransaction("blah", time.Minute, false, time.Minute) + require.NoErrorf(t, err, "StartTransaction(blah)") + require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID") + require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout") + require.Truef(t, trns.Active, "TranslateColumnKeys Active") + + trnsMap, err := cli.Transactions() + require.NoErrorf(t, err, "Transactions") + require.Equalf(t, 1, len(trnsMap), "Transactions len") + require.Truef(t, trnsMap["blah"].Active, "Transactions Active") + + trns, err = cli.GetTransaction("blah") + require.NoErrorf(t, err, "GetTransaction(blah)") + require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID") + require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout") + require.Truef(t, trns.Active, "TranslateColumnKeys Active") + + trns, err = cli.FinishTransaction("blah") + require.NoErrorf(t, err, "FinishTransaction(blah)") + require.Equalf(t, "blah", trns.ID, "TranslateColumnKeys ID") + require.Equalf(t, time.Minute, trns.Timeout, "TranslateColumnKeys Timeout") + require.Truef(t, trns.Active, "TranslateColumnKeys Active") + }) + + t.Run("ImportRoaringShard", func(t *testing.T) { + setup(t, cli) + + shardWidth := uint64(1 << shardwidth.Exponent) + bitmap := roaring.NewBitmap(1, shardWidth*2+1, shardWidth*3+1) + buf := &bytes.Buffer{} + _, err := bitmap.WriteTo(buf) + if err != nil { + t.Fatalf("serializing bitmap: %v", err) + } + request := &featurebase.ImportRoaringShardRequest{ + Remote: true, + Views: []featurebase.RoaringUpdate{ + { + Field: "test-field", + View: "standard", + Set: buf.Bytes(), + }, + { + Field: "test-field-timestamp", + View: "bsig_test-field-timestamp", + Set: buf.Bytes(), + }, + { + Field: "test-field-int", + View: "bsig_test-field-int", + Set: buf.Bytes(), + }, + }, + } + err = cli.ImportRoaringShard("test-index", 3, request) + if err != nil { + t.Fatalf("import-roaring-shard: %v", err) + } + if resp, err := cli.Query(testField.Row(2)); err != nil { + t.Fatalf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 { + t.Fatalf("unexpected result: %v", res) + } + if resp, err := cli.Query(testFieldInt.NotNull()); err != nil { + t.Fatalf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 { + t.Fatalf("unexpected result: %v", res) + } + if resp, err := cli.Query(testFieldTimestamp.NotNull()); err != nil { + t.Fatalf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 { + t.Fatalf("unexpected result: %v", res) + } + + if resp, err := cli.Query(testIndex.RawQuery("Row(test-field-timestamp>'1969-12-31T23:59:59Z')")); err != nil { + t.Fatalf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 1 || res[0] != shardWidth*3+1 { + t.Fatalf("unexpected result: %v", res) + } + + // now write more data + bitmap = roaring.NewBitmap(1, 2, shardWidth*3+1, shardWidth*3+2) + buf = &bytes.Buffer{} + _, err = bitmap.WriteTo(buf) + if err != nil { + t.Fatalf("serializing bitmap: %v", err) + } + request = &featurebase.ImportRoaringShardRequest{ + Remote: true, + Views: []featurebase.RoaringUpdate{ + { + Field: "test-field", + View: "standard", + Set: buf.Bytes(), + }, + { + Field: "test-field-timestamp", + View: "bsig_test-field-timestamp", + Set: buf.Bytes(), + }, + { + Field: "test-field-int", + View: "bsig_test-field-int", + Set: buf.Bytes(), + }, + }, + } + + err = cli.ImportRoaringShard("test-index", 3, request) + if err != nil { + t.Fatalf("import-roaring-shard: %v", err) + } + if resp, err := cli.Query(testField.Row(3)); err != nil { + t.Errorf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 { + t.Errorf("unexpected result: %v", res) + } + if resp, err := cli.Query(testFieldInt.NotNull()); err != nil { + t.Errorf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 { + t.Errorf("unexpected result: %v", res) + } + if resp, err := cli.Query(testFieldTimestamp.NotNull()); err != nil { + t.Errorf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 { + t.Errorf("unexpected result: %v", res) + } + + if resp, err := cli.Query(testIndex.RawQuery("Row(test-field-timestamp>'1969-12-31T23:59:59Z')")); err != nil { + t.Errorf("querying: %v", err) + } else if res := resp.ResultList[0].Row().Columns; len(res) != 2 || res[0] != shardWidth*3+1 || res[1] != shardWidth*3+2 { + t.Errorf("unexpected result: %v", res) + } + + }) + }) + } +} + +func assertGroupBy(t *testing.T, expected, results []GroupCount) { + t.Helper() + + require.Equalf(t, len(expected), len(results), "number of groupings mismatch") + + for i, result := range results { + require.Equalf(t, expected[i], result, "unexpected result at %d", i) + } +} diff --git a/client/client_test.go b/client/client_test.go new file mode 100644 index 000000000..ecce17031 --- /dev/null +++ b/client/client_test.go @@ -0,0 +1,210 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "crypto/tls" + "errors" + "reflect" + "testing" + + pnet "github.com/molecula/featurebase/v3/net" +) + +func TestQueryWithError(t *testing.T) { + var err error + client := DefaultClient() + index := NewIndex("foo") + invalid := NewPQLRowQuery("", index, errors.New("invalid")) + _, err = client.Query(invalid, nil) + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestClientOptions(t *testing.T) { + targets := []*ClientOptions{ + {SocketTimeout: 10}, + {ConnectTimeout: 5}, + {PoolSizePerRoute: 7}, + {TotalPoolSize: 17}, + {TLSConfig: &tls.Config{InsecureSkipVerify: true}}, + } + optionsList := [][]ClientOption{ + {OptClientSocketTimeout(10)}, + {OptClientConnectTimeout(5)}, + {OptClientPoolSizePerRoute(7)}, + {OptClientTotalPoolSize(17)}, + {OptClientTLSConfig(&tls.Config{InsecureSkipVerify: true})}, + } + + for i := 0; i < len(targets); i++ { + options := &ClientOptions{} + err := options.addOptions(optionsList[i]...) + if err != nil { + t.Fatal(err) + } + target := targets[i] + if !reflect.DeepEqual(target, options) { + t.Fatalf("%v != %v", target, options) + } + } +} + +func TestNewClientWithErrorredOption(t *testing.T) { + _, err := NewClient(":8888", ClientOptionErr(0)) + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestNewClient(t *testing.T) { + client, err := NewClient(":9999", OptClientManualServerAddress(true)) + if err != nil { + t.Fatal(err) + } + targetURI, err := pnet.NewURIFromAddress(":9999") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(targetURI, client.manualServerURI) { + t.Fatalf("%v != %v", targetURI, client.manualServerURI) + } + targetFragmentNode := &fragmentNode{ + Scheme: "http", + Host: "localhost", + Port: 9999, + } + if !reflect.DeepEqual(targetFragmentNode, client.manualFragmentNode) { + t.Fatalf("%v != %v", targetFragmentNode, client.manualFragmentNode) + } + client, err = NewClient(":9999") + if err != nil { + t.Fatal(err) + } + + targetURI, err = pnet.NewURIFromAddress(":9999") + if err != nil { + t.Fatal(err) + } + + target := []*pnet.URI{targetURI} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + client, err = NewClient([]string{":9999"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + + targetURI1, err := pnet.NewURIFromAddress(":8888") + if err != nil { + t.Fatal(err) + } + targetURI2, err := pnet.NewURIFromAddress(":9999") + if err != nil { + t.Fatal(err) + } + + client, err = NewClient([]*pnet.URI{targetURI1, targetURI2}) + if err != nil { + t.Fatal(err) + } + + target = []*pnet.URI{targetURI1, targetURI2} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + + client, err = NewClient([]*pnet.URI{targetURI}) + if err != nil { + t.Fatal(err) + } + target = []*pnet.URI{targetURI} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + + client, err = NewClient(DefaultCluster()) + if err != nil { + t.Fatal(err) + } + target = []*pnet.URI{} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } +} + +func TestNewClientWithInvalidAddr(t *testing.T) { + _, err := NewClient(10) + if err != ErrAddrURIClusterExpected { + t.Fatalf("%v != %v", ErrAddrURIClusterExpected, err) + } + _, err = NewClient(":invalid") + if err == nil { + t.Fatalf("should have failed: %+v", err) + } + _, err = NewClient([]string{"valid:8000", ":invalid"}) + if err != pnet.ErrInvalidAddress { + t.Fatalf("Should have failed '%v, got '%v'", pnet.ErrInvalidAddress, err) + } +} + +func TestNewClientManualAddressWithNoURIs(t *testing.T) { + _, err := NewClient([]string{}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } + _, err = NewClient([]*pnet.URI{}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } +} + +func TestNewClientManualAddressWithMultipleURIs(t *testing.T) { + _, err := NewClient([]string{":9000", ":5000"}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } + + targetURI1, err := pnet.NewURIFromAddress(":9000") + if err != nil { + t.Fatal(err) + } + targetURI2, err := pnet.NewURIFromAddress(":5000") + if err != nil { + t.Fatal(err) + } + + _, err = NewClient([]*pnet.URI{targetURI1, targetURI2}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } +} + +func ClientOptionErr(int) ClientOption { + return func(*ClientOptions) error { + return errors.New("Some error") + } +} + +func TestQueryOptionsError(t *testing.T) { + client := DefaultClient() + index := NewIndex("foo") + _, err := client.Query(index.RawQuery(""), QueryOptionErr(0)) + if err == nil { + t.Fatalf("should have failed") + } +} + +func QueryOptionErr(int) QueryOption { + return func(*QueryOptions) error { + return errors.New("Some error") + } +} diff --git a/client/cluster.go b/client/cluster.go new file mode 100644 index 000000000..ed7b81618 --- /dev/null +++ b/client/cluster.go @@ -0,0 +1,88 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "sync" + + pnet "github.com/molecula/featurebase/v3/net" +) + +// Cluster contains hosts in a Pilosa cluster. +type Cluster struct { + hosts []*pnet.URI + okList []bool + mutex *sync.RWMutex + lastHostIdx int +} + +// DefaultCluster returns the default Cluster. +func DefaultCluster() *Cluster { + return &Cluster{ + hosts: make([]*pnet.URI, 0), + okList: make([]bool, 0), + mutex: &sync.RWMutex{}, + } +} + +// NewClusterWithHost returns a cluster with the given URIs. +func NewClusterWithHost(hosts ...*pnet.URI) *Cluster { + cluster := DefaultCluster() + for _, host := range hosts { + cluster.AddHost(host) + } + return cluster +} + +// AddHost adds a host to the cluster. +func (c *Cluster) AddHost(address *pnet.URI) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.hosts = append(c.hosts, address) + c.okList = append(c.okList, true) +} + +// Host returns a host in the cluster. +func (c *Cluster) Host() *pnet.URI { + c.mutex.Lock() + var host *pnet.URI + for i := range c.okList { + idx := (i + c.lastHostIdx) % len(c.okList) + ok := c.okList[idx] + if ok { + host = c.hosts[idx] + break + } + } + c.lastHostIdx++ + c.mutex.Unlock() + if host != nil { + return host + } + c.reset() + return host +} + +// Hosts returns all available hosts in the cluster. +func (c *Cluster) Hosts() []pnet.URI { + c.mutex.RLock() + defer c.mutex.RUnlock() + hosts := make([]pnet.URI, 0, len(c.hosts)) + for i, host := range c.hosts { + if c.okList[i] { + hosts = append(hosts, *host) + } + } + return hosts +} + +func (c *Cluster) reset() { + c.mutex.Lock() + defer c.mutex.Unlock() + for i := range c.okList { + c.okList[i] = true + } +} diff --git a/client/cluster_test.go b/client/cluster_test.go new file mode 100644 index 000000000..67b245fdc --- /dev/null +++ b/client/cluster_test.go @@ -0,0 +1,52 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "testing" + + pnet "github.com/molecula/featurebase/v3/net" +) + +func TestNewClusterWithHost(t *testing.T) { + c := NewClusterWithHost(pnet.DefaultURI()) + hosts := c.Hosts() + if len(hosts) != 1 || !hosts[0].Equals(pnet.DefaultURI()) { + t.Fail() + } +} + +func TestAddHost(t *testing.T) { + const addr = "http://localhost:3000" + c := DefaultCluster() + if c.Hosts() == nil { + t.Fatalf("Hosts should not be nil") + } + uri, err := pnet.NewURIFromAddress(addr) + if err != nil { + t.Fatalf("Cannot parse address") + } + target, err := pnet.NewURIFromAddress(addr) + if err != nil { + t.Fatalf("Cannot parse address") + } + c.AddHost(uri) + hosts := c.Hosts() + if len(hosts) != 1 || !hosts[0].Equals(target) { + t.Fail() + } +} + +func TestHosts(t *testing.T) { + c := DefaultCluster() + if c.Host() != nil { + t.Fatalf("Hosts with empty cluster should return nil") + } + c = NewClusterWithHost(pnet.DefaultURI()) + if !c.Host().Equals(pnet.DefaultURI()) { + t.Fatalf("Host should return a value if there are hosts in the cluster") + } +} diff --git a/client/csv/csv.go b/client/csv/csv.go new file mode 100644 index 000000000..bb32dd75a --- /dev/null +++ b/client/csv/csv.go @@ -0,0 +1,182 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package csv + +import ( + "bufio" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/molecula/featurebase/v3/client" +) + +// Format is the format of the data in the CSV file. +type Format uint + +const ( + // RowIDColumnID formatted data is ROW_ID,COLUMN_ID. + RowIDColumnID Format = iota + // RowIDColumnKey formatted data is ROW_ID,COLUMN_KEY. + RowIDColumnKey + // RowKeyColumnID formatted data is ROW_KEY,COLUMN_ID. + RowKeyColumnID + // RowKeyColumnKey formatted data is ROW_KEY,COLUMN_ID. + RowKeyColumnKey + // ColumnID formatted data is COLUMN_ID. Valid only for value import. + ColumnID + // ColumnKey formatted data is COLUMN_KEY. Valud only for value import. + ColumnKey +) + +// ColumnUnmarshaller creates a RecordUnmarshaller for importing columns with the given format. +func ColumnUnmarshaller(format Format) RecordUnmarshaller { + return ColumnUnmarshallerWithTimestamp(format, "") +} + +// ColumnUnmarshallerWithTimestamp creates a RecordUnmarshaller for importing columns with the given format and timestamp format. +func ColumnUnmarshallerWithTimestamp(format Format, timestampFormat string) RecordUnmarshaller { + return func(text string) (client.Record, error) { + var err error + column := client.Column{} + parts := strings.Split(text, ",") + if len(parts) < 2 { + return nil, errors.New("invalid CSV line") + } + + hasRowKey := format == RowKeyColumnID || format == RowKeyColumnKey + hasColumnKey := format == RowIDColumnKey || format == RowKeyColumnKey + + if hasRowKey { + column.RowKey = parts[0] + } else { + column.RowID, err = strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return nil, errors.New("invalid row ID") + } + } + + if hasColumnKey { + column.ColumnKey = parts[1] + } else { + column.ColumnID, err = strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return nil, errors.New("invalid column ID") + } + } + + timestamp := int64(0) + if len(parts) == 3 { + if timestampFormat == "" { + if tsInt, err := strconv.Atoi(parts[2]); err != nil { + return nil, err + } else { + timestamp = int64(tsInt) + } + } else { + t, err := time.Parse(timestampFormat, parts[2]) + if err != nil { + return nil, err + } + timestamp = t.Unix() * int64(time.Second) // Casting a duration to int64 gives the number of nanoseconds in that duration. + } + } + column.Timestamp = timestamp + + return column, nil + } +} + +// RecordUnmarshaller is a function which creates a Record from a CSV file line with column data. +type RecordUnmarshaller func(text string) (client.Record, error) + +// Iterator reads records from a Reader. +// Each line should contain a single record in the following form: +// field1,field2,... +type Iterator struct { + reader io.Reader + line int + scanner *bufio.Scanner + unmarshaller RecordUnmarshaller +} + +// NewIterator creates a CSVIterator from a Reader. +func NewIterator(reader io.Reader, unmarshaller RecordUnmarshaller) *Iterator { + return &Iterator{ + reader: reader, + line: 0, + scanner: bufio.NewScanner(reader), + unmarshaller: unmarshaller, + } +} + +// NewColumnIterator creates a new iterator for column data. +func NewColumnIterator(format Format, reader io.Reader) *Iterator { + return NewIterator(reader, ColumnUnmarshaller(format)) +} + +// NewColumnIteratorWithTimestampFormat creates a new iterator for column data with timestamp. +func NewColumnIteratorWithTimestampFormat(format Format, reader io.Reader, timestampFormat string) *Iterator { + return NewIterator(reader, ColumnUnmarshallerWithTimestamp(format, timestampFormat)) +} + +// NewValueIterator creates a new iterator for value data. +func NewValueIterator(format Format, reader io.Reader) *Iterator { + return NewIterator(reader, FieldValueUnmarshaller(format)) +} + +// NextRecord iterates on lines of a Reader. +// Returns io.EOF on end of iteration. +func (c *Iterator) NextRecord() (client.Record, error) { + if ok := c.scanner.Scan(); ok { + c.line++ + text := strings.TrimSpace(c.scanner.Text()) + if text != "" { + rc, err := c.unmarshaller(text) + if err != nil { + return nil, fmt.Errorf("%s at line: %d", err.Error(), c.line) + } + return rc, nil + } + } + err := c.scanner.Err() + if err != nil { + return nil, err + } + return nil, io.EOF +} + +// FieldValueUnmarshaller is a function which creates a Record from a CSV file line with value data. +func FieldValueUnmarshaller(format Format) RecordUnmarshaller { + return func(text string) (client.Record, error) { + parts := strings.Split(text, ",") + if len(parts) < 2 { + return nil, errors.New("invalid CSV") + } + value, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, errors.New("invalid value") + } + switch format { + case ColumnID: + columnID, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return nil, errors.New("invalid column ID at line: %d") + } + return client.FieldValue{ + ColumnID: uint64(columnID), + Value: value, + }, nil + case ColumnKey: + return client.FieldValue{ + ColumnKey: parts[0], + Value: value, + }, nil + default: + return nil, fmt.Errorf("invalid format: %d", format) + } + } +} diff --git a/client/csv/csv_it_test.go b/client/csv/csv_it_test.go new file mode 100644 index 000000000..d0964dbbd --- /dev/null +++ b/client/csv/csv_it_test.go @@ -0,0 +1,49 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +//go:build integration +// +build integration + +package csv_test + +import ( + "io" + "reflect" + "strings" + "testing" + + "github.com/molecula/featurebase/v3/client" + "github.com/molecula/featurebase/v3/client/csv" +) + +func TestCSVIterate(t *testing.T) { + text := `10,7 + 10,5 + 2,3 + 7,1` + iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text)) + recs := consumeIterator(t, iterator) + target := []client.Record{ + client.Column{RowID: 10, ColumnID: 7}, + client.Column{RowID: 10, ColumnID: 5}, + client.Column{RowID: 2, ColumnID: 3}, + client.Column{RowID: 7, ColumnID: 1}, + } + if !reflect.DeepEqual(target, recs) { + t.Fatalf("%v != %v", target, recs) + } +} + +func consumeIterator(t *testing.T, it *csv.Iterator) []client.Record { + recs := []client.Record{} + for { + r, err := it.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + recs = append(recs, r) + } + return recs +} diff --git a/client/csv/csv_test.go b/client/csv/csv_test.go new file mode 100644 index 000000000..1726b14b2 --- /dev/null +++ b/client/csv/csv_test.go @@ -0,0 +1,255 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package csv_test + +import ( + "errors" + "io" + "reflect" + "strings" + "testing" + + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/client" + "github.com/molecula/featurebase/v3/client/csv" +) + +func TestCSVColumnIterator(t *testing.T) { + reader := strings.NewReader(`1,10,683793200 + 5,20,683793300 + 3,41,683793385`) + iterator := csv.NewColumnIterator(csv.RowIDColumnID, reader) + columns := []client.Record{} + for { + column, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + columns = append(columns, column) + } + if len(columns) != 3 { + t.Fatalf("There should be 3 columns") + } + target := []client.Column{ + {RowID: 1, ColumnID: 10, Timestamp: 683793200}, + {RowID: 5, ColumnID: 20, Timestamp: 683793300}, + {RowID: 3, ColumnID: 41, Timestamp: 683793385}, + } + for i := range target { + if !reflect.DeepEqual(target[i], columns[i]) { + t.Fatalf("%v != %v", target[i], columns[i]) + } + } +} + +func TestCSVColumnIteratorWithTimestampFormatRowIDColumnID(t *testing.T) { + format := "2006-01-02T03:04" + reader := strings.NewReader(`1,10,1991-09-02T09:33 + 5,20,1991-09-02T09:35 + 3,41,1991-09-02T09:36`) + iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format) + records := []client.Record{} + for { + record, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + records = append(records, record) + } + target := []client.Column{ + {RowID: 1, ColumnID: 10, Timestamp: 683803980000000000}, + {RowID: 5, ColumnID: 20, Timestamp: 683804100000000000}, + {RowID: 3, ColumnID: 41, Timestamp: 683804160000000000}, + } + if len(records) != len(target) { + t.Fatalf("There should be %d columns", len(target)) + } + for i := range target { + if !reflect.DeepEqual(target[i], records[i]) { + t.Fatalf("%v != %v", target[i], records[i]) + } + } +} + +func TestCSVColumnIteratorWithTimestampFormatRowKeyColumnKey(t *testing.T) { + format := "2006-01-02T03:04" + reader := strings.NewReader(`one,ten,1991-09-02T09:33 + five,twenty,1991-09-02T09:35 + three,forty-one,1991-09-02T09:36`) + iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowKeyColumnKey, reader, format) + records := []client.Record{} + for { + record, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + records = append(records, record) + } + target := []client.Column{ + {RowKey: "one", ColumnKey: "ten", Timestamp: 683803980000000000}, + {RowKey: "five", ColumnKey: "twenty", Timestamp: 683804100000000000}, + {RowKey: "three", ColumnKey: "forty-one", Timestamp: 683804160000000000}, + } + if len(records) != len(target) { + t.Fatalf("There should be %d columns", len(target)) + } + for i := range target { + if !reflect.DeepEqual(target[i], records[i]) { + t.Fatalf("%v != %v", target[i], records[i]) + } + } +} + +func TestCSVColumnIteratorWithTimestampFormatFail(t *testing.T) { + format := "2014-07-16" + reader := strings.NewReader(`1,10,X`) + iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestCSVValueIteratorWithColumnID(t *testing.T) { + reader := strings.NewReader(`1,10 + 5,-20 + 3,41 + `) + iterator := csv.NewValueIterator(csv.ColumnID, reader) + values := []client.Record{} + for { + value, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + values = append(values, value) + } + target := []pilosa.FieldValue{ + {ColumnID: 1, Value: 10}, + {ColumnID: 5, Value: -20}, + {ColumnID: 3, Value: 41}, + } + if len(values) != len(target) { + t.Fatalf("There should be %d values, got %d", len(target), len(values)) + } + for i := range target { + v := values[i].(client.FieldValue) + if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) { + t.Fatalf("'%+v' != '%+v'", target[i], values[i]) + } + } +} + +func TestCSVValueIteratorWithColumnKey(t *testing.T) { + reader := strings.NewReader(`one,10 + five,-20 + three,41 + `) + iterator := csv.NewValueIterator(csv.ColumnKey, reader) + values := []client.Record{} + for { + value, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + values = append(values, value) + } + target := []pilosa.FieldValue{ + {ColumnKey: "one", Value: 10}, + {ColumnKey: "five", Value: -20}, + {ColumnKey: "three", Value: 41}, + } + if len(values) != len(target) { + t.Fatalf("There should be %d values, got %d", len(target), len(values)) + } + for i := range target { + v := values[i].(client.FieldValue) + if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) { + t.Fatalf("%v != %v", target[i], values[i]) + } + } +} + +func TestCSValueIteratorWithInvalidFormat(t *testing.T) { + reader := strings.NewReader("1,2") + iterator := csv.NewValueIterator(csv.RowIDColumnID, reader) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("should have failed") + } +} + +func TestCSVColumnIteratorInvalidInput(t *testing.T) { + invalidInputs := []string{ + // less than 2 columns + "155", + // invalid row ID + "a5,155", + // invalid column ID + "155,a5", + // invalid timestamp + "155,255,a5", + } + for _, text := range invalidInputs { + iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text)) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("CSVColumnIterator input: %s should fail", text) + } + } +} + +func TestCSVValueIteratorInvalidInput(t *testing.T) { + invalidInputs := []string{ + // less than 2 columns + "155", + // invalid column ID + "a5,155", + // invalid value + "155,a5", + } + for _, text := range invalidInputs { + iterator := csv.NewValueIterator(csv.ColumnID, strings.NewReader(text)) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("CSVValueIterator input: %s should fail", text) + } + } +} + +func TestCSVColumnIteratorError(t *testing.T) { + iterator := csv.NewColumnIterator(csv.RowIDColumnID, &BrokenReader{}) + _, err := iterator.NextRecord() + if err == nil { + t.Fatal("CSVColumnIterator should fail with error") + } +} + +func TestCSVValueIteratorError(t *testing.T) { + iterator := csv.NewValueIterator(csv.ColumnID, &BrokenReader{}) + _, err := iterator.NextRecord() + if err == nil { + t.Fatal("CSVValueIterator should fail with error") + } +} + +type BrokenReader struct{} + +func (r BrokenReader) Read(p []byte) (n int, err error) { + return 0, errors.New("broken reader") +} diff --git a/client/doc.go b/client/doc.go new file mode 100644 index 000000000..0f5cc1f70 --- /dev/null +++ b/client/doc.go @@ -0,0 +1,56 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +/* +Package client enables querying a Pilosa server. + +This client uses Pilosa's http+protobuf API. + +Usage: + + import ( + "fmt" + "github.com/molecula/featurebase/v3/client" + ) + + // Create a Client instance + cli := client.DefaultClient() + + // Create a Schema instance + schema, err := cli.Schema() + if err != nil { + panic(err) + } + + // Create an Index instance + index, err := schema.Index("repository") + if err != nil { + panic(err) + } + + // Create a Field instance + stargazer, err := index.Field("stargazer") + if err != nil { + panic(err) + } + + // Sync the schema with the server-side, so non-existing indexes/fields are created on the server-side. + err = cli.SyncSchema(schema) + if err != nil { + panic(err) + } + + // Execute a query + response, err := cli.Query(stargazer.Row(5)) + if err != nil { + panic(err) + } + + // Act on the result + fmt.Println(response.Result()) + +See also https://www.pilosa.com/docs/api-reference/ and https://www.pilosa.com/docs/query-language/. +*/ +package client diff --git a/client/docs/data-model-queries.md b/client/docs/data-model-queries.md new file mode 100644 index 000000000..5ddcd572f --- /dev/null +++ b/client/docs/data-model-queries.md @@ -0,0 +1,149 @@ +# Data Model and Queries + +## Indexes and Fields + +*Index* and *field*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs/latest/data-model/) for more detail about the data model. + +The `schema.Index` function is used to create an index instance. Note that this does not create an index on the server; the index object simply defines the schema. + +```go +schema := client.NewSchema() +repository := schema.Index("repository") +``` + +You can pass options while creating index instances: +```go +repository := schema.Index("repository", pilosa.OptIndexKeys(true)) +``` + +Field definitions are created with a call to the `Field` function of an index: + +```go +stargazer := repository.Field("stargazer") +``` + +You can pass options to fields: + +```go +stargazer := repository.Field("stargazer", pilosa.OptFieldTypeTime(TimeQuantumYearMonthDay)) +``` + +In case the schema already exists on the server, you can retrieve that instead of creating the schema: +```go +cli := client.DefaultClient() +schema, err := cli.Schema() +if err != nil { + // act on the error +} +repository := schema.Index("repository") +``` + +## Queries + +Once you have indexes and field definitions, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows with related methods attached to fields. + +For instance, `Row` queries work on rows; use a `Field` object to create those queries: + +```go +rowQuery := stargazer.Row(1) // corresponds to PQL: Row(stargazer=1) +``` + +`Union` queries work on columns; use the index to create them: + +```go +query := repository.Union(rowQuery1, rowQuery2) +``` + +In order to increase throughput, you may want to batch queries sent to the Pilosa server. The `index.BatchQuery` function is used for that purpose: + +```go +query := repository.BatchQuery( + stargazer.Row(1), + repository.Union(stargazer.Row(100), stargazer.Row(5))) +``` + +The recommended way of creating query instances is using dedicated functions attached to index and field objects, but sometimes it would be desirable to send raw queries to Pilosa. You can use `index.RawQuery` method for that. Note that query string is not validated before sending to the server: + +```go +query := repository.RawQuery("Row(stargazer=5)") +``` + +Raw queries are only sent to the coordinator node of a Pilosa cluster, so currently there's a possible performance hit using them instead of ORM functions attached to index or field instances. + +This client supports [range queries using bit sliced indexes (BSI)](https://www.pilosa.com/docs/latest/query-language/#range-bsi). Read the [Range Encoded Bitmaps](https://www.pilosa.com/blog/range-encoded-bitmaps/) blog post for more information about the BSI implementation of range encoding in Pilosa. + +In order to use BSI range queries, an integer field should be created. The field should have its minimum and maximum set. Here's how you would do that: +```go +index := schema.Index("animals") +captivity := index.Field("captivity", pilosa.OptFieldTypeInt(0, 956)) +``` + +If the field with the necessary field already exists on the server, you don't need to create the field instance, `cli.SyncSchema(schema)` would load that to `schema`. You can then add some data: +```go +// Add the captivity values to the field. +data := []int{3, 392, 47, 956, 219, 14, 47, 504, 21, 0, 123, 318} +query := index.BatchQuery() +for i, x := range data { + column := uint64(i + 1) + query.Add(captivity.SetIntValue(column, x)) +} +cli.Query(query) +``` + +Let's write a range query: +```go +// Query for all animals with more than 100 specimens +response, _ := cli.Query(captivity.GT(100)) +fmt.Println(response.Result().Row().Columns) + +// Query for the total number of animals in captivity +response, _ = cli.Query(captivity.Sum(nil)) +fmt.Println(response.Result().Value()) +``` + +If you pass a row query to `Sum` as a filter, then only the columns matching the filter will be considered in the `Sum` calculation: +```go +// Let's run a few set queries first +cli.Query(index.BatchQuery( + field.Set(42, 1), + field.Set(42, 6))) +// Query for the total number of animals in captivity where row 42 is set +response, _ = cli.Query(captivity.Sum(field.Row(42))) +fmt.Println(response.Result().Value()) +``` + +See the functions further below for the list of functions that can be used with a `Field`. + +Please check [Pilosa documentation](https://www.pilosa.com/docs) for PQL details. Here is a list of methods corresponding to PQL calls: + +Index: + +* `Union(rows *PQLRowQuery...) *PQLRowQuery` +* `Intersect(rows *PQLRowQuery...) *PQLRowQuery` +* `Difference(rows *PQLRowQuery...) *PQLRowQuery` +* `Xor(rows ...*PQLRowQuery) *PQLRowQuery` +* `Not(row) *PQLRowQuery` +* `Count(row *PQLRowQuery) *PQLBaseQuery` +* `Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery` + +Field: + +* `Row(rowID uint64) *PQLRowQuery` +* `Set(rowID uint64, columnID uint64) *PQLBaseQuery` +* `SetTimestamp(rowID uint64, columnID uint64, timestamp time.Time) *PQLBaseQuery` +* `Clear(rowID uint64, columnID uint64) *PQLBaseQuery` +* `TopN(n uint64) *PQLRowQuery` +* `RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery` +* `Range(rowID uint64, start time.Time, end time.Time) *PQLRowQuery` +* `RowRange(rowID uint64, start time.Time, end time.Time) *PQLRowQuery` +* `ClearRow(rowIDOrKey interface{}) *PQLBaseQuery` +* `Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery` +* `LT(n int) *PQLRowQuery` +* `LTE(n int) *PQLRowQuery` +* `GT(n int) *PQLRowQuery` +* `GTE(n int) *PQLRowQuery` +* `Between(a int, b int) *PQLRowQuery` +* `Sum(row *PQLRowQuery) *PQLBaseQuery` +* `Min(row *PQLRowQuery) *PQLBaseQuery` +* `Max(row *PQLRowQuery) *PQLBaseQuery` +* `SetIntValue(columnID uint64, value int) *PQLBaseQuery` diff --git a/client/docs/server-interaction.md b/client/docs/server-interaction.md new file mode 100644 index 000000000..74ea5aacb --- /dev/null +++ b/client/docs/server-interaction.md @@ -0,0 +1,160 @@ +# Server Interaction + +## Pilosa URI + +A Pilosa URI has the `${SCHEME}://${HOST}:${PORT}` format: +* **Scheme**: Protocol of the URI. Default: `http`. +* **Host**: Hostname or ipv4/ipv6 IP address. Default: localhost. +* **Port**: Port number. Default: `10101`. + +All parts of the URI are optional, but at least one of them must be specified. The following are equivalent: + +* `http://localhost:10101` +* `http://localhost` +* `http://:10101` +* `localhost:10101` +* `localhost` +* `:10101` + +A Pilosa URI is represented by the `github.com/pilosa/pilosa/v2/net URI` struct. Below are a few ways to create `URI` objects: + +```go +import pnet "github.com/pilosa/pilosa/v2/net" + +// create the default URI: http://localhost:10101 +uri1 := pnet.DefaultURI() + +// create a URI from string address +uri2, err := pnet.NewURIFromAddress("index1.pilosa.com:20202"); + +// create a URI with the given host and port +uri3, err := pnet.NewURIFromHostPort("index1.pilosa.com", 20202); +``` + +## Pilosa Client + +In order to interact with a Pilosa server, an instance of `client.Client` should be created. The client is thread-safe and uses a pool of connections to the server, so we recommend creating a single instance of the client and sharing it when necessary. + +If the Pilosa server is running at the default address (`http://localhost:10101`) you can create the client with default options using: + +```go +import "github.com/pilosa/pilosa/v2/client" + +cli := client.DefaultClient() +``` + +To use a custom server address, use the `NewClient` function: + +```go +uri, err := pnet.NewURIFromAddress("http://index1.pilosa.com:15000") +if err != nil { + // Act on the error +} +cli, err := client.NewClient(uri) +``` + +Equivalently: +```go +cli, err := client.NewClient("http://index1.pilosa.com:15000") +``` + +If you are running a cluster of Pilosa servers, you can create a `Cluster` struct that keeps addresses of those servers: + +```go +uri1, err := pnet.NewURIFromAddress(":10101") +uri2, err := pnet.NewURIFromAddress(":10110") +uri3, err := pnet.NewURIFromAddress(":10111") +cluster := client.NewClusterWithHost(uri1, uri2, uri3) + +// Create a client with the cluster +cli, err := client.NewClient(cluster) +``` + +That is equivalent to: +```go +cli, err := client.NewClient([]string{":10101", ":10110", ":10111"}) + +``` + +It is possible to customize the behaviour of the underlying HTTP client by passing `ClientOption` structs to the `NewClient` function: + +```go +cli, err := client.NewClient(cluster, + client.OptClientConnectTimeout(1000), // if can't connect in a second, close the connection + client.OptClientSocketTimeout(10000), // if no response received in 10 seconds, close the connection + client.OptClientPoolSizePerRoute(3), // number of connections in the pool per host + client.OptClientTotalPoolSize(10)) // number of total connections in the pool +``` + +Once you create a client, you can create indexes, fields or start sending queries. + +Here is how you would create a index and field: + +```go +// materialize repository index definition and stargazer field definition initialized before +err := cli.SyncSchema(schema) +``` + +You can send queries to a Pilosa server using the `Query` function of the `Client` struct: + +```go +response, err := cli.Query(field.Row(5)); +``` + +## Server Response + +When a query is sent to a Pilosa server, the server either fulfills the query or sends an error message. In the case of an error, a `pilosa.Error` struct is returned, otherwise a `QueryResponse` struct is returned. + +A `QueryResponse` struct may contain zero or more results of `QueryResult` type. You can access all results using the `Results` function of `QueryResponse` (which returns a list of `QueryResult` objects), or you can use the `Result` method (which returns either the first result or `nil` if there are no results): + +```go +response, err := cli.Query(field.Row(5)) +if err != nil { + // Act on the error +} + +// check that there's a result and act on it +result := response.Result() +if result != nil { + // Act on the result +} + +// iterate over all results +for _, result := range response.Results() { + // Act on the result +} +``` + +`QueryResult` objects contain: + +* `Row()` function to retrieve a row result, +* `CountItems()` function to retrieve column count per row ID entries returned from `TopN` queries, +* `Count()` function to retrieve the number of rows per the given row ID returned from `Count` queries. +* `Value()` function to retrieve the result of `Min`, `Max` or `Sum` queries. +* `Changed()` function returns whether a `Set` or `Clear` query changed a column. + +```go +row := result.Row() +columns := row.Columns + +countItems := result.CountItems() + +count := result.Count() + +value := result.Value() + +changed := result.Changed() +``` + +## SSL/TLS + +Make sure the Pilosa server runs on a TLS address. [How To Set Up a Secure Cluster](https://www.pilosa.com/docs/latest/tutorials/#how-to-set-up-a-secure-cluster) tutorial explains how to do that. + +In order to enable TLS support on the client side, the scheme of the address should be explicitly specified as `https`, e.g.: `https://01.pilosa.local:10501` + +This client library uses the `net/http` module of Go standard library. You can pass a [tls.Config](https://golang.org/pkg/crypto/tls/#Config) struct in a `pilosa.TLSConfig` option to the client. If the Pilosa server is using a certificate from a recognized authority, you can use the defaults. + +If you are using a self signed certificate, just pass `pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true})` to `pilosa.NewClient` function: +```go +client, _ := pilosa.NewClient("https://01.pilosa.local:10501", pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true})) +``` diff --git a/client/docs/tracing.md b/client/docs/tracing.md new file mode 100644 index 000000000..868e374df --- /dev/null +++ b/client/docs/tracing.md @@ -0,0 +1,111 @@ +# Tracing + +Pilosa client supports distributed tracing via the [OpenTracing](https://opentracing.io/) API. + +In order to use a tracer with Go-Pilosa, you should: +1. Create the tracer, +2. Pass the `OptClientOption(tracer)` to `NewClient`. + +In this document, we will be using the [Jaeger](https://www.jaegertracing.io) tracer, but OpenTracing has support for [other tracing systems](https://opentracing.io/docs/supported-tracers/). + +## Running the Pilosa Server + +Let's run a temporary Pilosa container: + + $ docker run -it --rm -p 10101:10101 pilosa/pilosa:v1.2.0 + +Check that you can access Pilosa: + + $ curl localhost:10101 + Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information. + +## Running the Jaeger Server + +Let's run a Jaeger Server container: + + $ docker run -it --rm -p 5775:5775/udp -p 16686:16686 jaegertracing/all-in-one:latest + ...Jaeger UI... + +## Writing the Sample Code + +The sample code depdends on the Jaeger Go client, so let's install it first: + + $ go get -u github.com/uber/jaeger-client-go/ + +Save the following sample code as `gopilosa-tracing.go`: +```go +package main + +import ( + "log" + "time" + + "github.com/pilosa/pilosa/v2/client" + "github.com/uber/jaeger-client-go" + "github.com/uber/jaeger-client-go/config" +) + +func main() { + // Create the tracer. + cfg := config.Configuration{ + Sampler: &config.SamplerConfig{ + Type: "const", + Param: 1, + }, + Reporter: &config.ReporterConfig{ + LogSpans: true, + BufferFlushInterval: 1 * time.Second, + // Jaeger Server address + LocalAgentHostPort: "127.0.0.1:5775", + }, + } + tracer, closer, err := cfg.New( + "go_pilosa_test", + config.Logger(jaeger.StdLogger), + ) + + // Don't forget to close the tracer. + defer closer.Close() + + // Create the client, and pass the tracer. + cli, err := client.NewClient(":10101", pilosa.OptClientTracer(tracer)) + if err != nil { + log.Fatal(err) + } + + // Read the schema from the server. + // This should create a trace on the Jaeger server. + schema, err := cli.Schema() + if err != nil { + log.Fatal(err) + } + + // Create and sync the sample schema. + // This should create a trace on the Jaeger server. + myIndex := schema.Index("my-index") + myField := myIndex.Field("my-field") + err = cli.SyncSchema(schema) + if err != nil { + log.Fatal(err) + } + + // Run a query on Pilosa. + // This should create a trace on the Jaeger server. + _, err = cli.Query(myField.Set(1, 1000)) + if err != nil { + log.Fatal(err) + } +} +``` + +## Checking the Tracing Data + +Run the sample code: + + $ go run gopilosa-tracing.go + + +* Open http://localhost:16686 in your web browser to visit Jaeger UI. +* Click on the *Search* tab and select `go_pilosa_test` in the *Service* dropdown on the right. +* Click on *Find Traces* button at the bottom left. +* You should see a couple of traces, such as: `Client.Query`, `Client.CreateField`, `Client.Schema`, etc. diff --git a/client/egpool/egpool.go b/client/egpool/egpool.go new file mode 100644 index 000000000..51d6d0c5b --- /dev/null +++ b/client/egpool/egpool.go @@ -0,0 +1,111 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package egpool + +import ( + "errors" + "fmt" + "sync" +) + +type Group struct { + PoolSize int + + jobs chan func() error + + sema chan struct{} + errMu sync.Mutex + firstErr error + errs []error +} + +func (eg *Group) Go(f func() error) { + if eg.PoolSize <= 0 { + eg.PoolSize = 1 + } + + if eg.jobs == nil { + eg.jobs = make(chan func() error) + eg.sema = make(chan struct{}, eg.PoolSize) + } + + // Start the job in an idle worker if possible. + select { + case eg.jobs <- f: + return + default: + } + + // Start a new worker if necessary. + select { + case eg.jobs <- f: + // A worker finished its previous job and took this one over. + return + case eg.sema <- struct{}{}: + // Start a new worker. + go eg.processJobs() + eg.jobs <- f + } +} + +func (eg *Group) err(err error) { + eg.errMu.Lock() + defer eg.errMu.Unlock() + + if eg.firstErr == nil { + eg.firstErr = err + } + eg.errs = append(eg.errs, err) +} + +type ErrPanic struct { + Value interface{} +} + +func (p ErrPanic) Error() string { + return fmt.Sprintf("panic: %v", p.Value) +} + +var ErrGoexit = errors.New("runtime.Goexit used in job function") + +func (eg *Group) processJobs() { + // Notify pool of shutdown. + defer func() { <-eg.sema }() + + // Handle panic and Goexit. + var finished bool + defer func() { + if !finished { + if p := recover(); p != nil { + eg.err(ErrPanic{p}) + } else { + eg.err(ErrGoexit) + } + } + }() + + // Run jobs from queue. + for jobFn := range eg.jobs { + err := jobFn() + if err != nil { + eg.err(err) + } + } + + finished = true +} + +func (eg *Group) Wait() error { + if eg.jobs == nil { + return nil + } + close(eg.jobs) + for i := 0; i < eg.PoolSize; i++ { + eg.sema <- struct{}{} + } + return eg.firstErr +} + +func (eg *Group) Errors() []error { + return eg.errs +} diff --git a/client/egpool/egpool_test.go b/client/egpool/egpool_test.go new file mode 100644 index 000000000..c31bd2f5a --- /dev/null +++ b/client/egpool/egpool_test.go @@ -0,0 +1,38 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package egpool_test + +import ( + "errors" + "testing" + + "github.com/molecula/featurebase/v3/client/egpool" +) + +func TestEGPool(t *testing.T) { + eg := egpool.Group{} + + a := make([]int, 10) + + for i := 0; i < 10; i++ { + i := i + eg.Go(func() error { + a[i] = i + if i == 7 { + return errors.New("blah") + } + return nil + }) + } + + err := eg.Wait() + if err == nil || err.Error() != "blah" { + t.Errorf("expected err blah, got: %v", err) + } + + for i := 0; i < 10; i++ { + if a[i] != i { + t.Errorf("expected a[%d] to be %d, but is %d", i, i, a[i]) + } + } +} diff --git a/client/error.go b/client/error.go new file mode 100644 index 000000000..9387afbd1 --- /dev/null +++ b/client/error.go @@ -0,0 +1,26 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package client + +import "github.com/pkg/errors" + +// Predefined Pilosa errors. +var ( + ErrEmptyCluster = errors.New("No usable addresses in the cluster") + ErrIndexExists = errors.New("Index exists") + ErrFieldExists = errors.New("Field exists") + ErrInvalidIndexName = errors.New("Invalid index name") + ErrInvalidFieldName = errors.New("Invalid field name") + ErrInvalidLabel = errors.New("Invalid label") + ErrInvalidKey = errors.New("Invalid key") + ErrHTTPRequest = errors.New("Failed all HTTP retries") + ErrAddrURIClusterExpected = errors.New("Addresses, URIs or a cluster is expected") + ErrInvalidQueryOption = errors.New("Invalid query option") + ErrInvalidIndexOption = errors.New("Invalid index option") + ErrInvalidFieldOption = errors.New("Invalid field option") + ErrNoFragmentNodes = errors.New("No fragment nodes") + ErrNoShard = errors.New("Index has no shards") + ErrUnknownType = errors.New("Unknown type") + ErrSingleServerAddressRequired = errors.New("OptClientManualServerAddress requires a single URI or address") + ErrPreconditionFailed = errors.New("Precondition failed") +) diff --git a/client/ingest_api_batch.go b/client/ingest_api_batch.go new file mode 100644 index 000000000..a1ae7a1c5 --- /dev/null +++ b/client/ingest_api_batch.go @@ -0,0 +1,145 @@ +package client + +import ( + "time" + + "github.com/molecula/featurebase/v3/logger" + "github.com/pkg/errors" +) + +// NewIngestAPIBatch creates an alternate implementation of +// RecordBatch which exists to aid in testing the new Ingest API and +// is likely far slower than the Batch. +func NewIngestAPIBatch(client *Client, size int, logger logger.Logger, fields []*Field) *ingestAPIBatch { + if len(fields) == 0 { + return nil + } + + return &ingestAPIBatch{ + client: client, + log: logger, + fields: fields, + keyed: fields[0].index.Opts().Keys(), + index: fields[0].index.Name(), + batchSize: size, + + recordsK: make(map[string]map[string]interface{}), + records: make(map[uint64]map[string]interface{}), + } +} + +type ingestAPIBatch struct { + client *Client + log logger.Logger + batchSize int + + fields []*Field + keyed bool + index string + + // map[recordKey][fieldName]value + recordsK map[string]map[string]interface{} + records map[uint64]map[string]interface{} +} + +func (b *ingestAPIBatch) Add(row Row) error { + if len(row.Clears) > 0 { + return errors.New("ingest api batch does not support clears") + } + values := make(map[string]interface{}) + for i, val := range row.Values { + field := b.fields[i] + // val can be string, uint64, int64, []string, []uint64, nil + // TODO timestamp field might need special handling + // TODO check that the Row.Clears field is only used for packed bools, and then issue a warning/error (in IDK) if the ingest API mode is used in conjunction w/ packed bools. + if val == nil { + continue + } + zero := QuantizedTime{} + if field.Options().Type() == FieldTypeTime && row.Time != zero { + timeq, err := row.Time.Time() + if err != nil { + return errors.Wrap(err, "parsing row time") + } + values[field.Name()] = map[string]interface{}{"time": timeq.Format(time.RFC3339), "values": val} + } else { + values[field.Name()] = val + } + } + + if b.keyed { + switch rowID := row.ID.(type) { + case string: + b.recordsK[rowID] = values + case []byte: + b.recordsK[string(rowID)] = values + default: + return errors.Errorf("unsupported rowID %v of type %[1]T, must be string, or []byte for keyed index", rowID) + } + if len(b.recordsK) >= b.batchSize { + return ErrBatchNowFull + } + } else { + rowID, ok := row.ID.(uint64) + if !ok { + return errors.Errorf("unsupported rowID %v of type %[1]T, must be uint64 for unkeyed index", row.ID) + } + b.records[rowID] = values + if len(b.records) >= b.batchSize { + return ErrBatchNowFull + } + } + return nil +} + +func (b *ingestAPIBatch) Import() error { + if b.keyed { + return b.importKeyed() + } + return b.importUnkeyed() +} + +func (b *ingestAPIBatch) importKeyed() error { + req := []map[string]interface{}{ + { + "action": "set", + "records": b.recordsK, + }, + } + bod, err := b.client.IngestData(b.index, req) + if err != nil { + return errors.Wrapf(err, "importKeyed, body: %s", bod) + } + + for k := range b.recordsK { + delete(b.recordsK, k) + } + return nil +} + +func (b *ingestAPIBatch) importUnkeyed() error { + req := []map[string]interface{}{ + { + "action": "set", + "records": b.records, + }, + } + bod, err := b.client.IngestData(b.index, req) + if err != nil { + return errors.Wrapf(err, "importKeyed, body: %s", bod) + } + + for v := range b.records { + delete(b.records, v) + } + return nil +} + +func (b *ingestAPIBatch) Len() int { + if b.keyed { + return len(b.recordsK) + } + return len(b.records) +} + +func (b *ingestAPIBatch) Flush() error { return nil } diff --git a/client/ingest_api_batch_test.go b/client/ingest_api_batch_test.go new file mode 100644 index 000000000..bc6858c12 --- /dev/null +++ b/client/ingest_api_batch_test.go @@ -0,0 +1,305 @@ +package client + +import ( + "strings" + "testing" + "time" + + "github.com/molecula/featurebase/v3/logger" + "github.com/molecula/featurebase/v3/test" +) + +func TestIngestAPIBatchAdd(t *testing.T) { + t.Run("unkeyed", func(t *testing.T) { + batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{ + { + name: "a", + index: &Index{name: "idxname", options: &IndexOptions{}}, + options: &FieldOptions{ + fieldType: FieldTypeSet, + }, + }, + { + name: "b", + index: &Index{name: "idxname", options: &IndexOptions{}}, + options: &FieldOptions{ + fieldType: FieldTypeSet, + keys: true, + }, + }, + { + name: "c", + index: &Index{name: "idxname", options: &IndexOptions{}}, + options: &FieldOptions{ + fieldType: FieldTypeTime, + keys: true, + }, + }, + }) + qt := QuantizedTime{} + qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC)) + err := batch.Add(Row{ + ID: uint64(1), + Values: []interface{}{uint64(2), "bkey", "ckey"}, + Time: qt, + }) + if err != nil { + t.Fatalf("adding row to batch: %v", err) + } + + if batch.records[1]["a"] != uint64(2) { + t.Fatalf("unexpected batch.records: %+v", batch.records) + } + if batch.records[1]["b"] != "bkey" { + t.Fatalf("unexpected batch.records: %+v", batch.records) + } + if batch.records[1]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" { + t.Fatalf("unexpected batch.records: %+v", batch.records) + } + if batch.records[1]["c"].(map[string]interface{})["values"] != "ckey" { + t.Fatalf("unexpected batch.records: %+v", batch.records) + } + + }) + + t.Run("keyed", func(t *testing.T) { + batch := NewIngestAPIBatch(nil, 10, logger.NopLogger, []*Field{ + { + name: "a", + index: &Index{name: "idxname", options: &IndexOptions{keys: true}}, + options: &FieldOptions{ + fieldType: FieldTypeSet, + }, + }, + { + name: "b", + index: &Index{name: "idxname", options: &IndexOptions{keys: true}}, + options: &FieldOptions{ + fieldType: FieldTypeSet, + keys: true, + }, + }, + { + name: "c", + index: &Index{name: "idxname", options: &IndexOptions{keys: true}}, + options: &FieldOptions{ + fieldType: FieldTypeTime, + keys: true, + }, + }, + }) + qt := QuantizedTime{} + qt.Set(time.Date(2007, time.January, 1, 15, 0, 0, 0, time.UTC)) + err := batch.Add(Row{ + ID: "1", + Values: []interface{}{uint64(2), "bkey", "ckey"}, + Time: qt, + }) + + checkResult := func(batch *ingestAPIBatch, id string, err error) { + if err != nil { + t.Fatalf("adding row to batch: %v", err) + } + + if batch.recordsK[id]["a"] != uint64(2) { + t.Fatalf("unexpected batch.records: %+v", batch.recordsK) + } + if batch.recordsK[id]["b"] != "bkey" { + t.Fatalf("unexpected batch.records: %+v", batch.recordsK) + } + if batch.recordsK[id]["c"].(map[string]interface{})["time"] != "2007-01-01T15:00:00Z" { + t.Fatalf("unexpected batch.records: %+v", batch.recordsK) + } + if batch.recordsK[id]["c"].(map[string]interface{})["values"] != "ckey" { + t.Fatalf("unexpected batch.records: %+v", batch.recordsK) + } + } + checkResult(batch, "1", err) + + // test wrong type row ID + if err := batch.Add(Row{ID: 64.5}); !strings.Contains(err.Error(), "unsupported rowID") { + t.Fatalf("unexpected error w/ floating point rowID: %v", err) + } + + // test that byte slice ID works same as string + err = batch.Add(Row{ + ID: []byte("2"), + Values: []interface{}{uint64(2), "bkey", "ckey"}, + Time: qt, + }) + checkResult(batch, "2", err) + + }) +} + +func TestIngestAPIBatch(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + + urls := make([]string, len(c.Nodes)) + for i, n := range c.Nodes { + urls[i] = n.URL() + } + + // Create a new client for the cluster + cli, err := newClientFromAddresses(urls, &ClientOptions{}) + if err != nil { + t.Fatalf("getting new client: %v", err) + } + defer cli.Close() + + cli.IngestSchema(map[string]interface{}{ + "index-name": "test-1", + "index-action": "create", + "primary-key-type": "uint", + "field-action": "create", + "fields": []map[string]interface{}{ + { + "field-name": "astr", + "field-type": "string", + "field-options": map[string]interface{}{}, + }, + { + "field-name": "bint", + "field-type": "int", + "field-options": map[string]interface{}{}, + }, + { + "field-name": "cid", + "field-type": "id", + "field-options": map[string]interface{}{}, + }, + { + "field-name": "dtimestamp", + "field-type": "timestamp", + "field-options": map[string]interface{}{ + "unit": "s", + }, + }, + { + "field-name": "etime", + "field-type": "string", + "field-options": map[string]interface{}{ + "time-quantum": "YMD", + }, + }, + { + "field-name": "fdecimal", + "field-type": "decimal", + "field-options": map[string]interface{}{ + "scale": 3, + }, + }, + { + "field-name": "gbool", + "field-type": "bool", + "field-options": map[string]interface{}{}, + }, + }, + }) + + schema, err := cli.Schema() + if err != nil { + t.Fatalf("getting schema: %v", err) + } + index := schema.Index("test-1") + defer cli.DeleteIndex(index) + + batch := NewIngestAPIBatch(cli, 10, logger.NopLogger, []*Field{ + { + name: "astr", + index: &Index{name: "test-1", options: &IndexOptions{}}, + options: &FieldOptions{fieldType: FieldTypeSet, keys: true}, + }, + { + name: "bint", + options: &FieldOptions{fieldType: FieldTypeInt}, + }, + { + name: "cid", + options: &FieldOptions{fieldType: FieldTypeSet, keys: false}, + }, + { + name: "dtimestamp", + options: &FieldOptions{fieldType: FieldTypeTimestamp}, + }, + { + name: "etime", + options: &FieldOptions{fieldType: FieldTypeTime, keys: true, timeQuantum: TimeQuantumYearMonthDay}, + }, + { + name: "fdecimal", + options: &FieldOptions{fieldType: FieldTypeDecimal, scale: 3}, + }, + { + name: "gbool", + options: &FieldOptions{fieldType: FieldTypeBool}, + }, + }) + + qt0 := &QuantizedTime{} + qt0.Set(time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err := batch.Add(Row{ + ID: uint64(7), + Values: []interface{}{"a", -2, 9, 1287367623, "e", 1.2345, true}, + Time: *qt0, + }); err != nil { + t.Fatalf("adding row: %v", err) + } + + // test nil value case + if err := batch.Add(Row{ + ID: uint64(8), + Values: []interface{}{nil, nil, nil, nil, nil, nil, nil}, + Time: QuantizedTime{}, + }); err != nil { + t.Fatalf("error adding all nil batch which should affect nothing: %v", err) + } + + if err := batch.Import(); err != nil { + t.Fatalf("importing row: %v", err) + } + + if resp, err := cli.Query(NewPQLBaseQuery("Row(astr=a)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil { + t.Fatalf("querying: %v", err) + } else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) { + t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns) + } + + if resp, err := cli.Query(NewPQLBaseQuery("Row(bint==-2)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil { + t.Fatalf("querying: %v", err) + } else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) { + t.Fatalf("unexpected Row(bint==-2) result: %+v", resp.Result().Row().Columns) + } + + if resp, err := cli.Query(NewPQLBaseQuery("Row(cid=9)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil { + t.Fatalf("querying: %v", err) + } else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) { + t.Fatalf("unexpected Row(cid=9) result: %+v", resp.Result().Row().Columns) + } + + if resp, err := cli.Query(NewPQLBaseQuery("Row(dtimestamp=='2010-10-18T02:07:03Z')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil { + t.Fatalf("querying: %v", err) + } else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) { + t.Fatalf("unexpected Row(dtimestamp=='2010-10-18T02:07:03Z') result: %+v", resp.Result().Row().Columns) + } + + if resp, err := cli.Query(NewPQLBaseQuery("Row(etime=e, from='2010-01-01', to='2010-01-02')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil { + t.Fatalf("querying: %v", err) + } else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) { + t.Fatalf("unexpected Row(etime=e, from='2010-01-01', to='2010-01-02') result: %+v", resp.Result().Row().Columns) + } + + if resp, err := cli.Query(NewPQLBaseQuery("Row(fdecimal==1.234)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil { + t.Fatalf("querying: %v", err) + } else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) { + t.Fatalf("unexpected Row(fdecimal==1.234) result: %+v", resp.Result().Row().Columns) + } + + if resp, err := cli.Query(NewPQLBaseQuery("Row(gbool=true)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil { + t.Fatalf("querying: %v", err) + } else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) { + t.Fatalf("unexpected Row(gbool=true) result: %+v", resp.Result().Row().Columns) + } + +} diff --git a/client/logimport.go b/client/logimport.go new file mode 100644 index 000000000..117c20817 --- /dev/null +++ b/client/logimport.go @@ -0,0 +1,33 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package client + +import ( + "encoding/gob" + "io" +) + +type importLog struct { + Index string + Path string + Shard uint64 + IsRoaring bool + Timestamp int64 // Unix Nanoseconds + Data []byte +} + +type encoder interface { + Encode(thing interface{}) error +} + +func newImportLogEncoder(w io.Writer) encoder { + return gob.NewEncoder(w) +} + +type decoder interface { + Decode(thing interface{}) error +} + +func newImportLogDecoder(r io.Reader) decoder { + return gob.NewDecoder(r) +} diff --git a/client/logimport_test.go b/client/logimport_test.go new file mode 100644 index 000000000..6db0f41c4 --- /dev/null +++ b/client/logimport_test.go @@ -0,0 +1,143 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package client + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "reflect" + "testing" +) + +func TestEncodeDecode(t *testing.T) { + tests := []importLog{ + { + Index: "go-testindex", + Path: "/index/go-testindex/field/importfield-batchsize/import?clear=false", + Shard: 0, + Data: make([]byte, 3918), + }, + { + Index: "go-testindex", + Path: "/index/go-testindex/field/importfield-batchsize/import?clear=false", + Shard: 0, + Data: make([]byte, 3918), + }, + { + Index: "eheh", + Path: "blah", + Shard: 9, + Data: []byte("something"), + }, + { + Index: "", + Path: "", + Shard: 0, + Data: nil, + }, + { + Index: "eheh", + Path: "blah", + Shard: 10, + Data: []byte("blahaslkdjfeoiwujf"), + }, + { + Index: "eheh", + Path: "blah", + Shard: 10, + Data: make([]byte, 10000), + }, + { + Index: "zoop", + Path: "blah", + Shard: 8923734, + Data: []byte("blahaslkdjfeoiwujf"), + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + nl := importLog{ + Index: test.Index, + Path: test.Path, + Shard: test.Shard, + Data: make([]byte, len(test.Data)), + } + copy(nl.Data, test.Data) + buf := &bytes.Buffer{} + enc := newImportLogEncoder(buf) + err := enc.Encode(nl) + if err != nil { + t.Fatalf("writing to buf: %v", err) + } + dec := newImportLogDecoder(buf) + l2 := &importLog{} + err = dec.Decode(l2) + if err != nil { + t.Fatalf("reading from buf: %v", err) + } + if l2.Index != test.Index { + t.Errorf("indexes not equal:\n%s\n%s", test.Index, l2.Index) + } + if l2.Path != test.Path { + t.Errorf("paths not equal:\n%s\n%s", test.Path, l2.Path) + } + if l2.Shard != test.Shard { + t.Errorf("shards not equal exp: %d got %d", test.Shard, l2.Shard) + } + if !reflect.DeepEqual(test.Data, l2.Data) { + t.Errorf("data not equal \n%v\n%v", test.Data, l2.Data) + } + + }) + } + buf, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("getting temp file: %v", err) + } + enc := newImportLogEncoder(buf) + for _, test := range tests { + a := &test + err := enc.Encode(a) + if err != nil { + t.Errorf("encoding to buf: %v", err) + } + } + + name := buf.Name() + err = buf.Close() + if err != nil { + t.Fatalf("closing temp file: %v", err) + } + + buf, err = os.Open(name) + if err != nil { + t.Fatalf("reopening: %v", err) + } + + dec := newImportLogDecoder(buf) + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + l := &importLog{} + err := dec.Decode(l) + // err := l.ReadFrom(buf) + if err != nil { + t.Errorf("reading from buf: %v", err) + } + if l.Index != test.Index { + t.Errorf("indexes not equal:\n%s\n%s", test.Index, l.Index) + } + if l.Path != test.Path { + t.Errorf("paths not equal:\n%s\n%s", test.Path, l.Path) + } + if l.Shard != test.Shard { + t.Errorf("shards not equal exp: %d got %d", test.Shard, l.Shard) + } + if !reflect.DeepEqual(test.Data, l.Data) { + t.Errorf("data not equal \n%v\n%v", test.Data, l.Data) + } + }) + } +} diff --git a/client/metrics.go b/client/metrics.go new file mode 100644 index 000000000..d17917da2 --- /dev/null +++ b/client/metrics.go @@ -0,0 +1,28 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +package client + +const ( + // MetricBatchImportDurationSeconds records the full time of the + // RecordBatch.Import call. This includes starting and finishing a + // transaction, doing key translation, building fragments locally, + // importing all data, and resetting internal structures. + MetricBatchImportDurationSeconds = "batch_import_duration_seconds" + + // MetricBatchFlushDurationSeconds records the full time for + // RecordBatch.Flush (if splitBatchMode is in use). This includes + // starting and finishing a transaction, importing all data, and + // resetting internal structures. + MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds" + + // MetricBatchShardImportBuildRequestsSeconds is the time it takes + // after making fragments to build the shard-transactional request + // objects (but not actually import them or do any network activity). + MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds" + + // MetricBatchShardImportDurationSeconds is the time it takes to + // import all data for all shards in the batch using the + // shard-transactional endpoint. This does not include the time it + // takes to build the requests locally. + MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds" +) diff --git a/client/orm.go b/client/orm.go new file mode 100644 index 000000000..54e230334 --- /dev/null +++ b/client/orm.go @@ -0,0 +1,1586 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "sync" + "time" + + "github.com/molecula/featurebase/v3/pql" + "github.com/pkg/errors" +) + +const timeFormat = "2006-01-02T15:04" + +// Schema contains the index properties +type Schema struct { + mu sync.RWMutex + indexes map[string]*Index +} + +func (s *Schema) String() string { + s.mu.RLock() + defer s.mu.RUnlock() + return fmt.Sprintf("%s", s.indexes) +} + +// NewSchema creates a new Schema +func NewSchema() *Schema { + return &Schema{ + indexes: make(map[string]*Index), + } +} + +// Index returns an index with a name. +func (s *Schema) Index(name string, options ...IndexOption) *Index { + s.mu.Lock() + defer s.mu.Unlock() + if index, ok := s.indexes[name]; ok { + return index + } + indexOptions := &IndexOptions{} + indexOptions.addOptions(options...) + return s.indexWithOptions(name, 0, 0, indexOptions) +} + +func (s *Schema) indexWithOptions(name string, createdAt int64, shardWidth uint64, options *IndexOptions) *Index { + index := NewIndex(name) + if createdAt != 0 { + index.createdAt = createdAt + } + + index.options = options.withDefaults() + index.shardWidth = shardWidth + if index.Opts().TrackExistence() { + index.Field("_exists") + } + s.indexes[name] = index + return index +} + +// Indexes return a copy of the indexes in this schema +func (s *Schema) Indexes() map[string]*Index { + s.mu.RLock() + defer s.mu.RUnlock() + result := make(map[string]*Index) + for k, v := range s.indexes { + result[k] = v.copy() + } + return result +} + +// HasIndex returns true if the given index is in the schema. +func (s *Schema) HasIndex(indexName string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.indexes[indexName] + return ok +} + +func (s *Schema) diff(other *Schema) *Schema { + result := NewSchema() + for indexName, index := range s.indexes { + if otherIndex, ok := other.indexes[indexName]; !ok { + // if the index doesn't exist in the other schema, simply copy it + result.indexes[indexName] = index.copy() + } else { + // the index exists in the other schema; check the fields + resultIndex := NewIndex(indexName) + for fieldName, field := range index.fields { + if _, ok := otherIndex.fields[fieldName]; !ok { + // the field doesn't exist in the other schema, copy it + resultIndex.fields[fieldName] = field.copy() + } + } + // check whether we modified result index + if len(resultIndex.fields) > 0 { + // if so, move it to the result + result.indexes[indexName] = resultIndex + } + } + } + return result +} + +type SerializedQuery interface { + String() string + HasWriteKeys() bool +} + +type serializedQuery struct { + query string + hasWriteKeys bool +} + +func newSerializedQuery(query string, hasWriteKeys bool) serializedQuery { + return serializedQuery{ + query: query, + hasWriteKeys: hasWriteKeys, + } +} + +func (s serializedQuery) String() string { + return s.query +} + +func (s serializedQuery) HasWriteKeys() bool { + return s.hasWriteKeys +} + +// PQLQuery is an interface for PQL queries. +type PQLQuery interface { + Index() *Index + Serialize() SerializedQuery + Error() error +} + +// PQLBaseQuery is the base implementation for PQLQuery. +type PQLBaseQuery struct { + index *Index + pql string + err error + hasKeys bool +} + +// NewPQLBaseQuery creates a new PQLQuery with the given PQL and index. +func NewPQLBaseQuery(pql string, index *Index, err error) *PQLBaseQuery { + var hasKeys bool + if index != nil { + hasKeys = index.options.keys + } + return &PQLBaseQuery{ + index: index, + pql: pql, + err: err, + hasKeys: hasKeys, + } +} + +// Index returns the index for this query +func (q *PQLBaseQuery) Index() *Index { + return q.index +} + +func (q *PQLBaseQuery) Serialize() SerializedQuery { + return newSerializedQuery(q.pql, q.hasKeys) +} + +// Error returns the error or nil for this query. +func (q PQLBaseQuery) Error() error { + return q.err +} + +// PQLRowQuery is the return type for row queries. +type PQLRowQuery struct { + index *Index + pql string + err error + hasKeys bool +} + +// Index returns the index for this query/ +func (q *PQLRowQuery) Index() *Index { + return q.index +} + +func (q *PQLRowQuery) Serialize() SerializedQuery { + return q.serialize() +} + +func (q *PQLRowQuery) serialize() SerializedQuery { + return newSerializedQuery(q.pql, q.hasKeys) +} + +// Error returns the error or nil for this query. +func (q PQLRowQuery) Error() error { + return q.err +} + +// PQLBatchQuery contains a batch of PQL queries. +// Use Index.BatchQuery function to create an instance. +// +// Usage: +// +// repo, err := NewIndex("repository") +// stargazer, err := repo.Field("stargazer") +// query := repo.BatchQuery( +// stargazer.Row(5), +// stargazer.Row(15), +// repo.Union(stargazer.Row(20), stargazer.Row(25))) +type PQLBatchQuery struct { + index *Index + queries []string + err error + hasKeys bool +} + +// Index returns the index for this query. +func (q *PQLBatchQuery) Index() *Index { + return q.index +} + +func (q *PQLBatchQuery) Serialize() SerializedQuery { + query := strings.Join(q.queries, "") + return newSerializedQuery(query, q.hasKeys) +} + +func (q *PQLBatchQuery) Error() error { + return q.err +} + +// Add adds a query to the batch. +func (q *PQLBatchQuery) Add(query PQLQuery) { + err := query.Error() + if err != nil { + q.err = err + } + serializedQuery := query.Serialize() + q.hasKeys = q.hasKeys || serializedQuery.HasWriteKeys() + q.queries = append(q.queries, serializedQuery.String()) +} + +// NewPQLRowQuery creates a new PqlRowQuery. +func NewPQLRowQuery(pql string, index *Index, err error) *PQLRowQuery { + return &PQLRowQuery{ + index: index, + pql: pql, + err: err, + hasKeys: index.options.keys, + } +} + +// IndexOptions contains options to customize Index objects. +type IndexOptions struct { + keys bool + keysSet bool + trackExistence bool + trackExistenceSet bool +} + +func (io *IndexOptions) withDefaults() (updated *IndexOptions) { + // copy options so the original is not updated + updated = &IndexOptions{} + *updated = *io + if !updated.keysSet { + updated.keys = false + } + if !updated.trackExistenceSet { + updated.trackExistence = true + } + return +} + +// Keys return true if this index has keys. +func (io IndexOptions) Keys() bool { + return io.keys +} + +// TrackExistence returns true if existence is tracked for this index. +func (io IndexOptions) TrackExistence() bool { + return io.trackExistence +} + +// String serializes this index to a JSON string. +func (io IndexOptions) String() string { + mopt := map[string]interface{}{} + if io.keysSet { + mopt["keys"] = io.keys + } + if io.trackExistenceSet { + mopt["trackExistence"] = io.trackExistence + } + return fmt.Sprintf(`{"options":%s}`, encodeMap(mopt)) +} + +func (io *IndexOptions) addOptions(options ...IndexOption) { + for _, option := range options { + if option == nil { + continue + } + option(io) + } +} + +// IndexOption is used to pass an option to Index function. +type IndexOption func(options *IndexOptions) + +// OptIndexKeys sets whether index uses string keys. +func OptIndexKeys(keys bool) IndexOption { + return func(options *IndexOptions) { + options.keys = keys + options.keysSet = true + } +} + +// OptIndexTrackExistence enables keeping track of existence of columns. +func OptIndexTrackExistence(trackExistence bool) IndexOption { + return func(options *IndexOptions) { + options.trackExistence = trackExistence + options.trackExistenceSet = true + } +} + +// OptionsOptions is used to pass an option to Option call. +type OptionsOptions struct { + shards []uint64 +} + +func (oo OptionsOptions) marshal() string { + if oo.shards != nil { + shardsStr := make([]string, len(oo.shards)) + for i, shard := range oo.shards { + shardsStr[i] = strconv.FormatUint(shard, 10) + } + return fmt.Sprintf("shards=[%s]", strings.Join(shardsStr, ",")) + } + + return "" +} + +// OptionsOption is an option for Index.Options call. +type OptionsOption func(options *OptionsOptions) + +// OptOptionsShards run the query using only the data from the given shards. +// By default, the entire data set (i.e. data from all shards) is used. +func OptOptionsShards(shards ...uint64) OptionsOption { + return func(options *OptionsOptions) { + options.shards = shards + } +} + +// Index is a Pilosa index. The purpose of the Index is to represent a data namespace. +// You cannot perform cross-index queries. +type Index struct { + mu sync.RWMutex + name string + createdAt int64 + options *IndexOptions + fields map[string]*Field + shardWidth uint64 +} + +func (idx *Index) String() string { + return fmt.Sprintf(`{name: "%s", options: "%s", fields: %s, shardWidth: %d}`, idx.name, idx.options, idx.fields, idx.shardWidth) +} + +// NewIndex creates an index with a name. +func NewIndex(name string) *Index { + options := &IndexOptions{} + return &Index{ + name: name, + options: options.withDefaults(), + fields: map[string]*Field{}, + } +} + +func (idx *Index) ShardWidth() uint64 { + return idx.shardWidth +} + +// Fields return a copy of the fields in this index +func (idx *Index) Fields() map[string]*Field { + idx.mu.Lock() + defer idx.mu.Unlock() + result := make(map[string]*Field) + for k, v := range idx.fields { + result[k] = v.copy() + } + return result +} + +// HasFields returns true if the given field exists in the index. +func (idx *Index) HasField(fieldName string) bool { + idx.mu.Lock() + defer idx.mu.Unlock() + _, ok := idx.fields[fieldName] + return ok +} + +func (idx *Index) copy() *Index { + idx.mu.Lock() + defer idx.mu.Unlock() + fields := make(map[string]*Field) + for name, f := range idx.fields { + fields[name] = f.copy() + } + index := &Index{ + name: idx.name, + createdAt: idx.createdAt, + options: &IndexOptions{}, + fields: fields, + shardWidth: idx.shardWidth, + } + *index.options = *idx.options + return index +} + +// Name returns the name of this index. +func (idx *Index) Name() string { + return idx.name +} + +func (idx *Index) CreatedAt() int64 { + idx.mu.RLock() + defer idx.mu.RUnlock() + return idx.createdAt +} + +// Opts returns the options of this index. +func (idx *Index) Opts() IndexOptions { + return *idx.options +} + +// Field creates a Field struct with the specified name and defaults. +func (idx *Index) Field(name string, options ...FieldOption) *Field { + idx.mu.Lock() + defer idx.mu.Unlock() + if field, ok := idx.fields[name]; ok { + return field + } + fieldOptions := &FieldOptions{} + fieldOptions = fieldOptions.withDefaults() + fieldOptions.addOptions(options...) + return idx.fieldWithOptions(name, 0, fieldOptions) +} + +func (idx *Index) fieldWithOptions(name string, createdAt int64, fieldOptions *FieldOptions) *Field { + field := newField(name, idx) + if createdAt != 0 { + field.createdAt = createdAt + } + fieldOptions = fieldOptions.withDefaults() + field.options = fieldOptions + idx.fields[name] = field + return field +} + +// BatchQuery creates a batch query with the given queries. +func (idx *Index) BatchQuery(queries ...PQLQuery) *PQLBatchQuery { + stringQueries := make([]string, 0, len(queries)) + hasKeys := false + for _, query := range queries { + serializedQuery := query.Serialize() + hasKeys = hasKeys || serializedQuery.HasWriteKeys() + stringQueries = append(stringQueries, serializedQuery.String()) + } + return &PQLBatchQuery{ + index: idx, + queries: stringQueries, + hasKeys: hasKeys, + } +} + +// RawQuery creates a query with the given string. +// Note that the query is not validated before sending to the server. +func (idx *Index) RawQuery(query string) *PQLBaseQuery { + q := NewPQLBaseQuery(query, idx, nil) + // NOTE: raw queries always assumed to have keys set + q.hasKeys = true + return q +} + +// Union creates a Union query. +// Union performs a logical OR on the results of each ROW_CALL query passed to it. +func (idx *Index) Union(rows ...*PQLRowQuery) *PQLRowQuery { + return idx.rowOperation("Union", rows...) +} + +// Intersect creates an Intersect query. +// Intersect performs a logical AND on the results of each ROW_CALL query passed to it. +func (idx *Index) Intersect(rows ...*PQLRowQuery) *PQLRowQuery { + if len(rows) < 1 { + return NewPQLRowQuery("", idx, errors.New("Intersect operation requires at least 1 row")) + } + return idx.rowOperation("Intersect", rows...) +} + +// Difference creates an Intersect query. +// Difference returns all of the columns from the first ROW_CALL argument passed to it, without the columns from each subsequent ROW_CALL. +func (idx *Index) Difference(rows ...*PQLRowQuery) *PQLRowQuery { + if len(rows) < 1 { + return NewPQLRowQuery("", idx, errors.New("Difference operation requires at least 1 row")) + } + return idx.rowOperation("Difference", rows...) +} + +// Xor creates an Xor query. +func (idx *Index) Xor(rows ...*PQLRowQuery) *PQLRowQuery { + if len(rows) < 2 { + return NewPQLRowQuery("", idx, errors.New("Xor operation requires at least 2 rows")) + } + return idx.rowOperation("Xor", rows...) +} + +// Not creates a Not query. +func (idx *Index) Not(row *PQLRowQuery) *PQLRowQuery { + return NewPQLRowQuery(fmt.Sprintf("Not(%s)", row.serialize()), idx, row.Error()) +} + +// Count creates a Count query. +// Returns the number of set columns in the ROW_CALL passed in. +func (idx *Index) Count(row *PQLRowQuery) *PQLBaseQuery { + serializedQuery := row.serialize() + q := NewPQLBaseQuery(fmt.Sprintf("Count(%s)", serializedQuery.String()), idx, nil) + q.hasKeys = q.hasKeys || serializedQuery.HasWriteKeys() + return q +} + +// All creates an All query. +// Returns the set columns with existence true. +func (idx *Index) All() *PQLRowQuery { + q := NewPQLRowQuery("All()", idx, nil) + return q +} + +// TODO: impelement AllLimit(limit, offset uint64) *PQLRowQuery + +// Options creates an Options query. +func (idx *Index) Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery { + oo := &OptionsOptions{} + for _, opt := range opts { + opt(oo) + } + text := fmt.Sprintf("Options(%s,%s)", row.serialize(), oo.marshal()) + return NewPQLBaseQuery(text, idx, nil) +} + +type groupByBuilder struct { + rows []*PQLRowsQuery + limit int64 + filter *PQLRowQuery + aggregate *PQLBaseQuery + having *PQLBaseQuery +} + +// GroupByBuilderOption is a functional option type for index.GroupBy +type GroupByBuilderOption func(g *groupByBuilder) error + +// OptGroupByBuilderRows is a functional option on groupByBuilder +// used to set the rows. +func OptGroupByBuilderRows(rows ...*PQLRowsQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.rows = rows + return nil + } +} + +// OptGroupByBuilderLimit is a functional option on groupByBuilder +// used to set the limit. +func OptGroupByBuilderLimit(l int64) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.limit = l + return nil + } +} + +// OptGroupByBuilderFilter is a functional option on groupByBuilder +// used to set the filter. +func OptGroupByBuilderFilter(q *PQLRowQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.filter = q + return nil + } +} + +// OptGroupByBuilderAggregate is a functional option on groupByBuilder +// used to set the aggregate. +func OptGroupByBuilderAggregate(agg *PQLBaseQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.aggregate = agg + return nil + } +} + +// OptGroupByBuilderHaving is a functional option on groupByBuilder +// used to set the having clause. +func OptGroupByBuilderHaving(having *PQLBaseQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.having = having + return nil + } +} + +// GroupByBase creates a GroupBy query with the given functional options. +func (idx *Index) GroupByBase(opts ...GroupByBuilderOption) *PQLBaseQuery { + bldr := &groupByBuilder{} + for _, opt := range opts { + err := opt(bldr) + if err != nil { + return NewPQLBaseQuery("", idx, errors.Wrap(err, "applying option")) + } + } + + if len(bldr.rows) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + if bldr.limit < 0 { + return NewPQLBaseQuery("", idx, errors.New("limit must be non-negative")) + } + + // rows + text := fmt.Sprintf("GroupBy(%s", strings.Join(serializeGroupBy(bldr.rows...), ",")) + + // limit + if bldr.limit > 0 { + text += fmt.Sprintf(",limit=%d", bldr.limit) + } + + // filter + if bldr.filter != nil { + filterText := bldr.filter.serialize().String() + text += fmt.Sprintf(",filter=%s", filterText) + } + + // aggregate + if bldr.aggregate != nil { + aggregateText := bldr.aggregate.Serialize().String() + text += fmt.Sprintf(",aggregate=%s", aggregateText) + } + + // having + if bldr.having != nil { + havingText := bldr.having.Serialize().String() + text += fmt.Sprintf(",having=%s", havingText) + } + + text += ")" + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupBy creates a GroupBy query with the given Rows queries +func (idx *Index) GroupBy(rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + text := fmt.Sprintf("GroupBy(%s)", strings.Join(serializeGroupBy(rowsQueries...), ",")) + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupByLimit creates a GroupBy query with the given limit and Rows queries +func (idx *Index) GroupByLimit(limit int64, rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + if limit < 0 { + return NewPQLBaseQuery("", idx, errors.New("limit must be non-negative")) + } + text := fmt.Sprintf("GroupBy(%s,limit=%d)", strings.Join(serializeGroupBy(rowsQueries...), ","), limit) + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupByFilter creates a GroupBy query with the given filter and Rows queries +func (idx *Index) GroupByFilter(filterQuery *PQLRowQuery, rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + filterText := filterQuery.serialize().String() + text := fmt.Sprintf("GroupBy(%s,filter=%s)", strings.Join(serializeGroupBy(rowsQueries...), ","), filterText) + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupByLimitFilter creates a GroupBy query with the given filter and Rows queries +func (idx *Index) GroupByLimitFilter(limit int64, filterQuery *PQLRowQuery, rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + if limit < 0 { + return NewPQLBaseQuery("", idx, errors.New("limit must be non-negative")) + } + filterText := filterQuery.serialize().String() + text := fmt.Sprintf("GroupBy(%s,limit=%d,filter=%s)", strings.Join(serializeGroupBy(rowsQueries...), ","), limit, filterText) + return NewPQLBaseQuery(text, idx, nil) +} + +func (idx *Index) rowOperation(name string, rows ...*PQLRowQuery) *PQLRowQuery { + var err error + args := make([]string, 0, len(rows)) + for _, row := range rows { + if err = row.Error(); err != nil { + return NewPQLRowQuery("", idx, err) + } + args = append(args, row.serialize().String()) + } + query := NewPQLRowQuery(fmt.Sprintf("%s(%s)", name, strings.Join(args, ",")), idx, nil) + return query +} + +func serializeGroupBy(rowsQueries ...*PQLRowsQuery) []string { + qs := make([]string, 0, len(rowsQueries)) + for _, qry := range rowsQueries { + qs = append(qs, qry.serialize().String()) + } + return qs +} + +// FieldInfo represents schema information for a field. +type FieldInfo struct { + Name string `json:"name"` +} + +// FieldOptions contains options to customize Field objects and field queries. +type FieldOptions struct { + fieldType FieldType + timeQuantum TimeQuantum + ttl time.Duration + cacheType CacheType + cacheSize int + min pql.Decimal + max pql.Decimal + scale int64 + keys bool + noStandardView bool + foreignIndex string + timeUnit string + base int64 + epoch time.Time +} + +// Type returns the type of the field. Currently "set", "int", or "time". +func (fo FieldOptions) Type() FieldType { + return fo.fieldType +} + +// TimeQuantum returns the configured time quantum for a time field. Empty +// string otherwise. +func (fo FieldOptions) TimeQuantum() TimeQuantum { + return fo.timeQuantum +} + +// TTL returns the configured ttl for a time field. +func (fo FieldOptions) TTL() time.Duration { + return fo.ttl +} + +// CacheType returns the configured cache type for a "set" field. Empty string +// otherwise. +func (fo FieldOptions) CacheType() CacheType { + return fo.cacheType +} + +// CacheSize returns the cache size for a set field. Zero otherwise. +func (fo FieldOptions) CacheSize() int { + return fo.cacheSize +} + +// Min returns the minimum accepted value for an integer field. Zero otherwise. +func (fo FieldOptions) Min() pql.Decimal { + return fo.min +} + +// Max returns the maximum accepted value for an integer field. Zero otherwise. +func (fo FieldOptions) Max() pql.Decimal { + return fo.max +} + +// Scale returns the scale for a decimal field. +func (fo FieldOptions) Scale() int64 { + return fo.scale +} + +// Keys returns whether this field uses keys instead of IDs +func (fo FieldOptions) Keys() bool { + return fo.keys +} + +func (fo FieldOptions) ForeignIndex() string { + return fo.foreignIndex +} + +func (fo FieldOptions) TimeUnit() string { + return fo.timeUnit +} + +// NoStandardView suppresses creating the standard view for supported field types (currently, time) +func (fo FieldOptions) NoStandardView() bool { + return fo.noStandardView +} + +func (fo *FieldOptions) withDefaults() (updated *FieldOptions) { + // copy options so the original is not updated + updated = &FieldOptions{} + *updated = *fo + if updated.fieldType == "" { + updated.fieldType = FieldTypeSet + } + return +} + +func (fo FieldOptions) String() string { + mopt := map[string]interface{}{} + + switch fo.fieldType { + case FieldTypeSet, FieldTypeMutex: + if fo.cacheType != CacheTypeDefault { + mopt["cacheType"] = string(fo.cacheType) + } + if fo.cacheSize > 0 { + mopt["cacheSize"] = fo.cacheSize + } + case FieldTypeInt: + mopt["min"] = fo.min + mopt["max"] = fo.max + case FieldTypeDecimal: + mopt["min"] = fo.min + mopt["max"] = fo.max + mopt["scale"] = fo.scale + case FieldTypeTime: + mopt["timeQuantum"] = string(fo.timeQuantum) + mopt["noStandardView"] = fo.noStandardView + mopt["ttl"] = fo.ttl.String() + case FieldTypeTimestamp: + mopt["min"] = fo.min + mopt["max"] = fo.max + mopt["timeUnit"] = fo.timeUnit + mopt["epoch"] = fo.epoch + } + + if fo.fieldType != FieldTypeDefault { + mopt["type"] = string(fo.fieldType) + } + if fo.keys { + mopt["keys"] = fo.keys + } + if fo.foreignIndex != "" { + mopt["foreignIndex"] = fo.foreignIndex + } + return fmt.Sprintf(`{"options":%s}`, encodeMap(mopt)) +} + +func (fo *FieldOptions) addOptions(options ...FieldOption) { + for _, option := range options { + if option == nil { + continue + } + option(fo) + } +} + +// MinTimestamp returns the minimum value for a timestamp field. +func (o FieldOptions) MinTimestamp() time.Time { + return time.Unix(0, o.min.ToInt64(0)*int64(TimeUnitNano(o.TimeUnit()))) +} + +// MaxTimestamp returns the maxnimum value for a timestamp field. +func (o FieldOptions) MaxTimestamp() time.Time { + return time.Unix(0, o.max.ToInt64(0)*int64(TimeUnitNano(o.TimeUnit()))) +} + +// FieldOption is used to pass an option to index.Field function. +type FieldOption func(options *FieldOptions) + +// OptFieldTypeSet adds a set field. +// Specify CacheTypeDefault for the default cache type. +// Specify CacheSizeDefault for the default cache size. +func OptFieldTypeSet(cacheType CacheType, cacheSize int) FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeSet + options.cacheType = cacheType + options.cacheSize = cacheSize + } +} + +// OptFieldTypeInt adds an integer field. +// No arguments: min = min_int, max = max_int +// 1 argument: min = limit[0], max = max_int +// 2 or more arguments: min = limit[0], max = limit[1] +func OptFieldTypeInt(limits ...int64) FieldOption { + min := pql.NewDecimal(math.MinInt64, 0) + max := pql.NewDecimal(math.MaxInt64, 0) + + if len(limits) > 2 { + panic("error: OptFieldTypeInt accepts at most 2 arguments") + } + if len(limits) > 0 { + min = pql.NewDecimal(limits[0], 0) + } + if len(limits) > 1 { + max = pql.NewDecimal(limits[1], 0) + } + + return func(options *FieldOptions) { + options.fieldType = FieldTypeInt + options.min = min + options.max = max + } +} + +// OptFieldTypeTime adds a time field. +func OptFieldTypeTime(quantum TimeQuantum, opts ...bool) FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeTime + options.timeQuantum = quantum + if len(opts) > 0 && opts[0] { + options.noStandardView = true + } + } +} + +func OptFieldTTL(dur time.Duration) FieldOption { + return func(options *FieldOptions) { + options.ttl = dur + } +} + +// Timestamp field range. +var ( + DefaultEpoch = time.Unix(0, 0).UTC() // 1970-01-01T00:00:00Z + + MinTimestamp = time.Unix(-1<<32, 0).UTC() // 1833-11-24T17:31:44Z + MaxTimestamp = time.Unix(1<<32, 0).UTC() // 2106-02-07T06:28:16Z +) + +// TimeUnitNanos returns the number of nanoseconds in unit. +func TimeUnitNanos(unit string) int64 { + switch unit { + case TimeUnitSeconds: + return int64(time.Second) + case TimeUnitMilliseconds: + return int64(time.Millisecond) + case TimeUnitMicroseconds: + return int64(time.Microsecond) + default: + return int64(time.Nanosecond) + } +} + +func OptFieldTypeTimestamp(epoch time.Time, timeUnit string) FieldOption { + return func(fo *FieldOptions) { + epochValue := epoch.UnixNano() / TimeUnitNanos(timeUnit) + fo.fieldType = FieldTypeTimestamp + fo.timeUnit = timeUnit + fo.min = pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(timeUnit), 0) + fo.max = pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(timeUnit), 0) + fo.base = epochValue + fo.epoch = epoch + } +} + +// OptFieldTypeMutex adds a mutex field. +func OptFieldTypeMutex(cacheType CacheType, cacheSize int) FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeMutex + options.cacheType = cacheType + options.cacheSize = cacheSize + } +} + +// OptFieldTypeBool adds a bool field. +func OptFieldTypeBool() FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeBool + } +} + +func OptFieldTypeDecimal(scale int64, minmax ...pql.Decimal) FieldOption { + min, max := pql.MinMax(scale) + if len(minmax) > 2 { + panic("error: OptFieldTypeDecimal accepts at most 2 arguments") + } + if len(minmax) > 0 { + min = minmax[0] + } + if len(minmax) > 1 { + max = minmax[1] + } + return func(options *FieldOptions) { + options.fieldType = FieldTypeDecimal + options.scale = scale + options.min = min + options.max = max + } +} + +// OptFieldKeys sets whether field uses string keys. +func OptFieldKeys(keys bool) FieldOption { + return func(options *FieldOptions) { + options.keys = keys + } +} + +func OptFieldForeignIndex(index string) FieldOption { + return func(options *FieldOptions) { + options.foreignIndex = index + } +} + +// Field structs are used to segment and define different functional characteristics within your entire index. +// You can think of a Field as a table-like data partition within your Index. +type Field struct { + name string + createdAt int64 + index *Index + options *FieldOptions +} + +func (f *Field) String() string { + return fmt.Sprintf(`{name: "%s", index: "%s", options: "%s"}`, f.name, f.index.name, f.options) +} + +func newField(name string, index *Index) *Field { + return &Field{ + name: name, + index: index, + options: &FieldOptions{}, + } +} + +// Name returns the name of the field +func (f *Field) Name() string { + return f.name +} + +func (f *Field) CreatedAt() int64 { + return f.createdAt +} + +// Opts returns the options of the field +func (f *Field) Opts() FieldOptions { + return *f.options +} + +func (f *Field) copy() *Field { + field := newField(f.name, f.index) + field.createdAt = f.createdAt + *field.options = *f.options + return field +} + +// Row creates a Row query. +// Row retrieves the indices of all the set columns in a row. +func (f *Field) Row(rowIDOrKey interface{}) *PQLRowQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLRowQuery("", f.index, err) + } + text := fmt.Sprintf("Row(%s=%s)", f.name, rowStr) + q := NewPQLRowQuery(text, f.index, nil) + return q +} + +// Set creates a Set query. +// Set, assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given field with the given column. +func (f *Field) Set(rowIDOrKey, colIDOrKey interface{}) *PQLBaseQuery { + rowStr, colStr, err := formatRowColIDKey(rowIDOrKey, colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("Set(%s,%s=%s)", colStr, f.name, rowStr) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// SetTimestamp creates a Set query with timestamp. +// Set, assigns a value of 1 to a column in the binary matrix, +// thus associating the given row in the given field with the given column. +func (f *Field) SetTimestamp(rowIDOrKey, colIDOrKey interface{}, timestamp time.Time) *PQLBaseQuery { + rowStr, colStr, err := formatRowColIDKey(rowIDOrKey, colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("Set(%s,%s=%s,%s)", colStr, f.name, rowStr, timestamp.Format(timeFormat)) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// Clear creates a Clear query. +// Clear, assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column. +func (f *Field) Clear(rowIDOrKey, colIDOrKey interface{}) *PQLBaseQuery { + rowStr, colStr, err := formatRowColIDKey(rowIDOrKey, colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("Clear(%s,%s=%s)", colStr, f.name, rowStr) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// ClearRow creates a ClearRow query. +// ClearRow sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns. +func (f *Field) ClearRow(rowIDOrKey interface{}) *PQLBaseQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("ClearRow(%s=%s)", f.name, rowStr) + q := NewPQLBaseQuery(text, f.index, nil) + return q +} + +// TopN creates a TopN query with the given item count. +// Returns the id and count of the top n rows (by count of columns) in the field. +func (f *Field) TopN(n uint64) *PQLRowQuery { + q := NewPQLRowQuery(fmt.Sprintf("TopN(%s,n=%d)", f.name, n), f.index, nil) + return q +} + +// RowTopN creates a TopN query with the given item count and row. +// This variant supports customizing the row query. +func (f *Field) RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery { + q := NewPQLRowQuery(fmt.Sprintf("TopN(%s,%s,n=%d)", + f.name, row.serialize(), n), f.index, nil) + return q +} + +// Range creates a Range query. +// Similar to Row, but only returns columns which were set with timestamps between the given start and end timestamps. +// *Deprecated at Pilosa 1.3* +func (f *Field) Range(rowIDOrKey interface{}, start time.Time, end time.Time) *PQLRowQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLRowQuery("", f.index, err) + } + text := fmt.Sprintf("Range(%s=%s,%s,%s)", f.name, rowStr, start.Format(timeFormat), end.Format(timeFormat)) + q := NewPQLRowQuery(text, f.index, nil) + return q +} + +// RowRange creates a Row query with timestamps. +// Similar to Row, but only returns columns which were set with timestamps between the given start and end timestamps. +// *Introduced at Pilosa 1.3* +func (f *Field) RowRange(rowIDOrKey interface{}, start time.Time, end time.Time) *PQLRowQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLRowQuery("", f.index, err) + } + text := fmt.Sprintf("Row(%s=%s,from='%s',to='%s')", f.name, rowStr, start.Format(timeFormat), end.Format(timeFormat)) + q := NewPQLRowQuery(text, f.index, nil) + return q +} + +// Store creates a Store call. +// Store writes the result of the row query to the specified row. If the row already exists, it will be replaced. The destination field must be of field type set. +func (f *Field) Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + return NewPQLBaseQuery(fmt.Sprintf("Store(%s,%s=%s)", row.serialize().String(), f.name, rowStr), f.index, nil) +} + +func formatIDKey(idKey interface{}) (string, error) { + switch v := idKey.(type) { + case uint: + return strconv.FormatUint(uint64(v), 10), nil + case uint32: + return strconv.FormatUint(uint64(v), 10), nil + case uint64: + return strconv.FormatUint(v, 10), nil + case int: + return strconv.FormatInt(int64(v), 10), nil + case int32: + return strconv.FormatInt(int64(v), 10), nil + case int64: + return strconv.FormatInt(v, 10), nil + case string: + v = strings.ReplaceAll(v, `\`, `\\`) + return fmt.Sprintf(`'%s'`, strings.ReplaceAll(v, `'`, `\'`)), nil + default: + return "", errors.Errorf("id/key is not a string or integer type: %#v", idKey) + } +} + +func formatIDKeyBool(idKeyBool interface{}) (string, error) { + if b, ok := idKeyBool.(bool); ok { + return strconv.FormatBool(b), nil + } + if flt, ok := idKeyBool.(float64); ok { + return fmt.Sprintf("%f", flt), nil + } + return formatIDKey(idKeyBool) +} + +func formatRowColIDKey(rowIDOrKey, colIDOrKey interface{}) (string, string, error) { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return "", "", errors.Wrap(err, "formatting row") + } + colStr, err := formatIDKey(colIDOrKey) + if err != nil { + return "", "", errors.Wrap(err, "formatting column") + } + return rowStr, colStr, err +} + +// FieldType is the type of a field. +// See: https://www.pilosa.com/docs/latest/data-model/#field-type +type FieldType string + +const ( + // FieldTypeDefault is the default field type. + FieldTypeDefault FieldType = "" + // FieldTypeSet is the set field type. + // See: https://www.pilosa.com/docs/latest/data-model/#set + FieldTypeSet FieldType = "set" + // FieldTypeInt is the int field type. + // See: https://www.pilosa.com/docs/latest/data-model/#int + FieldTypeInt FieldType = "int" + // FieldTypeTime is the time field type. + // See: https://www.pilosa.com/docs/latest/data-model/#time + FieldTypeTime FieldType = "time" + // FieldTypeMutex is the mutex field type. + // See: https://www.pilosa.com/docs/latest/data-model/#mutex + FieldTypeMutex FieldType = "mutex" + // FieldTypeBool is the boolean field type. + // See: https://www.pilosa.com/docs/latest/data-model/#boolean + FieldTypeBool FieldType = "bool" + // FieldTypeDecimal can store floating point numbers as integers + // with a scale factor. This field type is only available in + // Molecula's Pilosa with enterprise extensions. + FieldTypeDecimal FieldType = "decimal" + FieldTypeTimestamp FieldType = "timestamp" +) + +// TimeQuantum type represents valid time quantum values time fields. +type TimeQuantum string + +// TimeQuantum constants +const ( + TimeQuantumNone TimeQuantum = "" + TimeQuantumYear TimeQuantum = "Y" + TimeQuantumMonth TimeQuantum = "M" + TimeQuantumDay TimeQuantum = "D" + TimeQuantumHour TimeQuantum = "H" + TimeQuantumYearMonth TimeQuantum = "YM" + TimeQuantumMonthDay TimeQuantum = "MD" + TimeQuantumDayHour TimeQuantum = "DH" + TimeQuantumYearMonthDay TimeQuantum = "YMD" + TimeQuantumMonthDayHour TimeQuantum = "MDH" + TimeQuantumYearMonthDayHour TimeQuantum = "YMDH" +) + +// List of time units. +const ( + TimeUnitSeconds = "s" + TimeUnitMilliseconds = "ms" + TimeUnitMicroseconds = "µs" + TimeUnitNanoseconds = "ns" +) + +// TimeUnitNano returns the number of nanoseconds in unit. +func TimeUnitNano(unit string) int64 { + switch unit { + case TimeUnitSeconds: + return int64(time.Second) + case TimeUnitMilliseconds: + return int64(time.Millisecond) + case TimeUnitMicroseconds: + return int64(time.Microsecond) + default: + return int64(time.Nanosecond) + } +} + +// CacheType represents cache type for a field +type CacheType string + +// CacheType constants +const ( + CacheTypeDefault CacheType = "" + CacheTypeLRU CacheType = "lru" + CacheTypeRanked CacheType = "ranked" + CacheTypeNone CacheType = "none" +) + +// CacheSizeDefault is the default cache size +const CacheSizeDefault = 0 + +// Options returns the options set for the field. Which fields of the +// FieldOptions struct are actually being used depends on the field's type. +// *DEPRECATED* +func (f *Field) Options() *FieldOptions { + return f.options +} + +type IntOrFloat interface{} + +type intOrFloatVal struct { + IntOrFloat +} + +func (i intOrFloatVal) String() string { + switch i.IntOrFloat.(type) { + case float64: + // In order to test expected values, we set the precision + // to 8. TODO: It's likely we'll need to address this + // at some point. + return fmt.Sprintf("%.8f", i.IntOrFloat) + default: + return fmt.Sprintf("%d", i.IntOrFloat) + } +} + +// LT creates a less than query. +func (f *Field) LT(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("<", n) +} + +// LTE creates a less than or equal query. +func (f *Field) LTE(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("<=", n) +} + +// GT creates a greater than query. +func (f *Field) GT(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation(">", n) +} + +// GTE creates a greater than or equal query. +func (f *Field) GTE(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation(">=", n) +} + +// Equals creates an equals query. +func (f *Field) Equals(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("==", n) +} + +// NotEquals creates a not equals query. +func (f *Field) NotEquals(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("!=", n) +} + +// NotNull creates a not equal to null query. +func (f *Field) NotNull() *PQLRowQuery { + text := fmt.Sprintf("Row(%s != null)", f.name) + q := NewPQLRowQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// Between creates a between query. +func (f *Field) Between(a IntOrFloat, b IntOrFloat) *PQLRowQuery { + text := fmt.Sprintf("Row(%s >< [%s,%s])", f.name, intOrFloatVal{a}, intOrFloatVal{b}) + q := NewPQLRowQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// Sum creates a sum query. +func (f *Field) Sum(row *PQLRowQuery) *PQLBaseQuery { + return f.valQuery("Sum", row) +} + +// Min creates a min query. +func (f *Field) Min(row *PQLRowQuery) *PQLBaseQuery { + return f.valQuery("Min", row) +} + +// Max creates a max query. +func (f *Field) Max(row *PQLRowQuery) *PQLBaseQuery { + return f.valQuery("Max", row) +} + +// MinRow creates a min row query. +func (f *Field) MinRow() *PQLBaseQuery { + q := fmt.Sprintf("MinRow(field='%s')", f.name) + return NewPQLBaseQuery(q, f.index, nil) +} + +// MaxRow creates a max row query. +func (f *Field) MaxRow() *PQLBaseQuery { + q := fmt.Sprintf("MaxRow(field='%s')", f.name) + return NewPQLBaseQuery(q, f.index, nil) +} + +// SetIntValue creates a Set query. +func (f *Field) SetIntValue(colIDOrKey interface{}, value int) *PQLBaseQuery { + colStr, err := formatIDKey(colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + q := fmt.Sprintf("Set(%s, %s=%d)", colStr, f.name, value) + return NewPQLBaseQuery(q, f.index, nil) +} + +// PQLRowsQuery is the return type for Rows calls. +type PQLRowsQuery struct { + index *Index + pql string + err error +} + +// NewPQLRowsQuery creates a new PQLRowsQuery. +func NewPQLRowsQuery(pql string, index *Index, err error) *PQLRowsQuery { + return &PQLRowsQuery{ + index: index, + pql: pql, + err: err, + } +} + +// Index returns the index for this query/ +func (q *PQLRowsQuery) Index() *Index { + return q.index +} + +func (q *PQLRowsQuery) Serialize() SerializedQuery { + return q.serialize() +} + +func (q *PQLRowsQuery) serialize() SerializedQuery { + return newSerializedQuery(q.pql, false) +} + +// Error returns the error or nil for this query. +func (q PQLRowsQuery) Error() error { + return q.err +} + +// Union returns the union of all matched rows. +func (q *PQLRowsQuery) Union() *PQLRowQuery { + return NewPQLRowQuery(fmt.Sprintf("UnionRows(%s)", q.serialize().String()), q.index, nil) +} + +// Rows creates a Rows query with defaults +func (f *Field) Rows() *PQLRowsQuery { + text := fmt.Sprintf("Rows(field='%s')", f.name) + return NewPQLRowsQuery(text, f.index, nil) +} + +// Like creates a Rows query filtered by a pattern. +// An underscore ('_') can be used as a placeholder for a single UTF-8 codepoint or a percent sign ('%') can be used as a placeholder for 0 or more codepoints. +// All other codepoints in the pattern are matched exactly. +func (f *Field) Like(pattern string) *PQLRowsQuery { + pattern = strings.ReplaceAll(pattern, `\`, `\\`) + pattern = strings.ReplaceAll(pattern, `'`, `\'`) + text := fmt.Sprintf("Rows(field='%s',like='%s')", f.name, pattern) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPrevious creates a Rows query with the given previous row ID/key +func (f *Field) RowsPrevious(rowIDOrKey interface{}) *PQLRowsQuery { + idKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s)", f.name, idKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsLimit creates a Rows query with the given limit +func (f *Field) RowsLimit(limit int64) *PQLRowsQuery { + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + text := fmt.Sprintf("Rows(field='%s',limit=%d)", f.name, limit) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsColumn creates a Rows query with the given column ID/key +func (f *Field) RowsColumn(columnIDOrKey interface{}) *PQLRowsQuery { + idKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',column=%s)", f.name, idKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPreviousLimit creates a Rows query with the given previous row ID/key and limit +func (f *Field) RowsPreviousLimit(rowIDOrKey interface{}, limit int64) *PQLRowsQuery { + idKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s,limit=%d)", f.name, idKey, limit) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPreviousColumn creates a Rows query with the given previous row ID/key and column ID/key +func (f *Field) RowsPreviousColumn(rowIDOrKey interface{}, columnIDOrKey interface{}) *PQLRowsQuery { + rowIDKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + columnIDKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s,column=%s)", f.name, rowIDKey, columnIDKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsLimitColumn creates a Row query with the given limit and column ID/key +func (f *Field) RowsLimitColumn(limit int64, columnIDOrKey interface{}) *PQLRowsQuery { + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + columnIDKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',limit=%d,column=%s)", f.name, limit, columnIDKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPreviousLimitColumn creates a Row query with the given previous row ID/key, limit and column ID/key +func (f *Field) RowsPreviousLimitColumn(rowIDOrKey interface{}, limit int64, columnIDOrKey interface{}) *PQLRowsQuery { + rowIDKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + columnIDKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s,limit=%d,column=%s)", f.name, rowIDKey, limit, columnIDKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// Distinct creates a Distinct query. +func (f *Field) Distinct() *PQLRowQuery { + text := fmt.Sprintf("Distinct(Row(%s!=null),index='%s',field='%s')", f.name, f.index.Name(), f.name) + return NewPQLRowQuery(text, f.index, nil) +} + +// RowDistinct creates a Distinct query with the given row filter. +func (f *Field) RowDistinct(row *PQLRowQuery) *PQLRowQuery { + text := fmt.Sprintf("Distinct(%s,index='%s',field='%s')", row.serialize(), f.index.Name(), f.name) + return NewPQLRowQuery(text, f.index, nil) +} + +func (f *Field) binaryOperation(op string, n IntOrFloat) *PQLRowQuery { + text := fmt.Sprintf("Row(%s %s %s)", f.name, op, intOrFloatVal{n}) + q := NewPQLRowQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +func (f *Field) valQuery(op string, row *PQLRowQuery) *PQLBaseQuery { + rowStr := "" + hasKeys := f.options.keys || f.index.options.keys + if row != nil { + serializedRow := row.serialize() + hasKeys = hasKeys || serializedRow.HasWriteKeys() + rowStr = fmt.Sprintf("%s,", serializedRow.String()) + } + text := fmt.Sprintf("%s(%sfield='%s')", op, rowStr, f.name) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = hasKeys + return q +} + +func encodeMap(m map[string]interface{}) string { + result, err := json.Marshal(m) + if err != nil { + panic(err) + } + return string(result) +} diff --git a/client/orm_test.go b/client/orm_test.go new file mode 100644 index 000000000..150cb231b --- /dev/null +++ b/client/orm_test.go @@ -0,0 +1,1235 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "fmt" + "math" + "reflect" + "sort" + "strings" + "testing" + "time" + + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/pql" + "github.com/pkg/errors" +) + +func TestORM(t *testing.T) { + var schema = NewSchema() + var sampleIndex = schema.Index("sample-index") + var sampleField = sampleIndex.Field("sample-field") + var projectIndex = schema.Index("project-index") + var collabField = projectIndex.Field("collaboration") + var b1 = sampleField.Row(10) + var b2 = sampleField.Row(20) + var b3 = sampleField.Row(42) + var b4 = collabField.Row(2) + + t.Run("SchemaDiff", func(t *testing.T) { + schema1 := NewSchema() + index11 := schema1.Index("diff-index1") + index11.Field("field1-1") + index11.Field("field1-2") + index12 := schema1.Index("diff-index2", OptIndexKeys(true), OptIndexTrackExistence(false)) + index12.Field("field2-1") + + schema2 := NewSchema() + index21 := schema2.Index("diff-index1") + index21.Field("another-field") + + targetDiff12 := NewSchema() + targetIndex1 := targetDiff12.Index("diff-index1", OptIndexTrackExistence(false)) + targetIndex1.Field("field1-1") + targetIndex1.Field("field1-2") + targetIndex2 := targetDiff12.Index("diff-index2", OptIndexKeys(true), OptIndexTrackExistence(false)) + targetIndex2.Field("field2-1") + targetIndex1.options = &IndexOptions{} + + diff12 := schema1.diff(schema2) + + strTargetDiff12 := fmt.Sprintf("%+v", targetDiff12.indexes) + strDiff12 := fmt.Sprintf("%+v", diff12.indexes) + if strDiff12 != strTargetDiff12 { + t.Fatalf("The diff must be correctly calculated, but exp/got\n%s\n%s", strTargetDiff12, strDiff12) + } + }) + + t.Run("SchemaIndexes", func(t *testing.T) { + schema1 := NewSchema() + index11 := schema1.Index("diff-index1") + index12 := schema1.Index("diff-index2") + indexes := schema1.Indexes() + target := map[string]*Index{ + "diff-index1": index11, + "diff-index2": index12, + } + if !reflect.DeepEqual(target, indexes) { + t.Fatalf("calling schema.Indexes should return indexes") + } + }) + + t.Run("SchemaToString", func(t *testing.T) { + schema1 := NewSchema() + _ = schema1.Index("test-index") + target := `map[test-index:{name: "test-index", options: "{"options":{}}", fields: map[_exists:{name: "_exists", index: "test-index", options: "{"options":{"type":"set"}}"}], shardWidth: 0}]` + if target != schema1.String() { + t.Fatalf("%s != %s", target, schema1.String()) + } + }) + + t.Run("NewIndex", func(t *testing.T) { + index1 := schema.Index("index-name") + if index1.Name() != "index-name" { + t.Fatalf("index name was not set") + } + // calling schema.Index again should return the same index + index2 := schema.Index("index-name") + if index1 != index2 { + t.Fatalf("calling schema.Index again should return the same index") + } + if !schema.HasIndex("index-name") { + t.Fatalf("HasIndex should return true") + } + if schema.HasIndex("index-x") { + t.Fatalf("HasIndex should return false") + } + }) + + t.Run("NewIndexCopy", func(t *testing.T) { + index := schema.Index("my-index-4copy", OptIndexKeys(true)) + index.Field("my-field-4copy", OptFieldTypeTime(TimeQuantumDayHour)) + copiedIndex := index.copy() + if !reflect.DeepEqual(index, copiedIndex) { + t.Fatalf("copied index should be equivalent") + } + }) + + t.Run("NewIndexOptions", func(t *testing.T) { + schemal := NewSchema() + // test the defaults + index := schemal.Index("index-default-options") + target := `{"options":{}}` + if target != index.options.String() { + t.Fatalf("%s != %s", target, index.options.String()) + } + + index = schemal.Index("index-keys", OptIndexKeys(true)) + if true != index.Opts().Keys() { + t.Fatalf("index keys %v != %v", true, index.Opts().Keys()) + } + target = `{"options":{"keys":true}}` + if target != index.options.String() { + t.Fatalf("%s != %s", target, index.options.String()) + } + + index = schemal.Index("index-trackexistence", OptIndexTrackExistence(false)) + if false != index.Opts().TrackExistence() { + t.Fatalf("index trackExistene %v != %v", true, index.Opts().TrackExistence()) + } + target = `{"options":{"trackExistence":false}}` + if target != index.options.String() { + t.Fatalf("%s != %s", target, index.options.String()) + } + }) + + t.Run("NilIndexOption", func(t *testing.T) { + schema.Index("index-with-nil-option", nil) + }) + + t.Run("IndexFields", func(t *testing.T) { + schema1 := NewSchema() + index11 := schema1.Index("diff-index1", OptIndexTrackExistence(false)) + field11 := index11.Field("field1-1") + field12 := index11.Field("field1-2") + fields := index11.Fields() + target := map[string]*Field{ + "field1-1": field11, + "field1-2": field12, + } + if !reflect.DeepEqual(target, fields) { + t.Fatalf("calling index.Fields should return fields") + } + if !index11.HasField("field1-1") { + t.Fatalf("HasField should return true") + } + if index11.HasField("field-x") { + t.Fatalf("HasField should return false") + } + }) + + t.Run("IndexToString", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + target := `{name: "test-index", options: "{"options":{}}", fields: map[_exists:{name: "_exists", index: "test-index", options: "{"options":{"type":"set"}}"}], shardWidth: 0}` + if target != index.String() { + t.Fatalf("indexes not equal exp/got:\n%s\n%s", target, index.String()) + } + }) + + t.Run("Field", func(t *testing.T) { + field1 := sampleIndex.Field("nonexistent-field") + field2 := sampleIndex.Field("nonexistent-field") + if field1 != field2 { + t.Fatalf("calling index.Field again should return the same field") + } + if field1.Name() != "nonexistent-field" { + t.Fatalf("calling field.Name should return field's name") + } + }) + + t.Run("FieldCopy", func(t *testing.T) { + field := sampleIndex.Field("my-field-4copy", OptFieldTypeSet(CacheTypeRanked, 123456)) + copiedField := field.copy() + if !reflect.DeepEqual(field, copiedField) { + t.Fatalf("copied field should be equivalent") + } + }) + + t.Run("FieldToString", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + field := index.Field("test-field") + target := `{name: "test-field", index: "test-index", options: "{"options":{"type":"set"}}"}` + if target != field.String() { + t.Fatalf("%s != %s", target, field.String()) + } + }) + + t.Run("NilFieldOption", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + index.Field("test-field-with-nil-option", nil) + }) + + t.Run("FieldSetType", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + field := index.Field("test-set-field", OptFieldTypeSet(CacheTypeLRU, 1000), OptFieldKeys(true)) + target := `{"options":{"type":"set","cacheType":"lru","cacheSize":1000,"keys":true}}` + if sortedString(target) != sortedString(field.options.String()) { + t.Fatalf("%s != %s", target, field.options.String()) + } + + field = index.Field("test-set-field2", OptFieldTypeSet(CacheTypeLRU, -10), OptFieldKeys(true)) + target = `{"options":{"type":"set","cacheType":"lru","keys":true}}` + if sortedString(target) != sortedString(field.options.String()) { + t.Fatalf("%s != %s", target, field.options.String()) + } + }) + + t.Run("Row", func(t *testing.T) { + comparePQL(t, + "Row(collaboration=5)", + collabField.Row(5)) + + comparePQL(t, + "Row(collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", + collabField.Row("b7feb014-8ea7-49a8-9cd8-19709161ab63")) + + q := collabField.Row(nil) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Set", func(t *testing.T) { + comparePQL(t, + "Set(10,collaboration=5)", + collabField.Set(5, 10)) + + comparePQL(t, + `Set('some_id',collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')`, + collabField.Set("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id")) + + q := collabField.Set(nil, 10) + if q.err == nil { + t.Fatalf("should have failed") + } + q = collabField.Set(5, false) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Timestamp", func(t *testing.T) { + timestamp := time.Date(2017, time.April, 24, 12, 14, 0, 0, time.UTC) + comparePQL(t, + "Set(20,collaboration=10,2017-04-24T12:14)", + collabField.SetTimestamp(10, 20, timestamp)) + + comparePQL(t, + "Set('mycol',collaboration='myrow',2017-04-24T12:14)", + collabField.SetTimestamp("myrow", "mycol", timestamp)) + + q := collabField.SetTimestamp(nil, 20, timestamp) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Clear", func(t *testing.T) { + comparePQL(t, + "Clear(10,collaboration=5)", + collabField.Clear(5, 10)) + + comparePQL(t, + "Clear('some_id',collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", + collabField.Clear("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id")) + comparePQL(t, + `Clear('bill\'s',collaboration='will\'s')`, + collabField.Clear("will's", "bill's")) + + q := collabField.Clear(nil, 10) + if q.err == nil { + t.Fatalf("should have failed") + } + q = collabField.Clear(5, false) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("ClearRow", func(t *testing.T) { + comparePQL(t, + "ClearRow(collaboration=5)", + collabField.ClearRow(5)) + + comparePQL(t, + "ClearRow(collaboration='five')", + collabField.ClearRow("five")) + + comparePQL(t, + "ClearRow(collaboration=true)", + collabField.ClearRow(true)) + + q := collabField.ClearRow(nil) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Union", func(t *testing.T) { + comparePQL(t, + "Union(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Union(b1, b2)) + comparePQL(t, + "Union(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Union(b1, b2, b3)) + comparePQL(t, + "Union(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Union(b1, b4)) + comparePQL(t, + "Union(Row(sample-field=10))", + sampleIndex.Union(b1)) + comparePQL(t, + "Union()", + sampleIndex.Union()) + }) + + t.Run("Intersect", func(t *testing.T) { + comparePQL(t, + "Intersect(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Intersect(b1, b2)) + comparePQL(t, + "Intersect(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Intersect(b1, b2, b3)) + comparePQL(t, + "Intersect(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Intersect(b1, b4)) + comparePQL(t, + "Intersect(Row(sample-field=10))", + sampleIndex.Intersect(b1)) + }) + + t.Run("Difference", func(t *testing.T) { + comparePQL(t, + "Difference(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Difference(b1, b2)) + comparePQL(t, + "Difference(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Difference(b1, b2, b3)) + comparePQL(t, + "Difference(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Difference(b1, b4)) + comparePQL(t, + "Difference(Row(sample-field=10))", + sampleIndex.Difference(b1)) + }) + + t.Run("Xor", func(t *testing.T) { + comparePQL(t, + "Xor(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Xor(b1, b2)) + comparePQL(t, + "Xor(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Xor(b1, b2, b3)) + comparePQL(t, + "Xor(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Xor(b1, b4)) + }) + + t.Run("Not", func(t *testing.T) { + comparePQL(t, + "Not(Row(sample-field=10))", + sampleIndex.Not(sampleField.Row(10))) + }) + + t.Run("TopN", func(t *testing.T) { + comparePQL(t, + "TopN(collaboration,n=27)", + collabField.TopN(27)) + comparePQL(t, + "TopN(collaboration,Row(collaboration=3),n=10)", + collabField.RowTopN(10, collabField.Row(3))) + }) + + t.Run("FieldLT", func(t *testing.T) { + comparePQL(t, + "Row(collaboration < 10)", + collabField.LT(10)) + comparePQL(t, + "Row(collaboration < 10.12300000)", + collabField.LT(10.123)) + }) + + t.Run("FieldLTE", func(t *testing.T) { + comparePQL(t, + "Row(collaboration <= 10)", + collabField.LTE(10)) + comparePQL(t, + "Row(collaboration <= 10.12300000)", + collabField.LTE(10.123)) + }) + + t.Run("FieldGT", func(t *testing.T) { + comparePQL(t, + "Row(collaboration > 10)", + collabField.GT(10)) + comparePQL(t, + "Row(collaboration > 10.12300000)", + collabField.GT(10.123)) + }) + + t.Run("FieldGTE", func(t *testing.T) { + comparePQL(t, + "Row(collaboration >= 10)", + collabField.GTE(10)) + comparePQL(t, + "Row(collaboration >= 10.12300000)", + collabField.GTE(10.123)) + }) + + t.Run("FieldEQ", func(t *testing.T) { + comparePQL(t, + "Row(collaboration == 10)", + collabField.Equals(10)) + comparePQL(t, + "Row(collaboration == 10.12300000)", + collabField.Equals(10.123)) + }) + + t.Run("FieldNEQ", func(t *testing.T) { + comparePQL(t, + "Row(collaboration != 10)", + collabField.NotEquals(10)) + comparePQL(t, + "Row(collaboration != 10.12300000)", + collabField.NotEquals(10.123)) + }) + + t.Run("FieldNotNull", func(t *testing.T) { + comparePQL(t, + "Row(collaboration != null)", + collabField.NotNull()) + }) + + t.Run("FieldBetween", func(t *testing.T) { + comparePQL(t, + "Row(collaboration >< [10,20])", + collabField.Between(10, 20)) + comparePQL(t, + "Row(collaboration >< [10.12300000,20.45600000])", + collabField.Between(10.123, 20.456)) + }) + + t.Run("FieldSum", func(t *testing.T) { + comparePQL(t, + "Sum(Row(collaboration=10),field='collaboration')", + collabField.Sum(collabField.Row(10))) + comparePQL(t, + "Sum(field='collaboration')", + collabField.Sum(nil)) + }) + + t.Run("FieldMinRow", func(t *testing.T) { + comparePQL(t, + "MinRow(field='sample-field')", + sampleField.MinRow()) + }) + + t.Run("FieldMaxRow", func(t *testing.T) { + comparePQL(t, + "MaxRow(field='sample-field')", + sampleField.MaxRow()) + }) + + t.Run("FieldSetValue", func(t *testing.T) { + comparePQL(t, + "Set(50, collaboration=15)", + collabField.SetIntValue(50, 15)) + + comparePQL(t, + "Set('mycol', sample-field=22)", + sampleField.SetIntValue("mycol", 22)) + + q := sampleField.SetIntValue(false, 22) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowOperationInvalidArg", func(t *testing.T) { + invalid := NewPQLRowQuery("", sampleIndex, errors.New("invalid")) + // invalid argument in pos 1 + q := sampleIndex.Union(invalid, b1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + // invalid argument in pos 2 + q = sampleIndex.Intersect(b1, invalid) + if q.Error() == nil { + t.Fatalf("should have failed") + } + // invalid argument in pos 3 + q = sampleIndex.Intersect(b1, b2, invalid) + if q.Error() == nil { + t.Fatalf("should have failed") + } + // not enough rows supplied + q = sampleIndex.Difference() + if q.Error() == nil { + t.Fatalf("should have failed") + } + // not enough rows supplied + q = sampleIndex.Intersect() + if q.Error() == nil { + t.Fatalf("should have failed") + } + + // not enough rows supplied + q = sampleIndex.Xor(b1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Store", func(t *testing.T) { + comparePQL(t, + "Store(Row(collaboration=5),sample-field=10)", + sampleField.Store(collabField.Row(5), 10)) + q := sampleField.Store(collabField.Row(5), nil) + if q.Error() == nil { + t.Fatalf("query error should be not nil") + } + }) + + t.Run("Options", func(t *testing.T) { + comparePQL(t, + "Options(Row(collaboration=5),shards=[1,3])", + sampleIndex.Options(collabField.Row(5), + OptOptionsShards(1, 3), + )) + }) + + t.Run("BatchQuery", func(t *testing.T) { + q := sampleIndex.BatchQuery() + if q.Index() != sampleIndex { + t.Fatalf("The correct index should be assigned") + } + q.Add(sampleField.Row(44)) + q.Add(sampleField.Row(10101)) + if q.Error() != nil { + t.Fatalf("Error should be nil") + } + comparePQL(t, "Row(sample-field=44)Row(sample-field=10101)", q) + + q2 := sampleField.Row(nil) + if q2.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("BatchQueryWithError", func(t *testing.T) { + q := sampleIndex.BatchQuery() + q.Add(NewPQLBaseQuery("", nil, errors.New("invalid"))) + if q.Error() == nil { + t.Fatalf("The error must be set") + } + }) + + t.Run("Count", func(t *testing.T) { + q := projectIndex.Count(collabField.Row(42)) + comparePQL(t, "Count(Row(collaboration=42))", q) + }) + + t.Run("Range", func(t *testing.T) { + start := time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2000, time.February, 2, 3, 4, 0, 0, time.UTC) + comparePQL(t, + "Range(collaboration=10,1970-01-01T00:00,2000-02-02T03:04)", + collabField.Range(10, start, end)) + + comparePQL(t, + "Range(collaboration='foo',1970-01-01T00:00,2000-02-02T03:04)", + collabField.Range("foo", start, end)) + + q := collabField.Range(nil, start, end) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowRange", func(t *testing.T) { + start := time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2000, time.February, 2, 3, 4, 0, 0, time.UTC) + comparePQL(t, + "Row(collaboration=10,from='1970-01-01T00:00',to='2000-02-02T03:04')", + collabField.RowRange(10, start, end)) + + comparePQL(t, + "Row(collaboration='foo',from='1970-01-01T00:00',to='2000-02-02T03:04')", + collabField.RowRange("foo", start, end)) + comparePQL(t, + `Row(collaboration='bill\'s',from='1970-01-01T00:00',to='2000-02-02T03:04')`, + collabField.RowRange("bill's", start, end)) + + q := collabField.RowRange(nil, start, end) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Rows", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration')", + collabField.Rows()) + }) + + t.Run("UnionRows", func(t *testing.T) { + comparePQL(t, + "UnionRows(Rows(field='collaboration'))", + collabField.Rows().Union()) + }) + + t.Run("Like", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',like='_')", + collabField.Like("_")) + comparePQL(t, + `Rows(field='collaboration',like='_\\')`, + collabField.Like(`_\`)) + comparePQL(t, + `Rows(field='collaboration',like='_\'')`, + collabField.Like(`_'`)) + }) + + t.Run("RowPrevious", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42)", + collabField.RowsPrevious(42)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two')", + collabField.RowsPrevious("forty-two")) + comparePQL(t, + `Rows(field='collaboration',previous='bill\'s')`, + collabField.RowsPrevious("bill's")) + q := collabField.RowsPrevious(1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowLimit", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',limit=10)", + collabField.RowsLimit(10)) + q := collabField.RowsLimit(-1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',column=1000)", + collabField.RowsColumn(1000)) + comparePQL(t, + "Rows(field='collaboration',column='one-thousand')", + collabField.RowsColumn("one-thousand")) + q := collabField.RowsColumn(1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsPreviousLimit", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42,limit=10)", + collabField.RowsPreviousLimit(42, 10)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two',limit=10)", + collabField.RowsPreviousLimit("forty-two", 10)) + q := collabField.RowsPreviousLimit(1.2, 10) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousLimit("forty-two", -1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsPreviousColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42,column=1000)", + collabField.RowsPreviousColumn(42, 1000)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two',column='one-thousand')", + collabField.RowsPreviousColumn("forty-two", "one-thousand")) + q := collabField.RowsPreviousColumn(1.2, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousColumn("forty-two", 1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("All", func(t *testing.T) { + comparePQL(t, + "All()", + projectIndex.All()) + }) + + t.Run("Distinct", func(t *testing.T) { + comparePQL(t, + "Distinct(Row(collaboration!=null),index='project-index',field='collaboration')", + collabField.Distinct()) + }) + + t.Run("RowDistinct", func(t *testing.T) { + comparePQL(t, + "Distinct(Row(sample-field=44),index='project-index',field='collaboration')", + collabField.RowDistinct(sampleField.Row(44))) + }) + + t.Run("RowLimitColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',limit=10,column=1000)", + collabField.RowsLimitColumn(10, 1000)) + comparePQL(t, + "Rows(field='collaboration',limit=10,column='one-thousand')", + collabField.RowsLimitColumn(10, "one-thousand")) + q := collabField.RowsLimitColumn(10, 1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsLimitColumn(-1, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsPreviousLimitColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42,limit=10,column=1000)", + collabField.RowsPreviousLimitColumn(42, 10, 1000)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two',limit=10,column='one-thousand')", + collabField.RowsPreviousLimitColumn("forty-two", 10, "one-thousand")) + q := collabField.RowsPreviousLimitColumn(1.2, 10, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousLimitColumn(42, -1, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousLimitColumn(42, 10, 1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupBy", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'))", + sampleIndex.GroupBy(collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'))", + sampleIndex.GroupBy(collabField.Rows(), field.Rows())) + q := sampleIndex.GroupBy() + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByLimit", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10)", + sampleIndex.GroupByLimit(10, collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10)", + sampleIndex.GroupByLimit(10, collabField.Rows(), field.Rows())) + q := sampleIndex.GroupByLimit(10) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = sampleIndex.GroupByLimit(-1, collabField.Rows()) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByFilter", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),filter=Row(test=5))", + sampleIndex.GroupByFilter(field.Row(5), collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),filter=Row(test=5))", + sampleIndex.GroupByFilter(field.Row(5), collabField.Rows(), field.Rows())) + q := sampleIndex.GroupByFilter(field.Row(5)) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByLimitFilter", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByLimitFilter(10, field.Row(5), collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByLimitFilter(10, field.Row(5), collabField.Rows(), field.Rows())) + q := sampleIndex.GroupByLimitFilter(10, field.Row(5)) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = sampleIndex.GroupByLimitFilter(-1, field.Row(5), collabField.Rows()) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByBase", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'))", + sampleIndex.GroupByBase( + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'))", + sampleIndex.GroupByBase( + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10)", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10)", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + comparePQL(t, + "GroupBy(Rows(field='collaboration'),filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + field2 := sampleIndex.Field("age") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),aggregate=Sum(Row(age=20),field='age'))", + sampleIndex.GroupByBase( + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + OptGroupByBuilderAggregate(field2.Sum(field2.Row(20))), + ), + ) + }) + + t.Run("FieldOptions", func(t *testing.T) { + field := sampleIndex.Field("foo", OptFieldKeys(true)) + if true != field.Opts().Keys() { + t.Fatalf("field keys: %v != %v", true, field.Opts().Keys()) + } + }) + + t.Run("SetFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("set-field", OptFieldTypeSet(CacheTypeRanked, 9999)) + jsonString := field.options.String() + targetString := `{"options":{"type":"set","cacheType":"ranked","cacheSize":9999}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeSet, + TimeQuantumNone, + CacheTypeRanked, + 9999, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "", + "", + 0) + }) + + t.Run("IntFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("int-field", OptFieldTypeInt(-10, 100)) + jsonString := field.options.String() + targetString := `{"options":{"type":"int","min":-10,"max":100}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(-10, 0), + pql.NewDecimal(100, 0), + "", + "", + 0) + + field = sampleIndex.Field("int-field2", OptFieldTypeInt(-10)) + jsonString = field.options.String() + targetString = fmt.Sprintf(`{"options":{"type":"int","min":-10,"max":%d}}`, math.MaxInt64) + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(-10, 0), + pql.NewDecimal(math.MaxInt64, 0), + "", + "", + 0) + field = sampleIndex.Field("int-field3", OptFieldTypeInt()) + jsonString = field.options.String() + targetString = fmt.Sprintf(`{"options":{"type":"int","min":%d,"max":%d}}`, math.MinInt64, math.MaxInt64) + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(math.MinInt64, 0), + pql.NewDecimal(math.MaxInt64, 0), + "", + "", + 0) + + field = sampleIndex.Field("int-field4", OptFieldTypeInt(), OptFieldForeignIndex("blerg")) + jsonString = field.options.String() + targetString = fmt.Sprintf(`{"options":{"type":"int","min":%d,"max":%d,"foreignIndex":"blerg"}}`, math.MinInt64, math.MaxInt64) + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(math.MinInt64, 0), + pql.NewDecimal(math.MaxInt64, 0), + "blerg", + "", + 0) + }) + + t.Run("TimeFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("time-field", OptFieldTypeTime(TimeQuantumDayHour, true)) + if true != field.Opts().NoStandardView() { + t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView()) + } + jsonString := field.options.String() + targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeTime, + TimeQuantumDayHour, + CacheTypeDefault, + 0, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "", + "", + 0) + }) + + t.Run("TTLOptions", func(t *testing.T) { + field := sampleIndex.Field("ttl-field", OptFieldTypeTime(TimeQuantumDayHour, true), OptFieldTTL(0)) + if true != field.Opts().NoStandardView() { + t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView()) + } + jsonString := field.options.String() + targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeTime, + TimeQuantumDayHour, + CacheTypeDefault, + 0, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "", + "", + 0) + }) + + t.Run("MutexFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("mutex-field", OptFieldTypeMutex(CacheTypeRanked, 9999)) + jsonString := field.options.String() + targetString := `{"options":{"type":"mutex","cacheType":"ranked","cacheSize":9999}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeMutex, + TimeQuantumNone, + CacheTypeRanked, + 9999, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "", + "", + 0) + }) + + t.Run("BoolFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("bool-field", OptFieldTypeBool()) + jsonString := field.options.String() + targetString := `{"options":{"type":"bool"}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeBool, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "", + "", + 0) + }) + + t.Run("DecimalFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("decimal-field", OptFieldTypeDecimal(3, pql.NewDecimal(7, 3), pql.NewDecimal(999, 3))) + jsonString := field.options.String() + targetString := `{"options":{"type":"decimal","scale":3,"max":0.999,"min":0.007}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeDecimal, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(7, 3), + pql.NewDecimal(999, 3), + "", + "", + 0) + }) + + t.Run("DecimalFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("decimal-field", OptFieldTypeDecimal(3, pql.NewDecimal(7, 3), pql.NewDecimal(999, 3))) + jsonString := field.options.String() + targetString := `{"options":{"type":"decimal","scale":3,"max":0.999,"min":0.007}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeDecimal, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(7, 3), + pql.NewDecimal(999, 3), + "", + "", + 0) + }) + + t.Run("TimestampFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("timestamp-field", OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds)) + + /*jsonString := field.options.String() + targetString := `{"options":{"type":"timestamp","timeUnit":"s","max":5000,"min":2}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + */ + compareFieldOptions(t, + field.Options(), + FieldTypeTimestamp, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0), + pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0), + "", + pilosa.TimeUnitSeconds, + 0) + + }) + + t.Run("EncodeMapPanicsOnMarshalFailure", func(t *testing.T) { + defer func() { + _ = recover() + }() + m := map[string]interface{}{ + "foo": func() {}, + } + encodeMap(m) + t.Fatal("Should have panicked") + }) + + t.Run("FormatIDKey", func(t *testing.T) { + testCase := [][]interface{}{ + {uint(42), "42", nil}, + {uint32(42), "42", nil}, + {uint64(42), "42", nil}, + {42, "42", nil}, + {int32(42), "42", nil}, + {int64(42), "42", nil}, + {"foo", `'foo'`, nil}, + {false, "", errors.New("error")}, + } + for i, item := range testCase { + s, err := formatIDKey(item[0]) + if item[2] != nil { + if err == nil { + t.Fatalf("Should have failed: %d", i) + } + continue + } + if item[1] != s { + t.Fatalf("%s != %s", item[1], s) + } + } + }) +} + +func comparePQL(t *testing.T, target string, q PQLQuery) { + t.Helper() + pql := q.Serialize().String() + if target != pql { + t.Fatalf("%s != %s", target, pql) + } +} + +func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string, timeUnit string, ttl time.Duration) { + if fieldType != opts.Type() { + t.Fatalf("%s != %s", fieldType, opts.Type()) + } + if timeQuantum != opts.TimeQuantum() { + t.Fatalf("%s != %s", timeQuantum, opts.TimeQuantum()) + } + if cacheType != opts.CacheType() { + t.Fatalf("%s != %s", cacheType, opts.CacheType()) + } + if cacheSize != opts.CacheSize() { + t.Fatalf("%d != %d", cacheSize, opts.CacheSize()) + } + if !min.EqualTo(opts.Min()) { + t.Fatalf("%v != %v", min, opts.Min()) + } + if !max.EqualTo(opts.Max()) { + t.Fatalf("%v != %v", max, opts.Max()) + } + if foreignIndex != opts.ForeignIndex() { + t.Fatalf("%s != %s", foreignIndex, opts.ForeignIndex()) + } + if timeUnit != opts.TimeUnit() { + t.Fatalf("%s != %s", timeUnit, opts.TimeUnit()) + } + if ttl != opts.TTL() { + t.Fatalf("%s != %s", ttl, opts.TTL()) + } +} + +func sortedString(s string) string { + arr := strings.Split(s, "") + sort.Strings(arr) + return strings.Join(arr, "") +} diff --git a/client/record.go b/client/record.go new file mode 100644 index 000000000..c38d1b23d --- /dev/null +++ b/client/record.go @@ -0,0 +1,63 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +// Record is a Column or a FieldValue. +type Record interface { + Shard(shardWidth uint64) uint64 + Less(other Record) bool +} + +// RecordIterator is an iterator for a record. +type RecordIterator interface { + NextRecord() (Record, error) +} + +// Column defines a single Pilosa column. +type Column struct { + RowID uint64 + ColumnID uint64 + RowKey string + ColumnKey string + Timestamp int64 +} + +// Shard returns the shard for this column. +func (b Column) Shard(shardWidth uint64) uint64 { + return b.ColumnID / shardWidth +} + +// Less returns true if this column sorts before the given Record. +func (b Column) Less(other Record) bool { + if ob, ok := other.(Column); ok { + if b.RowID == ob.RowID { + return b.ColumnID < ob.ColumnID + } + return b.RowID < ob.RowID + } + return false +} + +// FieldValue represents the value for a column within a +// range-encoded field. +type FieldValue struct { + ColumnID uint64 + ColumnKey string + Value int64 +} + +// Shard returns the shard for this field value. +func (v FieldValue) Shard(shardWidth uint64) uint64 { + return v.ColumnID / shardWidth +} + +// Less returns true if this field value sorts before the given Record. +func (v FieldValue) Less(other Record) bool { + if ov, ok := other.(FieldValue); ok { + return v.ColumnID < ov.ColumnID + } + return false +} diff --git a/client/record_test.go b/client/record_test.go new file mode 100644 index 000000000..2d75a464e --- /dev/null +++ b/client/record_test.go @@ -0,0 +1,71 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client_test + +import ( + "testing" + + "github.com/molecula/featurebase/v3/client" +) + +func TestColumnShard(t *testing.T) { + a := client.Column{RowID: 15, ColumnID: 55, Timestamp: 100101} + target := uint64(0) + if a.Shard(100) != target { + t.Fatalf("shard %d != %d", target, a.Shard(100)) + } + target = 5 + if a.Shard(10) != target { + t.Fatalf("shard %d != %d", target, a.Shard(10)) + } +} + +func TestColumnLess(t *testing.T) { + a := client.Column{RowID: 10, ColumnID: 200} + a2 := client.Column{RowID: 10, ColumnID: 1000} + b := client.Column{RowID: 200, ColumnID: 10} + c := client.FieldValue{ColumnID: 1} + if !a.Less(a2) { + t.Fatalf("%v should be less than %v", a, a2) + } + if !a.Less(b) { + t.Fatalf("%v should be less than %v", a, b) + } + if b.Less(a) { + t.Fatalf("%v should not be less than %v", b, a) + } + if c.Less(a) { + t.Fatalf("%v should not be less than %v", c, a) + } +} + +func TestFieldValueShard(t *testing.T) { + a := client.FieldValue{ColumnID: 55, Value: 125} + target := uint64(0) + if a.Shard(100) != target { + t.Fatalf("shard %d != %d", target, a.Shard(100)) + } + target = 5 + if a.Shard(10) != target { + t.Fatalf("shard %d != %d", target, a.Shard(10)) + } + +} + +func TestFieldValueLess(t *testing.T) { + a := client.FieldValue{ColumnID: 55, Value: 125} + b := client.FieldValue{ColumnID: 100, Value: 125} + c := client.Column{ColumnID: 1, RowID: 2} + if !a.Less(b) { + t.Fatalf("%v should be less than %v", a, b) + } + if b.Less(a) { + t.Fatalf("%v should not be less than %v", b, a) + } + if c.Less(a) { + t.Fatalf("%v should not be less than %v", c, a) + } +} diff --git a/client/response.go b/client/response.go new file mode 100644 index 000000000..b51f29940 --- /dev/null +++ b/client/response.go @@ -0,0 +1,496 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "encoding/json" + "fmt" + + "github.com/molecula/featurebase/v3/pb" +) + +// QueryResponse types. +const ( + QueryResultTypeNil uint32 = iota + QueryResultTypeRow + QueryResultTypePairs + QueryResultTypePairsField + QueryResultTypeValCount + QueryResultTypeUint64 + QueryResultTypeBool + QueryResultTypeRowIDs // this is not used by the client + QueryResultTypeGroupCounts + QueryResultTypeRowIdentifiers + QueryResultTypePair + QueryResultTypePairField + QueryResultTypeSignedRow +) + +// QueryResponse represents the response from a Pilosa query. +type QueryResponse struct { + ResultList []QueryResult `json:"results,omitempty"` + ErrorMessage string `json:"error-message,omitempty"` + Success bool `json:"success,omitempty"` +} + +func newQueryResponseFromInternal(response *pb.QueryResponse) (*QueryResponse, error) { + if response.Err != "" { + return &QueryResponse{ + ErrorMessage: response.Err, + Success: false, + }, nil + } + results := make([]QueryResult, 0, len(response.Results)) + for _, r := range response.Results { + result, err := newQueryResultFromInternal(r) + if err != nil { + return nil, err + } + results = append(results, result) + } + + return &QueryResponse{ + ResultList: results, + Success: true, + }, nil +} + +// Results returns all results in the response. +func (qr *QueryResponse) Results() []QueryResult { + return qr.ResultList +} + +// Result returns the first result or nil. +func (qr *QueryResponse) Result() QueryResult { + if len(qr.ResultList) == 0 { + return nil + } + return qr.ResultList[0] +} + +// QueryResult represents one of the results in the response. +type QueryResult interface { + Type() uint32 + Row() RowResult + CountItems() []CountResultItem + CountItem() CountResultItem + Count() int64 + Value() int64 + Changed() bool + GroupCounts() []GroupCount + RowIdentifiers() RowIdentifiersResult +} + +func newQueryResultFromInternal(result *pb.QueryResult) (QueryResult, error) { + switch result.Type { + case QueryResultTypeNil: + return NilResult{}, nil + case QueryResultTypeRow: + return newRowResultFromInternal(result.Row) + case QueryResultTypePairs: + return countItemsFromInternal(result.Pairs), nil + case QueryResultTypePairsField: + return countItemsFromInternal(result.PairsField.Pairs), nil + case QueryResultTypeValCount: + return &ValCountResult{ + Val: result.ValCount.Val, + Cnt: result.ValCount.Count, + }, nil + case QueryResultTypeUint64: + return IntResult(result.N), nil + case QueryResultTypeBool: + return BoolResult(result.Changed), nil + case QueryResultTypeRowIdentifiers: + return &RowIdentifiersResult{ + IDs: result.RowIdentifiers.Rows, + Keys: result.RowIdentifiers.Keys, + }, nil + case QueryResultTypeGroupCounts: + return groupCountsFromInternal(result.GroupCounts), nil + case QueryResultTypePair: + return CountItem{CountResultItem: countItemFromInternal(result.Pairs[0])}, nil + case QueryResultTypePairField: + return CountItem{CountResultItem: countItemFromInternal(result.PairField.Pair)}, nil + } + + return nil, ErrUnknownType +} + +// CountResultItem represents a result from TopN call. +type CountResultItem struct { + ID uint64 `json:"id"` + Key string `json:"key,omitempty"` + Count uint64 `json:"count"` +} + +func (c *CountResultItem) String() string { + if c.Key != "" { + return fmt.Sprintf("%s:%d", c.Key, c.Count) + } + return fmt.Sprintf("%d:%d", c.ID, c.Count) +} + +type CountItem struct { + CountResultItem +} + +// Type is the type of this result. +func (CountItem) Type() uint32 { return QueryResultTypePairField } + +// Row returns a RowResult. +func (CountItem) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (t CountItem) CountItems() []CountResultItem { return []CountResultItem{t.CountResultItem} } + +// CountItem returns a CountResultItem +func (t CountItem) CountItem() CountResultItem { return t.CountResultItem } + +// Count returns the result of a Count call. +func (CountItem) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (CountItem) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (CountItem) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (CountItem) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (CountItem) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +func countItemFromInternal(item *pb.Pair) CountResultItem { + return CountResultItem{ID: item.ID, Key: item.Key, Count: item.Count} +} + +func countItemsFromInternal(items []*pb.Pair) TopNResult { + result := make([]CountResultItem, 0, len(items)) + for _, v := range items { + result = append(result, countItemFromInternal(v)) + } + return TopNResult(result) +} + +// TopNResult is returned from TopN call. +type TopNResult []CountResultItem + +// Type is the type of this result. +func (TopNResult) Type() uint32 { return QueryResultTypePairsField } + +// Row returns a RowResult. +func (TopNResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (t TopNResult) CountItems() []CountResultItem { return t } + +// CountItem returns a CountResultItem +func (t TopNResult) CountItem() CountResultItem { + if len(t) >= 1 { + return t[0] + } + return CountResultItem{} +} + +// Count returns the result of a Count call. +func (TopNResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (TopNResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (TopNResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (TopNResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (TopNResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// RowResult represents a result from Row, Union, Intersect, Difference and Range PQL calls. +type RowResult struct { + Columns []uint64 `json:"columns"` + Keys []string `json:"keys"` +} + +func newRowResultFromInternal(row *pb.Row) (*RowResult, error) { + return &RowResult{ + Columns: row.Columns, + Keys: row.Keys, + }, nil +} + +// Type is the type of this result. +func (RowResult) Type() uint32 { return QueryResultTypeRow } + +// Row returns a RowResult. +func (b RowResult) Row() RowResult { return b } + +// CountItems returns a CountResultItem slice. +func (RowResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (RowResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (RowResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (RowResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (RowResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (RowResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (RowResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// MarshalJSON serializes this row result. +func (b RowResult) MarshalJSON() ([]byte, error) { + columns := b.Columns + if columns == nil { + columns = []uint64{} + } + keys := b.Keys + if keys == nil { + keys = []string{} + } + return json.Marshal(struct { + Columns []uint64 `json:"columns"` + Keys []string `json:"keys"` + }{ + Columns: columns, + Keys: keys, + }) +} + +// ValCountResult is returned from Min, Max and Sum calls. +type ValCountResult struct { + Val int64 `json:"val"` + Cnt int64 `json:"count"` +} + +// Type is the type of this result. +func (ValCountResult) Type() uint32 { return QueryResultTypeValCount } + +// Row returns a RowResult. +func (ValCountResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (ValCountResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (ValCountResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (c ValCountResult) Count() int64 { return c.Cnt } + +// Value returns the result of a Min, Max or Sum call. +func (c ValCountResult) Value() int64 { return c.Val } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (ValCountResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (ValCountResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (ValCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// IntResult is returned from Count call. +type IntResult int64 + +// Type is the type of this result. +func (IntResult) Type() uint32 { return QueryResultTypeUint64 } + +// Row returns a RowResult. +func (IntResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (IntResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (IntResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (i IntResult) Count() int64 { return int64(i) } + +// Value returns the result of a Min, Max or Sum call. +func (IntResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (IntResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (IntResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (IntResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// BoolResult is returned from Set and Clear calls. +type BoolResult bool + +// Type is the type of this result. +func (BoolResult) Type() uint32 { return QueryResultTypeBool } + +// Row returns a RowResult. +func (BoolResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (BoolResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (BoolResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (BoolResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (BoolResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (b BoolResult) Changed() bool { return bool(b) } + +// GroupCounts returns the result of a GroupBy call. +func (BoolResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (BoolResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// NilResult is returned from calls which don't return a value. +type NilResult struct{} + +// Type is the type of this result. +func (NilResult) Type() uint32 { return QueryResultTypeNil } + +// Row returns a RowResult. +func (NilResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (NilResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (NilResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (NilResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (NilResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (NilResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (NilResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (NilResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// FieldRow represents a Group in a GroupBy call result. +type FieldRow struct { + FieldName string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey"` + Value *int64 `json:"value,omitempty"` +} + +// GroupCount contains groups and their count in a GroupBy call result. +type GroupCount struct { + Groups []FieldRow `json:"groups"` + Count int64 `json:"count"` + Agg int64 `json:"agg"` +} + +// GroupCountResult is returned from GroupBy call. +type GroupCountResult []GroupCount + +// Type is the type of this result. +func (GroupCountResult) Type() uint32 { return QueryResultTypeGroupCounts } + +// Row returns a RowResult. +func (GroupCountResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (GroupCountResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (GroupCountResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (GroupCountResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (GroupCountResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (GroupCountResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (r GroupCountResult) GroupCounts() []GroupCount { return r } + +// RowIdentifiers returns the result of a Rows call. +func (GroupCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// RowIdentifiersResult is returned from a Rows call. +type RowIdentifiersResult struct { + IDs []uint64 `json:"ids"` + Keys []string `json:"keys,omitempty"` +} + +// Type is the type of this result. +func (RowIdentifiersResult) Type() uint32 { return QueryResultTypeRowIdentifiers } + +// Row returns a RowResult. +func (RowIdentifiersResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (RowIdentifiersResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (RowIdentifiersResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (RowIdentifiersResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (RowIdentifiersResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (RowIdentifiersResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (RowIdentifiersResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (r RowIdentifiersResult) RowIdentifiers() RowIdentifiersResult { return r } + +func groupCountsFromInternal(items *pb.GroupCounts) GroupCountResult { + result := make([]GroupCount, 0, len(items.Groups)) + for _, g := range items.Groups { + groups := make([]FieldRow, 0, len(g.Group)) + for _, f := range g.Group { + fr := FieldRow{ + FieldName: f.Field, + RowID: f.RowID, + RowKey: f.RowKey, + } + if f.Value != nil { + fr.Value = &f.Value.Value + } + groups = append(groups, fr) + } + result = append(result, GroupCount{ + Groups: groups, + Count: int64(g.Count), + Agg: int64(g.Agg), + }) + } + return GroupCountResult(result) +} diff --git a/client/response_test.go b/client/response_test.go new file mode 100644 index 000000000..e80430d2b --- /dev/null +++ b/client/response_test.go @@ -0,0 +1,273 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "encoding/json" + "fmt" + "log" + "reflect" + "testing" + + "github.com/molecula/featurebase/v3/pb" +) + +func TestNewRowResultFromInternal(t *testing.T) { + targetColumns := []uint64{5, 10} + row := &pb.Row{ + Columns: []uint64{5, 10}, + } + result, err := newRowResultFromInternal(row) + if err != nil { + t.Fatalf("Failed with error: %s", err) + } + if !reflect.DeepEqual(targetColumns, result.Columns) { + t.Fatal() + } +} + +func TestNewQueryResponseFromInternal(t *testing.T) { + targetColumns := []uint64{5, 10} + targetCountItems := []CountResultItem{ + {ID: 10, Count: 100}, + } + row := &pb.Row{ + Columns: []uint64{5, 10}, + } + pairs := []*pb.Pair{ + {ID: 10, Count: 100}, + } + response := &pb.QueryResponse{ + Results: []*pb.QueryResult{ + {Type: QueryResultTypeRow, Row: row}, + {Type: QueryResultTypePairs, Pairs: pairs}, + }, + Err: "", + } + qr, err := newQueryResponseFromInternal(response) + if err != nil { + t.Fatalf("Failed with error: %s", err) + } + if qr.ErrorMessage != "" { + t.Fatalf("ErrorMessage should be empty") + } + if !qr.Success { + t.Fatalf("IsSuccess should be true") + } + + results := qr.Results() + if len(results) != 2 { + t.Fatalf("Number of results should be 2") + } + if results[0] != qr.Result() { + t.Fatalf("Result() should return the first result") + } + if !reflect.DeepEqual(targetColumns, results[0].Row().Columns) { + t.Fatalf("The row result should contain the columns") + } + if !reflect.DeepEqual(targetCountItems, results[1].CountItems()) { + t.Fatalf("The response should include count items") + } +} + +func TestNewQueryResponseWithErrorFromInternal(t *testing.T) { + response := &pb.QueryResponse{ + Err: "some error", + } + qr, err := newQueryResponseFromInternal(response) + if err != nil { + t.Fatalf("Failed with error: %s", err) + } + if qr.ErrorMessage != "some error" { + t.Fatalf("The response should include the error message") + } + if qr.Success { + t.Fatalf("IsSuccess should be false") + } + if qr.Result() != nil { + t.Fatalf("If there are no results, Result should return nil") + } +} + +func TestCountResultItemToString(t *testing.T) { + tests := []struct { + item *CountResultItem + expected string + }{ + {item: &CountResultItem{ID: 100, Count: 50}, expected: "100:50"}, + {item: &CountResultItem{Key: "blah", Count: 50}, expected: "blah:50"}, + {item: &CountResultItem{Key: "blah", ID: 22, Count: 50}, expected: "blah:50"}, + {item: &CountResultItem{Key: "blah", ID: 22}, expected: "blah:0"}, + {item: &CountResultItem{}, expected: "0:0"}, + } + + for i, tst := range tests { + t.Run(fmt.Sprintf("%d: ", i), func(t *testing.T) { + if tst.expected != tst.item.String() { + t.Fatalf("%s != %s", tst.expected, tst.item.String()) + } + }) + } +} + +func TestMarshalResults(t *testing.T) { + row := &pb.Row{ + Columns: []uint64{5, 10}, + } + pairs := []*pb.Pair{ + {ID: 10, Count: 100}, + } + pbufResults := []*pb.QueryResult{ + {Type: QueryResultTypeRow, Row: row}, + {Type: QueryResultTypePairs, Pairs: pairs}, + } + resultJSONStrings := make([]string, len(pbufResults)) + for i, pr := range pbufResults { + r, err := newQueryResultFromInternal(pr) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + resultJSONStrings[i] = string(b) + } + targetJSON := []string{ + `{"columns":[5,10],"keys":[]}`, + `[{"id":10,"count":100}]`, + } + for i := range targetJSON { + if sortedString(targetJSON[i]) != sortedString(resultJSONStrings[i]) { + t.Fatalf("%v != %v ", targetJSON[i], resultJSONStrings[i]) + } + } + +} + +func TestUnknownQueryResultType(t *testing.T) { + result := &pb.QueryResult{ + Type: 999, + } + _, err := newQueryResultFromInternal(result) + if err != ErrUnknownType { + t.Fatalf("Should have failed with ErrUnknownType") + } +} + +func TestTopNResult(t *testing.T) { + result := TopNResult{ + CountResultItem{ID: 100, Count: 10}, + } + expectResult(t, result, QueryResultTypePairsField, RowResult{}, []CountResultItem{{100, "", 10}}, 0, 0, false, nil, RowIdentifiersResult{}) +} + +func TestRowResult(t *testing.T) { + result := RowResult{ + Columns: []uint64{1, 2, 3}, + } + targetBmp := RowResult{ + Columns: []uint64{1, 2, 3}, + } + expectResult(t, result, QueryResultTypeRow, targetBmp, nil, 0, 0, false, nil, RowIdentifiersResult{}) +} + +func TestRowResultNilColumns(t *testing.T) { + result := RowResult{ + Columns: nil, + } + _, err := result.MarshalJSON() + if err != nil { + t.Fatal(err) + } +} + +func TestSumCountResult(t *testing.T) { + result := ValCountResult{ + Val: 100, + Cnt: 50, + } + expectResult(t, result, QueryResultTypeValCount, RowResult{}, nil, 100, 50, false, nil, RowIdentifiersResult{}) +} + +func TestIntResult(t *testing.T) { + result := IntResult(11) + expectResult(t, result, QueryResultTypeUint64, RowResult{}, nil, 0, 11, false, nil, RowIdentifiersResult{}) +} + +func TestBoolResult(t *testing.T) { + result := BoolResult(true) + expectResult(t, result, QueryResultTypeBool, RowResult{}, nil, 0, 0, true, nil, RowIdentifiersResult{}) +} + +func TestNilResult(t *testing.T) { + result := NilResult{} + expectResult(t, result, QueryResultTypeNil, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{}) +} + +func TestGroupCountResult(t *testing.T) { + result := GroupCountResult{ + {Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2}, + {Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1}, + } + expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{ + {Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2}, + {Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1}, + }, RowIdentifiersResult{}) +} + +func TestGroupCountWithValueResult(t *testing.T) { + var a, b int64 = -1, 1 + + result := GroupCountResult{ + {Groups: []FieldRow{{FieldName: "f1", Value: &a}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "f1", Value: &b}}, Count: 1}, + } + + var aa, bb int64 = -1, 1 + expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{ + {Groups: []FieldRow{{FieldName: "f1", Value: &aa}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "f1", Value: &bb}}, Count: 1}, + }, RowIdentifiersResult{}) +} + +func TestRowIdentifiersResult(t *testing.T) { + result := RowIdentifiersResult{ + IDs: []uint64{1, 2, 3, 4}, + } + expectResult(t, result, QueryResultTypeRowIdentifiers, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{ + IDs: []uint64{1, 2, 3, 4}, + }) +} + +func expectResult(t *testing.T, r QueryResult, resultType uint32, bmp RowResult, + countItems []CountResultItem, sum int64, count int64, changed bool, + groupCounts []GroupCount, rowIdentifiers RowIdentifiersResult) { + if resultType != r.Type() { + log.Fatalf("Result type: %d != %d", resultType, r.Type()) + } + if !reflect.DeepEqual(bmp, r.Row()) { + log.Fatalf("Row: %v != %v", bmp, r.Row()) + } + if !reflect.DeepEqual(countItems, r.CountItems()) { + log.Fatalf("Count items: %v != %v", countItems, r.CountItems()) + } + if count != r.Count() { + log.Fatalf("Count: %d != %d", count, r.Count()) + } + if sum != r.Value() { + log.Fatalf("Sum: %d != %d", sum, r.Value()) + } + if changed != r.Changed() { + log.Fatalf("Changed: %v != %v", changed, r.Changed()) + } + if !reflect.DeepEqual(groupCounts, r.GroupCounts()) { + log.Fatalf("Group counts: %v != %v", groupCounts, r.GroupCounts()) + } + if !reflect.DeepEqual(rowIdentifiers, r.RowIdentifiers()) { + log.Fatalf("Row identifiers: %v != %v", rowIdentifiers, r.RowIdentifiers()) + } +} diff --git a/client/shardnodes.go b/client/shardnodes.go new file mode 100644 index 000000000..2ffc304b4 --- /dev/null +++ b/client/shardnodes.go @@ -0,0 +1,54 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "sync" + + pnet "github.com/molecula/featurebase/v3/net" +) + +type shardNodes struct { + data map[string]map[uint64][]*pnet.URI + mu *sync.RWMutex +} + +func newShardNodes() shardNodes { + return shardNodes{ + data: make(map[string]map[uint64][]*pnet.URI), + mu: &sync.RWMutex{}, + } +} + +func (s shardNodes) Get(index string, shard uint64) ([]*pnet.URI, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + if idx, ok := s.data[index]; ok { + if uris, ok := idx[shard]; ok { + return uris, true + } + } + return nil, false +} + +func (s shardNodes) Put(index string, shard uint64, uris []*pnet.URI) { + s.mu.Lock() + defer s.mu.Unlock() + idx, ok := s.data[index] + if !ok { + idx = make(map[uint64][]*pnet.URI) + } + idx[shard] = uris + s.data[index] = idx +} + +func (s shardNodes) Invalidate() { + s.mu.Lock() + defer s.mu.Unlock() + for k := range s.data { + delete(s.data, k) + } +} diff --git a/client/tracer.go b/client/tracer.go new file mode 100644 index 000000000..22b2022b0 --- /dev/null +++ b/client/tracer.go @@ -0,0 +1,77 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/log" +) + +type NoopTracer struct{} + +type NoopSpan struct{} + +func (s NoopSpan) Finish() { + // pass +} +func (s NoopSpan) FinishWithOptions(opts opentracing.FinishOptions) { + // pass +} + +func (s NoopSpan) Context() opentracing.SpanContext { + return nil +} +func (s NoopSpan) SetOperationName(operationName string) opentracing.Span { + return s +} + +func (s NoopSpan) SetTag(key string, value interface{}) opentracing.Span { + return s +} + +func (s NoopSpan) LogFields(fields ...log.Field) { + // pass +} + +func (s NoopSpan) LogKV(alternatingKeyValues ...interface{}) { + // pass +} + +func (s NoopSpan) SetBaggageItem(restrictedKey, value string) opentracing.Span { + return s +} + +func (s NoopSpan) BaggageItem(restrictedKey string) string { + return "" +} + +func (s NoopSpan) Tracer() opentracing.Tracer { + return nil +} + +func (s NoopSpan) LogEvent(event string) { + // pass +} + +func (s NoopSpan) LogEventWithPayload(event string, payload interface{}) { + // pass +} + +func (s NoopSpan) Log(data opentracing.LogData) { + // pass +} + +func (t NoopTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span { + return NoopSpan{} +} + +func (t NoopTracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error { + return nil +} + +func (t NoopTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) { + return nil, nil +} diff --git a/client/validate.go b/client/validate.go new file mode 100644 index 000000000..df2f24bdc --- /dev/null +++ b/client/validate.go @@ -0,0 +1,42 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "regexp" +) + +const ( + maxLabel = 64 + maxKey = 64 +) + +var labelRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9_-]*$") +var keyRegex = regexp.MustCompile("^[A-Za-z0-9_{}+/=.~%:-]*$") + +// ValidLabel returns true if the given label is valid, otherwise false. +func ValidLabel(label string) bool { + return len(label) <= maxLabel && labelRegex.Match([]byte(label)) +} + +// ValidKey returns true if the given key is valid, otherwise false. +func ValidKey(key string) bool { + return len(key) <= maxKey && keyRegex.Match([]byte(key)) +} + +func validateLabel(label string) error { + if ValidLabel(label) { + return nil + } + return ErrInvalidLabel +} + +func validateKey(key string) error { + if ValidKey(key) { + return nil + } + return ErrInvalidKey +} diff --git a/client/validate_test.go b/client/validate_test.go new file mode 100644 index 000000000..ec486dc5a --- /dev/null +++ b/client/validate_test.go @@ -0,0 +1,61 @@ +// Copyright 2022 Molecula Corp. (DBA FeatureBase). +// SPDX-License-Identifier: Apache-2.0 +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import "testing" + +func TestValidateLabel(t *testing.T) { + labels := []string{ + "a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, label := range labels { + if validateLabel(label) != nil { + t.Fatalf("Should be valid label: %s", label) + } + } +} + +func TestValidateLabelInvalid(t *testing.T) { + labels := []string{ + "", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, label := range labels { + if validateLabel(label) == nil { + t.Fatalf("Should be invalid label: %s", label) + } + } +} + +func TestValidateKey(t *testing.T) { + keys := []string{ + "", "1", "ab", "ab1", "b-c", "d_e", "pilosa.com", + "bbf8d41c-7dba-40c4-94dc-94677b43bcf3", // UUID + "{bbf8d41c-7dba-40c4-94dc-94677b43bcf3}", // Windows GUID + "https%3A//www.pilosa.com/about/%23contact", // escaped URL + "aHR0cHM6Ly93d3cucGlsb3NhLmNvbS9hYm91dC8jY29udGFjdA==", // base64 + "urn:isbn:1234567", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, key := range keys { + if validateKey(key) != nil { + t.Fatalf("Should be valid key: %s", key) + } + } +} + +func TestValidateKeyInvalid(t *testing.T) { + keys := []string{ + "\"", "'", "slice\\dice", "valid?no", "yüce", "*xyz", "with space", "