diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 269e47a19..d01a2465e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -58,6 +58,20 @@ build lattice: - job: install lattice allow_failure: true +run jest tests: + stage: test + image: node:14 + variables: + CI: "true" + script: + - echo "Testing lattice..." + - cd lattice + - npm install --force + - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter + artifacts: + paths: + - lattice/coverage/lcov.info + run go tests: stage: test image: golang:1.16.9 @@ -97,10 +111,11 @@ upload to sonarcloud: variables: SONAR_TOKEN: $SONAR_TOKEN script: - - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out + - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info needs: - job: run go tests - job: run go tests with output + - job: run jest tests build for linux amd64: stage: build diff --git a/README.md b/README.md index e221fbb1d..eba94455b 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,4 @@ See our [internal documentation](https://internal-docs.molecula.cloud), which includes all [external documentation](https://docs.molecula.cloud), plus many internal-only pages, listed under the "Internal" heading in the main navigation bar. Follow along with the [Sample Project](https://internal-docs.molecula.cloud/tutorials/getting-started) to get a better understanding of FeatureBase's capabilities. + diff --git a/api.go b/api.go index 7bcc0dda7..458164f6e 100644 --- a/api.go +++ b/api.go @@ -62,7 +62,8 @@ type API struct { importWorkerPoolSize int importWork chan importJob - usageCache *usageCache + usageCache *usageCache + schemaDetailsOn bool Serializer Serializer } @@ -84,6 +85,14 @@ func OptAPIServer(s *Server) apiOption { } } +// Used to configure API option: schemaDetailsOn +func OptAPISchemaDetailsOn(isOn bool) apiOption { + return func(a *API) error { + a.schemaDetailsOn = isOn + return nil + } +} + func OptAPIImportWorkerPoolSize(size int) apiOption { return func(a *API) error { a.importWorkerPoolSize = size @@ -118,6 +127,17 @@ func NewAPI(opts ...apiOption) (*API, error) { return api, nil } +// Setter for API options. +func (api *API) SetAPIOptions(opts ...apiOption) error { + for _, opt := range opts { + err := opt(api) + if err != nil { + return errors.Wrap(err, "setting API option") + } + } + return nil +} + // validAPIMethods specifies the api methods that are valid for each // cluster state. var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{ @@ -1237,7 +1257,8 @@ func (api *API) Schema(ctx context.Context, withViews bool) ([]*IndexInfo, error } // SchemaDetails returns information about each index in Pilosa including which -// fields they contain, and additional field information such as cardinality +// fields they contain. Additional field information such as cardinality unless +// turned off via the schemaDetailsOn cli option. func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") defer span.Finish() @@ -1245,6 +1266,9 @@ func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) { if err != nil { return nil, errors.Wrap(err, "getting schema") } + if !api.schemaDetailsOn { + return schema, nil + } for _, index := range schema { for _, field := range index.Fields { q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name) diff --git a/api_test.go b/api_test.go index 50a59a03e..378aee9fd 100644 --- a/api_test.go +++ b/api_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" @@ -846,6 +846,29 @@ func TestAPI_IDAlloc(t *testing.T) { }) } +func TestAPI_SchemaDetailsOff(t *testing.T) { + cluster := test.MustRunCluster(t, 2) + defer cluster.Close() + cmd := cluster.GetNode(0) + err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) + if err != nil { + t.Fatalf("could not toggle schema details to off: %v", err) + } + schema, err := cmd.API.SchemaDetails(context.Background()) + if err != nil { + t.Fatalf("getting schema: %v", err) + } + + for _, i := range schema { + for _, f := range i.Fields { + if f.Cardinality != nil { + t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality) + } + } + } + +} + type mutexCheckIndex struct { index *pilosa.Index indexName string diff --git a/ctl/server.go b/ctl/server.go index 71c26bb3c..d53bf81af 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -118,4 +118,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Future flags. flags.BoolVar(&srv.Config.Future.Rename, "future.rename", false, "Present application name as FeatureBase. Defaults to false, will default to true in an upcoming release.") + + // Toggle /schema/details endpoint. + flags.BoolVar(&srv.Config.SchemaDetailsOn, "schema-details-on", true, "Disable /schema/details endpoint") } diff --git a/lattice/package.json b/lattice/package.json index 15211e27f..48785d4c7 100644 --- a/lattice/package.json +++ b/lattice/package.json @@ -39,15 +39,14 @@ "devDependencies": { "@types/d3-array": "^2.0.0", "@types/jest": "^26.0.4", - "@types/node": "^14.0.20", - "@types/node-sass": "^4.11.0", "@types/react": "^16.9.41", "@types/react-beautiful-dnd": "^13.0.0", "@types/react-dom": "^16.9.8", "@types/react-router": "^5.1.8", "@types/react-router-dom": "^5.1.5", - "node-sass": "^4.12.0", - "react-scripts": "^4.0.0", + "jest-sonar-reporter": "^2.0.0", + "node-sass": "^4.14.1", + "react-scripts": "^4.0.3", "tslint": "^6.1.2", "typescript": "^4.2.2" }, diff --git a/lattice/src/App/QueryBuilder/utils.ts b/lattice/src/App/QueryBuilder/utils.ts index c90cafaed..8706e8140 100644 --- a/lattice/src/App/QueryBuilder/utils.ts +++ b/lattice/src/App/QueryBuilder/utils.ts @@ -58,7 +58,7 @@ export const stringifyRowData = (rowCalls: RowGrouping[], operator?: Operator) = .map((ip) => `Row(${field}="${ip}")`) .join(', '); rowString = `Union(${ipRows})`; - } catch (error) { + } catch (error: any) { return { error: true, queryString: error.message }; } } else if (rowOperator === 'like') { diff --git a/lattice/src/shared/DataTable/DataTable.tsx b/lattice/src/shared/DataTable/DataTable.tsx index 7d7b53f1a..94510f86d 100644 --- a/lattice/src/shared/DataTable/DataTable.tsx +++ b/lattice/src/shared/DataTable/DataTable.tsx @@ -1,7 +1,6 @@ import React, { FC, Fragment, useEffect, useRef, useState } from 'react'; import ArrowDropDownIcon from '@material-ui/icons/ArrowDropDown'; import classNames from 'classnames'; -import moment from 'moment'; import OrderBy from 'lodash/orderBy'; import Table from '@material-ui/core/Table'; import TableBody from '@material-ui/core/TableBody'; @@ -12,6 +11,7 @@ import TableRow from '@material-ui/core/TableRow'; import Typography from '@material-ui/core/Typography'; import { ColumnInfo } from 'proto/pilosa_pb'; import { Pager } from 'shared/Pager'; +import { formatTableCell } from 'shared/utils/formatTableCell'; import css from './DataTable.module.scss'; type TableProps = { @@ -70,6 +70,8 @@ export const DataTable: FC = ({ } }; + + return (
@@ -117,20 +119,8 @@ export const DataTable: FC = ({ key={`table-cell-${rowIdx}-${colIdx}`} className={css.tableCell} > - {typeof row[col.name] === 'object' ? ( - 
-                            {JSON.stringify(row[col.name], null, 2)}
-
- ) : row[col.name] !== undefined ? ( - - {col.datatype === 'timestamp' && row[col.name] - ? moment - .utc(row[col.name]) - .format('MM/DD/YYYY hh:mm:ss a') - : row[col.name].toLocaleString()} - - ) : null} - + {formatTableCell(row, col)} + ))} {autoWidth ? : null} diff --git a/lattice/src/shared/utils/formatTableCell.test.tsx b/lattice/src/shared/utils/formatTableCell.test.tsx new file mode 100644 index 000000000..5a8a0e211 --- /dev/null +++ b/lattice/src/shared/utils/formatTableCell.test.tsx @@ -0,0 +1,77 @@ +import { formatTableCell } from "./formatTableCell"; +import React from "react"; +import { render, unmountComponentAtNode } from "react-dom"; +import { act } from "react-dom/test-utils"; + +let container = null; +beforeEach(() => { + // setup a DOM element as a render target + container = document.createElement("div"); + document.body.appendChild(container); +}); + +afterEach(() => { + // cleanup on exiting + unmountComponentAtNode(container); + container.remove(); + container = null; +}); + +it("renders strings in quotes", () => { + let row = { thing: "quoted string!" }; + let col = { name: "thing", datatype: "[]string" }; + + act(() => { + render(formatTableCell(row, col), container); + }); + expect(container.textContent).toBe('"quoted string!"'); +}); + +it("renders objects as stringified", () => { + let row = { thing: { val: "quoted string!" } }; + let col = { name: "thing", datatype: "object" }; + + act(() => { + render(formatTableCell(row, col), container); + }); + expect(container.textContent).toBe(`{ + "val": "quoted string!" +}`); +}); + +it("puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => { + let row = { thing: 1635452050094 }; + let col = { name: "thing", datatype: "timestamp" }; + + act(() => { + render(formatTableCell(row, col), container); + }); + expect(container.textContent).toBe("10/28/2021 08:14:10 pm"); +}); + +it("renders timestamps with no value as LocaleString", () => { + let row = { thing: false }; + let col = { name: "thing", datatype: "timestamp" }; + + act(() => { + render(formatTableCell(row, col), container); + }); + expect(container.textContent).toBe(row.thing.toLocaleString()); +}); + +it("renders non-timestamp, non-string, non-objects as LocaleString", () => { + let row = { thing: "idk" }; + let col = { name: "thing", datatype: "idk" }; + + act(() => { + render(formatTableCell(row, col), container); + }); + expect(container.textContent).toBe(row.thing.toLocaleString()); +}); + +it("returns null for undefined objects", () => { + let row = {}; + let col = { name: "thing", datatype: "" }; + + expect(formatTableCell(row, col)).toBeNull(); +}); diff --git a/lattice/src/shared/utils/formatTableCell.tsx b/lattice/src/shared/utils/formatTableCell.tsx new file mode 100644 index 000000000..b21c10713 --- /dev/null +++ b/lattice/src/shared/utils/formatTableCell.tsx @@ -0,0 +1,22 @@ +import moment from "moment"; +import css from "../DataTable/DataTable.module.scss"; +export const formatTableCell = (row: any, col: any) => { + if (typeof row[col.name] === "object") { + return ( + 
+        {JSON.stringify(row[col.name], null, 2)}
+
+ ); + } else if (row[col.name] !== undefined) { + if (col.datatype === "[]string") { + return {'"' + row[col.name] + '"'}; + } + if (col.datatype === "timestamp" && row[col.name]) { + return ( + {moment.utc(row[col.name]).format("MM/DD/YYYY hh:mm:ss a")} + ); + } + return {row[col.name].toLocaleString()}; + } + return null; +}; diff --git a/lattice/yarn.lock b/lattice/yarn.lock index 335aa9264..914b40f4b 100644 --- a/lattice/yarn.lock +++ b/lattice/yarn.lock @@ -2443,23 +2443,11 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== -"@types/node-sass@^4.11.0": - version "4.11.1" - resolved "https://registry.yarnpkg.com/@types/node-sass/-/node-sass-4.11.1.tgz#bda27c5181cbf7c090c3058e119633dfb2b6504c" - integrity sha512-wPOmOEEtbwQiPTIgzUuRSQZ3H5YHinsxRGeZzPSDefAm4ylXWnZG9C0adses8ymyplKK0gwv3JkDNO8GGxnWfg== - dependencies: - "@types/node" "*" - "@types/node@*": version "13.9.2" resolved "https://registry.yarnpkg.com/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349" integrity sha512-bnoqK579sAYrQbp73wwglccjJ4sfRdKU7WNEZ5FW4K2U6Kc0/eZ5kvXG0JKsEKFB50zrFmfFt52/cvBbZa7eXg== -"@types/node@^14.0.20": - version "14.0.20" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.0.20.tgz#0da05cddbc761e1fa98af88a17244c8c1ff37231" - integrity sha512-MRn/NP3dee8yL5QhbSA6riuwkS+UOcsPUMOIOG3KMUQpuor/2TopdRBu8QaaB4fGU+gz/bzyDWt0FtUbeJ8H1A== - "@types/normalize-package-data@^2.4.0": version "2.4.0" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" @@ -7969,6 +7957,13 @@ jest-snapshot@^26.6.0, jest-snapshot@^26.6.2: pretty-format "^26.6.2" semver "^7.3.2" +jest-sonar-reporter@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/jest-sonar-reporter/-/jest-sonar-reporter-2.0.0.tgz#faa54a7d2af7198767ee246a82b78c576789cf08" + integrity sha512-ZervDCgEX5gdUbdtWsjdipLN3bKJwpxbvhkYNXTAYvAckCihobSLr9OT/IuyNIRT1EZMDDwR6DroWtrq+IL64w== + dependencies: + xml "^1.0.1" + jest-util@^26.6.0, jest-util@^26.6.2: version "26.6.2" resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.6.2.tgz#907535dbe4d5a6cb4c47ac9b926f6af29576cbc1" @@ -9139,7 +9134,7 @@ node-releases@^1.1.61, node-releases@^1.1.70: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.71.tgz#cb1334b179896b1c89ecfdd4b725fb7bbdfc7dbb" integrity sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg== -node-sass@^4.12.0: +node-sass@^4.14.1: version "4.14.1" resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-4.14.1.tgz#99c87ec2efb7047ed638fb4c9db7f3a42e2217b5" integrity sha512-sjCuOlvGyCJS40R8BscF5vhVlQjNN069NtQ1gSxyK1u9iqvn6tf7O1R4GNowVZfiZUCRt5MmMs1xd+4V/7Yr0g== @@ -10983,7 +10978,7 @@ react-router@5.2.0, react-router@^5.2.0: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" -react-scripts@^4.0.0: +react-scripts@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-4.0.3.tgz#b1cafed7c3fa603e7628ba0f187787964cb5d345" integrity sha512-S5eO4vjUzUisvkIPB7jVsKtuH2HhWcASREYWHAQ1FP5HyCv3xgn+wpILAEWkmy+A+tTNbSZClhxjT3qz6g4L1A== @@ -13767,6 +13762,11 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" + integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" diff --git a/server/config.go b/server/config.go index 344440ded..a42eb7917 100644 --- a/server/config.go +++ b/server/config.go @@ -237,6 +237,9 @@ type Config struct { // as FeatureBase instead of Pilosa. Rename bool `toml:"rename"` } `toml:"future"` + + // Toggles /schema/details endpoint. If off, it returns empty. + SchemaDetailsOn bool `toml:"schema-details-on"` } // Namespace returns the namespace to use based on the Future flag. @@ -386,6 +389,9 @@ func NewConfig() *Config { // Future flags. c.Future.Rename = false + // Schema Details Toggle + c.SchemaDetailsOn = true + return c } diff --git a/server/handler_test.go b/server/handler_test.go index 25fe90c99..0e76255c3 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -344,6 +344,38 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("SchemaDetailsOff", func(t *testing.T) { + err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) + if err != nil { + t.Fatalf("setting schema details option") + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var bodySchema pilosa.Schema + if err := json.Unmarshal(w.Body.Bytes(), + &bodySchema); err != nil { + t.Fatalf("unexpected unmarshalling error: %v", err) + + } + for _, i := range bodySchema.Indexes { + for _, f := range i.Fields { + if f.Cardinality != nil { + t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality) + } + } + } + + err = cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(true)) + if err != nil { + t.Fatalf("could not toggle schema details to on: %v", err) + } + }) + t.Run("Import", func(t *testing.T) { indexInfo, err := cmd.API.Schema(context.Background(), false) if err != nil { diff --git a/server/server.go b/server/server.go index 8d227ade7..929404852 100644 --- a/server/server.go +++ b/server/server.go @@ -511,6 +511,7 @@ func (m *Command) SetupServer() error { m.API, err = pilosa.NewAPI( pilosa.OptAPIServer(m.Server), pilosa.OptAPIImportWorkerPoolSize(m.Config.ImportWorkerPoolSize), + pilosa.OptAPISchemaDetailsOn(m.Config.SchemaDetailsOn), ) if err != nil { return errors.Wrap(err, "new api")