Merge branch 'golangci-lint' of github.com:molecula/pilosa into golangci-lint

.gitlab/.gitlab-ci.yml

@@ -58,6 +58,20 @@ build lattice:
     - job: install lattice
   allow_failure: true
 
+run jest tests:
+  stage: test
+  image: node:14
+  variables:
+    CI: "true"
+  script:
+    - echo "Testing lattice..."
+    - cd lattice
+    - npm install --force
+    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+  artifacts:
+    paths:
+      - lattice/coverage/lcov.info
+
 run go tests:
   stage: test
   image: golang:1.16.9
@@ -97,10 +111,11 @@ upload to sonarcloud:
   variables:
     SONAR_TOKEN: $SONAR_TOKEN
   script:
-    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out
+    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
   needs:
     - job: run go tests
     - job: run go tests with output
+    - job: run jest tests 
     
 build for linux amd64:
   stage: build

README.md

@@ -3,3 +3,4 @@
 See our [internal documentation](https://internal-docs.molecula.cloud), which includes all [external documentation](https://docs.molecula.cloud), plus many internal-only pages, listed under the "Internal" heading in the main navigation bar.
 
 Follow along with the [Sample Project](https://internal-docs.molecula.cloud/tutorials/getting-started) to get a better understanding of FeatureBase's capabilities.
+

api.go

@@ -62,7 +62,8 @@ type API struct {
 	importWorkerPoolSize int
 	importWork           chan importJob
 
-	usageCache *usageCache
+	usageCache      *usageCache
+	schemaDetailsOn bool
 
 	Serializer Serializer
 }
@@ -84,6 +85,14 @@ func OptAPIServer(s *Server) apiOption {
 	}
 }
 
+// Used to configure API option: schemaDetailsOn
+func OptAPISchemaDetailsOn(isOn bool) apiOption {
+	return func(a *API) error {
+		a.schemaDetailsOn = isOn
+		return nil
+	}
+}
+
 func OptAPIImportWorkerPoolSize(size int) apiOption {
 	return func(a *API) error {
 		a.importWorkerPoolSize = size
@@ -118,6 +127,17 @@ func NewAPI(opts ...apiOption) (*API, error) {
 	return api, nil
 }
 
+// Setter for API options.
+func (api *API) SetAPIOptions(opts ...apiOption) error {
+	for _, opt := range opts {
+		err := opt(api)
+		if err != nil {
+			return errors.Wrap(err, "setting API option")
+		}
+	}
+	return nil
+}
+
 // validAPIMethods specifies the api methods that are valid for each
 // cluster state.
 var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{
@@ -1237,7 +1257,8 @@ func (api *API) Schema(ctx context.Context, withViews bool) ([]*IndexInfo, error
 }
 
 // SchemaDetails returns information about each index in Pilosa including which
-// fields they contain, and additional field information such as cardinality
+// fields they contain. Additional field information such as cardinality unless
+// turned off via the schemaDetailsOn cli option.
 func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
 	defer span.Finish()
@@ -1245,6 +1266,9 @@ func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "getting schema")
 	}
+	if !api.schemaDetailsOn {
+		return schema, nil
+	}
 	for _, index := range schema {
 		for _, field := range index.Fields {
 			q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name)

api_test.go

@@ -26,7 +26,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/molecula/featurebase/v2"
+	pilosa "github.com/molecula/featurebase/v2"
 	"github.com/molecula/featurebase/v2/boltdb"
 	"github.com/molecula/featurebase/v2/http"
 	"github.com/molecula/featurebase/v2/server"
@@ -846,6 +846,29 @@ func TestAPI_IDAlloc(t *testing.T) {
 	})
 }
 
+func TestAPI_SchemaDetailsOff(t *testing.T) {
+	cluster := test.MustRunCluster(t, 2)
+	defer cluster.Close()
+	cmd := cluster.GetNode(0)
+	err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false))
+	if err != nil {
+		t.Fatalf("could not toggle schema details to off: %v", err)
+	}
+	schema, err := cmd.API.SchemaDetails(context.Background())
+	if err != nil {
+		t.Fatalf("getting schema: %v", err)
+	}
+
+	for _, i := range schema {
+		for _, f := range i.Fields {
+			if f.Cardinality != nil {
+				t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality)
+			}
+		}
+	}
+
+}
+
 type mutexCheckIndex struct {
 	index     *pilosa.Index
 	indexName string

ctl/server.go

@@ -118,4 +118,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 
 	// Future flags.
 	flags.BoolVar(&srv.Config.Future.Rename, "future.rename", false, "Present application name as FeatureBase. Defaults to false, will default to true in an upcoming release.")
+
+	// Toggle /schema/details endpoint.
+	flags.BoolVar(&srv.Config.SchemaDetailsOn, "schema-details-on", true, "Disable /schema/details endpoint")
 }

lattice/package.json

@@ -39,15 +39,14 @@
   "devDependencies": {
     "@types/d3-array": "^2.0.0",
     "@types/jest": "^26.0.4",
-    "@types/node": "^14.0.20",
-    "@types/node-sass": "^4.11.0",
     "@types/react": "^16.9.41",
     "@types/react-beautiful-dnd": "^13.0.0",
     "@types/react-dom": "^16.9.8",
     "@types/react-router": "^5.1.8",
     "@types/react-router-dom": "^5.1.5",
-    "node-sass": "^4.12.0",
-    "react-scripts": "^4.0.0",
+    "jest-sonar-reporter": "^2.0.0",
+    "node-sass": "^4.14.1",
+    "react-scripts": "^4.0.3",
     "tslint": "^6.1.2",
     "typescript": "^4.2.2"
   },

lattice/src/App/QueryBuilder/utils.ts

@@ -58,7 +58,7 @@ export const stringifyRowData = (rowCalls: RowGrouping[], operator?: Operator) =
               .map((ip) => `Row(${field}="${ip}")`)
               .join(', ');
             rowString = `Union(${ipRows})`;
-          } catch (error) {
+          } catch (error: any) {
             return { error: true, queryString: error.message };
           }
         } else if (rowOperator === 'like') {

lattice/src/shared/DataTable/DataTable.tsx

@@ -1,7 +1,6 @@
 import React, { FC, Fragment, useEffect, useRef, useState } from 'react';
 import ArrowDropDownIcon from '@material-ui/icons/ArrowDropDown';
 import classNames from 'classnames';
-import moment from 'moment';
 import OrderBy from 'lodash/orderBy';
 import Table from '@material-ui/core/Table';
 import TableBody from '@material-ui/core/TableBody';
@@ -12,6 +11,7 @@ import TableRow from '@material-ui/core/TableRow';
 import Typography from '@material-ui/core/Typography';
 import { ColumnInfo } from 'proto/pilosa_pb';
 import { Pager } from 'shared/Pager';
+import { formatTableCell } from 'shared/utils/formatTableCell';
 import css from './DataTable.module.scss';
 
 type TableProps = {
@@ -70,6 +70,8 @@ export const DataTable: FC<TableProps> = ({
     }
   };
 
+
+
   return (
     <Fragment>
       <div ref={resultsRef} />
@@ -117,20 +119,8 @@ export const DataTable: FC<TableProps> = ({
                         key={`table-cell-${rowIdx}-${colIdx}`}
                         className={css.tableCell}
                       >
-                        {typeof row[col.name] === 'object' ? (
-                          <pre className={css.preFormat}>
-                            {JSON.stringify(row[col.name], null, 2)}
-                          </pre>
-                        ) : row[col.name] !== undefined ? (
-                          <span>
-                            {col.datatype === 'timestamp' && row[col.name]
-                              ? moment
-                                  .utc(row[col.name])
-                                  .format('MM/DD/YYYY hh:mm:ss a')
-                              : row[col.name].toLocaleString()}
-                          </span>
-                        ) : null}
-                      </TableCell>
+                        {formatTableCell(row, col)}
+                            </TableCell>
                     ))}
                     {autoWidth ? <TableCell className={css.fillWidth} /> : null}
                   </TableRow>

lattice/src/shared/utils/formatTableCell.test.tsx

@@ -0,0 +1,77 @@
+import { formatTableCell } from "./formatTableCell";
+import React from "react";
+import { render, unmountComponentAtNode } from "react-dom";
+import { act } from "react-dom/test-utils";
+
+let container = null;
+beforeEach(() => {
+  // setup a DOM element as a render target
+  container = document.createElement("div");
+  document.body.appendChild(container);
+});
+
+afterEach(() => {
+  // cleanup on exiting
+  unmountComponentAtNode(container);
+  container.remove();
+  container = null;
+});
+
+it("renders strings in quotes", () => {
+  let row = { thing: "quoted string!" };
+  let col = { name: "thing", datatype: "[]string" };
+
+  act(() => {
+    render(formatTableCell(row, col), container);
+  });
+  expect(container.textContent).toBe('"quoted string!"');
+});
+
+it("renders objects as stringified", () => {
+  let row = { thing: { val: "quoted string!" } };
+  let col = { name: "thing", datatype: "object" };
+
+  act(() => {
+    render(formatTableCell(row, col), container);
+  });
+  expect(container.textContent).toBe(`{
+  "val": "quoted string!"
+}`);
+});
+
+it("puts timestamps in MM/DD/YYYY hh:mm:ss a format", () => {
+  let row = { thing: 1635452050094 };
+  let col = { name: "thing", datatype: "timestamp" };
+
+  act(() => {
+    render(formatTableCell(row, col), container);
+  });
+  expect(container.textContent).toBe("10/28/2021 08:14:10 pm");
+});
+
+it("renders timestamps with no value as LocaleString", () => {
+  let row = { thing: false };
+  let col = { name: "thing", datatype: "timestamp" };
+
+  act(() => {
+    render(formatTableCell(row, col), container);
+  });
+  expect(container.textContent).toBe(row.thing.toLocaleString());
+});
+
+it("renders non-timestamp, non-string, non-objects as LocaleString", () => {
+  let row = { thing: "idk" };
+  let col = { name: "thing", datatype: "idk" };
+
+  act(() => {
+    render(formatTableCell(row, col), container);
+  });
+  expect(container.textContent).toBe(row.thing.toLocaleString());
+});
+
+it("returns null for undefined objects", () => {
+  let row = {};
+  let col = { name: "thing", datatype: "" };
+
+  expect(formatTableCell(row, col)).toBeNull();
+});

lattice/src/shared/utils/formatTableCell.tsx

@@ -0,0 +1,22 @@
+import moment from "moment";
+import css from "../DataTable/DataTable.module.scss";
+export const formatTableCell = (row: any, col: any) => {
+  if (typeof row[col.name] === "object") {
+    return (
+      <pre className={css.preFormat}>
+        {JSON.stringify(row[col.name], null, 2)}
+      </pre>
+    );
+  } else if (row[col.name] !== undefined) {
+    if (col.datatype === "[]string") {
+      return <span>{'"' + row[col.name] + '"'}</span>;
+    }
+    if (col.datatype === "timestamp" && row[col.name]) {
+      return (
+        <span>{moment.utc(row[col.name]).format("MM/DD/YYYY hh:mm:ss a")}</span>
+      );
+    }
+    return <span>{row[col.name].toLocaleString()}</span>;
+  }
+  return null;
+};

lattice/yarn.lock

@@ -2443,23 +2443,11 @@
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
   integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
 
-"@types/node-sass@^4.11.0":
-  version "4.11.1"
-  resolved "https://registry.yarnpkg.com/@types/node-sass/-/node-sass-4.11.1.tgz#bda27c5181cbf7c090c3058e119633dfb2b6504c"
-  integrity sha512-wPOmOEEtbwQiPTIgzUuRSQZ3H5YHinsxRGeZzPSDefAm4ylXWnZG9C0adses8ymyplKK0gwv3JkDNO8GGxnWfg==
-  dependencies:
-    "@types/node" "*"
-
 "@types/node@*":
   version "13.9.2"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349"
   integrity sha512-bnoqK579sAYrQbp73wwglccjJ4sfRdKU7WNEZ5FW4K2U6Kc0/eZ5kvXG0JKsEKFB50zrFmfFt52/cvBbZa7eXg==
 
-"@types/node@^14.0.20":
-  version "14.0.20"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.0.20.tgz#0da05cddbc761e1fa98af88a17244c8c1ff37231"
-  integrity sha512-MRn/NP3dee8yL5QhbSA6riuwkS+UOcsPUMOIOG3KMUQpuor/2TopdRBu8QaaB4fGU+gz/bzyDWt0FtUbeJ8H1A==
-
 "@types/normalize-package-data@^2.4.0":
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
@@ -7969,6 +7957,13 @@ jest-snapshot@^26.6.0, jest-snapshot@^26.6.2:
     pretty-format "^26.6.2"
     semver "^7.3.2"
 
+jest-sonar-reporter@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/jest-sonar-reporter/-/jest-sonar-reporter-2.0.0.tgz#faa54a7d2af7198767ee246a82b78c576789cf08"
+  integrity sha512-ZervDCgEX5gdUbdtWsjdipLN3bKJwpxbvhkYNXTAYvAckCihobSLr9OT/IuyNIRT1EZMDDwR6DroWtrq+IL64w==
+  dependencies:
+    xml "^1.0.1"
+
 jest-util@^26.6.0, jest-util@^26.6.2:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.6.2.tgz#907535dbe4d5a6cb4c47ac9b926f6af29576cbc1"
@@ -9139,7 +9134,7 @@ node-releases@^1.1.61, node-releases@^1.1.70:
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.71.tgz#cb1334b179896b1c89ecfdd4b725fb7bbdfc7dbb"
   integrity sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg==
 
-node-sass@^4.12.0:
+node-sass@^4.14.1:
   version "4.14.1"
   resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-4.14.1.tgz#99c87ec2efb7047ed638fb4c9db7f3a42e2217b5"
   integrity sha512-sjCuOlvGyCJS40R8BscF5vhVlQjNN069NtQ1gSxyK1u9iqvn6tf7O1R4GNowVZfiZUCRt5MmMs1xd+4V/7Yr0g==
@@ -10983,7 +10978,7 @@ react-router@5.2.0, react-router@^5.2.0:
     tiny-invariant "^1.0.2"
     tiny-warning "^1.0.0"
 
-react-scripts@^4.0.0:
+react-scripts@^4.0.3:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-4.0.3.tgz#b1cafed7c3fa603e7628ba0f187787964cb5d345"
   integrity sha512-S5eO4vjUzUisvkIPB7jVsKtuH2HhWcASREYWHAQ1FP5HyCv3xgn+wpILAEWkmy+A+tTNbSZClhxjT3qz6g4L1A==
@@ -13767,6 +13762,11 @@ xml-name-validator@^3.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
   integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
 
+xml@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
+  integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
+
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"

server/config.go

@@ -237,6 +237,9 @@ type Config struct {
 		// as FeatureBase instead of Pilosa.
 		Rename bool `toml:"rename"`
 	} `toml:"future"`
+
+	// Toggles /schema/details endpoint. If off, it returns empty.
+	SchemaDetailsOn bool `toml:"schema-details-on"`
 }
 
 // Namespace returns the namespace to use based on the Future flag.
@@ -386,6 +389,9 @@ func NewConfig() *Config {
 	// Future flags.
 	c.Future.Rename = false
 
+	// Schema Details Toggle
+	c.SchemaDetailsOn = true
+
 	return c
 }
 

server/handler_test.go

@@ -344,6 +344,38 @@ func TestHandler_Endpoints(t *testing.T) {
 		}
 	})
 
+	t.Run("SchemaDetailsOff", func(t *testing.T) {
+		err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false))
+		if err != nil {
+			t.Fatalf("setting schema details option")
+		}
+
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil))
+		if w.Code != gohttp.StatusOK {
+			t.Fatalf("unexpected status code: %d", w.Code)
+		}
+
+		var bodySchema pilosa.Schema
+		if err := json.Unmarshal(w.Body.Bytes(),
+			&bodySchema); err != nil {
+			t.Fatalf("unexpected unmarshalling error: %v", err)
+
+		}
+		for _, i := range bodySchema.Indexes {
+			for _, f := range i.Fields {
+				if f.Cardinality != nil {
+					t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality)
+				}
+			}
+		}
+
+		err = cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(true))
+		if err != nil {
+			t.Fatalf("could not toggle schema details to on: %v", err)
+		}
+	})
+
 	t.Run("Import", func(t *testing.T) {
 		indexInfo, err := cmd.API.Schema(context.Background(), false)
 		if err != nil {

server/server.go

@@ -511,6 +511,7 @@ func (m *Command) SetupServer() error {
 	m.API, err = pilosa.NewAPI(
 		pilosa.OptAPIServer(m.Server),
 		pilosa.OptAPIImportWorkerPoolSize(m.Config.ImportWorkerPoolSize),
+		pilosa.OptAPISchemaDetailsOn(m.Config.SchemaDetailsOn),
 	)
 	if err != nil {
 		return errors.Wrap(err, "new api")