From fa353faf7220e4f4d8a16d82d6868e0c9f3d0a12 Mon Sep 17 00:00:00 2001 From: Stephanie Yang Date: Tue, 27 Jul 2021 11:22:08 -0500 Subject: [PATCH 1/4] initial refactor commit --- lattice/src/App.tsx | 4 +- .../QBuilder/CountBuilder/CountBuilder.tsx | 36 ++ .../src/App/QBuilder/CountBuilder/index.ts | 1 + .../ExtractBuilder/ExtractBuilder.module.scss | 21 + .../ExtractBuilder/ExtractBuilder.tsx | 92 +++++ .../src/App/QBuilder/ExtractBuilder/index.ts | 1 + .../GroupByBuilder/GroupByBuilder.module.scss | 32 ++ .../GroupByBuilder/GroupByBuilder.tsx | 179 ++++++++ .../src/App/QBuilder/GroupByBuilder/index.ts | 1 + lattice/src/App/QBuilder/QBuilder.module.scss | 52 +++ lattice/src/App/QBuilder/QBuilder.tsx | 388 ++++++++++++++++++ .../QBuilder/QBuilderContainer.module.scss | 66 +++ .../src/App/QBuilder/QBuilderContainer.tsx | 323 +++++++++++++++ .../RowCallBuilder/RowCall/helpers.ts | 52 +++ .../QBuilder/RowCallBuilder/RowCall/index.ts | 1 + .../RowCallBuilder/RowCallBuilder.module.scss | 40 ++ .../RowCallBuilder/RowCallBuilder.tsx | 180 ++++++++ .../src/App/QBuilder/RowCallBuilder/index.ts | 1 + lattice/src/App/QBuilder/index.ts | 2 + lattice/src/App/QBuilder/rowTypes.ts | 22 + lattice/src/App/QBuilder/utils.ts | 174 ++++++++ 21 files changed, 1666 insertions(+), 2 deletions(-) create mode 100644 lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx create mode 100644 lattice/src/App/QBuilder/CountBuilder/index.ts create mode 100644 lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss create mode 100644 lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx create mode 100644 lattice/src/App/QBuilder/ExtractBuilder/index.ts create mode 100644 lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss create mode 100644 lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx create mode 100644 lattice/src/App/QBuilder/GroupByBuilder/index.ts create mode 100644 lattice/src/App/QBuilder/QBuilder.module.scss create mode 100644 lattice/src/App/QBuilder/QBuilder.tsx create mode 100644 lattice/src/App/QBuilder/QBuilderContainer.module.scss create mode 100644 lattice/src/App/QBuilder/QBuilderContainer.tsx create mode 100644 lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts create mode 100644 lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts create mode 100644 lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss create mode 100644 lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx create mode 100644 lattice/src/App/QBuilder/RowCallBuilder/index.ts create mode 100644 lattice/src/App/QBuilder/index.ts create mode 100644 lattice/src/App/QBuilder/rowTypes.ts create mode 100644 lattice/src/App/QBuilder/utils.ts diff --git a/lattice/src/App.tsx b/lattice/src/App.tsx index f465bd480..89bff9fce 100644 --- a/lattice/src/App.tsx +++ b/lattice/src/App.tsx @@ -9,7 +9,7 @@ import { Nav } from 'shared/Nav'; import { NotFound } from 'App/NotFound'; import { MoleculaTablesContainer } from 'App/MoleculaTables'; import { QueryContainer } from 'App/Query'; -import { QueryBuilderContainer } from 'App/QueryBuilder'; +import { QBuilderContainer } from 'App/QBuilder'; import css from './App.module.scss'; const App = () => { @@ -45,7 +45,7 @@ const App = () => { - + diff --git a/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx b/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx new file mode 100644 index 000000000..03391ec02 --- /dev/null +++ b/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx @@ -0,0 +1,36 @@ +import React, { FC } from 'react'; +import { RowCallBuilder } from 'App/QBuilder/RowCallBuilder'; + +type CountBuilderProps = { + table: any; + query: any; + showInvalid: boolean; + onChange: (query: any) => void; +}; + +export const CountBuilder: FC = ({ + table, + query, + showInvalid, + onChange +}) => { + const { rowCalls } = query ? query : { rowCalls: [] }; + + return ( +
+ { + onChange({ + ...query, + rowCalls, + operator, + isInvalid + }); + }} + /> +
+ ); +}; diff --git a/lattice/src/App/QBuilder/CountBuilder/index.ts b/lattice/src/App/QBuilder/CountBuilder/index.ts new file mode 100644 index 000000000..b83081751 --- /dev/null +++ b/lattice/src/App/QBuilder/CountBuilder/index.ts @@ -0,0 +1 @@ +export * from './CountBuilder'; diff --git a/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss new file mode 100644 index 000000000..38dc38646 --- /dev/null +++ b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss @@ -0,0 +1,21 @@ +.columnsSelector { + margin: 4px 14px 4px 0; + display: flex; + text-align: center; + + .info { + margin-left: 4px; + fill: var(--text-secondary); + } +} + +.textLink { + color: var(--primary); + font-size: 12px; + + &:hover { + cursor: pointer; + text-decoration: underline; + color: var(--primary); + } +} diff --git a/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx new file mode 100644 index 000000000..eaae42714 --- /dev/null +++ b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx @@ -0,0 +1,92 @@ +import React, { FC, useState } from 'react'; +import InfoIcon from '@material-ui/icons/Info'; +import Tooltip from '@material-ui/core/Tooltip'; +import { ColumnSelector } from 'App/QueryBuilder/ColumnSelector'; +import { RowCallBuilder } from 'App/QBuilder/RowCallBuilder'; +import { useEffectOnce } from 'react-use'; +import css from './ExtractBuilder.module.scss'; + +type ExtractBuilderProps = { + table: any; + query: any; + showInvalid: boolean; + onChange: (query: any) => void; +}; + +export const ExtractBuilder: FC = ({ + table, + query, + showInvalid, + onChange +}) => { + const { columns, rowCalls } = query + ? query + : { + columns: [], + rowCalls: [] + }; + const [showColumnSelector, setShowColumnSelector] = useState(false); + + useEffectOnce(() => { + if (!query) { + onChange({ + columns: table.fields.map((field) => field.name), + operation: 'Extract', + rowCalls: [] + }); + } + }); + + return ( +
+
+ setShowColumnSelector(true)} + > + Configure result fields + + + + +
+ + ({ + name: field.name, + show: columns ? columns.includes(field.name) : true + }))} + onChange={(allColumns) => { + let cols: string[] = []; + allColumns.forEach((col) => { + if (col.show) { + cols.push(col.name); + } + }); + onChange({ ...query, columns: cols }); + }} + onClose={() => setShowColumnSelector(false)} + /> + + { + onChange({ + ...query, + rowCalls, + operator, + isInvalid + }); + }} + /> +
+ ); +}; diff --git a/lattice/src/App/QBuilder/ExtractBuilder/index.ts b/lattice/src/App/QBuilder/ExtractBuilder/index.ts new file mode 100644 index 000000000..a923f7767 --- /dev/null +++ b/lattice/src/App/QBuilder/ExtractBuilder/index.ts @@ -0,0 +1 @@ +export * from './ExtractBuilder'; diff --git a/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss new file mode 100644 index 000000000..68c527523 --- /dev/null +++ b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss @@ -0,0 +1,32 @@ +.groupBy { + .fieldSelector { + margin-bottom: 16px; + } + + .filtersHeader { + display: flex; + align-items: center; + justify-content: space-between; + margin-top: 8px; + } + + .filtersInfo { + vertical-align: text-top; + margin-left: 4px; + } +} + +.textLink { + color: var(--primary); + font-size: 12px; + + &:hover { + cursor: pointer; + text-decoration: underline; + color: var(--primary); + } +} + +.filtersSection { + margin-bottom: 16px; +} diff --git a/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx new file mode 100644 index 000000000..b53bf6c59 --- /dev/null +++ b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx @@ -0,0 +1,179 @@ +import React, { FC } from 'react'; +import InfoIcon from '@material-ui/icons/Info'; +import Tooltip from '@material-ui/core/Tooltip'; +import Typography from '@material-ui/core/Typography'; +import { GroupBySort } from 'App/QueryBuilder/GroupBySort'; +import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; +import { Select } from 'shared/Select'; +import { stringifyRowData } from 'App/QueryBuilder/stringifyRowData'; +import css from './GroupByBuilder.module.scss'; + +type GroupByBuilderProps = { + table: any; + query: any; + showInvalid: boolean; + onChange: (query: any) => void; +}; + +export const GroupByBuilder: FC = ({ + table, + query, + showInvalid, + onChange +}) => { + const { groupByCall, filter, sort } = query; + const filters = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ).filter( + (q) => + q.table === table.name && + q.operation === 'Extract' && + q.rowCalls.length > 0 + ); + + return ( +
+ + ['set', 'time', 'mutex', 'bool', 'int', 'timestamp'].includes( + f.options.type + ) + ) + .map((field) => { + return { + label: `${field.name} (${field.options.type})`, + value: field.name + }; + })} + onChange={(value) => + onChange({ + ...query, + groupByCall: { + ...query.groupByCall, + secondary: value.toString() + } + }) + } + /> + +
+
+ + Filter (optional) + {filters.length > 0 && ( + + + + )} + + {filter ? ( + onChange({ ...query, filter: undefined })} + > + Clear Filter + + ) : null} +
+ {filter ? ( + {filter} + ) : filters.length > 0 ? ( + { + const { rowCalls, operator } = filters[queryIdx]; + const res = stringifyRowData(rowCalls, operator); + if (!res.error) { + onChange({ ...query, filter: res.query }); + } + }} + /> + ) : ( +
+ No available filters. To use filters, save an Extract query for{' '} + {table.name} with at least one field constraint. +
+ )} +
+ +
+ + Sort (optional) + + { + let isInvalid = false; + if ( + value.length > 0 && + value[0].sortValue.includes('sum') && + !value[0].field + ) { + isInvalid = true; + } else if ( + value.length > 1 && + value[1].sortValue.includes('sum') && + !value[1].field + ) { + isInvalid = true; + } + onChange({ ...query, sort: value, isInvalid }); + }} + fields={table.fields + .filter((field) => field.options.type === 'int') + .map((field) => { + return { label: field.name, value: field.name }; + })} + showErrors={showInvalid} + /> +
+
+ ); +}; diff --git a/lattice/src/App/QBuilder/GroupByBuilder/index.ts b/lattice/src/App/QBuilder/GroupByBuilder/index.ts new file mode 100644 index 000000000..11f595080 --- /dev/null +++ b/lattice/src/App/QBuilder/GroupByBuilder/index.ts @@ -0,0 +1 @@ +export * from './GroupByBuilder'; diff --git a/lattice/src/App/QBuilder/QBuilder.module.scss b/lattice/src/App/QBuilder/QBuilder.module.scss new file mode 100644 index 000000000..33c34ede7 --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilder.module.scss @@ -0,0 +1,52 @@ +.sharedConfig { + padding: 32px 32px 0; + + .configSelector { + margin-bottom: 24px; + } +} + +.textLink { + color: var(--primary); + font-size: 12px; + + .icon { + margin-right: 4px; + vertical-align: text-top; + } + + &:hover { + cursor: pointer; + text-decoration: underline; + color: var(--primary); + } +} + +.builder { + display: flex; + flex-direction: column; + height: 100%; + + .queryMetadata { + border-bottom: 1px solid var(--divider); + padding-bottom: 16px; + margin-bottom: 16px; + } +} + +.config { + padding: 24px 32px 0; + flex-grow: 1; +} + +.builderActions { + padding: 16px 32px; + display: flex; + justify-content: space-between; + + .mainActions { + display: grid; + grid-template-columns: auto auto; + grid-gap: 8px; + } +} diff --git a/lattice/src/App/QBuilder/QBuilder.tsx b/lattice/src/App/QBuilder/QBuilder.tsx new file mode 100644 index 000000000..3e5e0a8be --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilder.tsx @@ -0,0 +1,388 @@ +import React, { FC, Fragment, useEffect, useState } from 'react'; +import ArrowBackIcon from '@material-ui/icons/ArrowBack'; +import Button from '@material-ui/core/Button'; +import CircularProgress from '@material-ui/core/CircularProgress'; +import Divider from '@material-ui/core/Divider'; +import Popover from '@material-ui/core/Popover'; +import TextField from '@material-ui/core/TextField'; +import Typography from '@material-ui/core/Typography'; +import { + cleanupRows, + stringifyCount, + stringifyExtract, + stringifyGroupBy +} from './utils'; +import { CountBuilder } from './CountBuilder'; +import { ExtractBuilder } from './ExtractBuilder'; +import { GroupByBuilder } from './GroupByBuilder'; +import { Select } from 'shared/Select'; +import css from './QBuilder.module.scss'; + +type QBuilderProps = { + tables: any[]; + savedQuery: number; + onRun: ( + table: string, + operation: string, + query: string, + countQuery?: string + ) => void; + onClear: () => void; + onExitEdit: () => void; + onSaveQuery: () => void; +}; + +export const QBuilder: FC = ({ + tables, + savedQuery, + onRun, + onClear, + onExitEdit, + onSaveQuery +}) => { + const [saveButtonEl, setSaveButtonEl] = useState(null); + const queriesList = JSON.parse(localStorage.getItem('saved-queries') || '[]'); + const [editMode, setEditMode] = useState(false); + const [selectedTable, setSelectedTable] = useState(tables[0]); + const [operation, setOperation] = useState('Extract'); + const [query, setQuery] = useState({ + table: tables[0].name, + operation: 'Extract', + columns: tables[0].fields.map((field) => field.name) + }); + const [showInvalid, setShowInvalid] = useState(false); + const [updating, setUpdating] = useState(false); + const [queryNameIdx, setQueryNameIdx] = useState(-1); + + useEffect(() => { + if (savedQuery > -1) { + const updatedList = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ); + setQuery(updatedList[savedQuery]); + setEditMode(true); + setOperation(updatedList[savedQuery].operation); + const table = tables.find( + (table) => table.name === updatedList[savedQuery].table + ); + setSelectedTable(table); + } + }, [savedQuery, tables]); + + useEffect(() => { + if (query?.name) { + setQueryNameIdx(queriesList.findIndex((q) => q.name === query.name)); + } + }, [query, queriesList]); + + const cleanRows = () => { + const cleanRows = cleanupRows(query?.rowCalls); + return { + ...query, + rowCalls: cleanRows.cleanRowCalls, + isInvalid: cleanRows.isInvalid + }; + }; + + const runQuery = () => { + setShowInvalid(true); + let queryString = ''; + let countQuery = ''; + + switch (operation) { + case 'Extract': + const cleanExtractQuery = cleanRows(); + setQuery(cleanExtractQuery); + const extract = stringifyExtract(cleanExtractQuery); + if (!extract.error && !cleanExtractQuery.isInvalid) { + queryString = extract.queryString; + countQuery = extract.countQuery; + } + break; + case 'GroupBy': + queryString = stringifyGroupBy(query); + break; + case 'Count': + const cleanCountQuery = cleanRows(); + setQuery(cleanCountQuery); + const count = stringifyCount(cleanCountQuery); + if (!count.error && !cleanCountQuery.isInvalid) { + queryString = count.queryString; + } + break; + default: + break; + } + + if (queryString && !query.isInvalid) { + setShowInvalid(false); + onRun(selectedTable.name, operation, queryString, countQuery); + } else { + setShowInvalid(true); + } + }; + + const reset = () => { + setQuery({ + table: selectedTable.name, + columns: selectedTable.fields.map(field => field.name), + operation, + isInvalid: false + }); + setOperation('Extract'); + setShowInvalid(false); + setEditMode(false); + onExitEdit(); + onClear(); + }; + + const onSave = () => { + setUpdating(true); + setTimeout(() => { + setUpdating(false); + }, 1500); + const cleanQuery = cleanRows(); + let queries = [...queriesList]; + + if (editMode && queries.length > 0) { + const clone = [...queries]; + clone.splice(savedQuery, 1, cleanQuery); + localStorage.setItem('saved-queries', JSON.stringify(clone)); + } else { + queries.push(cleanQuery); + localStorage.setItem('saved-queries', JSON.stringify(queries)); + } + onSaveQuery(); + }; + + const renderBuilder = () => { + switch (operation) { + case 'Extract': + return ( + setQuery(q)} + /> + ); + case 'GroupBy': + return ( + setQuery(q)} + /> + ); + case 'Count': + return ( + setQuery(q)} + /> + ); + default: + return
Invalid Operation
; + } + }; + + return ( +
+
+ {editMode ? ( + + + Back to new query + + ) : null} + + {editMode ? 'Edit Saved Query' : 'New Query'} + + + {editMode ? ( +
+ = 0 && queryNameIdx !== savedQuery) + } + helperText={ + queryNameIdx >= 0 && queryNameIdx !== savedQuery + ? 'Query name must be unique' + : '' + } + onChange={(event) => + setQuery({ ...query, name: event.target.value }) + } + margin="normal" + size="small" + required + fullWidth + /> + + + setQuery({ ...query, description: event.target.value }) + } + margin="normal" + size="small" + fullWidth + /> +
+ ) : null} + + { + setOperation(value); + setShowInvalid(false); + setQuery({ + table: selectedTable.name, + operation: value, + isInvalid: false + }); + }} + /> + +
+
{renderBuilder()}
+
+
+ + {editMode ? ( + + ) : ( + + + setSaveButtonEl(null)} + anchorOrigin={{ + vertical: 'top', + horizontal: 'left' + }} + transformOrigin={{ + vertical: 'bottom', + horizontal: 'left' + }} + > +
+ + Give your query a unique name and a short description to + help you identify it for future use. + + + = 0} + helperText={ + queryNameIdx >= 0 ? 'Query name must be unique' : '' + } + onChange={(event) => + setQuery({ ...query, name: event.target.value }) + } + margin="dense" + required + fullWidth + /> + + + setQuery({ ...query, description: event.target.value }) + } + margin="dense" + fullWidth + /> + +
+
+ + +
+
+
+ + + )} +
+ +
+
+ ); +}; diff --git a/lattice/src/App/QBuilder/QBuilderContainer.module.scss b/lattice/src/App/QBuilder/QBuilderContainer.module.scss new file mode 100644 index 000000000..a5cd6c77d --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilderContainer.module.scss @@ -0,0 +1,66 @@ +.builderColumn, +.results { + height: calc(100vh - 64px); + overflow: scroll; + position: relative; + + .openClose { + position: absolute; + right: 10px; + top: 10px; + } +} + +.split { + display: flex; +} + +.collapsedBuilder { + border-right: 1px solid rgba(var(--contrast-rgb), 0.1); + text-align: center; + padding-top: 10px; +} + +.resultsBlock { + height: 100%; +} + +.resultsHeader { + display: flex; + align-items: center; + justify-content: space-between; + + .download { + display: flex; + align-items: center; + } + + .downloadInfo { + margin-right: 8px; + } +} + +.infoMessage { + padding: 16px; + border: 1px solid rgba(var(--primary-rgb), 0.5); + background: rgba(var(--primary-rgb), 0.1); + border-radius: 4px; + margin: 4px 0 16px; +} + +.spacer{ + padding-bottom: 16px; +} + +.resultsLoading { + margin-left: 8px; +} + +.savedQueries { + margin-top: 48px; +} + +.noTables { + margin-top: 20px; + padding: 12px 16px; +} diff --git a/lattice/src/App/QBuilder/QBuilderContainer.tsx b/lattice/src/App/QBuilder/QBuilderContainer.tsx new file mode 100644 index 000000000..7161cb5a3 --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilderContainer.tsx @@ -0,0 +1,323 @@ +import React, { Fragment, useState } from 'react'; +import Alert from '@material-ui/lab/Alert'; +import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos'; +import Button from '@material-ui/core/Button'; +import classNames from 'classnames'; +import CloseIcon from '@material-ui/icons/Close'; +import IconButton from '@material-ui/core/IconButton'; +import InfoIcon from '@material-ui/icons/Info'; +import moment, { Moment } from 'moment'; +import Paper from '@material-ui/core/Paper'; +import Split from 'react-split'; +import Snackbar from '@material-ui/core/Snackbar'; +import Tooltip from '@material-ui/core/Tooltip'; +import Typography from '@material-ui/core/Typography'; +import { Block } from 'shared/Block'; +import { formatDuration } from 'shared/utils/formatDuration'; +import { grpc } from '@improbable-eng/grpc-web'; +import { pilosa } from 'services/eventServices'; +import { QBuilder } from './QBuilder'; +import { queryPQL } from 'services/grpcServices'; +import { QueryResults } from 'App/Query/QueryResults'; +import { ResultType } from 'App/Query/QueryContainer'; +import { RowResponse } from 'proto/pilosa_pb'; +import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; +import { useEffectOnce } from 'react-use'; +import css from './QBuilderContainer.module.scss'; + +let streamingResults: ResultType = { + query: '', + operation: '', + type: 'PQL', + headers: [], + rows: [], + roundtrip: 0, + error: '' +}; + +export const QBuilderContainer = () => { + let startTime: Moment; + let exportRows: any[] = []; + const colSizes = JSON.parse( + localStorage.getItem('builderColSizes') || '[25, 75]' + ); + const [queriesList, setQueriesList] = useState( + JSON.parse(localStorage.getItem('saved-queries') || '[]') + ); + + const [tables, setTables] = useState([]); + const [showBuilder, setShowBuilder] = useState(true); + const [results, setResults] = useState(); + const [fullCount, setFullCount] = useState(); + const [recordsCount, setRecordsCount] = useState(); + const [errorResult, setErrorResult] = useState(); + const [error, setError] = useState(''); + const [loading, setLoading] = useState(false); + const [savedQuery, setSavedQuery] = useState(-1); + + useEffectOnce(() => { + pilosa.get.schema().then((res) => { + setTables(res.data.indexes); + }); + }); + + const handleQueryMessages = (message: RowResponse) => { + const response = message.toObject(); + if (response.headersList.length > 0 && response.duration > 0) { + streamingResults.headers = response.headersList; + streamingResults.duration = response.duration; + } + streamingResults.rows.push(response.columnsList); + }; + + const handleQueryEnd = (status: grpc.Code, statusMessage: string) => { + if (status !== grpc.Code.OK) { + streamingResults.error = statusMessage; + setErrorResult(streamingResults); + } else { + streamingResults.roundtrip = moment + .duration(moment().diff(startTime)) + .as('milliseconds'); + setErrorResult(undefined); + setResults(streamingResults); + } + setLoading(false); + }; + + const handleExternalLookup = (message: RowResponse) => { + const response = message.toObject(); + let rowStr: string[] = []; + if (exportRows.length === 0) { + const headers = response.headersList.map((header) => header.name); + exportRows.push(headers.join('\t')); + } + response.headersList.forEach((header, idx) => + rowStr.push(response.columnsList[idx][`${header.datatype}val`]) + ); + exportRows.push(rowStr.join('\t')); + }; + + const handleExternalLookupEnd = ( + status: grpc.Code, + statusMessage: string + ) => { + if (status !== grpc.Code.OK) { + setError(statusMessage); + } else if (exportRows.length === 0) { + setError('No record attributes for current query.'); + } else { + const dateTime = moment().unix(); + const element = document.createElement('a'); + const file = new Blob([exportRows.join('\n')], { + type: 'text/plain;charset=utf-8' + }); + element.href = URL.createObjectURL(file); + element.download = `molecula-${results?.index}-${dateTime}.csv`; + document.body.appendChild(element); + element.click(); + exportRows = []; + } + }; + + const onRunQuery = ( + table: string, + operation: string, + query: string, + countQuery?: string + ) => { + streamingResults = { + query, + operation, + type: 'PQL', + headers: [], + rows: [], + index: table, + roundtrip: 0, + error: '' + }; + startTime = moment(); + setLoading(true); + + if (operation !== 'Count') { + if (countQuery) { + pilosa.post.query(table, countQuery).then((res) => { + setFullCount(res.data.results[0]); + }); + } + + pilosa.post + .query(table, `Count(All())`) + .then((res) => setRecordsCount(res.data.results[0])); + } else { + setFullCount(undefined); + } + + queryPQL(table, query, handleQueryMessages, handleQueryEnd); + }; + + const onClear = () => { + setResults(undefined); + setFullCount(undefined); + setErrorResult(undefined); + }; + + const onRemoveQuery = (queryIdx: number) => { + let queries = [...queriesList]; + queries.splice(queryIdx, 1); + localStorage.setItem('saved-queries', JSON.stringify(queries)); + setQueriesList(queries); + }; + + const onSaveQuery = () => { + const updatedList = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ); + if (savedQuery < 0) { + setSavedQuery(updatedList.length - 1); + } + setQueriesList(updatedList); + }; + + const onExportLogs = () => { + if (results) { + const columns = results.rows.map((row) => row[0].uint64val); + const table = results.index ? results.index : ''; + onExternalLookup(table, columns); + } + }; + + const onExternalLookup = (table: string, columns: number[]) => { + const cols = JSON.stringify(columns); + const query = `ExternalLookup(ConstRow(columns=${cols}), query='select id, "rawlog" from "${table}" where id = ANY($1)')`; + queryPQL(table, query, handleExternalLookup, handleExternalLookupEnd); + }; + + return ( + + {tables.length > 0 ? ( + + localStorage.setItem('builderColSizes', JSON.stringify(sizes)) + } + gutter={(_index, direction) => { + const gutter = document.createElement('div'); + gutter.className = `gutter gutter-${direction}`; + const dragbars = document.createElement('div'); + dragbars.className = 'dragBar'; + gutter.appendChild(dragbars); + return gutter; + }} + className={classNames(css.split, !showBuilder ? 'hide-gutter' : '')} + > + {showBuilder ? ( +
+
+ setShowBuilder(false)} size="small"> + + +
+ setSavedQuery(-1)} + onSaveQuery={onSaveQuery} + /> +
+ ) : ( +
+ setShowBuilder(true)} size="small"> + + +
+ )} +
+ +
+ + Results{' '} + + {results?.query.includes('Extract(') ? ( +
+ + + + +
+ ) : null} +
+ {loading ?
Loading...
: null} + {results && !loading ? ( + + {fullCount && recordsCount ? ( +
+ {results.duration ? ( +
+ {recordsCount.toLocaleString()} records scanned in{' '} + {formatDuration(results.duration, true)}. +
+ ) : null} +
+ Showing{' '} + {fullCount > 1000 ? 'first 1,000 rows of' : 'all'}{' '} + {fullCount.toLocaleString()} results. +
+
+ ) : null} + +
+ + ) : null} + {errorResult && !loading ?
{errorResult.error}
: null} + {!loading && !results && !error ? ( + + + Build a query to see results + + {queriesList.length > 0 ? ( +
+ + Saved Queries + + setSavedQuery(queryIdx)} + onRemoveQuery={onRemoveQuery} + /> +
+ ) : null} + + ) : null} + +
+ + setError('')}> + {error} + + + + ) : ( + + + Query Builder + + + + There are no tables to query. + + + + )} + + ); +}; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts new file mode 100644 index 000000000..ebbd97470 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts @@ -0,0 +1,52 @@ +export const operators = { + decimal: [ + { label: '>', value: '>' }, + { label: '<', value: '<' }, + { label: '>=', value: '>=' }, + { label: '<=', value: '<=' }, + { label: '==', value: '=' }, + { label: '!=', value: '!=' }, + ], + int: [ + { label: '>', value: '>' }, + { label: '<', value: '<' }, + { label: '>=', value: '>=' }, + { label: '<=', value: '<=' }, + { label: '==', value: '=' }, + { label: '!=', value: '!=' }, + ], + 'mutex-id': [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' } + ], + 'mutex-keys': [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'like', value: 'like' }, + { label: 'CIDR', value: 'cidr' } + ], + "set-id": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' } + ], + "set-keys": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'like', value: 'like' }, + { label: 'CIDR', value: 'cidr' } + ], + "time-id": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' } + ], + "time-keys": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'like', value: 'like' } + ], + timestamp: [ + { label: 'is before', value: '<' }, + { label: 'is after', value: '>' }, + { label: 'is', value: '=' }, + ] +} diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts new file mode 100644 index 000000000..c5f595cf9 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts @@ -0,0 +1 @@ +export * from './helpers'; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss new file mode 100644 index 000000000..326b78876 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss @@ -0,0 +1,40 @@ +.newGroup, +.operator { + display: flex; + align-items: center; + width: 100%; + + .line { + flex-grow: 1; + height: 1px; + background: var(--divider); + } + + .addButton { + margin: 0 8px; + } +} + +.groupBy { + .line { + flex-grow: 1; + height: 1px; + background: var(--divider); + } + + .fieldSelector { + margin-top: 16px; + } + + .filterHeader { + display: flex; + align-items: center; + justify-content: space-between; + margin-top: 8px; + } + + .filterInfo { + vertical-align: text-top; + margin-left: 4px; + } +} diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx new file mode 100644 index 000000000..48f8e81ea --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx @@ -0,0 +1,180 @@ +import React, { FC, Fragment, useState } from 'react'; +import AddIcon from '@material-ui/icons/Add'; +import Button from '@material-ui/core/Button'; +import IconButton from '@material-ui/core/IconButton'; +import Menu from '@material-ui/core/Menu'; +import MenuItem from '@material-ui/core/MenuItem'; +import { + groupOperators, + Operator, + RowCallType, + RowGrouping +} from 'App/QueryBuilder/rowTypes'; +import { RowCall } from 'App/QueryBuilder/RowCall'; +import css from './RowCallBuilder.module.scss'; + +type RowCallBuilderProps = { + rowCalls: RowGrouping[]; + fields: any[]; + showInvalid: boolean; + onChange: ( + rows: RowGrouping[], + operator?: Operator, + hasInvalid?: boolean + ) => void; +}; + +export const RowCallBuilder: FC = ({ + rowCalls = [], + fields, + showInvalid, + onChange +}) => { + const [newOperatorEl, setNewOperatorEl] = useState(null); + const [operatorEl, setOperatorEl] = useState(null); + const [operator, setOperator] = useState(); + + const onNewGroup = () => { + const newRow: RowCallType[] = [ + { + field: '', + rowOperator: '=', + value: '', + type: 'set', + keys: true + } + ]; + + onChange([...rowCalls, { row: newRow }]); + }; + + const onRemoveGroup = (groupIdx: number) => { + if (groupIdx === 0) { + onChange(rowCalls.slice(1)); + } else { + const updatedRowCalls = [...rowCalls]; + updatedRowCalls.splice(groupIdx, 1); + onChange(updatedRowCalls); + + if (updatedRowCalls.length < 1) { + setOperator(undefined); + } + } + }; + + const onUpdateRow = ( + groupIdx: number, + newRowData?: RowCallType[], + operator?: Operator, + isNot?: boolean + ) => { + const updatedRowCalls = [...rowCalls]; + if (newRowData) { + const clone: RowGrouping = { + ...rowCalls[groupIdx], + row: newRowData, + isNot + }; + if (newRowData.length <= 1) { + delete clone.operator; + } else if (operator) { + clone.operator = operator; + } + updatedRowCalls.splice(groupIdx, 1, clone); + } else { + updatedRowCalls.splice(groupIdx, 1); + } + onChange(updatedRowCalls); + }; + + return ( + + {rowCalls?.map((rowCall, idx) => ( + + {idx > 0 ? ( + +
+
+ +
+
+ setOperatorEl(null)} + > + {groupOperators.map((op: Operator) => ( + { + setOperator(op); + setOperatorEl(null); + }} + > + {op} + + ))} + + + ) : null} + + onUpdateRow(idx, updatedRow, operator, isNot) + } + onRemoveGroup={() => onRemoveGroup(idx)} + /> + + ))} + +
+
+ { + if (rowCalls.length === 0 || !!operator) { + onNewGroup(); + } else { + setNewOperatorEl(event.currentTarget); + } + }} + > + + +
+
+ + setNewOperatorEl(null)} + > + {groupOperators.map((op: Operator) => ( + { + setOperator(op); + onNewGroup(); + setNewOperatorEl(null); + }} + > + {op} + + ))} + + + ); +}; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/index.ts b/lattice/src/App/QBuilder/RowCallBuilder/index.ts new file mode 100644 index 000000000..2f2d839f8 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/index.ts @@ -0,0 +1 @@ +export * from './RowCallBuilder'; diff --git a/lattice/src/App/QBuilder/index.ts b/lattice/src/App/QBuilder/index.ts new file mode 100644 index 000000000..058f256f4 --- /dev/null +++ b/lattice/src/App/QBuilder/index.ts @@ -0,0 +1,2 @@ +export * from './QBuilderContainer'; +export * from './rowTypes'; diff --git a/lattice/src/App/QBuilder/rowTypes.ts b/lattice/src/App/QBuilder/rowTypes.ts new file mode 100644 index 000000000..7ed00293f --- /dev/null +++ b/lattice/src/App/QBuilder/rowTypes.ts @@ -0,0 +1,22 @@ +export type Operator = 'and' | 'or'; + +export const groupOperators: Operator[] = ['and', 'or']; + +export type RowGrouping = { + row: RowCallType[]; + isNot?: boolean; + operator?: Operator; +} + +export type RowCallType = { + field: string; + rowOperator: string; + value: string; + type: string; + keys?: boolean; +}; + +export type RowsCallType = { + primary: string; + secondary: string; +}; diff --git a/lattice/src/App/QBuilder/utils.ts b/lattice/src/App/QBuilder/utils.ts new file mode 100644 index 000000000..5722fb31f --- /dev/null +++ b/lattice/src/App/QBuilder/utils.ts @@ -0,0 +1,174 @@ +import { RowGrouping } from './rowTypes'; +import { getIPRange } from 'get-ip-range'; + +export const stringifyExtract = (query: any) => { + const { columns, operator, rowCalls } = query; + + const rowData = stringifyRowData(rowCalls, operator); + if (!rowData.error) { + const fields = columns.map((field) => `Rows(${field})`); + const allRows = fields.join(', '); + return { + error: false, + queryString: `Extract(Limit(${rowData.queryString}, limit=1000), ${allRows})`, + countQuery: `Count(${rowData.queryString})` + }; + } else { + return { ...rowData, countQuery: '' }; + } +} + +export const stringifyCount = (query: any) => { + const { operator, rowCalls } = query; + + const rowData = stringifyRowData(rowCalls, operator); + if (!rowData.error) { + return { error: false, queryString: `Count(${rowData.queryString})` }; + } else { + return rowData; + } +} + +const stringifyRowData = (rowCalls, operator) => { + let queryString = ''; + if (rowCalls.length === 0) { + queryString = 'All()'; + } else { + let rowsMap: string[][] = []; + rowCalls.forEach((group, groupIdx) => { + rowsMap.push([]); + group.row.forEach((row) => { + let rowString = ''; + const { field, rowOperator, value, type, keys } = row; + const isNegatory = rowOperator === '!='; + const isUnion = + ['=', '!='].includes(rowOperator) && value.split(',').length > 1; + const operator = isNegatory ? '=' : rowOperator; + if (isUnion) { + const values = value.split(','); + const unionRows = values + .map((v) => keys ? `Row(${field}="${v.trim()}")` : `Row(${field}=${v.trim()})`) + .join(', '); + rowString = `Union(${unionRows})`; + } else if (rowOperator === 'cidr') { + try { + const ipRange = getIPRange(value); + const ipRows = ipRange + .map((ip) => `Row(${field}="${ip}")`) + .join(', '); + rowString = `Union(${ipRows})`; + } catch (error) { + return { error: true, query: error.message }; + } + } else if (rowOperator === 'like') { + if (value.includes('%') || value.includes('_')) { + rowString = `UnionRows(Rows(field=${field}, like="${value}"))`; + } else { + rowString = `UnionRows(Rows(field=${field}, like="%${value}%"))`; + } + } else { + rowString = keys || type === 'timestamp' + ? `Row(${field}${operator}"${value}")` + : `Row(${field}${operator}${value})`; + } + + if (isNegatory) { + rowsMap[groupIdx].push(`Not(${rowString})`); + } else { + rowsMap[groupIdx].push(rowString); + } + }); + }); + + queryString = rowsMap + .map((group, idx) => { + let joined = ''; + if (group.length > 1) { + joined = group.map((r) => r).join(', '); + const operator = rowCalls[idx].operator; + if (operator === 'and') { + joined = `Intersect(${joined})`; + } else if (operator === 'or') { + joined = `Union(${joined})`; + } + } else { + joined = group[0]; + } + return rowCalls[idx].isNot ? `Not(${joined})` : joined; + }) + .join(', '); + + if (rowCalls.length > 1 && operator) { + if (operator === 'and') { + queryString = `Intersect(${queryString})`; + } else if (operator === 'or') { + queryString = `Union(${queryString})`; + } + } + } + + return { error: false, queryString }; +} + +export const stringifyGroupBy = (query: any) => { + const { groupByCall, filter, sort } = query; + + let sortString = ''; + let aggregateString = ''; + if (sort?.length > 0) { + sortString = sort[0].sortValue; + if (sort[0].sortValue.includes('sum')) { + aggregateString = `, aggregate=Sum(field=${sort[0].field})`; + } + if (sort.length > 1) { + sortString = `${sortString}, ${sort[1].sortValue}`; + if (sort[1].sortValue.includes('sum')) { + aggregateString = `, aggregate=Sum(field=${sort[1].field})`; + } + } + sortString = `, sort="${sortString}"`; + } + const filterString = filter ? `, filter=${filter}` : ''; + const queryString = groupByCall.secondary + ? `GroupBy(Rows(${groupByCall.primary}), Rows(${groupByCall.secondary})${filterString}${sortString}${aggregateString})` + : `GroupBy(Rows(${groupByCall.primary})${filterString}${sortString}${aggregateString})`; + + return queryString; +} + +export const cleanupRows = (rowCalls: RowGrouping[]) => { + // remove empty groups and row calls + let isInvalid = false; + let cleanRowCalls: RowGrouping[] = []; + let cleanGroups: RowGrouping[] = []; + + if (rowCalls?.length > 0) { + rowCalls.forEach((group, groupIdx) => { + cleanGroups.push({ ...group, row: [] }); + group.row.forEach((row) => { + const { field, value, rowOperator } = row; + if (field || value) { + cleanGroups[groupIdx].row.push(row); + + if (!field || !value) { + isInvalid = true; + } else if (rowOperator === 'cidr') { + try { + getIPRange(value); + } catch (err) { + isInvalid = true; + } + } + } + }); + }); + + cleanGroups.forEach((group) => { + if (group.row.length > 0) { + cleanRowCalls.push(group); + } + }); + } + + return { cleanRowCalls, isInvalid }; +}; From d8ac3251f6d3879bb8c1b81ca3ff7717c1a1de01 Mon Sep 17 00:00:00 2001 From: Stephanie Yang Date: Wed, 28 Jul 2021 10:51:35 -0500 Subject: [PATCH 2/4] replace old query builder --- lattice/src/App.tsx | 4 +- lattice/src/App/QBuilder/QBuilder.module.scss | 52 - lattice/src/App/QBuilder/QBuilder.tsx | 388 ------ .../QBuilder/QBuilderContainer.module.scss | 66 - .../src/App/QBuilder/QBuilderContainer.tsx | 323 ----- .../QBuilder/RowCallBuilder/RowCall/index.ts | 1 - lattice/src/App/QBuilder/index.ts | 2 - lattice/src/App/QBuilder/rowTypes.ts | 22 - .../CountBuilder/CountBuilder.tsx | 8 +- .../CountBuilder/index.ts | 0 .../ExtractBuilder/ExtractBuilder.module.scss | 0 .../ExtractBuilder/ExtractBuilder.tsx | 8 +- .../ExtractBuilder/index.ts | 0 .../GroupByBuilder/GroupByBuilder.module.scss | 0 .../GroupByBuilder/GroupByBuilder.tsx | 4 +- .../GroupByBuilder/index.ts | 0 .../App/QueryBuilder/QueryBuilder.module.scss | 183 +-- lattice/src/App/QueryBuilder/QueryBuilder.tsx | 1199 +++++------------ .../QueryBuilderContainer.module.scss | 62 + .../QueryBuilder/QueryBuilderContainer.tsx | 328 +++-- .../src/App/QueryBuilder/RowCall/helpers.ts | 52 - .../RowCall/RowCall.module.scss | 0 .../{ => RowCallBuilder}/RowCall/RowCall.tsx | 2 +- .../RowCallBuilder/RowCall/helpers.ts | 0 .../{ => RowCallBuilder}/RowCall/index.ts | 0 .../RowCallBuilder/RowCallBuilder.module.scss | 0 .../RowCallBuilder/RowCallBuilder.tsx | 28 +- .../RowCallBuilder/index.ts | 0 .../src/App/QueryBuilder/stringifyRowData.ts | 79 -- ...stringifyRowData.test.ts => utils.test.ts} | 234 +++- .../App/{QBuilder => QueryBuilder}/utils.ts | 53 +- 31 files changed, 851 insertions(+), 2247 deletions(-) delete mode 100644 lattice/src/App/QBuilder/QBuilder.module.scss delete mode 100644 lattice/src/App/QBuilder/QBuilder.tsx delete mode 100644 lattice/src/App/QBuilder/QBuilderContainer.module.scss delete mode 100644 lattice/src/App/QBuilder/QBuilderContainer.tsx delete mode 100644 lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts delete mode 100644 lattice/src/App/QBuilder/index.ts delete mode 100644 lattice/src/App/QBuilder/rowTypes.ts rename lattice/src/App/{QBuilder => QueryBuilder}/CountBuilder/CountBuilder.tsx (71%) rename lattice/src/App/{QBuilder => QueryBuilder}/CountBuilder/index.ts (100%) rename lattice/src/App/{QBuilder => QueryBuilder}/ExtractBuilder/ExtractBuilder.module.scss (100%) rename lattice/src/App/{QBuilder => QueryBuilder}/ExtractBuilder/ExtractBuilder.tsx (90%) rename lattice/src/App/{QBuilder => QueryBuilder}/ExtractBuilder/index.ts (100%) rename lattice/src/App/{QBuilder => QueryBuilder}/GroupByBuilder/GroupByBuilder.module.scss (100%) rename lattice/src/App/{QBuilder => QueryBuilder}/GroupByBuilder/GroupByBuilder.tsx (97%) rename lattice/src/App/{QBuilder => QueryBuilder}/GroupByBuilder/index.ts (100%) delete mode 100644 lattice/src/App/QueryBuilder/RowCall/helpers.ts rename lattice/src/App/QueryBuilder/{ => RowCallBuilder}/RowCall/RowCall.module.scss (100%) rename lattice/src/App/QueryBuilder/{ => RowCallBuilder}/RowCall/RowCall.tsx (99%) rename lattice/src/App/{QBuilder => QueryBuilder}/RowCallBuilder/RowCall/helpers.ts (100%) rename lattice/src/App/QueryBuilder/{ => RowCallBuilder}/RowCall/index.ts (100%) rename lattice/src/App/{QBuilder => QueryBuilder}/RowCallBuilder/RowCallBuilder.module.scss (100%) rename lattice/src/App/{QBuilder => QueryBuilder}/RowCallBuilder/RowCallBuilder.tsx (88%) rename lattice/src/App/{QBuilder => QueryBuilder}/RowCallBuilder/index.ts (100%) delete mode 100644 lattice/src/App/QueryBuilder/stringifyRowData.ts rename lattice/src/App/QueryBuilder/{stringifyRowData.test.ts => utils.test.ts} (61%) rename lattice/src/App/{QBuilder => QueryBuilder}/utils.ts (81%) diff --git a/lattice/src/App.tsx b/lattice/src/App.tsx index 89bff9fce..f465bd480 100644 --- a/lattice/src/App.tsx +++ b/lattice/src/App.tsx @@ -9,7 +9,7 @@ import { Nav } from 'shared/Nav'; import { NotFound } from 'App/NotFound'; import { MoleculaTablesContainer } from 'App/MoleculaTables'; import { QueryContainer } from 'App/Query'; -import { QBuilderContainer } from 'App/QBuilder'; +import { QueryBuilderContainer } from 'App/QueryBuilder'; import css from './App.module.scss'; const App = () => { @@ -45,7 +45,7 @@ const App = () => { - +
diff --git a/lattice/src/App/QBuilder/QBuilder.module.scss b/lattice/src/App/QBuilder/QBuilder.module.scss deleted file mode 100644 index 33c34ede7..000000000 --- a/lattice/src/App/QBuilder/QBuilder.module.scss +++ /dev/null @@ -1,52 +0,0 @@ -.sharedConfig { - padding: 32px 32px 0; - - .configSelector { - margin-bottom: 24px; - } -} - -.textLink { - color: var(--primary); - font-size: 12px; - - .icon { - margin-right: 4px; - vertical-align: text-top; - } - - &:hover { - cursor: pointer; - text-decoration: underline; - color: var(--primary); - } -} - -.builder { - display: flex; - flex-direction: column; - height: 100%; - - .queryMetadata { - border-bottom: 1px solid var(--divider); - padding-bottom: 16px; - margin-bottom: 16px; - } -} - -.config { - padding: 24px 32px 0; - flex-grow: 1; -} - -.builderActions { - padding: 16px 32px; - display: flex; - justify-content: space-between; - - .mainActions { - display: grid; - grid-template-columns: auto auto; - grid-gap: 8px; - } -} diff --git a/lattice/src/App/QBuilder/QBuilder.tsx b/lattice/src/App/QBuilder/QBuilder.tsx deleted file mode 100644 index 3e5e0a8be..000000000 --- a/lattice/src/App/QBuilder/QBuilder.tsx +++ /dev/null @@ -1,388 +0,0 @@ -import React, { FC, Fragment, useEffect, useState } from 'react'; -import ArrowBackIcon from '@material-ui/icons/ArrowBack'; -import Button from '@material-ui/core/Button'; -import CircularProgress from '@material-ui/core/CircularProgress'; -import Divider from '@material-ui/core/Divider'; -import Popover from '@material-ui/core/Popover'; -import TextField from '@material-ui/core/TextField'; -import Typography from '@material-ui/core/Typography'; -import { - cleanupRows, - stringifyCount, - stringifyExtract, - stringifyGroupBy -} from './utils'; -import { CountBuilder } from './CountBuilder'; -import { ExtractBuilder } from './ExtractBuilder'; -import { GroupByBuilder } from './GroupByBuilder'; -import { Select } from 'shared/Select'; -import css from './QBuilder.module.scss'; - -type QBuilderProps = { - tables: any[]; - savedQuery: number; - onRun: ( - table: string, - operation: string, - query: string, - countQuery?: string - ) => void; - onClear: () => void; - onExitEdit: () => void; - onSaveQuery: () => void; -}; - -export const QBuilder: FC = ({ - tables, - savedQuery, - onRun, - onClear, - onExitEdit, - onSaveQuery -}) => { - const [saveButtonEl, setSaveButtonEl] = useState(null); - const queriesList = JSON.parse(localStorage.getItem('saved-queries') || '[]'); - const [editMode, setEditMode] = useState(false); - const [selectedTable, setSelectedTable] = useState(tables[0]); - const [operation, setOperation] = useState('Extract'); - const [query, setQuery] = useState({ - table: tables[0].name, - operation: 'Extract', - columns: tables[0].fields.map((field) => field.name) - }); - const [showInvalid, setShowInvalid] = useState(false); - const [updating, setUpdating] = useState(false); - const [queryNameIdx, setQueryNameIdx] = useState(-1); - - useEffect(() => { - if (savedQuery > -1) { - const updatedList = JSON.parse( - localStorage.getItem('saved-queries') || '[]' - ); - setQuery(updatedList[savedQuery]); - setEditMode(true); - setOperation(updatedList[savedQuery].operation); - const table = tables.find( - (table) => table.name === updatedList[savedQuery].table - ); - setSelectedTable(table); - } - }, [savedQuery, tables]); - - useEffect(() => { - if (query?.name) { - setQueryNameIdx(queriesList.findIndex((q) => q.name === query.name)); - } - }, [query, queriesList]); - - const cleanRows = () => { - const cleanRows = cleanupRows(query?.rowCalls); - return { - ...query, - rowCalls: cleanRows.cleanRowCalls, - isInvalid: cleanRows.isInvalid - }; - }; - - const runQuery = () => { - setShowInvalid(true); - let queryString = ''; - let countQuery = ''; - - switch (operation) { - case 'Extract': - const cleanExtractQuery = cleanRows(); - setQuery(cleanExtractQuery); - const extract = stringifyExtract(cleanExtractQuery); - if (!extract.error && !cleanExtractQuery.isInvalid) { - queryString = extract.queryString; - countQuery = extract.countQuery; - } - break; - case 'GroupBy': - queryString = stringifyGroupBy(query); - break; - case 'Count': - const cleanCountQuery = cleanRows(); - setQuery(cleanCountQuery); - const count = stringifyCount(cleanCountQuery); - if (!count.error && !cleanCountQuery.isInvalid) { - queryString = count.queryString; - } - break; - default: - break; - } - - if (queryString && !query.isInvalid) { - setShowInvalid(false); - onRun(selectedTable.name, operation, queryString, countQuery); - } else { - setShowInvalid(true); - } - }; - - const reset = () => { - setQuery({ - table: selectedTable.name, - columns: selectedTable.fields.map(field => field.name), - operation, - isInvalid: false - }); - setOperation('Extract'); - setShowInvalid(false); - setEditMode(false); - onExitEdit(); - onClear(); - }; - - const onSave = () => { - setUpdating(true); - setTimeout(() => { - setUpdating(false); - }, 1500); - const cleanQuery = cleanRows(); - let queries = [...queriesList]; - - if (editMode && queries.length > 0) { - const clone = [...queries]; - clone.splice(savedQuery, 1, cleanQuery); - localStorage.setItem('saved-queries', JSON.stringify(clone)); - } else { - queries.push(cleanQuery); - localStorage.setItem('saved-queries', JSON.stringify(queries)); - } - onSaveQuery(); - }; - - const renderBuilder = () => { - switch (operation) { - case 'Extract': - return ( - setQuery(q)} - /> - ); - case 'GroupBy': - return ( - setQuery(q)} - /> - ); - case 'Count': - return ( - setQuery(q)} - /> - ); - default: - return
Invalid Operation
; - } - }; - - return ( -
-
- {editMode ? ( - - - Back to new query - - ) : null} - - {editMode ? 'Edit Saved Query' : 'New Query'} - - - {editMode ? ( -
- = 0 && queryNameIdx !== savedQuery) - } - helperText={ - queryNameIdx >= 0 && queryNameIdx !== savedQuery - ? 'Query name must be unique' - : '' - } - onChange={(event) => - setQuery({ ...query, name: event.target.value }) - } - margin="normal" - size="small" - required - fullWidth - /> - - - setQuery({ ...query, description: event.target.value }) - } - margin="normal" - size="small" - fullWidth - /> -
- ) : null} - - { - setOperation(value); - setShowInvalid(false); - setQuery({ - table: selectedTable.name, - operation: value, - isInvalid: false - }); - }} - /> - -
-
{renderBuilder()}
-
-
- - {editMode ? ( - - ) : ( - - - setSaveButtonEl(null)} - anchorOrigin={{ - vertical: 'top', - horizontal: 'left' - }} - transformOrigin={{ - vertical: 'bottom', - horizontal: 'left' - }} - > -
- - Give your query a unique name and a short description to - help you identify it for future use. - - - = 0} - helperText={ - queryNameIdx >= 0 ? 'Query name must be unique' : '' - } - onChange={(event) => - setQuery({ ...query, name: event.target.value }) - } - margin="dense" - required - fullWidth - /> - - - setQuery({ ...query, description: event.target.value }) - } - margin="dense" - fullWidth - /> - -
-
- - -
-
-
- - - )} -
- -
-
- ); -}; diff --git a/lattice/src/App/QBuilder/QBuilderContainer.module.scss b/lattice/src/App/QBuilder/QBuilderContainer.module.scss deleted file mode 100644 index a5cd6c77d..000000000 --- a/lattice/src/App/QBuilder/QBuilderContainer.module.scss +++ /dev/null @@ -1,66 +0,0 @@ -.builderColumn, -.results { - height: calc(100vh - 64px); - overflow: scroll; - position: relative; - - .openClose { - position: absolute; - right: 10px; - top: 10px; - } -} - -.split { - display: flex; -} - -.collapsedBuilder { - border-right: 1px solid rgba(var(--contrast-rgb), 0.1); - text-align: center; - padding-top: 10px; -} - -.resultsBlock { - height: 100%; -} - -.resultsHeader { - display: flex; - align-items: center; - justify-content: space-between; - - .download { - display: flex; - align-items: center; - } - - .downloadInfo { - margin-right: 8px; - } -} - -.infoMessage { - padding: 16px; - border: 1px solid rgba(var(--primary-rgb), 0.5); - background: rgba(var(--primary-rgb), 0.1); - border-radius: 4px; - margin: 4px 0 16px; -} - -.spacer{ - padding-bottom: 16px; -} - -.resultsLoading { - margin-left: 8px; -} - -.savedQueries { - margin-top: 48px; -} - -.noTables { - margin-top: 20px; - padding: 12px 16px; -} diff --git a/lattice/src/App/QBuilder/QBuilderContainer.tsx b/lattice/src/App/QBuilder/QBuilderContainer.tsx deleted file mode 100644 index 7161cb5a3..000000000 --- a/lattice/src/App/QBuilder/QBuilderContainer.tsx +++ /dev/null @@ -1,323 +0,0 @@ -import React, { Fragment, useState } from 'react'; -import Alert from '@material-ui/lab/Alert'; -import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos'; -import Button from '@material-ui/core/Button'; -import classNames from 'classnames'; -import CloseIcon from '@material-ui/icons/Close'; -import IconButton from '@material-ui/core/IconButton'; -import InfoIcon from '@material-ui/icons/Info'; -import moment, { Moment } from 'moment'; -import Paper from '@material-ui/core/Paper'; -import Split from 'react-split'; -import Snackbar from '@material-ui/core/Snackbar'; -import Tooltip from '@material-ui/core/Tooltip'; -import Typography from '@material-ui/core/Typography'; -import { Block } from 'shared/Block'; -import { formatDuration } from 'shared/utils/formatDuration'; -import { grpc } from '@improbable-eng/grpc-web'; -import { pilosa } from 'services/eventServices'; -import { QBuilder } from './QBuilder'; -import { queryPQL } from 'services/grpcServices'; -import { QueryResults } from 'App/Query/QueryResults'; -import { ResultType } from 'App/Query/QueryContainer'; -import { RowResponse } from 'proto/pilosa_pb'; -import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; -import { useEffectOnce } from 'react-use'; -import css from './QBuilderContainer.module.scss'; - -let streamingResults: ResultType = { - query: '', - operation: '', - type: 'PQL', - headers: [], - rows: [], - roundtrip: 0, - error: '' -}; - -export const QBuilderContainer = () => { - let startTime: Moment; - let exportRows: any[] = []; - const colSizes = JSON.parse( - localStorage.getItem('builderColSizes') || '[25, 75]' - ); - const [queriesList, setQueriesList] = useState( - JSON.parse(localStorage.getItem('saved-queries') || '[]') - ); - - const [tables, setTables] = useState([]); - const [showBuilder, setShowBuilder] = useState(true); - const [results, setResults] = useState(); - const [fullCount, setFullCount] = useState(); - const [recordsCount, setRecordsCount] = useState(); - const [errorResult, setErrorResult] = useState(); - const [error, setError] = useState(''); - const [loading, setLoading] = useState(false); - const [savedQuery, setSavedQuery] = useState(-1); - - useEffectOnce(() => { - pilosa.get.schema().then((res) => { - setTables(res.data.indexes); - }); - }); - - const handleQueryMessages = (message: RowResponse) => { - const response = message.toObject(); - if (response.headersList.length > 0 && response.duration > 0) { - streamingResults.headers = response.headersList; - streamingResults.duration = response.duration; - } - streamingResults.rows.push(response.columnsList); - }; - - const handleQueryEnd = (status: grpc.Code, statusMessage: string) => { - if (status !== grpc.Code.OK) { - streamingResults.error = statusMessage; - setErrorResult(streamingResults); - } else { - streamingResults.roundtrip = moment - .duration(moment().diff(startTime)) - .as('milliseconds'); - setErrorResult(undefined); - setResults(streamingResults); - } - setLoading(false); - }; - - const handleExternalLookup = (message: RowResponse) => { - const response = message.toObject(); - let rowStr: string[] = []; - if (exportRows.length === 0) { - const headers = response.headersList.map((header) => header.name); - exportRows.push(headers.join('\t')); - } - response.headersList.forEach((header, idx) => - rowStr.push(response.columnsList[idx][`${header.datatype}val`]) - ); - exportRows.push(rowStr.join('\t')); - }; - - const handleExternalLookupEnd = ( - status: grpc.Code, - statusMessage: string - ) => { - if (status !== grpc.Code.OK) { - setError(statusMessage); - } else if (exportRows.length === 0) { - setError('No record attributes for current query.'); - } else { - const dateTime = moment().unix(); - const element = document.createElement('a'); - const file = new Blob([exportRows.join('\n')], { - type: 'text/plain;charset=utf-8' - }); - element.href = URL.createObjectURL(file); - element.download = `molecula-${results?.index}-${dateTime}.csv`; - document.body.appendChild(element); - element.click(); - exportRows = []; - } - }; - - const onRunQuery = ( - table: string, - operation: string, - query: string, - countQuery?: string - ) => { - streamingResults = { - query, - operation, - type: 'PQL', - headers: [], - rows: [], - index: table, - roundtrip: 0, - error: '' - }; - startTime = moment(); - setLoading(true); - - if (operation !== 'Count') { - if (countQuery) { - pilosa.post.query(table, countQuery).then((res) => { - setFullCount(res.data.results[0]); - }); - } - - pilosa.post - .query(table, `Count(All())`) - .then((res) => setRecordsCount(res.data.results[0])); - } else { - setFullCount(undefined); - } - - queryPQL(table, query, handleQueryMessages, handleQueryEnd); - }; - - const onClear = () => { - setResults(undefined); - setFullCount(undefined); - setErrorResult(undefined); - }; - - const onRemoveQuery = (queryIdx: number) => { - let queries = [...queriesList]; - queries.splice(queryIdx, 1); - localStorage.setItem('saved-queries', JSON.stringify(queries)); - setQueriesList(queries); - }; - - const onSaveQuery = () => { - const updatedList = JSON.parse( - localStorage.getItem('saved-queries') || '[]' - ); - if (savedQuery < 0) { - setSavedQuery(updatedList.length - 1); - } - setQueriesList(updatedList); - }; - - const onExportLogs = () => { - if (results) { - const columns = results.rows.map((row) => row[0].uint64val); - const table = results.index ? results.index : ''; - onExternalLookup(table, columns); - } - }; - - const onExternalLookup = (table: string, columns: number[]) => { - const cols = JSON.stringify(columns); - const query = `ExternalLookup(ConstRow(columns=${cols}), query='select id, "rawlog" from "${table}" where id = ANY($1)')`; - queryPQL(table, query, handleExternalLookup, handleExternalLookupEnd); - }; - - return ( - - {tables.length > 0 ? ( - - localStorage.setItem('builderColSizes', JSON.stringify(sizes)) - } - gutter={(_index, direction) => { - const gutter = document.createElement('div'); - gutter.className = `gutter gutter-${direction}`; - const dragbars = document.createElement('div'); - dragbars.className = 'dragBar'; - gutter.appendChild(dragbars); - return gutter; - }} - className={classNames(css.split, !showBuilder ? 'hide-gutter' : '')} - > - {showBuilder ? ( -
-
- setShowBuilder(false)} size="small"> - - -
- setSavedQuery(-1)} - onSaveQuery={onSaveQuery} - /> -
- ) : ( -
- setShowBuilder(true)} size="small"> - - -
- )} -
- -
- - Results{' '} - - {results?.query.includes('Extract(') ? ( -
- - - - -
- ) : null} -
- {loading ?
Loading...
: null} - {results && !loading ? ( - - {fullCount && recordsCount ? ( -
- {results.duration ? ( -
- {recordsCount.toLocaleString()} records scanned in{' '} - {formatDuration(results.duration, true)}. -
- ) : null} -
- Showing{' '} - {fullCount > 1000 ? 'first 1,000 rows of' : 'all'}{' '} - {fullCount.toLocaleString()} results. -
-
- ) : null} - -
- - ) : null} - {errorResult && !loading ?
{errorResult.error}
: null} - {!loading && !results && !error ? ( - - - Build a query to see results - - {queriesList.length > 0 ? ( -
- - Saved Queries - - setSavedQuery(queryIdx)} - onRemoveQuery={onRemoveQuery} - /> -
- ) : null} - - ) : null} - -
- - setError('')}> - {error} - - - - ) : ( - - - Query Builder - - - - There are no tables to query. - - - - )} - - ); -}; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts deleted file mode 100644 index c5f595cf9..000000000 --- a/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './helpers'; diff --git a/lattice/src/App/QBuilder/index.ts b/lattice/src/App/QBuilder/index.ts deleted file mode 100644 index 058f256f4..000000000 --- a/lattice/src/App/QBuilder/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './QBuilderContainer'; -export * from './rowTypes'; diff --git a/lattice/src/App/QBuilder/rowTypes.ts b/lattice/src/App/QBuilder/rowTypes.ts deleted file mode 100644 index 7ed00293f..000000000 --- a/lattice/src/App/QBuilder/rowTypes.ts +++ /dev/null @@ -1,22 +0,0 @@ -export type Operator = 'and' | 'or'; - -export const groupOperators: Operator[] = ['and', 'or']; - -export type RowGrouping = { - row: RowCallType[]; - isNot?: boolean; - operator?: Operator; -} - -export type RowCallType = { - field: string; - rowOperator: string; - value: string; - type: string; - keys?: boolean; -}; - -export type RowsCallType = { - primary: string; - secondary: string; -}; diff --git a/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx b/lattice/src/App/QueryBuilder/CountBuilder/CountBuilder.tsx similarity index 71% rename from lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx rename to lattice/src/App/QueryBuilder/CountBuilder/CountBuilder.tsx index 03391ec02..edc708b83 100644 --- a/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx +++ b/lattice/src/App/QueryBuilder/CountBuilder/CountBuilder.tsx @@ -1,5 +1,6 @@ import React, { FC } from 'react'; -import { RowCallBuilder } from 'App/QBuilder/RowCallBuilder'; +import { Operator, RowGrouping } from 'App/QueryBuilder/rowTypes'; +import { RowCallBuilder } from 'App/QueryBuilder/RowCallBuilder'; type CountBuilderProps = { table: any; @@ -22,12 +23,11 @@ export const CountBuilder: FC = ({ rowCalls={rowCalls} fields={table.fields} showInvalid={showInvalid} - onChange={(rowCalls, operator, isInvalid) => { + onChange={(rowCalls: RowGrouping[], operator?: Operator) => { onChange({ ...query, rowCalls, - operator, - isInvalid + operator }); }} /> diff --git a/lattice/src/App/QBuilder/CountBuilder/index.ts b/lattice/src/App/QueryBuilder/CountBuilder/index.ts similarity index 100% rename from lattice/src/App/QBuilder/CountBuilder/index.ts rename to lattice/src/App/QueryBuilder/CountBuilder/index.ts diff --git a/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss b/lattice/src/App/QueryBuilder/ExtractBuilder/ExtractBuilder.module.scss similarity index 100% rename from lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss rename to lattice/src/App/QueryBuilder/ExtractBuilder/ExtractBuilder.module.scss diff --git a/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx b/lattice/src/App/QueryBuilder/ExtractBuilder/ExtractBuilder.tsx similarity index 90% rename from lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx rename to lattice/src/App/QueryBuilder/ExtractBuilder/ExtractBuilder.tsx index eaae42714..9c52f7f4b 100644 --- a/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx +++ b/lattice/src/App/QueryBuilder/ExtractBuilder/ExtractBuilder.tsx @@ -2,7 +2,8 @@ import React, { FC, useState } from 'react'; import InfoIcon from '@material-ui/icons/Info'; import Tooltip from '@material-ui/core/Tooltip'; import { ColumnSelector } from 'App/QueryBuilder/ColumnSelector'; -import { RowCallBuilder } from 'App/QBuilder/RowCallBuilder'; +import { Operator, RowGrouping } from 'App/QueryBuilder/rowTypes'; +import { RowCallBuilder } from 'App/QueryBuilder/RowCallBuilder'; import { useEffectOnce } from 'react-use'; import css from './ExtractBuilder.module.scss'; @@ -78,12 +79,11 @@ export const ExtractBuilder: FC = ({ rowCalls={rowCalls} fields={table.fields} showInvalid={showInvalid} - onChange={(rowCalls, operator, isInvalid) => { + onChange={(rowCalls: RowGrouping[], operator?: Operator) => { onChange({ ...query, rowCalls, - operator, - isInvalid + operator }); }} /> diff --git a/lattice/src/App/QBuilder/ExtractBuilder/index.ts b/lattice/src/App/QueryBuilder/ExtractBuilder/index.ts similarity index 100% rename from lattice/src/App/QBuilder/ExtractBuilder/index.ts rename to lattice/src/App/QueryBuilder/ExtractBuilder/index.ts diff --git a/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss b/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.module.scss similarity index 100% rename from lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss rename to lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.module.scss diff --git a/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx b/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx similarity index 97% rename from lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx rename to lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx index b53bf6c59..56a535d0f 100644 --- a/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx +++ b/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx @@ -5,7 +5,7 @@ import Typography from '@material-ui/core/Typography'; import { GroupBySort } from 'App/QueryBuilder/GroupBySort'; import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; import { Select } from 'shared/Select'; -import { stringifyRowData } from 'App/QueryBuilder/stringifyRowData'; +import { stringifyRowData } from 'App/QueryBuilder/utils'; import css from './GroupByBuilder.module.scss'; type GroupByBuilderProps = { @@ -131,7 +131,7 @@ export const GroupByBuilder: FC = ({ const { rowCalls, operator } = filters[queryIdx]; const res = stringifyRowData(rowCalls, operator); if (!res.error) { - onChange({ ...query, filter: res.query }); + onChange({ ...query, filter: res.queryString }); } }} /> diff --git a/lattice/src/App/QBuilder/GroupByBuilder/index.ts b/lattice/src/App/QueryBuilder/GroupByBuilder/index.ts similarity index 100% rename from lattice/src/App/QBuilder/GroupByBuilder/index.ts rename to lattice/src/App/QueryBuilder/GroupByBuilder/index.ts diff --git a/lattice/src/App/QueryBuilder/QueryBuilder.module.scss b/lattice/src/App/QueryBuilder/QueryBuilder.module.scss index b5bc0985a..33c34ede7 100644 --- a/lattice/src/App/QueryBuilder/QueryBuilder.module.scss +++ b/lattice/src/App/QueryBuilder/QueryBuilder.module.scss @@ -1,154 +1,12 @@ -.builderColumn, -.results { - height: calc(100vh - 64px); - overflow: scroll; - position: relative; - - .openClose { - position: absolute; - right: 10px; - top: 10px; - } -} - -.resultsBlock { - height: 100%; -} - -.builderColumn { - display: grid; - grid-template-rows: auto 1fr auto; -} - -.results { - padding-bottom: 20px; -} - -.resultsHeader { - display: flex; - align-items: center; - justify-content: space-between; - - .download { - display: flex; - align-items: center; - } - - .downloadInfo { - margin-right: 8px; - } -} - -.collapsedBuilder { - border-right: 1px solid rgba(var(--contrast-rgb), 0.1); - text-align: center; - padding-top: 10px; -} - -.builderHeader { +.sharedConfig { padding: 32px 32px 0; -} -.builder { - overflow-y: scroll; - padding: 20px 32px; - - .queryMetadata, - .tableSelector, - .operationSelector { - margin-bottom: 16px; - } - - .columnsSelector { - margin: 4px 14px; - display: flex; - text-align: center; - - .info { - margin-left: 4px; - fill: var(--text-secondary); - } - } - - .queryMetadata { - border-bottom: 1px solid var(--divider); - padding-bottom: 16px; + .configSelector { + margin-bottom: 24px; } } -.builderActions { - padding: 16px 32px; - display: flex; - justify-content: space-between; - - .mainActions { - display: grid; - grid-template-columns: auto auto; - grid-gap: 8px; - } -} - -.newGroup, -.operator { - display: flex; - align-items: center; - width: 100%; - - .line { - flex-grow: 1; - height: 1px; - background: var(--divider); - } - - .addButton { - margin: 0 8px; - } -} - -.groupBy { - .line { - flex-grow: 1; - height: 1px; - background: var(--divider); - } - - .fieldSelector { - margin-top: 16px; - } - - .filterHeader { - display: flex; - align-items: center; - justify-content: space-between; - margin-top: 8px; - } - - .filterInfo { - vertical-align: text-top; - margin-left: 4px; - } -} - -.savePopover { - width: 300px; - - .popoverActions { - display: flex; - } - - .saveActions { - display: grid; - grid-template-columns: auto auto; - grid-gap: 8px; - padding-top: 16px; - } -} - -.menuPaper { - padding: 0 !important; -} - -.link { +.textLink { color: var(--primary); font-size: 12px; @@ -164,18 +22,31 @@ } } -.infoMessage { - padding: 16px; - border: 1px solid rgba(var(--primary-rgb), 0.5); - background: rgba(var(--primary-rgb), 0.1); - border-radius: 4px; - margin: 4px 0 16px; +.builder { + display: flex; + flex-direction: column; + height: 100%; + + .queryMetadata { + border-bottom: 1px solid var(--divider); + padding-bottom: 16px; + margin-bottom: 16px; + } } -.resultsLoading { - margin-left: 8px; +.config { + padding: 24px 32px 0; + flex-grow: 1; } -.savedQueries { - margin-top: 48px; +.builderActions { + padding: 16px 32px; + display: flex; + justify-content: space-between; + + .mainActions { + display: grid; + grid-template-columns: auto auto; + grid-gap: 8px; + } } diff --git a/lattice/src/App/QueryBuilder/QueryBuilder.tsx b/lattice/src/App/QueryBuilder/QueryBuilder.tsx index 08a9f02de..5ab545f1f 100644 --- a/lattice/src/App/QueryBuilder/QueryBuilder.tsx +++ b/lattice/src/App/QueryBuilder/QueryBuilder.tsx @@ -1,276 +1,139 @@ import React, { FC, Fragment, useEffect, useState } from 'react'; -import AddIcon from '@material-ui/icons/Add'; -import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos'; import ArrowBackIcon from '@material-ui/icons/ArrowBack'; import Button from '@material-ui/core/Button'; import CircularProgress from '@material-ui/core/CircularProgress'; -import CloseIcon from '@material-ui/icons/Close'; -import IconButton from '@material-ui/core/IconButton'; -import InfoIcon from '@material-ui/icons/Info'; -import Menu from '@material-ui/core/Menu'; -import MenuItem from '@material-ui/core/MenuItem'; +import Divider from '@material-ui/core/Divider'; import Popover from '@material-ui/core/Popover'; -import Split from 'react-split'; import TextField from '@material-ui/core/TextField'; -import Tooltip from '@material-ui/core/Tooltip'; import Typography from '@material-ui/core/Typography'; -import { Block } from 'shared/Block'; -import { ColumnSelector } from './ColumnSelector'; -import { formatDuration } from 'shared/utils/formatDuration'; import { - Operator, - groupOperators, - RowCallType, - RowGrouping, - RowsCallType -} from './rowTypes'; -import { getIPRange } from 'get-ip-range'; -import { QueryResults } from 'App/Query/QueryResults'; -import { ResultType } from 'App/Query/QueryContainer'; -import { RowCall } from './RowCall'; -import { SavedQueries } from './SavedQueries'; + cleanupRows, + stringifyCount, + stringifyExtract, + stringifyGroupBy +} from './utils'; +import { CountBuilder } from './CountBuilder'; +import { ExtractBuilder } from './ExtractBuilder'; +import { GroupByBuilder } from './GroupByBuilder'; import { Select } from 'shared/Select'; -import { GroupBySort, SortOption } from './GroupBySort'; -import { stringifyRowData } from './stringifyRowData'; import css from './QueryBuilder.module.scss'; type QueryBuilderProps = { tables: any[]; - results?: ResultType; - fullResultsCount?: number; - fullRecordsCount?: number; - error?: ResultType; - loading: boolean; - onQuery: ( - table: any, + savedQuery: number; + onRun: ( + table: string, operation: string, - rowData: RowGrouping[], - columns: string[], - operator?: Operator + query: string, + countQuery?: string ) => void; - onRunGroupBy: ( - table: any, - rowsData: RowsCallType, - sort: SortOption[], - filter?: string - ) => void; - onExternalLookup: (table: string, columns: number[]) => void; onClear: () => void; + onExitEdit: () => void; + onSaveQuery: () => void; }; export const QueryBuilder: FC = ({ tables, - results, - fullResultsCount, - fullRecordsCount, - error, - loading, - onQuery, - onRunGroupBy, - onExternalLookup, - onClear + savedQuery, + onRun, + onClear, + onExitEdit, + onSaveQuery }) => { - const [newOperatorEl, setNewOperatorEl] = useState(null); - const [operatorEl, setOperatorEl] = useState(null); const [saveButtonEl, setSaveButtonEl] = useState(null); - const [showBuilder, setShowBuilder] = useState(true); - const [showColumnSelector, setShowColumnSelector] = useState(false); - const [hasInvalid, setHasInvalid] = useState(false); + const queriesList = JSON.parse(localStorage.getItem('saved-queries') || '[]'); + const [editMode, setEditMode] = useState(false); const [selectedTable, setSelectedTable] = useState(tables[0]); - const [selectedColumns, setSelectedColumns] = useState< - { name: string; show: boolean }[] - >( - tables[0].fields.map((field) => ({ - name: field.name, - show: true - })) - ); const [operation, setOperation] = useState('Extract'); - const [groupByCall, setGroupByCall] = useState({ - primary: '', - secondary: '' + const [query, setQuery] = useState({ + table: tables[0].name, + operation: 'Extract', + columns: tables[0].fields.map((field) => field.name) }); - const [rowCalls, setRowCalls] = useState([]); - const [operator, setOperator] = useState(); - const [editSavedIdx, setEditSavedIdx] = useState(-1); - const [queryName, setQueryName] = useState(''); - const [queryDescription, setQueryDescription] = useState(''); + const [showInvalid, setShowInvalid] = useState(false); const [updating, setUpdating] = useState(false); - const [queriesList, setQueriesList] = useState( - JSON.parse(localStorage.getItem('saved-queries') || '[]') - ); - const [groupByFilters, setGroupByFilters] = useState([]); - const [filter, setFilter] = useState(); - const [sort, setSort] = useState([]); - const colSizes = JSON.parse( - localStorage.getItem('builderColSizes') || '[25, 75]' - ); - const queryNameIdx = queriesList.findIndex((q) => q.name === queryName); - const inEditMode = editSavedIdx >= 0; + const [queryNameIdx, setQueryNameIdx] = useState(-1); useEffect(() => { - setGroupByFilters( - queriesList.filter( - (q) => - q.table === selectedTable.name && - q.operation === 'Extract' && - q.rowCalls.length > 0 - ) - ); - }, [selectedTable, queriesList]); + if (savedQuery > -1) { + const updatedList = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ); + setQuery(updatedList[savedQuery]); + setEditMode(true); + setOperation(updatedList[savedQuery].operation); + const table = tables.find( + (table) => table.name === updatedList[savedQuery].table + ); + setSelectedTable(table); + } + }, [savedQuery, tables]); - const getColumnsToShow = (cols?: { name: string; show: boolean }[]) => { - let columns: string[] = []; - const list = cols ? cols : selectedColumns; - list.forEach((field) => { - if (field.show) { - columns.push(field.name); - } - }); + useEffect(() => { + if (query.name) { + setQueryNameIdx(queriesList.findIndex((q) => q.name === query.name)); + } + }, [query, queriesList]); - return columns; + const cleanRows = () => { + const cleanRows = cleanupRows(query?.rowCalls); + return { + ...query, + rowCalls: cleanRows.cleanRowCalls, + isInvalid: cleanRows.isInvalid + }; }; - const onChangeColumns = (cols) => { - setSelectedColumns(cols); - if (results) { - const { cleanRowCalls, isInvalid } = cleanupRows(); + const runQuery = () => { + setShowInvalid(true); + let queryString = ''; + let countQuery = ''; - setHasInvalid(isInvalid); - setRowCalls(cleanRowCalls); - if (!isInvalid) { - let columns: string[] = []; - if (operation === 'Extract') { - columns = getColumnsToShow(cols); + switch (operation) { + case 'Extract': + const cleanExtractQuery = cleanRows(); + setQuery(cleanExtractQuery); + const extract = stringifyExtract(cleanExtractQuery); + if (!extract.error && !cleanExtractQuery.isInvalid) { + queryString = extract.queryString; + countQuery = extract.countQuery; } - onQuery(selectedTable, operation, cleanRowCalls, columns, operator); - } - } - }; - - const onNewGroup = () => { - const newRow: RowCallType[] = [ - { - field: '', - rowOperator: '=', - value: '', - type: 'set', - keys: true - } - ]; - - setRowCalls([...rowCalls, { row: newRow }]); - }; - - const onRemoveGroup = (groupIdx: number) => { - if (groupIdx === 0) { - setRowCalls(rowCalls.slice(1)); - } else { - const updatedRowCalls = [...rowCalls]; - updatedRowCalls.splice(groupIdx, 1); - setRowCalls(updatedRowCalls); - - if (updatedRowCalls.length < 1) { - setOperator(undefined); - } - } - }; - - const onUpdateRow = ( - groupIdx: number, - newRowData?: RowCallType[], - operator?: Operator, - isNot?: boolean - ) => { - const updatedRowCalls = [...rowCalls]; - if (newRowData) { - const clone: RowGrouping = { - ...rowCalls[groupIdx], - row: newRowData, - isNot - }; - if (newRowData.length <= 1) { - delete clone.operator; - } else if (operator) { - clone.operator = operator; - } - updatedRowCalls.splice(groupIdx, 1, clone); - } else { - updatedRowCalls.splice(groupIdx, 1); - } - setRowCalls(updatedRowCalls); - }; - - const cleanupRows = () => { - // remove empty groups and row calls - let isInvalid = false; - let cleanRowCalls: RowGrouping[] = []; - let cleanGroups: RowGrouping[] = []; - - rowCalls.forEach((group, groupIdx) => { - cleanGroups.push({ ...group, row: [] }); - group.row.forEach((row) => { - const { field, value, rowOperator } = row; - if (field || value) { - cleanGroups[groupIdx].row.push(row); - - if (!field || !value) { - isInvalid = true; - } else if (rowOperator === 'cidr') { - try { - getIPRange(value); - } catch (err) { - isInvalid = true; - } - } + break; + case 'GroupBy': + queryString = stringifyGroupBy(query); + break; + case 'Count': + const cleanCountQuery = cleanRows(); + setQuery(cleanCountQuery); + const count = stringifyCount(cleanCountQuery); + if (!count.error && !cleanCountQuery.isInvalid) { + queryString = count.queryString; } - }); - }); + break; + default: + break; + } - cleanGroups.forEach((group) => { - if (group.row.length > 0) { - cleanRowCalls.push(group); - } - }); - - return { cleanRowCalls, isInvalid }; + if (queryString && !query.isInvalid) { + setShowInvalid(false); + onRun(selectedTable.name, operation, queryString, countQuery); + } else { + setShowInvalid(true); + } }; - const onRunClick = () => { - if (operation === 'GroupBy') { - let isInvalid = false; - if (!groupByCall.primary) { - isInvalid = true; - } else if ( - sort.length > 0 && - sort[0].sortValue.includes('sum') && - !sort[0].field - ) { - isInvalid = true; - } else if ( - sort.length > 1 && - sort[1].sortValue.includes('sum') && - !sort[1].field - ) { - isInvalid = true; - } - setHasInvalid(isInvalid); - if (!isInvalid) { - onRunGroupBy(selectedTable, groupByCall, sort, filter); - } - } else { - const { cleanRowCalls, isInvalid } = cleanupRows(); - - setHasInvalid(isInvalid); - setRowCalls(cleanRowCalls); - if (!isInvalid) { - let columns: string[] = []; - if (operation === 'Extract') { - columns = getColumnsToShow(); - } - onQuery(selectedTable, operation, cleanRowCalls, columns, operator); - } - } + const reset = () => { + setQuery({ + table: selectedTable.name, + columns: selectedTable.fields.map(field => field.name), + operation, + isInvalid: false + }); + setShowInvalid(false); + setEditMode(false); + setQueryNameIdx(-1); + onExitEdit(); + onClear(); }; const onSave = () => { @@ -278,662 +141,248 @@ export const QueryBuilder: FC = ({ setTimeout(() => { setUpdating(false); }, 1500); - const { cleanRowCalls, isInvalid } = cleanupRows(); + const cleanQuery = cleanRows(); let queries = [...queriesList]; - const queryObj = { - name: queryName, - description: queryDescription, - table: selectedTable.name, - operation, - columns: operation === 'Extract' ? getColumnsToShow() : undefined, - operator, - rowCalls: cleanRowCalls, - groupByCall, - filter, - isInvalid - }; - if (inEditMode && queries.length > 0) { + if (editMode && queries.length > 0) { const clone = [...queries]; - clone.splice(editSavedIdx, 1, queryObj); + clone.splice(savedQuery, 1, cleanQuery); localStorage.setItem('saved-queries', JSON.stringify(clone)); - setQueriesList(clone); } else { - queries.push(queryObj); + queries.push(cleanQuery); localStorage.setItem('saved-queries', JSON.stringify(queries)); - setQueriesList(queries); - setEditSavedIdx(queries.length - 1); } + onSaveQuery(); }; - const onExportLogs = () => { - if (results) { - const columns = results.rows.map((row) => row[0].uint64val); - const table = results.index ? results.index : ''; - onExternalLookup(table, columns); + const renderBuilder = () => { + switch (operation) { + case 'Extract': + return ( + setQuery(q)} + /> + ); + case 'GroupBy': + return ( + setQuery(q)} + /> + ); + case 'Count': + return ( + setQuery(q)} + /> + ); + default: + return
Invalid Operation
; } }; - const onSavedQueryClick = (idx: number) => { - const { - name, - description, - table, - operation, - columns, - operator, - rowCalls, - groupByCall, - filter, - isInvalid - } = queriesList[idx]; - const tableDetails = tables.find((t) => t.name === table); - setEditSavedIdx(idx); - setSelectedTable(tableDetails); - if (columns) { - setSelectedColumns( - tableDetails.fields.map((field) => - columns.includes(field.name) - ? { name: field.name, show: true } - : { name: field.name, show: false } - ) - ); - } - setOperation(operation); - setOperator(operator); - setRowCalls(rowCalls); - setGroupByCall(groupByCall); - setQueryName(name); - setQueryDescription(description || ''); - setHasInvalid(!!isInvalid); - setFilter(filter); - - if (!isInvalid) { - if (operation === 'GroupBy') { - onRunGroupBy(tableDetails, groupByCall, sort, filter); - } else { - onQuery(tableDetails, operation, rowCalls, columns, operator); - } - } - }; - - const onRemoveQuery = (queryIdx: number) => { - let queries = [...queriesList]; - queries.splice(queryIdx, 1); - localStorage.setItem('saved-queries', JSON.stringify(queries)); - setQueriesList(queries); - }; - - const reset = () => { - setEditSavedIdx(-1); - setSelectedTable(tables[0]); - setSelectedColumns( - tables[0].fields.map((field) => ({ - name: field.name, - show: true - })) - ); - setOperation('Extract'); - setRowCalls([]); - setQueryName(''); - setQueryDescription(''); - setHasInvalid(false); - setGroupByCall({ - primary: '', - secondary: '' - }); - setSort([]); - setFilter(undefined); - onClear(); - }; - return ( - - localStorage.setItem('builderColSizes', JSON.stringify(sizes)) - } - gutter={(_index, direction) => { - const gutter = document.createElement('div'); - gutter.className = `gutter gutter-${direction}`; - const dragbars = document.createElement('div'); - dragbars.className = 'dragBar'; - gutter.appendChild(dragbars); - return gutter; - }} - style={{ display: 'flex' }} - className={!showBuilder ? 'hide-gutter' : undefined} - > - {showBuilder ? ( -
-
- setShowBuilder(false)} size="small"> - - +
+
+ {editMode ? ( + + + Back to new query + + ) : null} + + {editMode ? 'Edit Saved Query' : 'New Query'} + + + {editMode ? ( +
+ = 0 && queryNameIdx !== savedQuery) + } + helperText={ + queryNameIdx >= 0 && queryNameIdx !== savedQuery + ? 'Query name must be unique' + : '' + } + onChange={(event) => + setQuery({ ...query, name: event.target.value }) + } + margin="normal" + size="small" + required + fullWidth + /> + + + setQuery({ ...query, description: event.target.value }) + } + margin="normal" + size="small" + fullWidth + />
-
- {inEditMode ? ( - - - Back to new query - - ) : null} - - {inEditMode ? 'Edit Saved Query' : 'New Query'} - -
-
- {inEditMode ? ( -
- = 0 && queryNameIdx !== editSavedIdx) - } - helperText={ - queryNameIdx >= 0 && queryNameIdx !== editSavedIdx - ? 'Query name must be unique' - : '' - } - onChange={(event) => setQueryName(event.target.value)} - margin="normal" - size="small" - required - fullWidth - /> + ) : null} - setQueryDescription(event.target.value)} - margin="normal" - size="small" - fullWidth - /> -
- ) : null} -
- { + return { label: table.name, value: table.name }; + })} + onChange={(value) => { + const table = tables.find((t) => t.name === value); + setSelectedTable(table); + setShowInvalid(false); + setQuery({ + table: table.name, + columns: table.fields.map((field) => field.name), + operation: query.operation, + isInvalid: false + }); + }} + /> -
- - [ - 'set', - 'time', - 'mutex', - 'bool', - 'int', - 'timestamp' - ].includes(f.options.type) - ) - .map((field) => { - return { - label: `${field.name} (${field.options.type})`, - value: field.name - }; - })} - onChange={(value) => - setGroupByCall({ - ...groupByCall, - primary: value.toString() - }) - } - error={hasInvalid && !groupByCall.primary} - /> - { + setOperation(value); + setShowInvalid(false); + setQuery({ + table: selectedTable.name, + operation: value, + isInvalid: false + }); + }} + /> +
- +
{renderBuilder()}
+
+
+ + {editMode ? ( + + ) : ( + + + setSaveButtonEl(null)} + anchorOrigin={{ + vertical: 'top', + horizontal: 'left' + }} + transformOrigin={{ + vertical: 'bottom', + horizontal: 'left' + }} + > +
+ + Give your query a unique name and a short description to + help you identify it for future use. + + + = 0} + helperText={ + queryNameIdx >= 0 ? 'Query name must be unique' : '' + } + onChange={(event) => + setQuery({ ...query, name: event.target.value }) + } + margin="dense" + required + fullWidth + /> + + + setQuery({ ...query, description: event.target.value }) + } + margin="dense" + fullWidth + /> + +
+
+ + +
+
+
+ + + )} +
+ +
+
); }; diff --git a/lattice/src/App/QueryBuilder/QueryBuilderContainer.module.scss b/lattice/src/App/QueryBuilder/QueryBuilderContainer.module.scss index cd406d4e8..a5cd6c77d 100644 --- a/lattice/src/App/QueryBuilder/QueryBuilderContainer.module.scss +++ b/lattice/src/App/QueryBuilder/QueryBuilderContainer.module.scss @@ -1,3 +1,65 @@ +.builderColumn, +.results { + height: calc(100vh - 64px); + overflow: scroll; + position: relative; + + .openClose { + position: absolute; + right: 10px; + top: 10px; + } +} + +.split { + display: flex; +} + +.collapsedBuilder { + border-right: 1px solid rgba(var(--contrast-rgb), 0.1); + text-align: center; + padding-top: 10px; +} + +.resultsBlock { + height: 100%; +} + +.resultsHeader { + display: flex; + align-items: center; + justify-content: space-between; + + .download { + display: flex; + align-items: center; + } + + .downloadInfo { + margin-right: 8px; + } +} + +.infoMessage { + padding: 16px; + border: 1px solid rgba(var(--primary-rgb), 0.5); + background: rgba(var(--primary-rgb), 0.1); + border-radius: 4px; + margin: 4px 0 16px; +} + +.spacer{ + padding-bottom: 16px; +} + +.resultsLoading { + margin-left: 8px; +} + +.savedQueries { + margin-top: 48px; +} + .noTables { margin-top: 20px; padding: 12px 16px; diff --git a/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx b/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx index 40e42fb8c..ef9f9062a 100644 --- a/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx +++ b/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx @@ -1,20 +1,28 @@ import React, { Fragment, useState } from 'react'; -import moment, { Moment } from 'moment'; import Alert from '@material-ui/lab/Alert'; +import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos'; +import Button from '@material-ui/core/Button'; +import classNames from 'classnames'; +import CloseIcon from '@material-ui/icons/Close'; +import IconButton from '@material-ui/core/IconButton'; +import InfoIcon from '@material-ui/icons/Info'; +import moment, { Moment } from 'moment'; import Paper from '@material-ui/core/Paper'; +import Split from 'react-split'; import Snackbar from '@material-ui/core/Snackbar'; +import Tooltip from '@material-ui/core/Tooltip'; import Typography from '@material-ui/core/Typography'; import { Block } from 'shared/Block'; -import { Operator, RowGrouping, RowsCallType } from './rowTypes'; +import { formatDuration } from 'shared/utils/formatDuration'; +import { grpc } from '@improbable-eng/grpc-web'; import { pilosa } from 'services/eventServices'; import { QueryBuilder } from './QueryBuilder'; -import { useEffectOnce } from 'react-use'; -import { ResultType } from 'App/Query/QueryContainer'; import { queryPQL } from 'services/grpcServices'; -import { grpc } from '@improbable-eng/grpc-web'; +import { QueryResults } from 'App/Query/QueryResults'; +import { ResultType } from 'App/Query/QueryContainer'; import { RowResponse } from 'proto/pilosa_pb'; -import { SortOption } from './GroupBySort'; -import { stringifyRowData } from './stringifyRowData'; +import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; +import { useEffectOnce } from 'react-use'; import css from './QueryBuilderContainer.module.scss'; let streamingResults: ResultType = { @@ -30,13 +38,22 @@ let streamingResults: ResultType = { export const QueryBuilderContainer = () => { let startTime: Moment; let exportRows: any[] = []; - const [tables, setTables] = useState(); + const colSizes = JSON.parse( + localStorage.getItem('builderColSizes') || '[25, 75]' + ); + const [queriesList, setQueriesList] = useState( + JSON.parse(localStorage.getItem('saved-queries') || '[]') + ); + + const [tables, setTables] = useState([]); + const [showBuilder, setShowBuilder] = useState(true); const [results, setResults] = useState(); const [fullCount, setFullCount] = useState(); const [recordsCount, setRecordsCount] = useState(); const [errorResult, setErrorResult] = useState(); const [error, setError] = useState(''); const [loading, setLoading] = useState(false); + const [savedQuery, setSavedQuery] = useState(-1); useEffectOnce(() => { pilosa.get.schema().then((res) => { @@ -67,62 +84,6 @@ export const QueryBuilderContainer = () => { setLoading(false); }; - const onRunQuery = ( - table: any, - operation: string, - rowData: RowGrouping[], - columnsList: string[], - operator?: Operator - ) => { - streamingResults = { - query: '', - operation, - type: 'PQL', - headers: [], - rows: [], - index: table.name, - roundtrip: 0, - error: '' - }; - startTime = moment(); - setLoading(true); - let query: string = ''; - if (rowData.length === 0) { - query = 'All()'; - } else { - const res = stringifyRowData(rowData, operator); - if (res.error) { - streamingResults.error = res.query; - } else { - query = res.query; - } - } - - if (operation !== 'Count') { - const countQuery = `Count(${query})`; - pilosa.post - .query(table.name, countQuery) - .then((res) => setFullCount(res.data.results[0])); - - pilosa.post - .query(table.name, `Count(All())`) - .then((res) => setRecordsCount(res.data.results[0])); - } else { - setFullCount(undefined); - } - - if (operation === 'Extract') { - const fields = columnsList.map((field) => `Rows(${field})`); - const allRows = fields.join(', '); - query = `${operation}(Limit(${query}, limit=1000), ${allRows})`; - } else { - query = `${operation}(${query})`; - } - - streamingResults.query = query; - queryPQL(table.name, query, handleQueryMessages, handleQueryEnd); - }; - const handleExternalLookup = (message: RowResponse) => { const response = message.toObject(); let rowStr: string[] = []; @@ -158,46 +119,71 @@ export const QueryBuilderContainer = () => { } }; - const onRunGroupBy = ( - table: any, - rowsData: RowsCallType, - sort: SortOption[], - filter?: string + const onRunQuery = ( + table: string, + operation: string, + query: string, + countQuery?: string ) => { streamingResults = { - query: '', - operation: 'GroupBy', + query, + operation, type: 'PQL', headers: [], rows: [], - index: table.name, + index: table, roundtrip: 0, error: '' }; startTime = moment(); setLoading(true); - let sortString = ''; - let aggregateString = ''; - if (sort.length > 0) { - sortString = sort[0].sortValue; - if (sort[0].sortValue.includes('sum')) { - aggregateString = `, aggregate=Sum(field=${sort[0].field})`; + + if (operation !== 'Count') { + if (countQuery) { + pilosa.post.query(table, countQuery).then((res) => { + setFullCount(res.data.results[0]); + }); } - if (sort.length > 1) { - sortString = `${sortString}, ${sort[1].sortValue}`; - if (sort[1].sortValue.includes('sum')) { - aggregateString = `, aggregate=Sum(field=${sort[1].field})`; - } - } - sortString = `, sort="${sortString}"`; + + pilosa.post + .query(table, `Count(All())`) + .then((res) => setRecordsCount(res.data.results[0])); + } else { + setFullCount(undefined); + } + + queryPQL(table, query, handleQueryMessages, handleQueryEnd); + }; + + const onClear = () => { + setResults(undefined); + setFullCount(undefined); + setErrorResult(undefined); + }; + + const onRemoveQuery = (queryIdx: number) => { + let queries = [...queriesList]; + queries.splice(queryIdx, 1); + localStorage.setItem('saved-queries', JSON.stringify(queries)); + setQueriesList(queries); + }; + + const onSaveQuery = () => { + const updatedList = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ); + if (savedQuery < 0) { + setSavedQuery(updatedList.length - 1); + } + setQueriesList(updatedList); + }; + + const onExportLogs = () => { + if (results) { + const columns = results.rows.map((row) => row[0].uint64val); + const table = results.index ? results.index : ''; + onExternalLookup(table, columns); } - const filterString = - filter && filter.length > 0 ? `, filter=${filter}` : ''; - const query = rowsData.secondary - ? `GroupBy(Rows(${rowsData.primary}), Rows(${rowsData.secondary})${filterString}${sortString}${aggregateString})` - : `GroupBy(Rows(${rowsData.primary})${filterString}${sortString}${aggregateString})`; - streamingResults.query = query; - queryPQL(table.name, query, handleQueryMessages, handleQueryEnd); }; const onExternalLookup = (table: string, columns: number[]) => { @@ -206,40 +192,132 @@ export const QueryBuilderContainer = () => { queryPQL(table, query, handleExternalLookup, handleExternalLookupEnd); }; - return tables && tables.length > 0 ? ( + return ( - { - setResults(undefined); - setFullCount(undefined); - setErrorResult(undefined); - }} - /> - - setError('')}> - {error} - - + {tables.length > 0 ? ( + + localStorage.setItem('builderColSizes', JSON.stringify(sizes)) + } + gutter={(_index, direction) => { + const gutter = document.createElement('div'); + gutter.className = `gutter gutter-${direction}`; + const dragbars = document.createElement('div'); + dragbars.className = 'dragBar'; + gutter.appendChild(dragbars); + return gutter; + }} + className={classNames(css.split, !showBuilder ? 'hide-gutter' : '')} + > + {showBuilder ? ( +
+
+ setShowBuilder(false)} size="small"> + + +
+ setSavedQuery(-1)} + onSaveQuery={onSaveQuery} + /> +
+ ) : ( +
+ setShowBuilder(true)} size="small"> + + +
+ )} +
+ +
+ + Results{' '} + + {results?.query.includes('Extract(') ? ( +
+ + + + +
+ ) : null} +
+ {loading ?
Loading...
: null} + {results && !loading ? ( + + {fullCount && recordsCount ? ( +
+ {results.duration ? ( +
+ {recordsCount.toLocaleString()} records scanned in{' '} + {formatDuration(results.duration, true)}. +
+ ) : null} +
+ Showing{' '} + {fullCount > 1000 ? 'first 1,000 rows of' : 'all'}{' '} + {fullCount.toLocaleString()} results. +
+
+ ) : null} + +
+ + ) : null} + {errorResult && !loading ?
{errorResult.error}
: null} + {!loading && !results && !error ? ( + + + Build a query to see results + + {queriesList.length > 0 ? ( +
+ + Saved Queries + + setSavedQuery(queryIdx)} + onRemoveQuery={onRemoveQuery} + /> +
+ ) : null} + + ) : null} + +
+ + setError('')}> + {error} + + + + ) : ( + + + Query Builder + + + + There are no tables to query. + + + + )} - ) : ( - - - Query Builder - - - - There are no tables to query. - - - ); }; diff --git a/lattice/src/App/QueryBuilder/RowCall/helpers.ts b/lattice/src/App/QueryBuilder/RowCall/helpers.ts deleted file mode 100644 index ebbd97470..000000000 --- a/lattice/src/App/QueryBuilder/RowCall/helpers.ts +++ /dev/null @@ -1,52 +0,0 @@ -export const operators = { - decimal: [ - { label: '>', value: '>' }, - { label: '<', value: '<' }, - { label: '>=', value: '>=' }, - { label: '<=', value: '<=' }, - { label: '==', value: '=' }, - { label: '!=', value: '!=' }, - ], - int: [ - { label: '>', value: '>' }, - { label: '<', value: '<' }, - { label: '>=', value: '>=' }, - { label: '<=', value: '<=' }, - { label: '==', value: '=' }, - { label: '!=', value: '!=' }, - ], - 'mutex-id': [ - { label: 'is', value: '=' }, - { label: 'is not', value: '!=' } - ], - 'mutex-keys': [ - { label: 'is', value: '=' }, - { label: 'is not', value: '!=' }, - { label: 'like', value: 'like' }, - { label: 'CIDR', value: 'cidr' } - ], - "set-id": [ - { label: 'is', value: '=' }, - { label: 'is not', value: '!=' } - ], - "set-keys": [ - { label: 'is', value: '=' }, - { label: 'is not', value: '!=' }, - { label: 'like', value: 'like' }, - { label: 'CIDR', value: 'cidr' } - ], - "time-id": [ - { label: 'is', value: '=' }, - { label: 'is not', value: '!=' } - ], - "time-keys": [ - { label: 'is', value: '=' }, - { label: 'is not', value: '!=' }, - { label: 'like', value: 'like' } - ], - timestamp: [ - { label: 'is before', value: '<' }, - { label: 'is after', value: '>' }, - { label: 'is', value: '=' }, - ] -} diff --git a/lattice/src/App/QueryBuilder/RowCall/RowCall.module.scss b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/RowCall.module.scss similarity index 100% rename from lattice/src/App/QueryBuilder/RowCall/RowCall.module.scss rename to lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/RowCall.module.scss diff --git a/lattice/src/App/QueryBuilder/RowCall/RowCall.tsx b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/RowCall.tsx similarity index 99% rename from lattice/src/App/QueryBuilder/RowCall/RowCall.tsx rename to lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/RowCall.tsx index cd0501060..a5f35f971 100644 --- a/lattice/src/App/QueryBuilder/RowCall/RowCall.tsx +++ b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/RowCall.tsx @@ -10,7 +10,7 @@ import TextField from '@material-ui/core/TextField'; import Typography from '@material-ui/core/Typography'; import { getIPRange } from 'get-ip-range'; import { operators } from './helpers'; -import { groupOperators, Operator, RowCallType } from '../rowTypes'; +import { groupOperators, Operator, RowCallType } from 'App/QueryBuilder/rowTypes'; import { Select } from 'shared/Select'; import MomentUtils from '@date-io/moment'; import { MuiPickersUtilsProvider, DateTimePicker } from '@material-ui/pickers'; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/helpers.ts similarity index 100% rename from lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts rename to lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/helpers.ts diff --git a/lattice/src/App/QueryBuilder/RowCall/index.ts b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/index.ts similarity index 100% rename from lattice/src/App/QueryBuilder/RowCall/index.ts rename to lattice/src/App/QueryBuilder/RowCallBuilder/RowCall/index.ts diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.module.scss similarity index 100% rename from lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss rename to lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.module.scss diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx similarity index 88% rename from lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx rename to lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx index 48f8e81ea..ba8a1ad8f 100644 --- a/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx +++ b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx @@ -10,18 +10,15 @@ import { RowCallType, RowGrouping } from 'App/QueryBuilder/rowTypes'; -import { RowCall } from 'App/QueryBuilder/RowCall'; +import { RowCall } from './RowCall'; import css from './RowCallBuilder.module.scss'; +import { useEffect } from 'react'; type RowCallBuilderProps = { rowCalls: RowGrouping[]; fields: any[]; showInvalid: boolean; - onChange: ( - rows: RowGrouping[], - operator?: Operator, - hasInvalid?: boolean - ) => void; + onChange: (rows: RowGrouping[], operator?: Operator) => void; }; export const RowCallBuilder: FC = ({ @@ -34,7 +31,7 @@ export const RowCallBuilder: FC = ({ const [operatorEl, setOperatorEl] = useState(null); const [operator, setOperator] = useState(); - const onNewGroup = () => { + const onNewGroup = (op?: Operator) => { const newRow: RowCallType[] = [ { field: '', @@ -45,19 +42,20 @@ export const RowCallBuilder: FC = ({ } ]; - onChange([...rowCalls, { row: newRow }]); + onChange([...rowCalls, { row: newRow }], op ? op : operator); }; const onRemoveGroup = (groupIdx: number) => { if (groupIdx === 0) { - onChange(rowCalls.slice(1)); + onChange(rowCalls.slice(1), operator); } else { const updatedRowCalls = [...rowCalls]; updatedRowCalls.splice(groupIdx, 1); - onChange(updatedRowCalls); + onChange(updatedRowCalls, operator); if (updatedRowCalls.length < 1) { setOperator(undefined); + onChange(updatedRowCalls); } } }; @@ -65,7 +63,7 @@ export const RowCallBuilder: FC = ({ const onUpdateRow = ( groupIdx: number, newRowData?: RowCallType[], - operator?: Operator, + rowGroupOperator?: Operator, isNot?: boolean ) => { const updatedRowCalls = [...rowCalls]; @@ -77,14 +75,14 @@ export const RowCallBuilder: FC = ({ }; if (newRowData.length <= 1) { delete clone.operator; - } else if (operator) { - clone.operator = operator; + } else if (rowGroupOperator) { + clone.operator = rowGroupOperator; } updatedRowCalls.splice(groupIdx, 1, clone); } else { updatedRowCalls.splice(groupIdx, 1); } - onChange(updatedRowCalls); + onChange(updatedRowCalls, operator); }; return ( @@ -167,7 +165,7 @@ export const RowCallBuilder: FC = ({ key={`operator-${op}`} onClick={() => { setOperator(op); - onNewGroup(); + onNewGroup(op); setNewOperatorEl(null); }} > diff --git a/lattice/src/App/QBuilder/RowCallBuilder/index.ts b/lattice/src/App/QueryBuilder/RowCallBuilder/index.ts similarity index 100% rename from lattice/src/App/QBuilder/RowCallBuilder/index.ts rename to lattice/src/App/QueryBuilder/RowCallBuilder/index.ts diff --git a/lattice/src/App/QueryBuilder/stringifyRowData.ts b/lattice/src/App/QueryBuilder/stringifyRowData.ts deleted file mode 100644 index 0faa642c5..000000000 --- a/lattice/src/App/QueryBuilder/stringifyRowData.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { Operator, RowGrouping } from './rowTypes'; -import { getIPRange } from 'get-ip-range'; - -export const stringifyRowData = (rowData: RowGrouping[], operator?: Operator) => { - let query = ''; - let rowsMap: string[][] = []; - rowData.forEach((group, groupIdx) => { - rowsMap.push([]); - group.row.forEach((row) => { - let rowString = ''; - const { field, rowOperator, value, type, keys } = row; - const isNegatory = rowOperator === '!='; - const isUnion = - ['=', '!='].includes(rowOperator) && value.split(',').length > 1; - const operator = isNegatory ? '=' : rowOperator; - if (isUnion) { - const values = value.split(','); - const unionRows = values - .map((v) => keys ? `Row(${field}="${v.trim()}")` : `Row(${field}=${v.trim()})`) - .join(', '); - rowString = `Union(${unionRows})`; - } else if (rowOperator === 'cidr') { - try { - const ipRange = getIPRange(value); - const ipRows = ipRange - .map((ip) => `Row(${field}="${ip}")`) - .join(', '); - rowString = `Union(${ipRows})`; - } catch (error) { - return { error: true, query: error.message }; - } - } else if (rowOperator === 'like') { - if (value.includes('%') || value.includes('_')) { - rowString = `UnionRows(Rows(field=${field}, like="${value}"))`; - } else { - rowString = `UnionRows(Rows(field=${field}, like="%${value}%"))`; - } - } else { - rowString = keys || type === 'timestamp' - ? `Row(${field}${operator}"${value}")` - : `Row(${field}${operator}${value})`; - } - - if (isNegatory) { - rowsMap[groupIdx].push(`Not(${rowString})`); - } else { - rowsMap[groupIdx].push(rowString); - } - }); - }); - - query = rowsMap - .map((group, idx) => { - let joined = ''; - if (group.length > 1) { - joined = group.map((r) => r).join(', '); - const operator = rowData[idx].operator; - if (operator === 'and') { - joined = `Intersect(${joined})`; - } else if (operator === 'or') { - joined = `Union(${joined})`; - } - } else { - joined = group[0]; - } - return rowData[idx].isNot ? `Not(${joined})` : joined; - }) - .join(', '); - - if (rowData.length > 1 && operator) { - if (operator === 'and') { - query = `Intersect(${query})`; - } else if (operator === 'or') { - query = `Union(${query})`; - } - } - - return { error: false, query }; -} diff --git a/lattice/src/App/QueryBuilder/stringifyRowData.test.ts b/lattice/src/App/QueryBuilder/utils.test.ts similarity index 61% rename from lattice/src/App/QueryBuilder/stringifyRowData.test.ts rename to lattice/src/App/QueryBuilder/utils.test.ts index 22f439e82..d34e15ef9 100644 --- a/lattice/src/App/QueryBuilder/stringifyRowData.test.ts +++ b/lattice/src/App/QueryBuilder/utils.test.ts @@ -1,4 +1,9 @@ -import { stringifyRowData } from './stringifyRowData'; +import { + stringifyRowData, + stringifyExtract, + stringifyCount, + stringifyGroupBy +} from './utils'; describe('decimal types', () => { it('stringifies = operator', () => { @@ -10,7 +15,7 @@ describe('decimal types', () => { "type": "decimal", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName=0)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName=0)' }) }); it('stringifies != operator', () => { @@ -22,7 +27,7 @@ describe('decimal types', () => { "type": "decimal", "keys": undefined }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName=0))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName=0))' }) }); it('stringifies > operator', () => { @@ -34,7 +39,7 @@ describe('decimal types', () => { "type": "decimal", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName>0)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName>0)' }) }); it('stringifies >= operator', () => { @@ -46,7 +51,7 @@ describe('decimal types', () => { "type": "decimal", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName>=0)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName>=0)' }) }); it('stringifies negated operations', () => { @@ -59,7 +64,7 @@ describe('decimal types', () => { "keys": undefined }], isNot: true - }])).toEqual({ error: false, query: 'Not(Row(fieldName>=0))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName>=0))' }) }); }); @@ -73,7 +78,7 @@ describe('int types', () => { "type": "int", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName=0)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName=0)' }) }); it('stringifies != operator', () => { @@ -85,7 +90,7 @@ describe('int types', () => { "type": "int", "keys": undefined }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName=0))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName=0))' }) }); it('stringifies > operator', () => { @@ -97,7 +102,7 @@ describe('int types', () => { "type": "int", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName>0)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName>0)' }) }); it('stringifies >= operator', () => { @@ -109,7 +114,7 @@ describe('int types', () => { "type": "int", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName>=0)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName>=0)' }) }); it('stringifies negated operations', () => { @@ -122,7 +127,7 @@ describe('int types', () => { "keys": undefined }], isNot: true - }])).toEqual({ error: false, query: 'Not(Row(fieldName>=0))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName>=0))' }) }); }); @@ -136,7 +141,7 @@ describe('mutex (id) types', () => { "type": "mutex", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName=1234)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName=1234)' }) }); it('stringifies is operator for multiple values', () => { @@ -148,7 +153,7 @@ describe('mutex (id) types', () => { "type": "mutex", "keys": undefined }] - }])).toEqual({ error: false, query: 'Union(Row(fieldName=1234), Row(fieldName=5678))' }) + }])).toEqual({ error: false, queryString: 'Union(Row(fieldName=1234), Row(fieldName=5678))' }) }); it('stringifies is not operator', () => { @@ -160,7 +165,7 @@ describe('mutex (id) types', () => { "type": "mutex", "keys": undefined }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName=1234))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName=1234))' }) }); it('stringifies is not operator for multiple values', () => { @@ -172,7 +177,7 @@ describe('mutex (id) types', () => { "type": "mutex", "keys": undefined }] - }])).toEqual({ error: false, query: 'Not(Union(Row(fieldName=1234), Row(fieldName=5678)))' }) + }])).toEqual({ error: false, queryString: 'Not(Union(Row(fieldName=1234), Row(fieldName=5678)))' }) }); }); @@ -186,7 +191,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'Row(fieldName="value")' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName="value")' }) }); it('stringifies is operator for multiple values', () => { @@ -198,7 +203,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'Union(Row(fieldName="one"), Row(fieldName="two"))' }) + }])).toEqual({ error: false, queryString: 'Union(Row(fieldName="one"), Row(fieldName="two"))' }) }); it('stringifies is not operator', () => { @@ -210,7 +215,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName="value"))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName="value"))' }) }); it('stringifies is not operator for multiple values', () => { @@ -222,7 +227,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'Not(Union(Row(fieldName="one"), Row(fieldName="two")))' }) + }])).toEqual({ error: false, queryString: 'Not(Union(Row(fieldName="one"), Row(fieldName="two")))' }) }); it('stringifies like operator with a single-codepoint key', () => { @@ -234,7 +239,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="_alue"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="_alue"))' }) }); it('stringifies like operator with a wildcard', () => { @@ -246,7 +251,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="value%"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="value%"))' }) }); it('stringifies like operator without a single-codepoint key or wildcard', () => { @@ -258,7 +263,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="%value%"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="%value%"))' }) }); it('stringifies CIDR operator', () => { @@ -270,7 +275,7 @@ describe('mutex (keys) types', () => { "type": "mutex", "keys": true }] - }])).toEqual({ error: false, query: 'Union(Row(fieldName="10.164.124.32"), Row(fieldName="10.164.124.33"), Row(fieldName="10.164.124.34"), Row(fieldName="10.164.124.35"))' }) + }])).toEqual({ error: false, queryString: 'Union(Row(fieldName="10.164.124.32"), Row(fieldName="10.164.124.33"), Row(fieldName="10.164.124.34"), Row(fieldName="10.164.124.35"))' }) }); it('stringifies negated operations', () => { @@ -283,7 +288,7 @@ describe('mutex (keys) types', () => { "keys": true }], isNot: true - }])).toEqual({ error: false, query: 'Not(UnionRows(Rows(field=fieldName, like="%value%")))' }) + }])).toEqual({ error: false, queryString: 'Not(UnionRows(Rows(field=fieldName, like="%value%")))' }) }); }); @@ -297,7 +302,7 @@ describe('set (id) types', () => { "type": "set", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName=1234)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName=1234)' }) }); it('stringifies is operator for multiple values', () => { @@ -309,7 +314,7 @@ describe('set (id) types', () => { "type": "set", "keys": undefined }] - }])).toEqual({ error: false, query: 'Union(Row(fieldName=1234), Row(fieldName=5678))' }) + }])).toEqual({ error: false, queryString: 'Union(Row(fieldName=1234), Row(fieldName=5678))' }) }); it('stringifies is not operator', () => { @@ -321,7 +326,7 @@ describe('set (id) types', () => { "type": "set", "keys": undefined }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName=1234))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName=1234))' }) }); it('stringifies is not operator for multiple values', () => { @@ -333,7 +338,7 @@ describe('set (id) types', () => { "type": "set", "keys": undefined }] - }])).toEqual({ error: false, query: 'Not(Union(Row(fieldName=1234), Row(fieldName=5678)))' }) + }])).toEqual({ error: false, queryString: 'Not(Union(Row(fieldName=1234), Row(fieldName=5678)))' }) }); }); @@ -347,7 +352,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'Row(fieldName="value")' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName="value")' }) }); it('stringifies is operator for multiple values', () => { @@ -359,7 +364,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'Union(Row(fieldName="one"), Row(fieldName="two"))' }) + }])).toEqual({ error: false, queryString: 'Union(Row(fieldName="one"), Row(fieldName="two"))' }) }); it('stringifies is not operator', () => { @@ -371,7 +376,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName="value"))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName="value"))' }) }); it('stringifies is not operator for multiple values', () => { @@ -383,7 +388,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'Not(Union(Row(fieldName="one"), Row(fieldName="two")))' }) + }])).toEqual({ error: false, queryString: 'Not(Union(Row(fieldName="one"), Row(fieldName="two")))' }) }); it('stringifies like operator with a single-codepoint key', () => { @@ -395,7 +400,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="_alue"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="_alue"))' }) }); it('stringifies like operator with a wildcard', () => { @@ -407,7 +412,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="value%"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="value%"))' }) }); it('stringifies like operator without a single-codepoint key or wildcard', () => { @@ -419,7 +424,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="%value%"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="%value%"))' }) }); it('stringifies CIDR operator', () => { @@ -431,7 +436,7 @@ describe('set (keys) types', () => { "type": "set", "keys": true }] - }])).toEqual({ error: false, query: 'Union(Row(fieldName="10.164.124.32"), Row(fieldName="10.164.124.33"), Row(fieldName="10.164.124.34"), Row(fieldName="10.164.124.35"))' }) + }])).toEqual({ error: false, queryString: 'Union(Row(fieldName="10.164.124.32"), Row(fieldName="10.164.124.33"), Row(fieldName="10.164.124.34"), Row(fieldName="10.164.124.35"))' }) }); it('stringifies negated operations', () => { @@ -444,7 +449,7 @@ describe('set (keys) types', () => { "keys": true }], isNot: true - }])).toEqual({ error: false, query: 'Not(UnionRows(Rows(field=fieldName, like="%value%")))' }) + }])).toEqual({ error: false, queryString: 'Not(UnionRows(Rows(field=fieldName, like="%value%")))' }) }); }); @@ -458,7 +463,7 @@ describe('time (ID) types', () => { "type": "time", "keys": undefined }] - }])).toEqual({ error: false, query: 'Row(fieldName=1234)' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName=1234)' }) }); it('stringifies is not operators', () => { @@ -470,7 +475,7 @@ describe('time (ID) types', () => { "type": "time", "keys": undefined }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName=1234))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName=1234))' }) }); it('stringifies negated operators', () => { @@ -483,7 +488,7 @@ describe('time (ID) types', () => { "keys": undefined }], isNot: true - }])).toEqual({ error: false, query: 'Not(Row(fieldName=1234))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName=1234))' }) }); }); @@ -497,7 +502,7 @@ describe('time (keys) types', () => { "type": "time", "keys": true }] - }])).toEqual({ error: false, query: 'Row(fieldName="1234")' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName="1234")' }) }); it('stringifies is not operators', () => { @@ -509,7 +514,7 @@ describe('time (keys) types', () => { "type": "time", "keys": true }] - }])).toEqual({ error: false, query: 'Not(Row(fieldName="1234"))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName="1234"))' }) }); it('stringifies like operators with a single-codepoint key', () => { @@ -521,7 +526,7 @@ describe('time (keys) types', () => { "type": "time", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="123_"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="123_"))' }) }); it('stringifies like operators with a wildcard', () => { @@ -533,7 +538,7 @@ describe('time (keys) types', () => { "type": "time", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="%1234"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="%1234"))' }) }); it('stringifies like operators without a single-codepoint key or wildcard', () => { @@ -545,7 +550,7 @@ describe('time (keys) types', () => { "type": "time", "keys": true }] - }])).toEqual({ error: false, query: 'UnionRows(Rows(field=fieldName, like="%1234%"))' }) + }])).toEqual({ error: false, queryString: 'UnionRows(Rows(field=fieldName, like="%1234%"))' }) }); it('stringifies negated operators', () => { @@ -558,7 +563,7 @@ describe('time (keys) types', () => { "keys": true }], isNot: true - }])).toEqual({ error: false, query: 'Not(Row(fieldName="1234"))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName="1234"))' }) }); }); @@ -572,7 +577,7 @@ describe('timestamp types', () => { "type": "timestamp", "keys": undefined }], - }])).toEqual({ error: false, query: 'Row(fieldName="2021-06-04T12:00:00Z")' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName="2021-06-04T12:00:00Z")' }) }); it('stringifies is before operators', () => { @@ -584,7 +589,7 @@ describe('timestamp types', () => { "type": "timestamp", "keys": undefined }], - }])).toEqual({ error: false, query: 'Row(fieldName<"2021-06-04T12:00:00Z")' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName<"2021-06-04T12:00:00Z")' }) }); it('stringifies is after operators', () => { @@ -596,7 +601,7 @@ describe('timestamp types', () => { "type": "timestamp", "keys": undefined }], - }])).toEqual({ error: false, query: 'Row(fieldName>"2021-06-04T12:00:00Z")' }) + }])).toEqual({ error: false, queryString: 'Row(fieldName>"2021-06-04T12:00:00Z")' }) }); it('stringifies negated operators', () => { @@ -609,7 +614,7 @@ describe('timestamp types', () => { "keys": undefined }], isNot: true - }])).toEqual({ error: false, query: 'Not(Row(fieldName="2021-06-04T12:00:00Z"))' }) + }])).toEqual({ error: false, queryString: 'Not(Row(fieldName="2021-06-04T12:00:00Z"))' }) }); }); @@ -630,7 +635,7 @@ describe('mixed types', () => { "keys": undefined }], operator: "and" - }])).toEqual({ error: false, query: 'Intersect(UnionRows(Rows(field=stringType, like="%value%")), Row(intType=0))' }) + }])).toEqual({ error: false, queryString: 'Intersect(UnionRows(Rows(field=stringType, like="%value%")), Row(intType=0))' }) }); it('stringifies multiple or-ed rows', () => { @@ -649,7 +654,7 @@ describe('mixed types', () => { "keys": undefined }], operator: "or" - }])).toEqual({ error: false, query: 'Union(UnionRows(Rows(field=stringType, like="%value%")), Row(intType=0))' }) + }])).toEqual({ error: false, queryString: 'Union(UnionRows(Rows(field=stringType, like="%value%")), Row(intType=0))' }) }); it('stringifies negated multiple rows', () => { @@ -669,6 +674,129 @@ describe('mixed types', () => { }], operator: "or", isNot: true - }])).toEqual({ error: false, query: 'Not(Union(UnionRows(Rows(field=stringType, like="%value%")), Row(intType=0)))' }) + }])).toEqual({ error: false, queryString: 'Not(Union(UnionRows(Rows(field=stringType, like="%value%")), Row(intType=0)))' }) }); }) + +describe('Query types', () => { + it('stringifies Extract queries', () => { + expect(stringifyExtract({ + columns: ['one', 'two', 'three'], + rowCalls: [{ + row: [{ + "field": "stringType", + "rowOperator": "like", + "value": "value", + "type": "set", + "keys": true + }, { + "field": "stringType2", + "rowOperator": "is", + "value": "value2", + "type": "set", + "keys": true + }], + operator: 'or' + }, { + row: [{ + "field": "intType", + "rowOperator": "=", + "value": "0", + "type": "int", + "keys": undefined + }] + }], + operator: "and" + })).toEqual({ error: false, queryString: 'Extract(Limit(Intersect(Union(UnionRows(Rows(field=stringType, like=\"%value%\")), Row(stringType2is\"value2\")), Row(intType=0)), limit=1000), Rows(one), Rows(two), Rows(three))', countQuery: 'Count(Intersect(Union(UnionRows(Rows(field=stringType, like=\"%value%\")), Row(stringType2is\"value2\")), Row(intType=0)))' }) + }); + + it('stringifies Count queries', () => { + expect(stringifyCount({ + columns: ['one', 'two', 'three'], + rowCalls: [{ + row: [{ + "field": "stringType", + "rowOperator": "like", + "value": "value", + "type": "set", + "keys": true + }, { + "field": "stringType2", + "rowOperator": "is", + "value": "value2", + "type": "set", + "keys": true + }], + operator: 'or' + }, { + row: [{ + "field": "intType", + "rowOperator": "=", + "value": "0", + "type": "int", + "keys": undefined + }] + }], + operator: "and" + })).toEqual({ error: false, queryString: 'Count(Intersect(Union(UnionRows(Rows(field=stringType, like=\"%value%\")), Row(stringType2is\"value2\")), Row(intType=0)))' }) + }); + + it('stringifies basic GroupBy queries', () => { + expect(stringifyGroupBy({ + groupByCall: { + primary: 'primary' + } + })).toEqual('GroupBy(Rows(primary))'); + }); + + it('stringifies GroupBy queries with secondary grouping', () => { + expect(stringifyGroupBy({ + groupByCall: { + primary: 'primary', + secondary: 'secondary' + } + })).toEqual('GroupBy(Rows(primary), Rows(secondary))'); + }); + + it('stringifies GroupBy queries with a filter', () => { + expect(stringifyGroupBy({ + groupByCall: { + primary: 'primary' + }, + filter: 'filterString' + })).toEqual('GroupBy(Rows(primary), filter=filterString)'); + }); + + it('stringifies GroupBy queries with a sort', () => { + expect(stringifyGroupBy({ + groupByCall: { + primary: 'primary' + }, + sort: [{ sortValue: 'count desc' }] + })).toEqual('GroupBy(Rows(primary), sort=\"count desc\")'); + }); + + it('stringifies GroupBy queries with a secondary sum sort', () => { + expect(stringifyGroupBy({ + groupByCall: { + primary: 'primary' + }, + sort: [ + { sortValue: 'count desc' }, + { sortValue: 'sum asc', field: 'field' } + ] + })).toEqual('GroupBy(Rows(primary), sort=\"count desc, sum asc\", aggregate=Sum(field=field))'); + }); + + it('stringifies GroupBy queries with a filter and sort', () => { + expect(stringifyGroupBy({ + groupByCall: { + primary: 'primary' + }, + filter: 'filterString', + sort: [ + { sortValue: 'sum asc', field: 'field' } + ] + })).toEqual('GroupBy(Rows(primary), filter=filterString, sort=\"sum asc\", aggregate=Sum(field=field))'); + }); +}); diff --git a/lattice/src/App/QBuilder/utils.ts b/lattice/src/App/QueryBuilder/utils.ts similarity index 81% rename from lattice/src/App/QBuilder/utils.ts rename to lattice/src/App/QueryBuilder/utils.ts index 5722fb31f..441b0ab8d 100644 --- a/lattice/src/App/QBuilder/utils.ts +++ b/lattice/src/App/QueryBuilder/utils.ts @@ -1,4 +1,4 @@ -import { RowGrouping } from './rowTypes'; +import { Operator, RowGrouping } from './rowTypes'; import { getIPRange } from 'get-ip-range'; export const stringifyExtract = (query: any) => { @@ -29,8 +29,9 @@ export const stringifyCount = (query: any) => { } } -const stringifyRowData = (rowCalls, operator) => { +export const stringifyRowData = (rowCalls: RowGrouping[], operator?: Operator) => { let queryString = ''; + let error = false; if (rowCalls.length === 0) { queryString = 'All()'; } else { @@ -58,7 +59,7 @@ const stringifyRowData = (rowCalls, operator) => { .join(', '); rowString = `Union(${ipRows})`; } catch (error) { - return { error: true, query: error.message }; + return { error: true, queryString: error.message }; } } else if (rowOperator === 'like') { if (value.includes('%') || value.includes('_')) { @@ -80,34 +81,36 @@ const stringifyRowData = (rowCalls, operator) => { }); }); - queryString = rowsMap - .map((group, idx) => { - let joined = ''; - if (group.length > 1) { - joined = group.map((r) => r).join(', '); - const operator = rowCalls[idx].operator; - if (operator === 'and') { - joined = `Intersect(${joined})`; - } else if (operator === 'or') { - joined = `Union(${joined})`; + if (!error) { + queryString = rowsMap + .map((group, idx) => { + let joined = ''; + if (group.length > 1) { + joined = group.map((r) => r).join(', '); + const operator = rowCalls[idx].operator; + if (operator === 'and') { + joined = `Intersect(${joined})`; + } else if (operator === 'or') { + joined = `Union(${joined})`; + } + } else { + joined = group[0]; } - } else { - joined = group[0]; - } - return rowCalls[idx].isNot ? `Not(${joined})` : joined; - }) - .join(', '); + return rowCalls[idx].isNot ? `Not(${joined})` : joined; + }) + .join(', '); - if (rowCalls.length > 1 && operator) { - if (operator === 'and') { - queryString = `Intersect(${queryString})`; - } else if (operator === 'or') { - queryString = `Union(${queryString})`; + if (rowCalls.length > 1 && operator) { + if (operator === 'and') { + queryString = `Intersect(${queryString})`; + } else if (operator === 'or') { + queryString = `Union(${queryString})`; + } } } } - return { error: false, queryString }; + return { error, queryString }; } export const stringifyGroupBy = (query: any) => { From 8dc2105b63a1f266c71f069951698474b78d3072 Mon Sep 17 00:00:00 2001 From: Stephanie Yang Date: Wed, 28 Jul 2021 11:38:13 -0500 Subject: [PATCH 3/4] add clear link for sort --- .../GroupByBuilder/GroupByBuilder.module.scss | 2 +- .../GroupByBuilder/GroupByBuilder.tsx | 16 ++++++++++++---- .../App/QueryBuilder/GroupBySort/GroupBySort.tsx | 4 +++- .../RowCallBuilder/RowCallBuilder.tsx | 1 - 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.module.scss b/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.module.scss index 68c527523..74cdd6f6e 100644 --- a/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.module.scss +++ b/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.module.scss @@ -3,7 +3,7 @@ margin-bottom: 16px; } - .filtersHeader { + .header { display: flex; align-items: center; justify-content: space-between; diff --git a/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx b/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx index 56a535d0f..c0486352f 100644 --- a/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx +++ b/lattice/src/App/QueryBuilder/GroupByBuilder/GroupByBuilder.tsx @@ -98,7 +98,7 @@ export const GroupByBuilder: FC = ({ />
-
+
Filter (optional) {filters.length > 0 && ( @@ -144,9 +144,17 @@ export const GroupByBuilder: FC = ({
- - Sort (optional) - +
+ + Sort (optional) + + onChange({ ...query, sort: [] })} + > + Clear + +
{ diff --git a/lattice/src/App/QueryBuilder/GroupBySort/GroupBySort.tsx b/lattice/src/App/QueryBuilder/GroupBySort/GroupBySort.tsx index fca83a954..c75f9320b 100644 --- a/lattice/src/App/QueryBuilder/GroupBySort/GroupBySort.tsx +++ b/lattice/src/App/QueryBuilder/GroupBySort/GroupBySort.tsx @@ -20,7 +20,7 @@ export const GroupBySort: FC = ({ showErrors, onUpdate }) => { - const hasPrimary = sort.length > 0; + const hasPrimary = sort.length > 0 && sort[0].sortValue; const primary = hasPrimary ? sort[0].sortValue.split(' ')[0] : ''; const hasSecondary = sort.length > 1; const secondary = hasSecondary ? sort[1].sortValue.split(' ')[0] : ''; @@ -76,6 +76,7 @@ export const GroupBySort: FC = ({ value={hasPrimary ? sort[0].sortValue : ''} options={sortOptions} onChange={(value) => onPrimaryChange(value)} + allowEmpty={true} /> {primary.includes('sum') ? ( @@ -99,6 +100,7 @@ export const GroupBySort: FC = ({ (option) => !option.value.includes(primary) )} onChange={(value) => onSecondaryChange(value)} + allowEmpty={true} /> {secondary.includes('sum') ? ( diff --git a/lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx index ba8a1ad8f..7ba6fc7ae 100644 --- a/lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx +++ b/lattice/src/App/QueryBuilder/RowCallBuilder/RowCallBuilder.tsx @@ -12,7 +12,6 @@ import { } from 'App/QueryBuilder/rowTypes'; import { RowCall } from './RowCall'; import css from './RowCallBuilder.module.scss'; -import { useEffect } from 'react'; type RowCallBuilderProps = { rowCalls: RowGrouping[]; From 2947583a959a7df87f8ad2e24ca83058c3c9e247 Mon Sep 17 00:00:00 2001 From: Stephanie Yang Date: Thu, 29 Jul 2021 12:03:32 -0500 Subject: [PATCH 4/4] fix issue with editing saved queries --- lattice/src/App/QueryBuilder/QueryBuilder.tsx | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/lattice/src/App/QueryBuilder/QueryBuilder.tsx b/lattice/src/App/QueryBuilder/QueryBuilder.tsx index 5ab545f1f..d4c890324 100644 --- a/lattice/src/App/QueryBuilder/QueryBuilder.tsx +++ b/lattice/src/App/QueryBuilder/QueryBuilder.tsx @@ -125,7 +125,7 @@ export const QueryBuilder: FC = ({ const reset = () => { setQuery({ table: selectedTable.name, - columns: selectedTable.fields.map(field => field.name), + columns: selectedTable.fields.map((field) => field.name), operation, isInvalid: false }); @@ -253,9 +253,15 @@ export const QueryBuilder: FC = ({ setSelectedTable(table); setShowInvalid(false); setQuery({ + ...query, table: table.name, columns: table.fields.map((field) => field.name), operation: query.operation, + rowCalls: [], + operator: undefined, + groupByCall: undefined, + sort: undefined, + filter: undefined, isInvalid: false }); }} @@ -273,11 +279,27 @@ export const QueryBuilder: FC = ({ onChange={(value) => { setOperation(value); setShowInvalid(false); - setQuery({ - table: selectedTable.name, - operation: value, - isInvalid: false - }); + if ( + ['Extract', 'Count'].includes(operation) && + ['Extract', 'Count'].includes(value) + ) { + setQuery({ + ...query, + operation: value + }); + } else { + setQuery({ + ...query, + operation: value, + columns: selectedTable.fields.map((field) => field.name), + rowCalls: [], + operator: undefined, + groupByCall: undefined, + sort: undefined, + filter: undefined, + isInvalid: false + }); + } }} />