diff --git a/lattice/src/App.tsx b/lattice/src/App.tsx index f465bd480..89bff9fce 100644 --- a/lattice/src/App.tsx +++ b/lattice/src/App.tsx @@ -9,7 +9,7 @@ import { Nav } from 'shared/Nav'; import { NotFound } from 'App/NotFound'; import { MoleculaTablesContainer } from 'App/MoleculaTables'; import { QueryContainer } from 'App/Query'; -import { QueryBuilderContainer } from 'App/QueryBuilder'; +import { QBuilderContainer } from 'App/QBuilder'; import css from './App.module.scss'; const App = () => { @@ -45,7 +45,7 @@ const App = () => { - + diff --git a/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx b/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx new file mode 100644 index 000000000..03391ec02 --- /dev/null +++ b/lattice/src/App/QBuilder/CountBuilder/CountBuilder.tsx @@ -0,0 +1,36 @@ +import React, { FC } from 'react'; +import { RowCallBuilder } from 'App/QBuilder/RowCallBuilder'; + +type CountBuilderProps = { + table: any; + query: any; + showInvalid: boolean; + onChange: (query: any) => void; +}; + +export const CountBuilder: FC = ({ + table, + query, + showInvalid, + onChange +}) => { + const { rowCalls } = query ? query : { rowCalls: [] }; + + return ( +
+ { + onChange({ + ...query, + rowCalls, + operator, + isInvalid + }); + }} + /> +
+ ); +}; diff --git a/lattice/src/App/QBuilder/CountBuilder/index.ts b/lattice/src/App/QBuilder/CountBuilder/index.ts new file mode 100644 index 000000000..b83081751 --- /dev/null +++ b/lattice/src/App/QBuilder/CountBuilder/index.ts @@ -0,0 +1 @@ +export * from './CountBuilder'; diff --git a/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss new file mode 100644 index 000000000..38dc38646 --- /dev/null +++ b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.module.scss @@ -0,0 +1,21 @@ +.columnsSelector { + margin: 4px 14px 4px 0; + display: flex; + text-align: center; + + .info { + margin-left: 4px; + fill: var(--text-secondary); + } +} + +.textLink { + color: var(--primary); + font-size: 12px; + + &:hover { + cursor: pointer; + text-decoration: underline; + color: var(--primary); + } +} diff --git a/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx new file mode 100644 index 000000000..eaae42714 --- /dev/null +++ b/lattice/src/App/QBuilder/ExtractBuilder/ExtractBuilder.tsx @@ -0,0 +1,92 @@ +import React, { FC, useState } from 'react'; +import InfoIcon from '@material-ui/icons/Info'; +import Tooltip from '@material-ui/core/Tooltip'; +import { ColumnSelector } from 'App/QueryBuilder/ColumnSelector'; +import { RowCallBuilder } from 'App/QBuilder/RowCallBuilder'; +import { useEffectOnce } from 'react-use'; +import css from './ExtractBuilder.module.scss'; + +type ExtractBuilderProps = { + table: any; + query: any; + showInvalid: boolean; + onChange: (query: any) => void; +}; + +export const ExtractBuilder: FC = ({ + table, + query, + showInvalid, + onChange +}) => { + const { columns, rowCalls } = query + ? query + : { + columns: [], + rowCalls: [] + }; + const [showColumnSelector, setShowColumnSelector] = useState(false); + + useEffectOnce(() => { + if (!query) { + onChange({ + columns: table.fields.map((field) => field.name), + operation: 'Extract', + rowCalls: [] + }); + } + }); + + return ( +
+
+ setShowColumnSelector(true)} + > + Configure result fields + + + + +
+ + ({ + name: field.name, + show: columns ? columns.includes(field.name) : true + }))} + onChange={(allColumns) => { + let cols: string[] = []; + allColumns.forEach((col) => { + if (col.show) { + cols.push(col.name); + } + }); + onChange({ ...query, columns: cols }); + }} + onClose={() => setShowColumnSelector(false)} + /> + + { + onChange({ + ...query, + rowCalls, + operator, + isInvalid + }); + }} + /> +
+ ); +}; diff --git a/lattice/src/App/QBuilder/ExtractBuilder/index.ts b/lattice/src/App/QBuilder/ExtractBuilder/index.ts new file mode 100644 index 000000000..a923f7767 --- /dev/null +++ b/lattice/src/App/QBuilder/ExtractBuilder/index.ts @@ -0,0 +1 @@ +export * from './ExtractBuilder'; diff --git a/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss new file mode 100644 index 000000000..68c527523 --- /dev/null +++ b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.module.scss @@ -0,0 +1,32 @@ +.groupBy { + .fieldSelector { + margin-bottom: 16px; + } + + .filtersHeader { + display: flex; + align-items: center; + justify-content: space-between; + margin-top: 8px; + } + + .filtersInfo { + vertical-align: text-top; + margin-left: 4px; + } +} + +.textLink { + color: var(--primary); + font-size: 12px; + + &:hover { + cursor: pointer; + text-decoration: underline; + color: var(--primary); + } +} + +.filtersSection { + margin-bottom: 16px; +} diff --git a/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx new file mode 100644 index 000000000..b53bf6c59 --- /dev/null +++ b/lattice/src/App/QBuilder/GroupByBuilder/GroupByBuilder.tsx @@ -0,0 +1,179 @@ +import React, { FC } from 'react'; +import InfoIcon from '@material-ui/icons/Info'; +import Tooltip from '@material-ui/core/Tooltip'; +import Typography from '@material-ui/core/Typography'; +import { GroupBySort } from 'App/QueryBuilder/GroupBySort'; +import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; +import { Select } from 'shared/Select'; +import { stringifyRowData } from 'App/QueryBuilder/stringifyRowData'; +import css from './GroupByBuilder.module.scss'; + +type GroupByBuilderProps = { + table: any; + query: any; + showInvalid: boolean; + onChange: (query: any) => void; +}; + +export const GroupByBuilder: FC = ({ + table, + query, + showInvalid, + onChange +}) => { + const { groupByCall, filter, sort } = query; + const filters = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ).filter( + (q) => + q.table === table.name && + q.operation === 'Extract' && + q.rowCalls.length > 0 + ); + + return ( +
+ + ['set', 'time', 'mutex', 'bool', 'int', 'timestamp'].includes( + f.options.type + ) + ) + .map((field) => { + return { + label: `${field.name} (${field.options.type})`, + value: field.name + }; + })} + onChange={(value) => + onChange({ + ...query, + groupByCall: { + ...query.groupByCall, + secondary: value.toString() + } + }) + } + /> + +
+
+ + Filter (optional) + {filters.length > 0 && ( + + + + )} + + {filter ? ( + onChange({ ...query, filter: undefined })} + > + Clear Filter + + ) : null} +
+ {filter ? ( + {filter} + ) : filters.length > 0 ? ( + { + const { rowCalls, operator } = filters[queryIdx]; + const res = stringifyRowData(rowCalls, operator); + if (!res.error) { + onChange({ ...query, filter: res.query }); + } + }} + /> + ) : ( +
+ No available filters. To use filters, save an Extract query for{' '} + {table.name} with at least one field constraint. +
+ )} +
+ +
+ + Sort (optional) + + { + let isInvalid = false; + if ( + value.length > 0 && + value[0].sortValue.includes('sum') && + !value[0].field + ) { + isInvalid = true; + } else if ( + value.length > 1 && + value[1].sortValue.includes('sum') && + !value[1].field + ) { + isInvalid = true; + } + onChange({ ...query, sort: value, isInvalid }); + }} + fields={table.fields + .filter((field) => field.options.type === 'int') + .map((field) => { + return { label: field.name, value: field.name }; + })} + showErrors={showInvalid} + /> +
+
+ ); +}; diff --git a/lattice/src/App/QBuilder/GroupByBuilder/index.ts b/lattice/src/App/QBuilder/GroupByBuilder/index.ts new file mode 100644 index 000000000..11f595080 --- /dev/null +++ b/lattice/src/App/QBuilder/GroupByBuilder/index.ts @@ -0,0 +1 @@ +export * from './GroupByBuilder'; diff --git a/lattice/src/App/QBuilder/QBuilder.module.scss b/lattice/src/App/QBuilder/QBuilder.module.scss new file mode 100644 index 000000000..33c34ede7 --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilder.module.scss @@ -0,0 +1,52 @@ +.sharedConfig { + padding: 32px 32px 0; + + .configSelector { + margin-bottom: 24px; + } +} + +.textLink { + color: var(--primary); + font-size: 12px; + + .icon { + margin-right: 4px; + vertical-align: text-top; + } + + &:hover { + cursor: pointer; + text-decoration: underline; + color: var(--primary); + } +} + +.builder { + display: flex; + flex-direction: column; + height: 100%; + + .queryMetadata { + border-bottom: 1px solid var(--divider); + padding-bottom: 16px; + margin-bottom: 16px; + } +} + +.config { + padding: 24px 32px 0; + flex-grow: 1; +} + +.builderActions { + padding: 16px 32px; + display: flex; + justify-content: space-between; + + .mainActions { + display: grid; + grid-template-columns: auto auto; + grid-gap: 8px; + } +} diff --git a/lattice/src/App/QBuilder/QBuilder.tsx b/lattice/src/App/QBuilder/QBuilder.tsx new file mode 100644 index 000000000..3e5e0a8be --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilder.tsx @@ -0,0 +1,388 @@ +import React, { FC, Fragment, useEffect, useState } from 'react'; +import ArrowBackIcon from '@material-ui/icons/ArrowBack'; +import Button from '@material-ui/core/Button'; +import CircularProgress from '@material-ui/core/CircularProgress'; +import Divider from '@material-ui/core/Divider'; +import Popover from '@material-ui/core/Popover'; +import TextField from '@material-ui/core/TextField'; +import Typography from '@material-ui/core/Typography'; +import { + cleanupRows, + stringifyCount, + stringifyExtract, + stringifyGroupBy +} from './utils'; +import { CountBuilder } from './CountBuilder'; +import { ExtractBuilder } from './ExtractBuilder'; +import { GroupByBuilder } from './GroupByBuilder'; +import { Select } from 'shared/Select'; +import css from './QBuilder.module.scss'; + +type QBuilderProps = { + tables: any[]; + savedQuery: number; + onRun: ( + table: string, + operation: string, + query: string, + countQuery?: string + ) => void; + onClear: () => void; + onExitEdit: () => void; + onSaveQuery: () => void; +}; + +export const QBuilder: FC = ({ + tables, + savedQuery, + onRun, + onClear, + onExitEdit, + onSaveQuery +}) => { + const [saveButtonEl, setSaveButtonEl] = useState(null); + const queriesList = JSON.parse(localStorage.getItem('saved-queries') || '[]'); + const [editMode, setEditMode] = useState(false); + const [selectedTable, setSelectedTable] = useState(tables[0]); + const [operation, setOperation] = useState('Extract'); + const [query, setQuery] = useState({ + table: tables[0].name, + operation: 'Extract', + columns: tables[0].fields.map((field) => field.name) + }); + const [showInvalid, setShowInvalid] = useState(false); + const [updating, setUpdating] = useState(false); + const [queryNameIdx, setQueryNameIdx] = useState(-1); + + useEffect(() => { + if (savedQuery > -1) { + const updatedList = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ); + setQuery(updatedList[savedQuery]); + setEditMode(true); + setOperation(updatedList[savedQuery].operation); + const table = tables.find( + (table) => table.name === updatedList[savedQuery].table + ); + setSelectedTable(table); + } + }, [savedQuery, tables]); + + useEffect(() => { + if (query?.name) { + setQueryNameIdx(queriesList.findIndex((q) => q.name === query.name)); + } + }, [query, queriesList]); + + const cleanRows = () => { + const cleanRows = cleanupRows(query?.rowCalls); + return { + ...query, + rowCalls: cleanRows.cleanRowCalls, + isInvalid: cleanRows.isInvalid + }; + }; + + const runQuery = () => { + setShowInvalid(true); + let queryString = ''; + let countQuery = ''; + + switch (operation) { + case 'Extract': + const cleanExtractQuery = cleanRows(); + setQuery(cleanExtractQuery); + const extract = stringifyExtract(cleanExtractQuery); + if (!extract.error && !cleanExtractQuery.isInvalid) { + queryString = extract.queryString; + countQuery = extract.countQuery; + } + break; + case 'GroupBy': + queryString = stringifyGroupBy(query); + break; + case 'Count': + const cleanCountQuery = cleanRows(); + setQuery(cleanCountQuery); + const count = stringifyCount(cleanCountQuery); + if (!count.error && !cleanCountQuery.isInvalid) { + queryString = count.queryString; + } + break; + default: + break; + } + + if (queryString && !query.isInvalid) { + setShowInvalid(false); + onRun(selectedTable.name, operation, queryString, countQuery); + } else { + setShowInvalid(true); + } + }; + + const reset = () => { + setQuery({ + table: selectedTable.name, + columns: selectedTable.fields.map(field => field.name), + operation, + isInvalid: false + }); + setOperation('Extract'); + setShowInvalid(false); + setEditMode(false); + onExitEdit(); + onClear(); + }; + + const onSave = () => { + setUpdating(true); + setTimeout(() => { + setUpdating(false); + }, 1500); + const cleanQuery = cleanRows(); + let queries = [...queriesList]; + + if (editMode && queries.length > 0) { + const clone = [...queries]; + clone.splice(savedQuery, 1, cleanQuery); + localStorage.setItem('saved-queries', JSON.stringify(clone)); + } else { + queries.push(cleanQuery); + localStorage.setItem('saved-queries', JSON.stringify(queries)); + } + onSaveQuery(); + }; + + const renderBuilder = () => { + switch (operation) { + case 'Extract': + return ( + setQuery(q)} + /> + ); + case 'GroupBy': + return ( + setQuery(q)} + /> + ); + case 'Count': + return ( + setQuery(q)} + /> + ); + default: + return
Invalid Operation
; + } + }; + + return ( +
+
+ {editMode ? ( + + + Back to new query + + ) : null} + + {editMode ? 'Edit Saved Query' : 'New Query'} + + + {editMode ? ( +
+ = 0 && queryNameIdx !== savedQuery) + } + helperText={ + queryNameIdx >= 0 && queryNameIdx !== savedQuery + ? 'Query name must be unique' + : '' + } + onChange={(event) => + setQuery({ ...query, name: event.target.value }) + } + margin="normal" + size="small" + required + fullWidth + /> + + + setQuery({ ...query, description: event.target.value }) + } + margin="normal" + size="small" + fullWidth + /> +
+ ) : null} + + { + setOperation(value); + setShowInvalid(false); + setQuery({ + table: selectedTable.name, + operation: value, + isInvalid: false + }); + }} + /> + +
+
{renderBuilder()}
+
+
+ + {editMode ? ( + + ) : ( + + + setSaveButtonEl(null)} + anchorOrigin={{ + vertical: 'top', + horizontal: 'left' + }} + transformOrigin={{ + vertical: 'bottom', + horizontal: 'left' + }} + > +
+ + Give your query a unique name and a short description to + help you identify it for future use. + + + = 0} + helperText={ + queryNameIdx >= 0 ? 'Query name must be unique' : '' + } + onChange={(event) => + setQuery({ ...query, name: event.target.value }) + } + margin="dense" + required + fullWidth + /> + + + setQuery({ ...query, description: event.target.value }) + } + margin="dense" + fullWidth + /> + +
+
+ + +
+
+
+ + + )} +
+ +
+
+ ); +}; diff --git a/lattice/src/App/QBuilder/QBuilderContainer.module.scss b/lattice/src/App/QBuilder/QBuilderContainer.module.scss new file mode 100644 index 000000000..a5cd6c77d --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilderContainer.module.scss @@ -0,0 +1,66 @@ +.builderColumn, +.results { + height: calc(100vh - 64px); + overflow: scroll; + position: relative; + + .openClose { + position: absolute; + right: 10px; + top: 10px; + } +} + +.split { + display: flex; +} + +.collapsedBuilder { + border-right: 1px solid rgba(var(--contrast-rgb), 0.1); + text-align: center; + padding-top: 10px; +} + +.resultsBlock { + height: 100%; +} + +.resultsHeader { + display: flex; + align-items: center; + justify-content: space-between; + + .download { + display: flex; + align-items: center; + } + + .downloadInfo { + margin-right: 8px; + } +} + +.infoMessage { + padding: 16px; + border: 1px solid rgba(var(--primary-rgb), 0.5); + background: rgba(var(--primary-rgb), 0.1); + border-radius: 4px; + margin: 4px 0 16px; +} + +.spacer{ + padding-bottom: 16px; +} + +.resultsLoading { + margin-left: 8px; +} + +.savedQueries { + margin-top: 48px; +} + +.noTables { + margin-top: 20px; + padding: 12px 16px; +} diff --git a/lattice/src/App/QBuilder/QBuilderContainer.tsx b/lattice/src/App/QBuilder/QBuilderContainer.tsx new file mode 100644 index 000000000..7161cb5a3 --- /dev/null +++ b/lattice/src/App/QBuilder/QBuilderContainer.tsx @@ -0,0 +1,323 @@ +import React, { Fragment, useState } from 'react'; +import Alert from '@material-ui/lab/Alert'; +import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos'; +import Button from '@material-ui/core/Button'; +import classNames from 'classnames'; +import CloseIcon from '@material-ui/icons/Close'; +import IconButton from '@material-ui/core/IconButton'; +import InfoIcon from '@material-ui/icons/Info'; +import moment, { Moment } from 'moment'; +import Paper from '@material-ui/core/Paper'; +import Split from 'react-split'; +import Snackbar from '@material-ui/core/Snackbar'; +import Tooltip from '@material-ui/core/Tooltip'; +import Typography from '@material-ui/core/Typography'; +import { Block } from 'shared/Block'; +import { formatDuration } from 'shared/utils/formatDuration'; +import { grpc } from '@improbable-eng/grpc-web'; +import { pilosa } from 'services/eventServices'; +import { QBuilder } from './QBuilder'; +import { queryPQL } from 'services/grpcServices'; +import { QueryResults } from 'App/Query/QueryResults'; +import { ResultType } from 'App/Query/QueryContainer'; +import { RowResponse } from 'proto/pilosa_pb'; +import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; +import { useEffectOnce } from 'react-use'; +import css from './QBuilderContainer.module.scss'; + +let streamingResults: ResultType = { + query: '', + operation: '', + type: 'PQL', + headers: [], + rows: [], + roundtrip: 0, + error: '' +}; + +export const QBuilderContainer = () => { + let startTime: Moment; + let exportRows: any[] = []; + const colSizes = JSON.parse( + localStorage.getItem('builderColSizes') || '[25, 75]' + ); + const [queriesList, setQueriesList] = useState( + JSON.parse(localStorage.getItem('saved-queries') || '[]') + ); + + const [tables, setTables] = useState([]); + const [showBuilder, setShowBuilder] = useState(true); + const [results, setResults] = useState(); + const [fullCount, setFullCount] = useState(); + const [recordsCount, setRecordsCount] = useState(); + const [errorResult, setErrorResult] = useState(); + const [error, setError] = useState(''); + const [loading, setLoading] = useState(false); + const [savedQuery, setSavedQuery] = useState(-1); + + useEffectOnce(() => { + pilosa.get.schema().then((res) => { + setTables(res.data.indexes); + }); + }); + + const handleQueryMessages = (message: RowResponse) => { + const response = message.toObject(); + if (response.headersList.length > 0 && response.duration > 0) { + streamingResults.headers = response.headersList; + streamingResults.duration = response.duration; + } + streamingResults.rows.push(response.columnsList); + }; + + const handleQueryEnd = (status: grpc.Code, statusMessage: string) => { + if (status !== grpc.Code.OK) { + streamingResults.error = statusMessage; + setErrorResult(streamingResults); + } else { + streamingResults.roundtrip = moment + .duration(moment().diff(startTime)) + .as('milliseconds'); + setErrorResult(undefined); + setResults(streamingResults); + } + setLoading(false); + }; + + const handleExternalLookup = (message: RowResponse) => { + const response = message.toObject(); + let rowStr: string[] = []; + if (exportRows.length === 0) { + const headers = response.headersList.map((header) => header.name); + exportRows.push(headers.join('\t')); + } + response.headersList.forEach((header, idx) => + rowStr.push(response.columnsList[idx][`${header.datatype}val`]) + ); + exportRows.push(rowStr.join('\t')); + }; + + const handleExternalLookupEnd = ( + status: grpc.Code, + statusMessage: string + ) => { + if (status !== grpc.Code.OK) { + setError(statusMessage); + } else if (exportRows.length === 0) { + setError('No record attributes for current query.'); + } else { + const dateTime = moment().unix(); + const element = document.createElement('a'); + const file = new Blob([exportRows.join('\n')], { + type: 'text/plain;charset=utf-8' + }); + element.href = URL.createObjectURL(file); + element.download = `molecula-${results?.index}-${dateTime}.csv`; + document.body.appendChild(element); + element.click(); + exportRows = []; + } + }; + + const onRunQuery = ( + table: string, + operation: string, + query: string, + countQuery?: string + ) => { + streamingResults = { + query, + operation, + type: 'PQL', + headers: [], + rows: [], + index: table, + roundtrip: 0, + error: '' + }; + startTime = moment(); + setLoading(true); + + if (operation !== 'Count') { + if (countQuery) { + pilosa.post.query(table, countQuery).then((res) => { + setFullCount(res.data.results[0]); + }); + } + + pilosa.post + .query(table, `Count(All())`) + .then((res) => setRecordsCount(res.data.results[0])); + } else { + setFullCount(undefined); + } + + queryPQL(table, query, handleQueryMessages, handleQueryEnd); + }; + + const onClear = () => { + setResults(undefined); + setFullCount(undefined); + setErrorResult(undefined); + }; + + const onRemoveQuery = (queryIdx: number) => { + let queries = [...queriesList]; + queries.splice(queryIdx, 1); + localStorage.setItem('saved-queries', JSON.stringify(queries)); + setQueriesList(queries); + }; + + const onSaveQuery = () => { + const updatedList = JSON.parse( + localStorage.getItem('saved-queries') || '[]' + ); + if (savedQuery < 0) { + setSavedQuery(updatedList.length - 1); + } + setQueriesList(updatedList); + }; + + const onExportLogs = () => { + if (results) { + const columns = results.rows.map((row) => row[0].uint64val); + const table = results.index ? results.index : ''; + onExternalLookup(table, columns); + } + }; + + const onExternalLookup = (table: string, columns: number[]) => { + const cols = JSON.stringify(columns); + const query = `ExternalLookup(ConstRow(columns=${cols}), query='select id, "rawlog" from "${table}" where id = ANY($1)')`; + queryPQL(table, query, handleExternalLookup, handleExternalLookupEnd); + }; + + return ( + + {tables.length > 0 ? ( + + localStorage.setItem('builderColSizes', JSON.stringify(sizes)) + } + gutter={(_index, direction) => { + const gutter = document.createElement('div'); + gutter.className = `gutter gutter-${direction}`; + const dragbars = document.createElement('div'); + dragbars.className = 'dragBar'; + gutter.appendChild(dragbars); + return gutter; + }} + className={classNames(css.split, !showBuilder ? 'hide-gutter' : '')} + > + {showBuilder ? ( +
+
+ setShowBuilder(false)} size="small"> + + +
+ setSavedQuery(-1)} + onSaveQuery={onSaveQuery} + /> +
+ ) : ( +
+ setShowBuilder(true)} size="small"> + + +
+ )} +
+ +
+ + Results{' '} + + {results?.query.includes('Extract(') ? ( +
+ + + + +
+ ) : null} +
+ {loading ?
Loading...
: null} + {results && !loading ? ( + + {fullCount && recordsCount ? ( +
+ {results.duration ? ( +
+ {recordsCount.toLocaleString()} records scanned in{' '} + {formatDuration(results.duration, true)}. +
+ ) : null} +
+ Showing{' '} + {fullCount > 1000 ? 'first 1,000 rows of' : 'all'}{' '} + {fullCount.toLocaleString()} results. +
+
+ ) : null} + +
+ + ) : null} + {errorResult && !loading ?
{errorResult.error}
: null} + {!loading && !results && !error ? ( + + + Build a query to see results + + {queriesList.length > 0 ? ( +
+ + Saved Queries + + setSavedQuery(queryIdx)} + onRemoveQuery={onRemoveQuery} + /> +
+ ) : null} + + ) : null} + +
+ + setError('')}> + {error} + + + + ) : ( + + + Query Builder + + + + There are no tables to query. + + + + )} + + ); +}; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts new file mode 100644 index 000000000..ebbd97470 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/helpers.ts @@ -0,0 +1,52 @@ +export const operators = { + decimal: [ + { label: '>', value: '>' }, + { label: '<', value: '<' }, + { label: '>=', value: '>=' }, + { label: '<=', value: '<=' }, + { label: '==', value: '=' }, + { label: '!=', value: '!=' }, + ], + int: [ + { label: '>', value: '>' }, + { label: '<', value: '<' }, + { label: '>=', value: '>=' }, + { label: '<=', value: '<=' }, + { label: '==', value: '=' }, + { label: '!=', value: '!=' }, + ], + 'mutex-id': [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' } + ], + 'mutex-keys': [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'like', value: 'like' }, + { label: 'CIDR', value: 'cidr' } + ], + "set-id": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' } + ], + "set-keys": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'like', value: 'like' }, + { label: 'CIDR', value: 'cidr' } + ], + "time-id": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' } + ], + "time-keys": [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'like', value: 'like' } + ], + timestamp: [ + { label: 'is before', value: '<' }, + { label: 'is after', value: '>' }, + { label: 'is', value: '=' }, + ] +} diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts new file mode 100644 index 000000000..c5f595cf9 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCall/index.ts @@ -0,0 +1 @@ +export * from './helpers'; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss new file mode 100644 index 000000000..326b78876 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.module.scss @@ -0,0 +1,40 @@ +.newGroup, +.operator { + display: flex; + align-items: center; + width: 100%; + + .line { + flex-grow: 1; + height: 1px; + background: var(--divider); + } + + .addButton { + margin: 0 8px; + } +} + +.groupBy { + .line { + flex-grow: 1; + height: 1px; + background: var(--divider); + } + + .fieldSelector { + margin-top: 16px; + } + + .filterHeader { + display: flex; + align-items: center; + justify-content: space-between; + margin-top: 8px; + } + + .filterInfo { + vertical-align: text-top; + margin-left: 4px; + } +} diff --git a/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx new file mode 100644 index 000000000..48f8e81ea --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/RowCallBuilder.tsx @@ -0,0 +1,180 @@ +import React, { FC, Fragment, useState } from 'react'; +import AddIcon from '@material-ui/icons/Add'; +import Button from '@material-ui/core/Button'; +import IconButton from '@material-ui/core/IconButton'; +import Menu from '@material-ui/core/Menu'; +import MenuItem from '@material-ui/core/MenuItem'; +import { + groupOperators, + Operator, + RowCallType, + RowGrouping +} from 'App/QueryBuilder/rowTypes'; +import { RowCall } from 'App/QueryBuilder/RowCall'; +import css from './RowCallBuilder.module.scss'; + +type RowCallBuilderProps = { + rowCalls: RowGrouping[]; + fields: any[]; + showInvalid: boolean; + onChange: ( + rows: RowGrouping[], + operator?: Operator, + hasInvalid?: boolean + ) => void; +}; + +export const RowCallBuilder: FC = ({ + rowCalls = [], + fields, + showInvalid, + onChange +}) => { + const [newOperatorEl, setNewOperatorEl] = useState(null); + const [operatorEl, setOperatorEl] = useState(null); + const [operator, setOperator] = useState(); + + const onNewGroup = () => { + const newRow: RowCallType[] = [ + { + field: '', + rowOperator: '=', + value: '', + type: 'set', + keys: true + } + ]; + + onChange([...rowCalls, { row: newRow }]); + }; + + const onRemoveGroup = (groupIdx: number) => { + if (groupIdx === 0) { + onChange(rowCalls.slice(1)); + } else { + const updatedRowCalls = [...rowCalls]; + updatedRowCalls.splice(groupIdx, 1); + onChange(updatedRowCalls); + + if (updatedRowCalls.length < 1) { + setOperator(undefined); + } + } + }; + + const onUpdateRow = ( + groupIdx: number, + newRowData?: RowCallType[], + operator?: Operator, + isNot?: boolean + ) => { + const updatedRowCalls = [...rowCalls]; + if (newRowData) { + const clone: RowGrouping = { + ...rowCalls[groupIdx], + row: newRowData, + isNot + }; + if (newRowData.length <= 1) { + delete clone.operator; + } else if (operator) { + clone.operator = operator; + } + updatedRowCalls.splice(groupIdx, 1, clone); + } else { + updatedRowCalls.splice(groupIdx, 1); + } + onChange(updatedRowCalls); + }; + + return ( + + {rowCalls?.map((rowCall, idx) => ( + + {idx > 0 ? ( + +
+
+ +
+
+ setOperatorEl(null)} + > + {groupOperators.map((op: Operator) => ( + { + setOperator(op); + setOperatorEl(null); + }} + > + {op} + + ))} + + + ) : null} + + onUpdateRow(idx, updatedRow, operator, isNot) + } + onRemoveGroup={() => onRemoveGroup(idx)} + /> + + ))} + +
+
+ { + if (rowCalls.length === 0 || !!operator) { + onNewGroup(); + } else { + setNewOperatorEl(event.currentTarget); + } + }} + > + + +
+
+ + setNewOperatorEl(null)} + > + {groupOperators.map((op: Operator) => ( + { + setOperator(op); + onNewGroup(); + setNewOperatorEl(null); + }} + > + {op} + + ))} + + + ); +}; diff --git a/lattice/src/App/QBuilder/RowCallBuilder/index.ts b/lattice/src/App/QBuilder/RowCallBuilder/index.ts new file mode 100644 index 000000000..2f2d839f8 --- /dev/null +++ b/lattice/src/App/QBuilder/RowCallBuilder/index.ts @@ -0,0 +1 @@ +export * from './RowCallBuilder'; diff --git a/lattice/src/App/QBuilder/index.ts b/lattice/src/App/QBuilder/index.ts new file mode 100644 index 000000000..058f256f4 --- /dev/null +++ b/lattice/src/App/QBuilder/index.ts @@ -0,0 +1,2 @@ +export * from './QBuilderContainer'; +export * from './rowTypes'; diff --git a/lattice/src/App/QBuilder/rowTypes.ts b/lattice/src/App/QBuilder/rowTypes.ts new file mode 100644 index 000000000..7ed00293f --- /dev/null +++ b/lattice/src/App/QBuilder/rowTypes.ts @@ -0,0 +1,22 @@ +export type Operator = 'and' | 'or'; + +export const groupOperators: Operator[] = ['and', 'or']; + +export type RowGrouping = { + row: RowCallType[]; + isNot?: boolean; + operator?: Operator; +} + +export type RowCallType = { + field: string; + rowOperator: string; + value: string; + type: string; + keys?: boolean; +}; + +export type RowsCallType = { + primary: string; + secondary: string; +}; diff --git a/lattice/src/App/QBuilder/utils.ts b/lattice/src/App/QBuilder/utils.ts new file mode 100644 index 000000000..5722fb31f --- /dev/null +++ b/lattice/src/App/QBuilder/utils.ts @@ -0,0 +1,174 @@ +import { RowGrouping } from './rowTypes'; +import { getIPRange } from 'get-ip-range'; + +export const stringifyExtract = (query: any) => { + const { columns, operator, rowCalls } = query; + + const rowData = stringifyRowData(rowCalls, operator); + if (!rowData.error) { + const fields = columns.map((field) => `Rows(${field})`); + const allRows = fields.join(', '); + return { + error: false, + queryString: `Extract(Limit(${rowData.queryString}, limit=1000), ${allRows})`, + countQuery: `Count(${rowData.queryString})` + }; + } else { + return { ...rowData, countQuery: '' }; + } +} + +export const stringifyCount = (query: any) => { + const { operator, rowCalls } = query; + + const rowData = stringifyRowData(rowCalls, operator); + if (!rowData.error) { + return { error: false, queryString: `Count(${rowData.queryString})` }; + } else { + return rowData; + } +} + +const stringifyRowData = (rowCalls, operator) => { + let queryString = ''; + if (rowCalls.length === 0) { + queryString = 'All()'; + } else { + let rowsMap: string[][] = []; + rowCalls.forEach((group, groupIdx) => { + rowsMap.push([]); + group.row.forEach((row) => { + let rowString = ''; + const { field, rowOperator, value, type, keys } = row; + const isNegatory = rowOperator === '!='; + const isUnion = + ['=', '!='].includes(rowOperator) && value.split(',').length > 1; + const operator = isNegatory ? '=' : rowOperator; + if (isUnion) { + const values = value.split(','); + const unionRows = values + .map((v) => keys ? `Row(${field}="${v.trim()}")` : `Row(${field}=${v.trim()})`) + .join(', '); + rowString = `Union(${unionRows})`; + } else if (rowOperator === 'cidr') { + try { + const ipRange = getIPRange(value); + const ipRows = ipRange + .map((ip) => `Row(${field}="${ip}")`) + .join(', '); + rowString = `Union(${ipRows})`; + } catch (error) { + return { error: true, query: error.message }; + } + } else if (rowOperator === 'like') { + if (value.includes('%') || value.includes('_')) { + rowString = `UnionRows(Rows(field=${field}, like="${value}"))`; + } else { + rowString = `UnionRows(Rows(field=${field}, like="%${value}%"))`; + } + } else { + rowString = keys || type === 'timestamp' + ? `Row(${field}${operator}"${value}")` + : `Row(${field}${operator}${value})`; + } + + if (isNegatory) { + rowsMap[groupIdx].push(`Not(${rowString})`); + } else { + rowsMap[groupIdx].push(rowString); + } + }); + }); + + queryString = rowsMap + .map((group, idx) => { + let joined = ''; + if (group.length > 1) { + joined = group.map((r) => r).join(', '); + const operator = rowCalls[idx].operator; + if (operator === 'and') { + joined = `Intersect(${joined})`; + } else if (operator === 'or') { + joined = `Union(${joined})`; + } + } else { + joined = group[0]; + } + return rowCalls[idx].isNot ? `Not(${joined})` : joined; + }) + .join(', '); + + if (rowCalls.length > 1 && operator) { + if (operator === 'and') { + queryString = `Intersect(${queryString})`; + } else if (operator === 'or') { + queryString = `Union(${queryString})`; + } + } + } + + return { error: false, queryString }; +} + +export const stringifyGroupBy = (query: any) => { + const { groupByCall, filter, sort } = query; + + let sortString = ''; + let aggregateString = ''; + if (sort?.length > 0) { + sortString = sort[0].sortValue; + if (sort[0].sortValue.includes('sum')) { + aggregateString = `, aggregate=Sum(field=${sort[0].field})`; + } + if (sort.length > 1) { + sortString = `${sortString}, ${sort[1].sortValue}`; + if (sort[1].sortValue.includes('sum')) { + aggregateString = `, aggregate=Sum(field=${sort[1].field})`; + } + } + sortString = `, sort="${sortString}"`; + } + const filterString = filter ? `, filter=${filter}` : ''; + const queryString = groupByCall.secondary + ? `GroupBy(Rows(${groupByCall.primary}), Rows(${groupByCall.secondary})${filterString}${sortString}${aggregateString})` + : `GroupBy(Rows(${groupByCall.primary})${filterString}${sortString}${aggregateString})`; + + return queryString; +} + +export const cleanupRows = (rowCalls: RowGrouping[]) => { + // remove empty groups and row calls + let isInvalid = false; + let cleanRowCalls: RowGrouping[] = []; + let cleanGroups: RowGrouping[] = []; + + if (rowCalls?.length > 0) { + rowCalls.forEach((group, groupIdx) => { + cleanGroups.push({ ...group, row: [] }); + group.row.forEach((row) => { + const { field, value, rowOperator } = row; + if (field || value) { + cleanGroups[groupIdx].row.push(row); + + if (!field || !value) { + isInvalid = true; + } else if (rowOperator === 'cidr') { + try { + getIPRange(value); + } catch (err) { + isInvalid = true; + } + } + } + }); + }); + + cleanGroups.forEach((group) => { + if (group.row.length > 0) { + cleanRowCalls.push(group); + } + }); + } + + return { cleanRowCalls, isInvalid }; +};