= ({ results }) => {
+ const theme = useTheme();
+ const isDark = theme.palette.type === 'dark';
+ const { headers, rows } = results;
+ let uniqueKeys: string[] = [];
+ const grouped = groupBy(rows, (row) => row[0].stringval);
+ const data = Object.keys(grouped).map((key) => {
+ let groupData = {};
+ grouped[key].forEach((row) => {
+ const secondaryValue = row[1][`${headers[1].datatype}val`];
+ if (headers.length > 2) {
+ groupData = {
+ ...groupData,
+ [headers[0].name]: row[0][`${headers[0].datatype}val`],
+ [secondaryValue]: row[2][`${headers[2].datatype}val`]
+ };
+
+ if (!uniqueKeys.includes(secondaryValue.toString())) {
+ uniqueKeys.push(secondaryValue.toString());
+ }
+ } else {
+ groupData = {
+ ...groupData,
+ [headers[0].name]: row[0][`${headers[0].datatype}val`],
+ value: secondaryValue
+ };
+ }
+ });
+
+ return groupData;
+ });
+
+ return (
+
+ 0 ? uniqueKeys : undefined}
+ indexBy={headers[0].name}
+ margin={{ top: 50, right: 130, bottom: 100, left: 60 }}
+ padding={0.3}
+ valueScale={{ type: 'linear' }}
+ indexScale={{ type: 'band', round: true }}
+ colors={schemeTableau10}
+ enableLabel={false}
+ theme={
+ isDark
+ ? {
+ textColor: 'var(--text-secondary)',
+ axis: {
+ domain: { line: { stroke: 'rgba(255, 255, 255, 0.1)' } }
+ },
+ grid: { line: { stroke: 'rgba(255, 255, 255, 0.1)' } },
+ tooltip: { container: { background: '#1c2022' } }
+ }
+ : {
+ axis: {
+ domain: { line: { stroke: '#dddddd' } }
+ }
+ }
+ }
+ groupMode="grouped"
+ axisBottom={{
+ tickSize: 5,
+ tickPadding: 5,
+ tickRotation: -40,
+ legend: headers[0].name,
+ legendPosition: 'middle',
+ legendOffset: 80
+ }}
+ axisLeft={{
+ tickSize: 5,
+ tickPadding: 5,
+ tickRotation: 0,
+ legend: 'count',
+ legendPosition: 'middle',
+ legendOffset: -40
+ }}
+ tooltip={({ id, value, color }) => (
+
+ {id}: {value}
+
+ )}
+ labelSkipWidth={12}
+ labelSkipHeight={12}
+ labelTextColor={{ from: 'color', modifiers: [['darker', 1.6]] }}
+ legends={[
+ {
+ dataFrom: 'keys',
+ anchor: 'bottom-right',
+ direction: 'column',
+ justify: false,
+ translateX: 120,
+ translateY: 0,
+ itemsSpacing: 2,
+ itemWidth: 120,
+ itemHeight: 20,
+ itemDirection: 'left-to-right',
+ itemOpacity: 0.85,
+ symbolShape: 'circle',
+ symbolSize: 20,
+ effects: [
+ {
+ on: 'hover',
+ style: {
+ itemOpacity: 1
+ }
+ }
+ ]
+ }
+ ]}
+ animate={true}
+ motionStiffness={90}
+ motionDamping={15}
+ />
+
+ );
+};
diff --git a/lattice/src/App/QueryBuilder/GroupByChart/index.ts b/lattice/src/App/QueryBuilder/GroupByChart/index.ts
new file mode 100644
index 000000000..98412a988
--- /dev/null
+++ b/lattice/src/App/QueryBuilder/GroupByChart/index.ts
@@ -0,0 +1 @@
+export * from './GroupByChart';
diff --git a/lattice/src/App/QueryBuilder/QueryBuilder.module.scss b/lattice/src/App/QueryBuilder/QueryBuilder.module.scss
new file mode 100644
index 000000000..af2358697
--- /dev/null
+++ b/lattice/src/App/QueryBuilder/QueryBuilder.module.scss
@@ -0,0 +1,169 @@
+.builderColumn,
+.results {
+ height: calc(100vh - 64px);
+ overflow: scroll;
+ position: relative;
+
+ .openClose {
+ position: absolute;
+ right: 10px;
+ top: 10px;
+ }
+}
+
+.resultsBlock {
+ height: 100%;
+}
+
+.builderColumn {
+ display: grid;
+ grid-template-rows: auto 1fr auto;
+}
+
+.results {
+ padding-bottom: 20px;
+}
+
+.resultsHeader {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+
+ .download {
+ display: flex;
+ align-items: center;
+ }
+
+ .downloadInfo {
+ margin-right: 8px;
+ }
+}
+
+.collapsedBuilder {
+ border-right: 1px solid rgba(var(--contrast-rgb), 0.1);
+ text-align: center;
+ padding-top: 10px;
+}
+
+.builderHeader {
+ padding: 32px 32px 0;
+}
+
+.builder {
+ overflow-y: scroll;
+ padding: 20px 32px;
+
+ .queryMetadata,
+ .tableSelector,
+ .operationSelector {
+ margin-bottom: 16px;
+ }
+
+ .columnsSelector {
+ margin: 4px 14px;
+ display: flex;
+ text-align: center;
+
+ .info {
+ margin-left: 4px;
+ fill: var(--text-secondary);
+ }
+ }
+
+ .queryMetadata {
+ border-bottom: 1px solid var(--divider);
+ padding-bottom: 16px;
+ }
+}
+
+.builderActions {
+ padding: 16px 32px;
+ display: flex;
+ justify-content: space-between;
+
+ .mainActions {
+ display: grid;
+ grid-template-columns: auto auto;
+ grid-gap: 8px;
+ }
+}
+
+.newGroup,
+.operator {
+ display: flex;
+ align-items: center;
+ width: 100%;
+
+ .line {
+ flex-grow: 1;
+ height: 1px;
+ background: var(--divider);
+ }
+
+ .addButton {
+ margin: 0 8px;
+ }
+}
+
+.groupBy {
+ .line {
+ flex-grow: 1;
+ height: 1px;
+ background: var(--divider);
+ }
+
+ .fieldSelector {
+ margin-top: 16px;
+ }
+}
+
+.savePopover {
+ width: 300px;
+
+ .popoverActions {
+ display: flex;
+ }
+
+ .saveActions {
+ display: grid;
+ grid-template-columns: auto auto;
+ grid-gap: 8px;
+ padding-top: 16px;
+ }
+}
+
+.menuPaper {
+ padding: 0 !important;
+}
+
+.link {
+ color: var(--primary);
+ font-size: 12px;
+
+ .icon {
+ margin-right: 4px;
+ vertical-align: text-top;
+ }
+
+ &:hover {
+ cursor: pointer;
+ text-decoration: underline;
+ color: var(--primary);
+ }
+}
+
+.infoMessage {
+ padding: 16px;
+ border: 1px solid rgba(var(--primary-rgb), 0.5);
+ background: rgba(var(--primary-rgb), 0.1);
+ border-radius: 4px;
+ margin: 4px 0 16px;
+}
+
+.resultsLoading {
+ margin-left: 8px;
+}
+
+.savedQueries {
+ margin-top: 48px;
+}
diff --git a/lattice/src/App/QueryBuilder/QueryBuilder.tsx b/lattice/src/App/QueryBuilder/QueryBuilder.tsx
new file mode 100644
index 000000000..ec495294b
--- /dev/null
+++ b/lattice/src/App/QueryBuilder/QueryBuilder.tsx
@@ -0,0 +1,804 @@
+import React, { FC, Fragment, useState } from 'react';
+import AddIcon from '@material-ui/icons/Add';
+import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos';
+import ArrowBackIcon from '@material-ui/icons/ArrowBack';
+import Button from '@material-ui/core/Button';
+import CircularProgress from '@material-ui/core/CircularProgress';
+import CloseIcon from '@material-ui/icons/Close';
+import IconButton from '@material-ui/core/IconButton';
+import InfoIcon from '@material-ui/icons/Info';
+import Menu from '@material-ui/core/Menu';
+import MenuItem from '@material-ui/core/MenuItem';
+import Popover from '@material-ui/core/Popover';
+import Split from 'react-split';
+import TextField from '@material-ui/core/TextField';
+import Tooltip from '@material-ui/core/Tooltip';
+import Typography from '@material-ui/core/Typography';
+import { Block } from 'shared/Block';
+import { ColumnSelector } from './ColumnSelector';
+import { formatDuration } from 'shared/utils/formatDuration';
+import {
+ Operator,
+ groupOperators,
+ RowCallType,
+ RowGrouping,
+ RowsCallType
+} from './rowTypes';
+import { getIPRange } from 'get-ip-range';
+import { QueryResults } from 'App/Query/QueryResults';
+import { ResultType } from 'App/Query/QueryContainer';
+import { RowCall } from './RowCall';
+import { SavedQueries } from './SavedQueries';
+import { Select } from 'shared/Select';
+import css from './QueryBuilder.module.scss';
+
+type QueryBuilderProps = {
+ tables: any[];
+ results?: ResultType;
+ fullResultsCount?: number;
+ fullRecordsCount?: number;
+ error?: ResultType;
+ loading: boolean;
+ onQuery: (
+ table: any,
+ operation: string,
+ rowData: RowGrouping[],
+ columns: string[],
+ operator?: Operator
+ ) => void;
+ onRunGroupBy: (table: any, rowsData: RowsCallType) => void;
+ onExternalLookup: (table: string, columns: number[]) => void;
+ onClear: () => void;
+};
+
+export const QueryBuilder: FC = ({
+ tables,
+ results,
+ fullResultsCount,
+ fullRecordsCount,
+ error,
+ loading,
+ onQuery,
+ onRunGroupBy,
+ onExternalLookup,
+ onClear
+}) => {
+ const [newOperatorEl, setNewOperatorEl] = useState(null);
+ const [operatorEl, setOperatorEl] = useState(null);
+ const [saveButtonEl, setSaveButtonEl] = useState(null);
+ const [showBuilder, setShowBuilder] = useState(true);
+ const [showColumnSelector, setShowColumnSelector] = useState(false);
+ const [hasInvalid, setHasInvalid] = useState(false);
+ const [selectedTable, setSelectedTable] = useState(tables[0]);
+ const [selectedColumns, setSelectedColumns] = useState<
+ { name: string; show: boolean }[]
+ >(
+ tables[0].fields.map((field) => ({
+ name: field.name,
+ show: true
+ }))
+ );
+ const [operation, setOperation] = useState('Extract');
+ const [groupByCall, setGroupByCall] = useState({
+ primary: '',
+ secondary: ''
+ });
+ const [rowCalls, setRowCalls] = useState([]);
+ const [operator, setOperator] = useState();
+ const [editSavedIdx, setEditSavedIdx] = useState(-1);
+ const [queryName, setQueryName] = useState('');
+ const [queryDescription, setQueryDescription] = useState('');
+ const [updating, setUpdating] = useState(false);
+ const [queriesList, setQueriesList] = useState(
+ JSON.parse(localStorage.getItem('saved-queries') || '[]')
+ );
+ const colSizes = JSON.parse(
+ localStorage.getItem('builderColSizes') || '[25, 75]'
+ );
+ const queryNameIdx = queriesList.findIndex((q) => q.name === queryName);
+ const inEditMode = editSavedIdx >= 0;
+
+ const getColumnsToShow = (cols?: { name: string; show: boolean }[]) => {
+ let columns: string[] = [];
+ const list = cols ? cols : selectedColumns;
+ list.forEach((field) => {
+ if (field.show) {
+ columns.push(field.name);
+ }
+ });
+
+ return columns;
+ };
+
+ const onChangeColumns = (cols) => {
+ setSelectedColumns(cols);
+ if (results) {
+ const { cleanRowCalls, isInvalid } = cleanupRows();
+
+ setHasInvalid(isInvalid);
+ setRowCalls(cleanRowCalls);
+ if (!isInvalid) {
+ let columns: string[] = [];
+ if (operation === 'Extract') {
+ columns = getColumnsToShow(cols);
+ }
+ onQuery(selectedTable, operation, cleanRowCalls, columns, operator);
+ }
+ }
+ };
+
+ const onNewGroup = () => {
+ const newRow: RowCallType[] = [
+ {
+ field: '',
+ rowOperator: '=',
+ value: '',
+ type: 'set'
+ }
+ ];
+
+ setRowCalls([...rowCalls, { row: newRow }]);
+ };
+
+ const onRemoveGroup = (groupIdx: number) => {
+ if (groupIdx === 0) {
+ setRowCalls(rowCalls.slice(1));
+ } else {
+ const updatedRowCalls = [...rowCalls];
+ updatedRowCalls.splice(groupIdx, 1);
+ setRowCalls(updatedRowCalls);
+
+ if (updatedRowCalls.length < 1) {
+ setOperator(undefined);
+ }
+ }
+ };
+
+ const onUpdateRow = (
+ groupIdx: number,
+ newRowData?: RowCallType[],
+ operator?: Operator,
+ isNot?: boolean
+ ) => {
+ const updatedRowCalls = [...rowCalls];
+ if (newRowData) {
+ const clone: RowGrouping = {
+ ...rowCalls[groupIdx],
+ row: newRowData,
+ isNot
+ };
+ if (newRowData.length <= 1) {
+ delete clone.operator;
+ } else if (operator) {
+ clone.operator = operator;
+ }
+ updatedRowCalls.splice(groupIdx, 1, clone);
+ } else {
+ updatedRowCalls.splice(groupIdx, 1);
+ }
+ setRowCalls(updatedRowCalls);
+ };
+
+ const cleanupRows = () => {
+ // remove empty groups and row calls
+ let isInvalid = false;
+ let cleanRowCalls: RowGrouping[] = [];
+ let cleanGroups: RowGrouping[] = [];
+
+ rowCalls.forEach((group, groupIdx) => {
+ cleanGroups.push({ ...group, row: [] });
+ group.row.forEach((row) => {
+ const { field, value, rowOperator } = row;
+ if (field || value) {
+ cleanGroups[groupIdx].row.push(row);
+
+ if (!field || !value) {
+ isInvalid = true;
+ } else if (rowOperator === 'cidr') {
+ try {
+ getIPRange(value);
+ } catch (err) {
+ isInvalid = true;
+ }
+ }
+ }
+ });
+ });
+
+ cleanGroups.forEach((group) => {
+ if (group.row.length > 0) {
+ cleanRowCalls.push(group);
+ }
+ });
+
+ return { cleanRowCalls, isInvalid };
+ };
+
+ const onRunClick = () => {
+ if (operation === 'GroupBy') {
+ onRunGroupBy(selectedTable, groupByCall);
+ } else {
+ const { cleanRowCalls, isInvalid } = cleanupRows();
+
+ setHasInvalid(isInvalid);
+ setRowCalls(cleanRowCalls);
+ if (!isInvalid) {
+ let columns: string[] = [];
+ if (operation === 'Extract') {
+ columns = getColumnsToShow();
+ }
+ onQuery(selectedTable, operation, cleanRowCalls, columns, operator);
+ }
+ }
+ };
+
+ const onSave = () => {
+ setUpdating(true);
+ setTimeout(() => {
+ setUpdating(false);
+ }, 1500);
+ const { cleanRowCalls, isInvalid } = cleanupRows();
+ let queries = [...queriesList];
+ const queryObj = {
+ name: queryName,
+ description: queryDescription,
+ table: selectedTable.name,
+ operation,
+ columns: operation === 'Extract' ? getColumnsToShow() : undefined,
+ operator,
+ rowCalls: cleanRowCalls,
+ groupByCall,
+ isInvalid
+ };
+
+ if (inEditMode && queries.length > 0) {
+ const clone = [...queries];
+ clone.splice(editSavedIdx, 1, queryObj);
+ localStorage.setItem('saved-queries', JSON.stringify(clone));
+ setQueriesList(clone);
+ } else {
+ queries.push(queryObj);
+ localStorage.setItem('saved-queries', JSON.stringify(queries));
+ setQueriesList(queries);
+ setEditSavedIdx(queries.length - 1);
+ }
+ };
+
+ const onExportLogs = () => {
+ if (results) {
+ const columns = results.rows.map((row) => row[0].uint64val);
+ const table = results.index ? results.index : '';
+ onExternalLookup(table, columns);
+ }
+ };
+
+ const onSavedQueryClick = (idx: number) => {
+ const {
+ name,
+ description,
+ table,
+ operation,
+ columns,
+ operator,
+ rowCalls,
+ groupByCall,
+ isInvalid
+ } = queriesList[idx];
+ const tableDetails = tables.find((t) => t.name === table);
+ setEditSavedIdx(idx);
+ setSelectedTable(tableDetails);
+ if (columns) {
+ setSelectedColumns(
+ tableDetails.fields.map((field) =>
+ columns.includes(field.name)
+ ? { name: field.name, show: true }
+ : { name: field.name, show: false }
+ )
+ );
+ }
+ setOperation(operation);
+ setOperator(operator);
+ setRowCalls(rowCalls);
+ setGroupByCall(groupByCall);
+ setQueryName(name);
+ setQueryDescription(description || '');
+ setHasInvalid(!!isInvalid);
+
+ if (!isInvalid) {
+ if (operation === 'GroupBy') {
+ onRunGroupBy(tableDetails, groupByCall);
+ } else {
+ onQuery(tableDetails, operation, rowCalls, columns, operator);
+ }
+ }
+ };
+
+ const onRemoveQuery = (queryIdx: number) => {
+ let queries = [...queriesList];
+ queries.splice(queryIdx, 1);
+ localStorage.setItem('saved-queries', JSON.stringify(queries));
+ setQueriesList(queries);
+ };
+
+ const reset = () => {
+ setEditSavedIdx(-1);
+ setSelectedTable(tables[0]);
+ setSelectedColumns(
+ tables[0].fields.map((field) => ({
+ name: field.name,
+ show: true
+ }))
+ );
+ setOperation('Extract');
+ setRowCalls([]);
+ setQueryName('');
+ setQueryDescription('');
+ setHasInvalid(false);
+ setGroupByCall({
+ primary: '',
+ secondary: ''
+ });
+ onClear();
+ };
+
+ return (
+
+ localStorage.setItem('builderColSizes', JSON.stringify(sizes))
+ }
+ gutter={(_index, direction) => {
+ const gutter = document.createElement('div');
+ gutter.className = `gutter gutter-${direction}`;
+ const dragbars = document.createElement('div');
+ dragbars.className = 'dragBar';
+ gutter.appendChild(dragbars);
+ return gutter;
+ }}
+ style={{ display: 'flex' }}
+ className={!showBuilder ? 'hide-gutter' : undefined}
+ >
+ {showBuilder ? (
+
+
+ setShowBuilder(false)} size="small">
+
+
+
+
+ {inEditMode ? (
+
+
+ Back to new query
+
+ ) : null}
+
+ {inEditMode ? 'Edit Saved Query' : 'New Query'}
+
+
+
+ {inEditMode ? (
+
+ = 0 && queryNameIdx !== editSavedIdx)
+ }
+ helperText={
+ queryNameIdx >= 0 && queryNameIdx !== editSavedIdx
+ ? 'Query name must be unique'
+ : ''
+ }
+ onChange={(event) => setQueryName(event.target.value)}
+ margin="normal"
+ size="small"
+ required
+ fullWidth
+ />
+
+ setQueryDescription(event.target.value)}
+ margin="normal"
+ size="small"
+ fullWidth
+ />
+
+ ) : null}
+
+
+
+
+
+
+ {selectedTable
+ ? rowCalls.map((rowCall, idx) => (
+
+ {idx > 0 ? (
+
+
+
+
+
+
+ ))
+ : null}
+
+ {selectedTable && operation === 'GroupBy' ? (
+
+
+
+ ) : (
+
+
+
{
+ if (rowCalls.length === 0 || !!operator) {
+ onNewGroup();
+ } else {
+ setNewOperatorEl(event.currentTarget);
+ }
+ }}
+ >
+
+
+
+
+ )}
+
+
+
+
+
+ {inEditMode ? (
+
+ ) : (
+
+
+ setSaveButtonEl(null)}
+ anchorOrigin={{
+ vertical: 'top',
+ horizontal: 'left'
+ }}
+ transformOrigin={{
+ vertical: 'bottom',
+ horizontal: 'left'
+ }}
+ >
+
+
+ Give your query a unique name and a short description to
+ help you identify it for future use.
+
+
+
= 0}
+ helperText={
+ queryNameIdx >= 0 ? 'Query name must be unique' : ''
+ }
+ onChange={(event) => setQueryName(event.target.value)}
+ margin="dense"
+ required
+ fullWidth
+ />
+
+
+ setQueryDescription(event.target.value)
+ }
+ margin="dense"
+ fullWidth
+ />
+
+
+
+
+
+
+
+ )}
+
+
+
+
+
setShowBuilder(true)} size="small">
+
+
+
+
+
+
+ Results{' '}
+
+ {results?.query.includes('Extract(') ? (
+
+
+
+
+
+
+ ) : null}
+
Loading...
: null}
+ {results && !error && !loading ? (
+
+ {fullResultsCount && fullRecordsCount ? (
+
+ {results.duration ? (
+
+ {fullRecordsCount.toLocaleString()} records scanned in{' '}
+ {formatDuration(results.duration, true)}.
+
+ ) : null}
+
+ Showing{' '}
+ {fullResultsCount > 1000 ? 'first 1,000 rows of' : 'all'}{' '}
+ {fullResultsCount.toLocaleString()} results.
+
+
{error.error}
: null}
+ {!loading && !results && !error ? (
+
+
+ Build a query to see results
+
+ {queriesList.length > 0 ? (
+
+
+ Saved Queries
+
+
+
+ ) : null}
+
+ ) : null}
+
+
+
+
+
+
+
+ {isNot ? (
+ onUpdate(rowData, operator, !isNot)}
+ />
+ ) : (
+ onUpdate(rowData, operator, !isNot)}
+ />
+ )}
+ onUpdate(rowData, operator, !isNot)}
+ >
+ Not
+
+
+
+ {rowData.map((row, idx) => {
+ const { field, rowOperator, value, type } = row;
+ let isInvalidValue = !value;
+ if (rowOperator === 'cidr') {
+ try {
+ getIPRange(value);
+ } catch (err) {
+ isInvalidValue = true;
+ }
+ }
+
+ return (
+
+ {idx > 0 && operator ? (
+
+ setActiveOperatorEl(event.currentTarget)}
+ >
+ {operator}
+
+
+
+ ) : null}
+
+