Merge pull request #1608 from molecula/feature/syang/groupby-filter

CLOUD-59: Allow filter parameter query builder GroupBy
2026-09-09 06:31:02 +00:00 · 2021-05-24 23:06:10 -05:00 · 2021-05-24 23:06:10 -05:00 · 17a58f1020
commit 17a58f1020
parent aed8727aea 3edac67c0c
5 changed files with 180 additions and 81 deletions
--- a/lattice/src/App/QueryBuilder/QueryBuilder.module.scss
+++ b/lattice/src/App/QueryBuilder/QueryBuilder.module.scss
@ -115,6 +115,18 @@
  .fieldSelector {
    margin-top: 16px;
  }
+
+  .filterHeader {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    margin-top: 8px;
+  }
+
+  .filterInfo {
+    vertical-align: text-top;
+    margin-left: 4px;
+  }
 }

 .savePopover {
--- a/lattice/src/App/QueryBuilder/QueryBuilder.tsx
+++ b/lattice/src/App/QueryBuilder/QueryBuilder.tsx
@ -1,4 +1,4 @@
-import React, { FC, Fragment, useState } from 'react';
+import React, { FC, Fragment, useEffect, useState } from 'react';
 import AddIcon from '@material-ui/icons/Add';
 import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos';
 import ArrowBackIcon from '@material-ui/icons/ArrowBack';
@ -30,6 +30,7 @@ import { ResultType } from 'App/Query/QueryContainer';
 import { RowCall } from './RowCall';
 import { SavedQueries } from './SavedQueries';
 import { Select } from 'shared/Select';
+import { stringifyRowData } from './utils';
 import css from './QueryBuilder.module.scss';

 type QueryBuilderProps = {
@ -46,7 +47,7 @@ type QueryBuilderProps = {
    columns: string[],
    operator?: Operator
  ) => void;
-  onRunGroupBy: (table: any, rowsData: RowsCallType) => void;
+  onRunGroupBy: (table: any, rowsData: RowsCallType, filter?: string) => void;
  onExternalLookup: (table: string, columns: number[]) => void;
  onClear: () => void;
 };
@ -92,12 +93,22 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
  const [queriesList, setQueriesList] = useState<any[]>(
    JSON.parse(localStorage.getItem('saved-queries') || '[]')
  );
+  const [groupByFilters, setGroupByFilters] = useState<any[]>([]);
+  const [filter, setFilter] = useState<string>();
  const colSizes = JSON.parse(
    localStorage.getItem('builderColSizes') || '[25, 75]'
  );
  const queryNameIdx = queriesList.findIndex((q) => q.name === queryName);
  const inEditMode = editSavedIdx >= 0;

+  useEffect(() => {
+    setGroupByFilters(
+      queriesList.filter(
+        (q) => q.table === selectedTable.name && q.operation === 'Extract'
+      )
+    );
+  }, [selectedTable]);
+
  const getColumnsToShow = (cols?: { name: string; show: boolean }[]) => {
    let columns: string[] = [];
    const list = cols ? cols : selectedColumns;
@ -216,7 +227,11 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({

  const onRunClick = () => {
    if (operation === 'GroupBy') {
-      onRunGroupBy(selectedTable, groupByCall);
+      const isInvalid = groupByCall.primary ? false : true;
+      setHasInvalid(isInvalid);
+      if (!isInvalid) {
+        onRunGroupBy(selectedTable, groupByCall, filter);
+      }
    } else {
      const { cleanRowCalls, isInvalid } = cleanupRows();

@ -248,6 +263,7 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
      operator,
      rowCalls: cleanRowCalls,
      groupByCall,
+      filter,
      isInvalid
    };

@ -282,6 +298,7 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
      operator,
      rowCalls,
      groupByCall,
+      filter,
      isInvalid
    } = queriesList[idx];
    const tableDetails = tables.find((t) => t.name === table);
@ -303,10 +320,11 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
    setQueryName(name);
    setQueryDescription(description || '');
    setHasInvalid(!!isInvalid);
+    setFilter(filter);

    if (!isInvalid) {
      if (operation === 'GroupBy') {
-        onRunGroupBy(tableDetails, groupByCall);
+        onRunGroupBy(tableDetails, groupByCall, filter);
      } else {
        onQuery(tableDetails, operation, rowCalls, columns, operator);
      }
@ -338,6 +356,7 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
      primary: '',
      secondary: ''
    });
+    setFilter(undefined);
    onClear();
  };

@ -474,7 +493,7 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
              />
            </div>

-            {selectedTable
+            {selectedTable && operation !== 'GroupBy'
              ? rowCalls.map((rowCall, idx) => (
                  <Fragment key={`group-${idx}`}>
                    {idx > 0 ? (
@ -546,6 +565,7 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
                      primary: value.toString()
                    })
                  }
+                  error={hasInvalid && !groupByCall.primary}
                />
                <Select
                  className={css.fieldSelector}
@ -565,6 +585,50 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
                    })
                  }
                />
+
+                <div className={css.filterHeader}>
+                  <Typography variant="caption" color="textSecondary">
+                    Filter (optional)
+                    {groupByFilters.length > 0 && (
+                      <Tooltip
+                        className={css.filterInfo}
+                        title={`To use filters, save an Extract query for ${selectedTable.name} with at least one field constraint.`}
+                        placement="top"
+                        arrow
+                      >
+                        <InfoIcon fontSize="inherit" />
+                      </Tooltip>
+                    )}
+                  </Typography>
+                  {filter ? (
+                    <span
+                      className={css.link}
+                      onClick={() => setFilter(undefined)}
+                    >
+                      Clear Filter
+                    </span>
+                  ) : null}
+                </div>
+                {filter ? (
+                  <Typography variant="caption">{filter}</Typography>
+                ) : groupByFilters.length > 0 ? (
+                  <SavedQueries
+                    queries={groupByFilters}
+                    tables={tables}
+                    onClickSaved={(queryIdx) => {
+                      const { rowCalls, operator } = groupByFilters[queryIdx];
+                      const res = stringifyRowData(rowCalls, operator);
+                      if (!res.error) {
+                        setFilter(res.query);
+                      }
+                    }}
+                  />
+                ) : (
+                  <div className={css.infoMessage}>
+                    No available filters. To use filters, save an Extract query
+                    for {selectedTable.name} with at least one field constraint.
+                  </div>
+                )}
              </div>
            ) : (
              <div className={css.newGroup}>
@ -721,6 +785,7 @@ export const QueryBuilder: FC<QueryBuilderProps> = ({
                  primary: '',
                  secondary: ''
                });
+                setFilter(undefined);
                onClear();
              }}
            >
--- a/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx
+++ b/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx
@ -13,7 +13,7 @@ import { ResultType } from 'App/Query/QueryContainer';
 import { queryPQL } from 'services/grpcServices';
 import { grpc } from '@improbable-eng/grpc-web';
 import { RowResponse } from 'proto/pilosa_pb';
-import { getIPRange } from 'get-ip-range';
+import { stringifyRowData } from './utils';
 import css from './QueryBuilderContainer.module.scss';

 let streamingResults: ResultType = {
@ -89,70 +89,11 @@ export const QueryBuilderContainer = () => {
    if (rowData.length === 0) {
      query = 'All()';
    } else {
-      let rowsMap: string[][] = [];
-      rowData.forEach((group, groupIdx) => {
-        rowsMap.push([]);
-        group.row.forEach((row) => {
-          let rowString = '';
-          const { field, rowOperator, value, type } = row;
-          const isNegatory = rowOperator === '!=';
-          const isUnion =
-            ['=', '!='].includes(rowOperator) && value.split(',').length > 1;
-          const operator = isNegatory ? '=' : rowOperator;
-          if (isUnion) {
-            const values = value.split(',');
-            const unionRows = values
-              .map((v) => `Row(${field}="${v.trim()}")`)
-              .join(', ');
-            rowString = `Union(${unionRows})`;
-          } else if (rowOperator === 'cidr') {
-            try {
-              const ipRange = getIPRange(value);
-              const ipRows = ipRange
-                .map((ip) => `Row(${field}="${ip}")`)
-                .join(', ');
-              rowString = `Union(${ipRows})`;
-            } catch (error) {
-              streamingResults.error = error.message;
-            }
-          } else {
-            rowString = ['set', 'timestamp'].includes(type)
-              ? `Row(${field}${operator}"${value}")`
-              : `Row(${field}${operator}${value})`;
-          }
-
-          if (isNegatory) {
-            rowsMap[groupIdx].push(`Not(${rowString})`);
-          } else {
-            rowsMap[groupIdx].push(rowString);
-          }
-        });
-      });
-
-      query = rowsMap
-        .map((group, idx) => {
-          let joined = '';
-          if (group.length > 1) {
-            joined = group.map((r) => r).join(', ');
-            const operator = rowData[idx].operator;
-            if (operator === 'and') {
-              joined = `Intersect(${joined})`;
-            } else if (operator === 'or') {
-              joined = `Union(${joined})`;
-            }
-          } else {
-            joined = group[0];
-          }
-          return rowData[idx].isNot ? `Not(${joined})` : joined;
-        })
-        .join(', ');
-
-      if (rowData.length > 1 && operator) {
-        if (operator === 'and') {
-          query = `Intersect(${query})`;
-        } else if (operator === 'or') {
-          query = `Union(${query})`;
-        }
+      const res = stringifyRowData(rowData, operator);
+      if (res.error) {
+        streamingResults.error = res.query;
+      } else {
+        query = res.query;
      }
    }

@ -216,7 +157,11 @@ export const QueryBuilderContainer = () => {
    }
  };

-  const onRunGroupBy = (table: any, rowsData: RowsCallType) => {
+  const onRunGroupBy = (
+    table: any,
+    rowsData: RowsCallType,
+    filter?: string
+  ) => {
    streamingResults = {
      query: '',
      operation: 'GroupBy',
@ -229,9 +174,11 @@ export const QueryBuilderContainer = () => {
    };
    startTime = moment();
    setLoading(true);
+    const filterString =
+      filter && filter.length > 0 ? `, filter=${filter}` : '';
    const query = rowsData.secondary
-      ? `GroupBy(Rows(${rowsData.primary}), Rows(${rowsData.secondary}))`
-      : `GroupBy(Rows(${rowsData.primary}))`;
+      ? `GroupBy(Rows(${rowsData.primary}), Rows(${rowsData.secondary})${filterString})`
+      : `GroupBy(Rows(${rowsData.primary})${filterString})`;
    streamingResults.query = query;
    queryPQL(table.name, query, handleQueryMessages, handleQueryEnd);
  };
--- a/lattice/src/App/QueryBuilder/SavedQueries/SavedQueries.tsx
+++ b/lattice/src/App/QueryBuilder/SavedQueries/SavedQueries.tsx
@ -23,7 +23,7 @@ type SavedQueriesProps = {
  }[];
  tables: any[];
  onClickSaved: (queryIdx: number) => void;
-  onRemoveQuery: (queryIdx: number) => void;
+  onRemoveQuery?: (queryIdx: number) => void;
 };

 export const SavedQueries: FC<SavedQueriesProps> = ({
@ -70,13 +70,15 @@ export const SavedQueries: FC<SavedQueriesProps> = ({
                          </span>
                        </Tooltip>
                      ) : null}
-                      <IconButton
-                        edge="end"
-                        aria-label="Delete saved query"
-                        onClick={() => onRemoveQuery(idx)}
-                      >
-                        <DeleteIcon />
-                      </IconButton>
+                      {onRemoveQuery ? (
+                        <IconButton
+                          edge="end"
+                          aria-label="Delete saved query"
+                          onClick={() => onRemoveQuery(idx)}
+                        >
+                          <DeleteIcon />
+                        </IconButton>
+                      ) : null}
                    </ListItemSecondaryAction>
                  </ListItem>
                </MotionSlideItem>
--- a/lattice/src/App/QueryBuilder/utils.ts
+++ b/lattice/src/App/QueryBuilder/utils.ts
@ -0,0 +1,73 @@
+import { Operator, RowGrouping } from './rowTypes';
+import { getIPRange } from 'get-ip-range';
+
+export const stringifyRowData = (rowData: RowGrouping[], operator?: Operator) => {
+  let query = '';
+  let rowsMap: string[][] = [];
+  rowData.forEach((group, groupIdx) => {
+    rowsMap.push([]);
+    group.row.forEach((row) => {
+      let rowString = '';
+      const { field, rowOperator, value, type } = row;
+      const isNegatory = rowOperator === '!=';
+      const isUnion =
+        ['=', '!='].includes(rowOperator) && value.split(',').length > 1;
+      const operator = isNegatory ? '=' : rowOperator;
+      if (isUnion) {
+        const values = value.split(',');
+        const unionRows = values
+          .map((v) => `Row(${field}="${v.trim()}")`)
+          .join(', ');
+        rowString = `Union(${unionRows})`;
+      } else if (rowOperator === 'cidr') {
+        try {
+          const ipRange = getIPRange(value);
+          const ipRows = ipRange
+            .map((ip) => `Row(${field}="${ip}")`)
+            .join(', ');
+          rowString = `Union(${ipRows})`;
+        } catch (error) {
+          return { error: true, query: error.message };
+        }
+      } else {
+        rowString = ['set', 'timestamp'].includes(type)
+          ? `Row(${field}${operator}"${value}")`
+          : `Row(${field}${operator}${value})`;
+      }
+
+      if (isNegatory) {
+        rowsMap[groupIdx].push(`Not(${rowString})`);
+      } else {
+        rowsMap[groupIdx].push(rowString);
+      }
+    });
+  });
+
+  query = rowsMap
+    .map((group, idx) => {
+      let joined = '';
+      if (group.length > 1) {
+        joined = group.map((r) => r).join(', ');
+        const operator = rowData[idx].operator;
+        if (operator === 'and') {
+          joined = `Intersect(${joined})`;
+        } else if (operator === 'or') {
+          joined = `Union(${joined})`;
+        }
+      } else {
+        joined = group[0];
+      }
+      return rowData[idx].isNot ? `Not(${joined})` : joined;
+    })
+    .join(', ');
+
+  if (rowData.length > 1 && operator) {
+    if (operator === 'and') {
+      query = `Intersect(${query})`;
+    } else if (operator === 'or') {
+      query = `Union(${query})`;
+    }
+  }
+
+  return { error: false, query };
+}