From ad2f5c5ed4cc005c7fb2f31f2ef46d29506f96b7 Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 2 Nov 2021 16:03:43 -0500 Subject: [PATCH] limit the number of results that we keep also tells the user how many results there were, but they only get to see 1000 of them --- lattice/src/App/Query/QueryContainer.tsx | 85 +++++++------ .../App/Query/QueryResults/QueryResults.tsx | 68 ++++++---- .../GroupByChart/GroupByChart.tsx | 73 +++++------ .../QueryBuilder/QueryBuilderContainer.tsx | 116 +++++++++--------- 4 files changed, 189 insertions(+), 153 deletions(-) diff --git a/lattice/src/App/Query/QueryContainer.tsx b/lattice/src/App/Query/QueryContainer.tsx index 7c102509c..4623cfc3c 100644 --- a/lattice/src/App/Query/QueryContainer.tsx +++ b/lattice/src/App/Query/QueryContainer.tsx @@ -1,37 +1,42 @@ -import React, { FC, useState } from 'react'; -import moment, { Moment } from 'moment'; -import uniqBy from 'lodash/uniqBy'; -import { Query } from './Query'; -import { pilosa } from 'services/eventServices'; -import { useEffectOnce } from 'react-use'; -import { grpc } from '@improbable-eng/grpc-web'; -import { queryPQL, querySQL } from 'services/grpcServices'; -import { ColumnInfo, ColumnResponse, RowResponse } from 'proto/pilosa_pb'; +import React, { FC, useState } from "react"; +import moment, { Moment } from "moment"; +import uniqBy from "lodash/uniqBy"; +import { Query } from "./Query"; +import { pilosa } from "services/eventServices"; +import { useEffectOnce } from "react-use"; +import { grpc } from "@improbable-eng/grpc-web"; +import { queryPQL, querySQL } from "services/grpcServices"; +import { ColumnInfo, ColumnResponse, RowResponse } from "proto/pilosa_pb"; export type ResultType = { query: string; operation: string; - type: 'PQL' | 'SQL'; + type: "PQL" | "SQL"; headers: ColumnInfo.AsObject[]; rows: ColumnResponse.AsObject[][]; duration?: number; roundtrip: number; index?: string; error: string; + totalMessageCount: number; }; let streamingResults: ResultType = { - query: '', - operation: '', - type: 'SQL', + query: "", + operation: "", + type: "SQL", headers: [], rows: [], roundtrip: 0, - error: '' + error: "", + totalMessageCount: 0, }; export const QueryContainer: FC<{}> = () => { let startTime: Moment; + + const MAX_MESSAGES = 1000; // same limit as in query builder + const [indexes, setIndexes] = useState(); const [results, setResults] = useState([]); const [errorResult, setErrorResult] = useState(); @@ -44,12 +49,15 @@ export const QueryContainer: FC<{}> = () => { }); const handleQueryMessages = (message: RowResponse) => { - const response = message.toObject(); - if (response.headersList.length > 0) { - streamingResults.headers = response.headersList; - streamingResults.duration = response.duration; + if (streamingResults.totalMessageCount < MAX_MESSAGES) { + const response = message.toObject(); + if (response.headersList.length > 0) { + streamingResults.headers = response.headersList; + streamingResults.duration = response.duration; + } + streamingResults.rows.push(response.columnsList); } - streamingResults.rows.push(response.columnsList); + streamingResults.totalMessageCount += 1; }; const handleQueryEnd = (status: grpc.Code, statusMessage: string) => { @@ -58,71 +66,72 @@ export const QueryContainer: FC<{}> = () => { setErrorResult(streamingResults); } else { let recentQueries = JSON.parse( - localStorage.getItem('recent-queries') || '[]' + localStorage.getItem("recent-queries") || "[]" ); - const lastQuery = localStorage.getItem('last-query'); + const lastQuery = localStorage.getItem("last-query"); recentQueries.unshift(lastQuery); recentQueries = uniqBy(recentQueries); if (recentQueries.length > 10) { localStorage.setItem( - 'recent-queries', + "recent-queries", JSON.stringify(recentQueries.slice(0, 9)) ); } else { - localStorage.setItem('recent-queries', JSON.stringify(recentQueries)); + localStorage.setItem("recent-queries", JSON.stringify(recentQueries)); } streamingResults.roundtrip = moment .duration(moment().diff(startTime)) - .as('milliseconds'); + .as("milliseconds"); setErrorResult(undefined); setResults([streamingResults, ...results]); } setLoading(false); }; - const onQuery = (query: string, type: 'PQL' | 'SQL', index?: string) => { + const onQuery = (query: string, type: "PQL" | "SQL", index?: string) => { streamingResults = { query, - operation: '', + operation: "", type, headers: [], rows: [], index, roundtrip: 0, - error: '' + error: "", + totalMessageCount: 0, }; startTime = moment(); if (query) { setLoading(true); localStorage.setItem( - 'last-query', - type === 'PQL' ? `[${index}]${query}` : query + "last-query", + type === "PQL" ? `[${index}]${query}` : query ); - if (type === 'PQL') { + if (type === "PQL") { if (index) { queryPQL(index, query, handleQueryMessages, handleQueryEnd); } else { - streamingResults.error = 'missing index'; + streamingResults.error = "missing index"; setErrorResult(streamingResults); setLoading(false); } } else { - let queryArr = query.split(' '); + let queryArr = query.split(" "); queryArr.forEach((word, idx) => { - if (word.includes('-')) { - let wordArr = word.split('.'); + if (word.includes("-")) { + let wordArr = word.split("."); wordArr.forEach((section, idx) => { - if(section.includes('-') && !word.includes('`')) { + if (section.includes("-") && !word.includes("`")) { wordArr[idx] = `\`${wordArr[idx]}\``; } - }) - queryArr[idx] = wordArr.join('.'); + }); + queryArr[idx] = wordArr.join("."); } }); - querySQL(queryArr.join(' '), handleQueryMessages, handleQueryEnd); + querySQL(queryArr.join(" "), handleQueryMessages, handleQueryEnd); } } }; diff --git a/lattice/src/App/Query/QueryResults/QueryResults.tsx b/lattice/src/App/Query/QueryResults/QueryResults.tsx index abab622e0..e940a7ab6 100644 --- a/lattice/src/App/Query/QueryResults/QueryResults.tsx +++ b/lattice/src/App/Query/QueryResults/QueryResults.tsx @@ -1,15 +1,15 @@ -import React, { FC, Fragment, useState } from 'react'; -import CloseIcon from '@material-ui/icons/Close'; -import copy from 'copy-to-clipboard'; -import FileCopySharpIcon from '@material-ui/icons/FileCopySharp'; -import IconButton from '@material-ui/core/IconButton'; -import Tooltip from '@material-ui/core/Tooltip'; -import Typography from '@material-ui/core/Typography'; -import { DataTable } from 'shared/DataTable'; -import { formatDuration } from 'shared/utils/formatDuration'; -import { GroupByChart } from 'App/QueryBuilder/GroupByChart'; -import { ResultType } from '../QueryContainer'; -import css from './QueryResults.module.scss'; +import React, { FC, Fragment, useState } from "react"; +import CloseIcon from "@material-ui/icons/Close"; +import copy from "copy-to-clipboard"; +import FileCopySharpIcon from "@material-ui/icons/FileCopySharp"; +import IconButton from "@material-ui/core/IconButton"; +import Tooltip from "@material-ui/core/Tooltip"; +import Typography from "@material-ui/core/Typography"; +import { DataTable } from "shared/DataTable"; +import { formatDuration } from "shared/utils/formatDuration"; +import { GroupByChart } from "App/QueryBuilder/GroupByChart"; +import { ResultType } from "../QueryContainer"; +import css from "./QueryResults.module.scss"; type QueryResultsProps = { collapsibleQuery?: boolean; @@ -20,12 +20,12 @@ type QueryResultsProps = { export const QueryResults: FC = ({ collapsibleQuery = true, results, - onRemoveResult + onRemoveResult, }) => { const [showQuery, setShowQuery] = useState(false); - const [copyTooltip, setCopyTooltip] = useState('Copy Query'); + const [copyTooltip, setCopyTooltip] = useState("Copy Query"); const queryString = - results.type === 'PQL' + results.type === "PQL" ? `[${results.index}]${results.query}` : results.query; @@ -34,10 +34,10 @@ export const QueryResults: FC = ({ let rowData = {}; row.forEach((col, colIdx) => { const header = headers[colIdx]; - if (header.datatype.includes('[]')) { + if (header.datatype.includes("[]")) { const dataTypeVal = `${header.datatype.slice(2)}arrayval`; - rowData[header.name] = col[dataTypeVal].valsList.join(', '); - } else if (header.datatype === 'decimal') { + rowData[header.name] = col[dataTypeVal].valsList.join(", "); + } else if (header.datatype === "decimal") { const decimalVal = col[`${header.datatype}val`]; if (decimalVal) { const { value, scale } = decimalVal; @@ -55,9 +55,9 @@ export const QueryResults: FC = ({ const onCopyQuery = () => { copy(queryString); - setCopyTooltip('Copied!'); + setCopyTooltip("Copied!"); setTimeout(() => { - setCopyTooltip('Copy Query'); + setCopyTooltip("Copy Query"); }, 1500); }; @@ -75,7 +75,7 @@ export const QueryResults: FC = ({ onClick={() => setShowQuery(!showQuery)} className={css.link} > - {showQuery ? 'Hide' : 'Show'} Query + {showQuery ? "Hide" : "Show"} Query )} @@ -136,10 +136,32 @@ export const QueryResults: FC = ({ {!collapsibleQuery || showQuery ? ( - {queryString} +
+ {queryString} + + {results.totalMessageCount > 1000 ? ( + + + NOTE + + + ) : null} +
) : null} - {results.operation === 'GroupBy' && results.rows.length <= 50 ? ( + {results.operation === "GroupBy" && results.rows.length <= 50 ? ( ) : ( diff --git a/lattice/src/App/QueryBuilder/GroupByChart/GroupByChart.tsx b/lattice/src/App/QueryBuilder/GroupByChart/GroupByChart.tsx index fb323264e..90973a231 100644 --- a/lattice/src/App/QueryBuilder/GroupByChart/GroupByChart.tsx +++ b/lattice/src/App/QueryBuilder/GroupByChart/GroupByChart.tsx @@ -1,9 +1,9 @@ -import React, { FC } from 'react'; -import groupBy from 'lodash/groupBy'; -import { ResponsiveBar } from '@nivo/bar'; -import { ResultType } from '../../Query'; -import { schemeTableau10 } from 'd3-scale-chromatic'; -import { useTheme } from '@material-ui/core/styles'; +import React, { FC } from "react"; +import groupBy from "lodash/groupBy"; +import { ResponsiveBar } from "@nivo/bar"; +import { ResultType } from "../../Query"; +import { schemeTableau10 } from "d3-scale-chromatic"; +import { useTheme } from "@material-ui/core/styles"; type GroupByChartType = { results: ResultType; @@ -11,7 +11,7 @@ type GroupByChartType = { export const GroupByChart: FC = ({ results }) => { const theme = useTheme(); - const isDark = theme.palette.type === 'dark'; + const isDark = theme.palette.type === "dark"; const { headers, rows } = results; let uniqueKeys: string[] = []; const grouped = groupBy(rows, (row) => row[0].stringval); @@ -23,7 +23,7 @@ export const GroupByChart: FC = ({ results }) => { groupData = { ...groupData, [headers[0].name]: row[0][`${headers[0].datatype}val`], - [secondaryValue]: row[2][`${headers[2].datatype}val`] + [secondaryValue]: row[2][`${headers[2].datatype}val`], }; if (!uniqueKeys.includes(secondaryValue.toString())) { @@ -33,7 +33,7 @@ export const GroupByChart: FC = ({ results }) => { groupData = { ...groupData, [headers[0].name]: row[0][`${headers[0].datatype}val`], - value: secondaryValue + value: secondaryValue, }; } }); @@ -42,31 +42,34 @@ export const GroupByChart: FC = ({ results }) => { }); return ( -
+
+ This query has {results.totalMessageCount} total results. Due to browser + memory limitations, you will only be able to see {results.rows.length}{" "} + results. 0 ? uniqueKeys : undefined} indexBy={headers[0].name} margin={{ top: 50, right: 130, bottom: 100, left: 60 }} padding={0.3} - valueScale={{ type: 'linear' }} - indexScale={{ type: 'band', round: true }} + valueScale={{ type: "linear" }} + indexScale={{ type: "band", round: true }} colors={schemeTableau10} enableLabel={false} theme={ isDark ? { - textColor: 'var(--text-secondary)', + textColor: "var(--text-secondary)", axis: { - domain: { line: { stroke: 'rgba(255, 255, 255, 0.1)' } } + domain: { line: { stroke: "rgba(255, 255, 255, 0.1)" } }, }, - grid: { line: { stroke: 'rgba(255, 255, 255, 0.1)' } }, - tooltip: { container: { background: '#1c2022' } } + grid: { line: { stroke: "rgba(255, 255, 255, 0.1)" } }, + tooltip: { container: { background: "#1c2022" } }, } : { axis: { - domain: { line: { stroke: '#dddddd' } } - } + domain: { line: { stroke: "#dddddd" } }, + }, } } groupMode="grouped" @@ -75,16 +78,16 @@ export const GroupByChart: FC = ({ results }) => { tickPadding: 5, tickRotation: -40, legend: headers[0].name, - legendPosition: 'middle', - legendOffset: 80 + legendPosition: "middle", + legendOffset: 80, }} axisLeft={{ tickSize: 5, tickPadding: 5, tickRotation: 0, - legend: 'count', - legendPosition: 'middle', - legendOffset: -40 + legend: "count", + legendPosition: "middle", + legendOffset: -40, }} tooltip={({ id, value, color }) => ( @@ -93,31 +96,31 @@ export const GroupByChart: FC = ({ results }) => { )} labelSkipWidth={12} labelSkipHeight={12} - labelTextColor={{ from: 'color', modifiers: [['darker', 1.6]] }} + labelTextColor={{ from: "color", modifiers: [["darker", 1.6]] }} legends={[ { - dataFrom: 'keys', - anchor: 'bottom-right', - direction: 'column', + dataFrom: "keys", + anchor: "bottom-right", + direction: "column", justify: false, translateX: 120, translateY: 0, itemsSpacing: 2, itemWidth: 120, itemHeight: 20, - itemDirection: 'left-to-right', + itemDirection: "left-to-right", itemOpacity: 0.85, - symbolShape: 'circle', + symbolShape: "circle", symbolSize: 20, effects: [ { - on: 'hover', + on: "hover", style: { - itemOpacity: 1 - } - } - ] - } + itemOpacity: 1, + }, + }, + ], + }, ]} animate={true} motionStiffness={90} diff --git a/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx b/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx index ef9f9062a..a526fd372 100644 --- a/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx +++ b/lattice/src/App/QueryBuilder/QueryBuilderContainer.tsx @@ -1,48 +1,49 @@ -import React, { Fragment, useState } from 'react'; -import Alert from '@material-ui/lab/Alert'; -import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos'; -import Button from '@material-ui/core/Button'; -import classNames from 'classnames'; -import CloseIcon from '@material-ui/icons/Close'; -import IconButton from '@material-ui/core/IconButton'; -import InfoIcon from '@material-ui/icons/Info'; -import moment, { Moment } from 'moment'; -import Paper from '@material-ui/core/Paper'; -import Split from 'react-split'; -import Snackbar from '@material-ui/core/Snackbar'; -import Tooltip from '@material-ui/core/Tooltip'; -import Typography from '@material-ui/core/Typography'; -import { Block } from 'shared/Block'; -import { formatDuration } from 'shared/utils/formatDuration'; -import { grpc } from '@improbable-eng/grpc-web'; -import { pilosa } from 'services/eventServices'; -import { QueryBuilder } from './QueryBuilder'; -import { queryPQL } from 'services/grpcServices'; -import { QueryResults } from 'App/Query/QueryResults'; -import { ResultType } from 'App/Query/QueryContainer'; -import { RowResponse } from 'proto/pilosa_pb'; -import { SavedQueries } from 'App/QueryBuilder/SavedQueries'; -import { useEffectOnce } from 'react-use'; -import css from './QueryBuilderContainer.module.scss'; +import React, { Fragment, useState } from "react"; +import Alert from "@material-ui/lab/Alert"; +import ArrowForwardIosIcon from "@material-ui/icons/ArrowForwardIos"; +import Button from "@material-ui/core/Button"; +import classNames from "classnames"; +import CloseIcon from "@material-ui/icons/Close"; +import IconButton from "@material-ui/core/IconButton"; +import InfoIcon from "@material-ui/icons/Info"; +import moment, { Moment } from "moment"; +import Paper from "@material-ui/core/Paper"; +import Split from "react-split"; +import Snackbar from "@material-ui/core/Snackbar"; +import Tooltip from "@material-ui/core/Tooltip"; +import Typography from "@material-ui/core/Typography"; +import { Block } from "shared/Block"; +import { formatDuration } from "shared/utils/formatDuration"; +import { grpc } from "@improbable-eng/grpc-web"; +import { pilosa } from "services/eventServices"; +import { QueryBuilder } from "./QueryBuilder"; +import { queryPQL } from "services/grpcServices"; +import { QueryResults } from "App/Query/QueryResults"; +import { ResultType } from "App/Query/QueryContainer"; +import { RowResponse } from "proto/pilosa_pb"; +import { SavedQueries } from "App/QueryBuilder/SavedQueries"; +import { useEffectOnce } from "react-use"; +import css from "./QueryBuilderContainer.module.scss"; let streamingResults: ResultType = { - query: '', - operation: '', - type: 'PQL', + query: "", + operation: "", + type: "PQL", headers: [], rows: [], roundtrip: 0, - error: '' + error: "", + totalMessageCount: 0, }; export const QueryBuilderContainer = () => { let startTime: Moment; let exportRows: any[] = []; const colSizes = JSON.parse( - localStorage.getItem('builderColSizes') || '[25, 75]' + localStorage.getItem("builderColSizes") || "[25, 75]" ); const [queriesList, setQueriesList] = useState( - JSON.parse(localStorage.getItem('saved-queries') || '[]') + JSON.parse(localStorage.getItem("saved-queries") || "[]") ); const [tables, setTables] = useState([]); @@ -51,7 +52,7 @@ export const QueryBuilderContainer = () => { const [fullCount, setFullCount] = useState(); const [recordsCount, setRecordsCount] = useState(); const [errorResult, setErrorResult] = useState(); - const [error, setError] = useState(''); + const [error, setError] = useState(""); const [loading, setLoading] = useState(false); const [savedQuery, setSavedQuery] = useState(-1); @@ -77,7 +78,7 @@ export const QueryBuilderContainer = () => { } else { streamingResults.roundtrip = moment .duration(moment().diff(startTime)) - .as('milliseconds'); + .as("milliseconds"); setErrorResult(undefined); setResults(streamingResults); } @@ -89,12 +90,12 @@ export const QueryBuilderContainer = () => { let rowStr: string[] = []; if (exportRows.length === 0) { const headers = response.headersList.map((header) => header.name); - exportRows.push(headers.join('\t')); + exportRows.push(headers.join("\t")); } response.headersList.forEach((header, idx) => rowStr.push(response.columnsList[idx][`${header.datatype}val`]) ); - exportRows.push(rowStr.join('\t')); + exportRows.push(rowStr.join("\t")); }; const handleExternalLookupEnd = ( @@ -104,12 +105,12 @@ export const QueryBuilderContainer = () => { if (status !== grpc.Code.OK) { setError(statusMessage); } else if (exportRows.length === 0) { - setError('No record attributes for current query.'); + setError("No record attributes for current query."); } else { const dateTime = moment().unix(); - const element = document.createElement('a'); - const file = new Blob([exportRows.join('\n')], { - type: 'text/plain;charset=utf-8' + const element = document.createElement("a"); + const file = new Blob([exportRows.join("\n")], { + type: "text/plain;charset=utf-8", }); element.href = URL.createObjectURL(file); element.download = `molecula-${results?.index}-${dateTime}.csv`; @@ -128,17 +129,18 @@ export const QueryBuilderContainer = () => { streamingResults = { query, operation, - type: 'PQL', + type: "PQL", headers: [], rows: [], index: table, roundtrip: 0, - error: '' + error: "", + totalMessageCount: 0, }; startTime = moment(); setLoading(true); - if (operation !== 'Count') { + if (operation !== "Count") { if (countQuery) { pilosa.post.query(table, countQuery).then((res) => { setFullCount(res.data.results[0]); @@ -164,13 +166,13 @@ export const QueryBuilderContainer = () => { const onRemoveQuery = (queryIdx: number) => { let queries = [...queriesList]; queries.splice(queryIdx, 1); - localStorage.setItem('saved-queries', JSON.stringify(queries)); + localStorage.setItem("saved-queries", JSON.stringify(queries)); setQueriesList(queries); }; const onSaveQuery = () => { const updatedList = JSON.parse( - localStorage.getItem('saved-queries') || '[]' + localStorage.getItem("saved-queries") || "[]" ); if (savedQuery < 0) { setSavedQuery(updatedList.length - 1); @@ -181,7 +183,7 @@ export const QueryBuilderContainer = () => { const onExportLogs = () => { if (results) { const columns = results.rows.map((row) => row[0].uint64val); - const table = results.index ? results.index : ''; + const table = results.index ? results.index : ""; onExternalLookup(table, columns); } }; @@ -200,17 +202,17 @@ export const QueryBuilderContainer = () => { cursor="col-resize" minSize={showBuilder ? 350 : 50} onDragEnd={(sizes) => - localStorage.setItem('builderColSizes', JSON.stringify(sizes)) + localStorage.setItem("builderColSizes", JSON.stringify(sizes)) } gutter={(_index, direction) => { - const gutter = document.createElement('div'); + const gutter = document.createElement("div"); gutter.className = `gutter gutter-${direction}`; - const dragbars = document.createElement('div'); - dragbars.className = 'dragBar'; + const dragbars = document.createElement("div"); + dragbars.className = "dragBar"; gutter.appendChild(dragbars); return gutter; }} - className={classNames(css.split, !showBuilder ? 'hide-gutter' : '')} + className={classNames(css.split, !showBuilder ? "hide-gutter" : "")} > {showBuilder ? (
@@ -239,9 +241,9 @@ export const QueryBuilderContainer = () => {
- Results{' '} + Results{" "} - {results?.query.includes('Extract(') ? ( + {results?.query.includes("Extract(") ? (
{
{results.duration ? (
- {recordsCount.toLocaleString()} records scanned in{' '} + {recordsCount.toLocaleString()} records scanned in{" "} {formatDuration(results.duration, true)}.
) : null}
- Showing{' '} - {fullCount > 1000 ? 'first 1,000 rows of' : 'all'}{' '} + Showing{" "} + {fullCount > 1000 ? "first 1,000 rows of" : "all"}{" "} {fullCount.toLocaleString()} results.
@@ -301,7 +303,7 @@ export const QueryBuilderContainer = () => {
- setError('')}> + setError("")}> {error}