limit the number of results that we keep

also tells the user how many results there were, but they only
get to see 1000 of them
This commit is contained in:
reesporte 2021-11-02 16:03:43 -05:00
parent 4fd2a4fedd
commit ad2f5c5ed4
4 changed files with 189 additions and 153 deletions

View file

@ -1,37 +1,42 @@
import React, { FC, useState } from 'react';
import moment, { Moment } from 'moment';
import uniqBy from 'lodash/uniqBy';
import { Query } from './Query';
import { pilosa } from 'services/eventServices';
import { useEffectOnce } from 'react-use';
import { grpc } from '@improbable-eng/grpc-web';
import { queryPQL, querySQL } from 'services/grpcServices';
import { ColumnInfo, ColumnResponse, RowResponse } from 'proto/pilosa_pb';
import React, { FC, useState } from "react";
import moment, { Moment } from "moment";
import uniqBy from "lodash/uniqBy";
import { Query } from "./Query";
import { pilosa } from "services/eventServices";
import { useEffectOnce } from "react-use";
import { grpc } from "@improbable-eng/grpc-web";
import { queryPQL, querySQL } from "services/grpcServices";
import { ColumnInfo, ColumnResponse, RowResponse } from "proto/pilosa_pb";
export type ResultType = {
query: string;
operation: string;
type: 'PQL' | 'SQL';
type: "PQL" | "SQL";
headers: ColumnInfo.AsObject[];
rows: ColumnResponse.AsObject[][];
duration?: number;
roundtrip: number;
index?: string;
error: string;
totalMessageCount: number;
};
let streamingResults: ResultType = {
query: '',
operation: '',
type: 'SQL',
query: "",
operation: "",
type: "SQL",
headers: [],
rows: [],
roundtrip: 0,
error: ''
error: "",
totalMessageCount: 0,
};
export const QueryContainer: FC<{}> = () => {
let startTime: Moment;
const MAX_MESSAGES = 1000; // same limit as in query builder
const [indexes, setIndexes] = useState<any>();
const [results, setResults] = useState<ResultType[]>([]);
const [errorResult, setErrorResult] = useState<ResultType>();
@ -44,12 +49,15 @@ export const QueryContainer: FC<{}> = () => {
});
const handleQueryMessages = (message: RowResponse) => {
const response = message.toObject();
if (response.headersList.length > 0) {
streamingResults.headers = response.headersList;
streamingResults.duration = response.duration;
if (streamingResults.totalMessageCount < MAX_MESSAGES) {
const response = message.toObject();
if (response.headersList.length > 0) {
streamingResults.headers = response.headersList;
streamingResults.duration = response.duration;
}
streamingResults.rows.push(response.columnsList);
}
streamingResults.rows.push(response.columnsList);
streamingResults.totalMessageCount += 1;
};
const handleQueryEnd = (status: grpc.Code, statusMessage: string) => {
@ -58,71 +66,72 @@ export const QueryContainer: FC<{}> = () => {
setErrorResult(streamingResults);
} else {
let recentQueries = JSON.parse(
localStorage.getItem('recent-queries') || '[]'
localStorage.getItem("recent-queries") || "[]"
);
const lastQuery = localStorage.getItem('last-query');
const lastQuery = localStorage.getItem("last-query");
recentQueries.unshift(lastQuery);
recentQueries = uniqBy(recentQueries);
if (recentQueries.length > 10) {
localStorage.setItem(
'recent-queries',
"recent-queries",
JSON.stringify(recentQueries.slice(0, 9))
);
} else {
localStorage.setItem('recent-queries', JSON.stringify(recentQueries));
localStorage.setItem("recent-queries", JSON.stringify(recentQueries));
}
streamingResults.roundtrip = moment
.duration(moment().diff(startTime))
.as('milliseconds');
.as("milliseconds");
setErrorResult(undefined);
setResults([streamingResults, ...results]);
}
setLoading(false);
};
const onQuery = (query: string, type: 'PQL' | 'SQL', index?: string) => {
const onQuery = (query: string, type: "PQL" | "SQL", index?: string) => {
streamingResults = {
query,
operation: '',
operation: "",
type,
headers: [],
rows: [],
index,
roundtrip: 0,
error: ''
error: "",
totalMessageCount: 0,
};
startTime = moment();
if (query) {
setLoading(true);
localStorage.setItem(
'last-query',
type === 'PQL' ? `[${index}]${query}` : query
"last-query",
type === "PQL" ? `[${index}]${query}` : query
);
if (type === 'PQL') {
if (type === "PQL") {
if (index) {
queryPQL(index, query, handleQueryMessages, handleQueryEnd);
} else {
streamingResults.error = 'missing index';
streamingResults.error = "missing index";
setErrorResult(streamingResults);
setLoading(false);
}
} else {
let queryArr = query.split(' ');
let queryArr = query.split(" ");
queryArr.forEach((word, idx) => {
if (word.includes('-')) {
let wordArr = word.split('.');
if (word.includes("-")) {
let wordArr = word.split(".");
wordArr.forEach((section, idx) => {
if(section.includes('-') && !word.includes('`')) {
if (section.includes("-") && !word.includes("`")) {
wordArr[idx] = `\`${wordArr[idx]}\``;
}
})
queryArr[idx] = wordArr.join('.');
});
queryArr[idx] = wordArr.join(".");
}
});
querySQL(queryArr.join(' '), handleQueryMessages, handleQueryEnd);
querySQL(queryArr.join(" "), handleQueryMessages, handleQueryEnd);
}
}
};

View file

@ -1,15 +1,15 @@
import React, { FC, Fragment, useState } from 'react';
import CloseIcon from '@material-ui/icons/Close';
import copy from 'copy-to-clipboard';
import FileCopySharpIcon from '@material-ui/icons/FileCopySharp';
import IconButton from '@material-ui/core/IconButton';
import Tooltip from '@material-ui/core/Tooltip';
import Typography from '@material-ui/core/Typography';
import { DataTable } from 'shared/DataTable';
import { formatDuration } from 'shared/utils/formatDuration';
import { GroupByChart } from 'App/QueryBuilder/GroupByChart';
import { ResultType } from '../QueryContainer';
import css from './QueryResults.module.scss';
import React, { FC, Fragment, useState } from "react";
import CloseIcon from "@material-ui/icons/Close";
import copy from "copy-to-clipboard";
import FileCopySharpIcon from "@material-ui/icons/FileCopySharp";
import IconButton from "@material-ui/core/IconButton";
import Tooltip from "@material-ui/core/Tooltip";
import Typography from "@material-ui/core/Typography";
import { DataTable } from "shared/DataTable";
import { formatDuration } from "shared/utils/formatDuration";
import { GroupByChart } from "App/QueryBuilder/GroupByChart";
import { ResultType } from "../QueryContainer";
import css from "./QueryResults.module.scss";
type QueryResultsProps = {
collapsibleQuery?: boolean;
@ -20,12 +20,12 @@ type QueryResultsProps = {
export const QueryResults: FC<QueryResultsProps> = ({
collapsibleQuery = true,
results,
onRemoveResult
onRemoveResult,
}) => {
const [showQuery, setShowQuery] = useState<boolean>(false);
const [copyTooltip, setCopyTooltip] = useState<string>('Copy Query');
const [copyTooltip, setCopyTooltip] = useState<string>("Copy Query");
const queryString =
results.type === 'PQL'
results.type === "PQL"
? `[${results.index}]${results.query}`
: results.query;
@ -34,10 +34,10 @@ export const QueryResults: FC<QueryResultsProps> = ({
let rowData = {};
row.forEach((col, colIdx) => {
const header = headers[colIdx];
if (header.datatype.includes('[]')) {
if (header.datatype.includes("[]")) {
const dataTypeVal = `${header.datatype.slice(2)}arrayval`;
rowData[header.name] = col[dataTypeVal].valsList.join(', ');
} else if (header.datatype === 'decimal') {
rowData[header.name] = col[dataTypeVal].valsList.join(", ");
} else if (header.datatype === "decimal") {
const decimalVal = col[`${header.datatype}val`];
if (decimalVal) {
const { value, scale } = decimalVal;
@ -55,9 +55,9 @@ export const QueryResults: FC<QueryResultsProps> = ({
const onCopyQuery = () => {
copy(queryString);
setCopyTooltip('Copied!');
setCopyTooltip("Copied!");
setTimeout(() => {
setCopyTooltip('Copy Query');
setCopyTooltip("Copy Query");
}, 1500);
};
@ -75,7 +75,7 @@ export const QueryResults: FC<QueryResultsProps> = ({
onClick={() => setShowQuery(!showQuery)}
className={css.link}
>
{showQuery ? 'Hide' : 'Show'} Query
{showQuery ? "Hide" : "Show"} Query
</span>
)}
<Tooltip title={copyTooltip} placement="top" arrow>
@ -136,10 +136,32 @@ export const QueryResults: FC<QueryResultsProps> = ({
</div>
</div>
{!collapsibleQuery || showQuery ? (
<code className={css.queryString}>{queryString}</code>
<div>
<code className={css.queryString}>{queryString}</code>
{results.totalMessageCount > 1000 ? (
<Tooltip
title={
"This query has " +
results.totalMessageCount +
" results, but you will only be able to view " +
1000 +
" of them."
}
>
<Typography
color="textSecondary"
variant="caption"
component="div"
>
NOTE
</Typography>
</Tooltip>
) : null}
</div>
) : null}
</div>
{results.operation === 'GroupBy' && results.rows.length <= 50 ? (
{results.operation === "GroupBy" && results.rows.length <= 50 ? (
<GroupByChart results={results} />
) : (
<DataTable headers={headers} data={data} autoWidth={true} />

View file

@ -1,9 +1,9 @@
import React, { FC } from 'react';
import groupBy from 'lodash/groupBy';
import { ResponsiveBar } from '@nivo/bar';
import { ResultType } from '../../Query';
import { schemeTableau10 } from 'd3-scale-chromatic';
import { useTheme } from '@material-ui/core/styles';
import React, { FC } from "react";
import groupBy from "lodash/groupBy";
import { ResponsiveBar } from "@nivo/bar";
import { ResultType } from "../../Query";
import { schemeTableau10 } from "d3-scale-chromatic";
import { useTheme } from "@material-ui/core/styles";
type GroupByChartType = {
results: ResultType;
@ -11,7 +11,7 @@ type GroupByChartType = {
export const GroupByChart: FC<GroupByChartType> = ({ results }) => {
const theme = useTheme();
const isDark = theme.palette.type === 'dark';
const isDark = theme.palette.type === "dark";
const { headers, rows } = results;
let uniqueKeys: string[] = [];
const grouped = groupBy(rows, (row) => row[0].stringval);
@ -23,7 +23,7 @@ export const GroupByChart: FC<GroupByChartType> = ({ results }) => {
groupData = {
...groupData,
[headers[0].name]: row[0][`${headers[0].datatype}val`],
[secondaryValue]: row[2][`${headers[2].datatype}val`]
[secondaryValue]: row[2][`${headers[2].datatype}val`],
};
if (!uniqueKeys.includes(secondaryValue.toString())) {
@ -33,7 +33,7 @@ export const GroupByChart: FC<GroupByChartType> = ({ results }) => {
groupData = {
...groupData,
[headers[0].name]: row[0][`${headers[0].datatype}val`],
value: secondaryValue
value: secondaryValue,
};
}
});
@ -42,31 +42,34 @@ export const GroupByChart: FC<GroupByChartType> = ({ results }) => {
});
return (
<div style={{ height: '80%' }}>
<div style={{ height: "80%" }}>
This query has {results.totalMessageCount} total results. Due to browser
memory limitations, you will only be able to see {results.rows.length}{" "}
results.
<ResponsiveBar
data={data}
keys={uniqueKeys.length > 0 ? uniqueKeys : undefined}
indexBy={headers[0].name}
margin={{ top: 50, right: 130, bottom: 100, left: 60 }}
padding={0.3}
valueScale={{ type: 'linear' }}
indexScale={{ type: 'band', round: true }}
valueScale={{ type: "linear" }}
indexScale={{ type: "band", round: true }}
colors={schemeTableau10}
enableLabel={false}
theme={
isDark
? {
textColor: 'var(--text-secondary)',
textColor: "var(--text-secondary)",
axis: {
domain: { line: { stroke: 'rgba(255, 255, 255, 0.1)' } }
domain: { line: { stroke: "rgba(255, 255, 255, 0.1)" } },
},
grid: { line: { stroke: 'rgba(255, 255, 255, 0.1)' } },
tooltip: { container: { background: '#1c2022' } }
grid: { line: { stroke: "rgba(255, 255, 255, 0.1)" } },
tooltip: { container: { background: "#1c2022" } },
}
: {
axis: {
domain: { line: { stroke: '#dddddd' } }
}
domain: { line: { stroke: "#dddddd" } },
},
}
}
groupMode="grouped"
@ -75,16 +78,16 @@ export const GroupByChart: FC<GroupByChartType> = ({ results }) => {
tickPadding: 5,
tickRotation: -40,
legend: headers[0].name,
legendPosition: 'middle',
legendOffset: 80
legendPosition: "middle",
legendOffset: 80,
}}
axisLeft={{
tickSize: 5,
tickPadding: 5,
tickRotation: 0,
legend: 'count',
legendPosition: 'middle',
legendOffset: -40
legend: "count",
legendPosition: "middle",
legendOffset: -40,
}}
tooltip={({ id, value, color }) => (
<strong style={{ color }}>
@ -93,31 +96,31 @@ export const GroupByChart: FC<GroupByChartType> = ({ results }) => {
)}
labelSkipWidth={12}
labelSkipHeight={12}
labelTextColor={{ from: 'color', modifiers: [['darker', 1.6]] }}
labelTextColor={{ from: "color", modifiers: [["darker", 1.6]] }}
legends={[
{
dataFrom: 'keys',
anchor: 'bottom-right',
direction: 'column',
dataFrom: "keys",
anchor: "bottom-right",
direction: "column",
justify: false,
translateX: 120,
translateY: 0,
itemsSpacing: 2,
itemWidth: 120,
itemHeight: 20,
itemDirection: 'left-to-right',
itemDirection: "left-to-right",
itemOpacity: 0.85,
symbolShape: 'circle',
symbolShape: "circle",
symbolSize: 20,
effects: [
{
on: 'hover',
on: "hover",
style: {
itemOpacity: 1
}
}
]
}
itemOpacity: 1,
},
},
],
},
]}
animate={true}
motionStiffness={90}

View file

@ -1,48 +1,49 @@
import React, { Fragment, useState } from 'react';
import Alert from '@material-ui/lab/Alert';
import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos';
import Button from '@material-ui/core/Button';
import classNames from 'classnames';
import CloseIcon from '@material-ui/icons/Close';
import IconButton from '@material-ui/core/IconButton';
import InfoIcon from '@material-ui/icons/Info';
import moment, { Moment } from 'moment';
import Paper from '@material-ui/core/Paper';
import Split from 'react-split';
import Snackbar from '@material-ui/core/Snackbar';
import Tooltip from '@material-ui/core/Tooltip';
import Typography from '@material-ui/core/Typography';
import { Block } from 'shared/Block';
import { formatDuration } from 'shared/utils/formatDuration';
import { grpc } from '@improbable-eng/grpc-web';
import { pilosa } from 'services/eventServices';
import { QueryBuilder } from './QueryBuilder';
import { queryPQL } from 'services/grpcServices';
import { QueryResults } from 'App/Query/QueryResults';
import { ResultType } from 'App/Query/QueryContainer';
import { RowResponse } from 'proto/pilosa_pb';
import { SavedQueries } from 'App/QueryBuilder/SavedQueries';
import { useEffectOnce } from 'react-use';
import css from './QueryBuilderContainer.module.scss';
import React, { Fragment, useState } from "react";
import Alert from "@material-ui/lab/Alert";
import ArrowForwardIosIcon from "@material-ui/icons/ArrowForwardIos";
import Button from "@material-ui/core/Button";
import classNames from "classnames";
import CloseIcon from "@material-ui/icons/Close";
import IconButton from "@material-ui/core/IconButton";
import InfoIcon from "@material-ui/icons/Info";
import moment, { Moment } from "moment";
import Paper from "@material-ui/core/Paper";
import Split from "react-split";
import Snackbar from "@material-ui/core/Snackbar";
import Tooltip from "@material-ui/core/Tooltip";
import Typography from "@material-ui/core/Typography";
import { Block } from "shared/Block";
import { formatDuration } from "shared/utils/formatDuration";
import { grpc } from "@improbable-eng/grpc-web";
import { pilosa } from "services/eventServices";
import { QueryBuilder } from "./QueryBuilder";
import { queryPQL } from "services/grpcServices";
import { QueryResults } from "App/Query/QueryResults";
import { ResultType } from "App/Query/QueryContainer";
import { RowResponse } from "proto/pilosa_pb";
import { SavedQueries } from "App/QueryBuilder/SavedQueries";
import { useEffectOnce } from "react-use";
import css from "./QueryBuilderContainer.module.scss";
let streamingResults: ResultType = {
query: '',
operation: '',
type: 'PQL',
query: "",
operation: "",
type: "PQL",
headers: [],
rows: [],
roundtrip: 0,
error: ''
error: "",
totalMessageCount: 0,
};
export const QueryBuilderContainer = () => {
let startTime: Moment;
let exportRows: any[] = [];
const colSizes = JSON.parse(
localStorage.getItem('builderColSizes') || '[25, 75]'
localStorage.getItem("builderColSizes") || "[25, 75]"
);
const [queriesList, setQueriesList] = useState(
JSON.parse(localStorage.getItem('saved-queries') || '[]')
JSON.parse(localStorage.getItem("saved-queries") || "[]")
);
const [tables, setTables] = useState<any[]>([]);
@ -51,7 +52,7 @@ export const QueryBuilderContainer = () => {
const [fullCount, setFullCount] = useState<number>();
const [recordsCount, setRecordsCount] = useState<number>();
const [errorResult, setErrorResult] = useState<ResultType>();
const [error, setError] = useState<string>('');
const [error, setError] = useState<string>("");
const [loading, setLoading] = useState<boolean>(false);
const [savedQuery, setSavedQuery] = useState<number>(-1);
@ -77,7 +78,7 @@ export const QueryBuilderContainer = () => {
} else {
streamingResults.roundtrip = moment
.duration(moment().diff(startTime))
.as('milliseconds');
.as("milliseconds");
setErrorResult(undefined);
setResults(streamingResults);
}
@ -89,12 +90,12 @@ export const QueryBuilderContainer = () => {
let rowStr: string[] = [];
if (exportRows.length === 0) {
const headers = response.headersList.map((header) => header.name);
exportRows.push(headers.join('\t'));
exportRows.push(headers.join("\t"));
}
response.headersList.forEach((header, idx) =>
rowStr.push(response.columnsList[idx][`${header.datatype}val`])
);
exportRows.push(rowStr.join('\t'));
exportRows.push(rowStr.join("\t"));
};
const handleExternalLookupEnd = (
@ -104,12 +105,12 @@ export const QueryBuilderContainer = () => {
if (status !== grpc.Code.OK) {
setError(statusMessage);
} else if (exportRows.length === 0) {
setError('No record attributes for current query.');
setError("No record attributes for current query.");
} else {
const dateTime = moment().unix();
const element = document.createElement('a');
const file = new Blob([exportRows.join('\n')], {
type: 'text/plain;charset=utf-8'
const element = document.createElement("a");
const file = new Blob([exportRows.join("\n")], {
type: "text/plain;charset=utf-8",
});
element.href = URL.createObjectURL(file);
element.download = `molecula-${results?.index}-${dateTime}.csv`;
@ -128,17 +129,18 @@ export const QueryBuilderContainer = () => {
streamingResults = {
query,
operation,
type: 'PQL',
type: "PQL",
headers: [],
rows: [],
index: table,
roundtrip: 0,
error: ''
error: "",
totalMessageCount: 0,
};
startTime = moment();
setLoading(true);
if (operation !== 'Count') {
if (operation !== "Count") {
if (countQuery) {
pilosa.post.query(table, countQuery).then((res) => {
setFullCount(res.data.results[0]);
@ -164,13 +166,13 @@ export const QueryBuilderContainer = () => {
const onRemoveQuery = (queryIdx: number) => {
let queries = [...queriesList];
queries.splice(queryIdx, 1);
localStorage.setItem('saved-queries', JSON.stringify(queries));
localStorage.setItem("saved-queries", JSON.stringify(queries));
setQueriesList(queries);
};
const onSaveQuery = () => {
const updatedList = JSON.parse(
localStorage.getItem('saved-queries') || '[]'
localStorage.getItem("saved-queries") || "[]"
);
if (savedQuery < 0) {
setSavedQuery(updatedList.length - 1);
@ -181,7 +183,7 @@ export const QueryBuilderContainer = () => {
const onExportLogs = () => {
if (results) {
const columns = results.rows.map((row) => row[0].uint64val);
const table = results.index ? results.index : '';
const table = results.index ? results.index : "";
onExternalLookup(table, columns);
}
};
@ -200,17 +202,17 @@ export const QueryBuilderContainer = () => {
cursor="col-resize"
minSize={showBuilder ? 350 : 50}
onDragEnd={(sizes) =>
localStorage.setItem('builderColSizes', JSON.stringify(sizes))
localStorage.setItem("builderColSizes", JSON.stringify(sizes))
}
gutter={(_index, direction) => {
const gutter = document.createElement('div');
const gutter = document.createElement("div");
gutter.className = `gutter gutter-${direction}`;
const dragbars = document.createElement('div');
dragbars.className = 'dragBar';
const dragbars = document.createElement("div");
dragbars.className = "dragBar";
gutter.appendChild(dragbars);
return gutter;
}}
className={classNames(css.split, !showBuilder ? 'hide-gutter' : '')}
className={classNames(css.split, !showBuilder ? "hide-gutter" : "")}
>
{showBuilder ? (
<div className={css.builderColumn}>
@ -239,9 +241,9 @@ export const QueryBuilderContainer = () => {
<Block className={css.resultsBlock}>
<div className={css.resultsHeader}>
<Typography variant="h5" color="textSecondary">
Results{' '}
Results{" "}
</Typography>
{results?.query.includes('Extract(') ? (
{results?.query.includes("Extract(") ? (
<div className={css.download}>
<Tooltip
className={css.downloadInfo}
@ -262,13 +264,13 @@ export const QueryBuilderContainer = () => {
<div className={css.infoMessage}>
{results.duration ? (
<div>
{recordsCount.toLocaleString()} records scanned in{' '}
{recordsCount.toLocaleString()} records scanned in{" "}
{formatDuration(results.duration, true)}.
</div>
) : null}
<div>
Showing{' '}
{fullCount > 1000 ? 'first 1,000 rows of' : 'all'}{' '}
Showing{" "}
{fullCount > 1000 ? "first 1,000 rows of" : "all"}{" "}
{fullCount.toLocaleString()} results.
</div>
</div>
@ -301,7 +303,7 @@ export const QueryBuilderContainer = () => {
</Block>
</div>
<Snackbar open={!!error}>
<Alert severity="info" onClose={() => setError('')}>
<Alert severity="info" onClose={() => setError("")}>
{error}
</Alert>
</Snackbar>