Skip to content

Commit 365e9e5

Browse files
Cheaper stats: single pass, column selection and sampling; tables --no-count.
Signed-off-by: Carlos Andrés Planchón Prestes <carlosandresplanchonprestes@gmail.com>
1 parent 88d4634 commit 365e9e5

6 files changed

Lines changed: 218 additions & 32 deletions

File tree

‎README.md‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,14 @@ raw SQL condition.
5555

5656
`stats` reports, for each column, the declared type, how many NULLs, how many
5757
distinct values, the minimum, the maximum and the most frequent values
58-
(`--top N`). `search` looks for a case-insensitive substring in every non-BLOB
59-
column of every table (`--table` restricts it, `--limit` stops early) and
60-
prints the table, column, rowid and value of each match.
58+
(`--top N`). Everything but the frequent values comes from a single pass over
59+
the table; on big tables use `--top 0` to skip them, `--columns` to analyse
60+
only some columns, or `--sample N` to work on a random sample of N rows (the
61+
output then says so). Likewise `tables --no-count` skips the row counts.
62+
63+
`search` looks for a case-insensitive substring in every non-BLOB column of
64+
every table (`--table` restricts it, `--limit` stops early) and prints the
65+
table, column, rowid and value of each match. Each table is scanned once.
6166

6267
## Output options
6368

‎sqlitexplorer/cli.py‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -309,12 +309,20 @@ def _split_list(value: str | None) -> list[str] | None:
309309
def tables(
310310
database: DatabaseArg,
311311
include_internal: AllOption = False,
312+
count: Annotated[
313+
bool,
314+
typer.Option(
315+
"--count/--no-count",
316+
help="Count the rows of every table and view (--no-count is faster on big ones).",
317+
),
318+
] = True,
312319
*,
313320
options: OutputOptions,
314321
) -> None:
315322
"""List the tables and views of the database with their row counts."""
316323
with _reporting_errors(), open_database(database) as db:
317-
emit(db.tables(include_internal=include_internal), options, empty="(no tables)")
324+
result = db.tables(include_internal=include_internal, count=count)
325+
emit(result, options, empty="(no tables)")
318326

319327

320328
@app.command()
@@ -439,14 +447,36 @@ def stats(
439447
database: DatabaseArg,
440448
table: TableArg,
441449
top: Annotated[
442-
int, typer.Option("--top", min=0, help="How many frequent values to list per column.")
450+
int,
451+
typer.Option(
452+
"--top",
453+
min=0,
454+
help="How many frequent values to list per column; 0 skips them, which is much "
455+
"faster on big tables.",
456+
),
443457
] = 3,
458+
columns: Annotated[
459+
str | None,
460+
typer.Option("--columns", "-c", help="Comma-separated list of columns to analyse."),
461+
] = None,
462+
sample: Annotated[
463+
int | None,
464+
typer.Option(
465+
"--sample",
466+
min=1,
467+
help="Compute on a random sample of N rows instead of the whole table.",
468+
),
469+
] = None,
444470
*,
445471
options: OutputOptions,
446472
) -> None:
447473
"""Per-column statistics: nulls, distinct values, min, max and most frequent values."""
448474
with _reporting_errors(), open_database(database) as db:
449-
emit(db.stats(table, top=top), options)
475+
report = db.stats(table, top=top, columns=_split_list(columns) or None, sample=sample)
476+
emit(report.result, options)
477+
if report.sampled:
478+
plural = "" if report.rows == 1 else "s"
479+
typer.echo(f"computed on a random sample of {report.rows} row{plural}", err=True)
450480

451481

452482
@app.command()

‎sqlitexplorer/core.py‎

Lines changed: 112 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
"ReadOnlyError",
2424
"ResultSet",
2525
"RowStream",
26+
"StatsReport",
2627
"open_database",
2728
"plan_tree",
2829
"quote_identifier",
@@ -90,6 +91,14 @@ def collect(self) -> ResultSet:
9091
return ResultSet(columns=self.columns, rows=list(self.rows), rowcount=self.rowcount)
9192

9293

94+
class StatsReport(NamedTuple):
95+
"""Per-column statistics, plus how many rows they were computed on."""
96+
97+
result: ResultSet
98+
rows: int
99+
sampled: bool
100+
101+
93102
def _rows_of(cursor: sqlite3.Cursor) -> Iterator[tuple]:
94103
try:
95104
yield from cursor
@@ -271,13 +280,16 @@ def objects(self, *, include_internal: bool = False) -> list[tuple[str, str, str
271280
sql += " ORDER BY rowid"
272281
return [(kind, name, statement) for kind, name, statement in self.execute(sql).rows]
273282

274-
def tables(self, *, include_internal: bool = False) -> ResultSet:
275-
"""List tables and views together with their row counts."""
283+
def tables(self, *, include_internal: bool = False, count: bool = True) -> ResultSet:
284+
"""List tables and views, with their row counts unless *count* is false."""
276285
sql = "SELECT type, name FROM sqlite_master WHERE type IN ('table', 'view')"
277286
if not include_internal:
278287
sql += " AND name NOT LIKE 'sqlite_%'"
279288
sql += " ORDER BY type, name"
280-
rows = [(kind, name, self._count_rows(name)) for kind, name in self.execute(sql).rows]
289+
objects = self.execute(sql).rows
290+
if not count:
291+
return ResultSet(columns=("type", "name"), rows=[tuple(row) for row in objects])
292+
rows = [(kind, name, self._count_rows(name)) for kind, name in objects]
281293
return ResultSet(columns=("type", "name", "rows"), rows=rows)
282294

283295
def _count_rows(self, name: str) -> int | str:
@@ -488,26 +500,99 @@ def stream_rows(
488500
query = f"SELECT {selection} {source}{order} LIMIT ? OFFSET ?"
489501
return self.stream(query, (-1 if limit is None else limit, offset))
490502

491-
def stats(self, table: str, *, top: int = 3) -> ResultSet:
492-
"""Per-column summary of *table*: nulls, distinct values, min, max, top values."""
493-
q_table = quote_identifier(table)
494-
rows = []
495-
for _cid, name, declared, *_ in self.columns(table).rows:
496-
q_col = quote_identifier(name)
497-
nulls, distinct, minimum, maximum, min_type = self.execute(
498-
f"SELECT COUNT(*) - COUNT({q_col}), COUNT(DISTINCT {q_col}),"
499-
f" MIN({q_col}), MAX({q_col}), typeof(MIN({q_col})) FROM {q_table}"
500-
).rows[0]
501-
frequent = self.execute(
502-
f"SELECT quote({q_col}), COUNT(*) AS n FROM {q_table}"
503-
f" WHERE {q_col} IS NOT NULL GROUP BY {q_col} ORDER BY n DESC, 1 LIMIT ?",
504-
(top,),
505-
).rows
506-
summary = ", ".join(f"{value} ({count})" for value, count in frequent)
507-
rows.append((name, declared or min_type, nulls, distinct, minimum, maximum, summary))
508-
return ResultSet(
509-
columns=("column", "type", "nulls", "distinct", "min", "max", "top"), rows=rows
503+
def stats(
504+
self,
505+
table: str,
506+
*,
507+
top: int = 3,
508+
columns: Sequence[str] | None = None,
509+
sample: int | None = None,
510+
) -> StatsReport:
511+
"""Per-column summary of *table*: nulls, distinct values, min, max, top values.
512+
513+
Nulls, distinct values, minimum and maximum of every column come from a
514+
single pass over the table; the *top* most frequent values need one
515+
``GROUP BY`` per column, so ``top=0`` is much cheaper on big tables.
516+
With *sample*, everything is computed on a random sample of that many
517+
rows copied to a temporary table.
518+
"""
519+
info = self.columns(table).rows
520+
known = [row[1] for row in info]
521+
declared = {row[1]: row[2] for row in info}
522+
if columns is None:
523+
selected = known
524+
else:
525+
selected = [self._resolve_column(table, name, known) for name in columns]
526+
if sample is None:
527+
return self._collect_stats(quote_identifier(table), selected, declared, top, False)
528+
source = self._sample(table, selected, sample)
529+
try:
530+
return self._collect_stats(source, selected, declared, top, True)
531+
finally:
532+
self._connection.execute(f"DROP TABLE IF EXISTS {source}")
533+
534+
def _sample(self, table: str, columns: Sequence[str], size: int) -> str:
535+
"""Copy a random sample of *table* into a temporary table; return its name."""
536+
selection = ", ".join(quote_identifier(name) for name in columns)
537+
self._connection.execute(f"DROP TABLE IF EXISTS {_SAMPLE_TABLE}")
538+
self._connection.execute(
539+
f"CREATE TEMP TABLE {_SAMPLE_NAME} AS SELECT {selection}"
540+
f" FROM {quote_identifier(table)} ORDER BY random() LIMIT ?",
541+
(size,),
510542
)
543+
return _SAMPLE_TABLE
544+
545+
def _collect_stats(
546+
self,
547+
source: str,
548+
columns: Sequence[str],
549+
declared: Mapping[str, str],
550+
top: int,
551+
sampled: bool,
552+
) -> StatsReport:
553+
rows = []
554+
examined = 0
555+
for start in range(0, len(columns), _STATS_CHUNK):
556+
chunk = columns[start : start + _STATS_CHUNK]
557+
expressions = ["COUNT(*)"]
558+
for name in chunk:
559+
q_col = quote_identifier(name)
560+
expressions += [
561+
f"COUNT({q_col})",
562+
f"COUNT(DISTINCT {q_col})",
563+
f"MIN({q_col})",
564+
f"MAX({q_col})",
565+
f"typeof(MIN({q_col}))",
566+
]
567+
values = self.execute(f"SELECT {', '.join(expressions)} FROM {source}").rows[0]
568+
examined = values[0]
569+
for index, name in enumerate(chunk):
570+
filled, distinct, minimum, maximum, min_type = values[1 + index * 5 : 6 + index * 5]
571+
rows.append(
572+
(
573+
name,
574+
declared.get(name) or min_type,
575+
examined - filled,
576+
distinct,
577+
minimum,
578+
maximum,
579+
self._top_values(source, name, top),
580+
)
581+
)
582+
if not columns:
583+
examined = self.execute(f"SELECT COUNT(*) FROM {source}").rows[0][0]
584+
return StatsReport(ResultSet(columns=_STATS_COLUMNS, rows=rows), examined, sampled)
585+
586+
def _top_values(self, source: str, column: str, top: int) -> str:
587+
if top <= 0:
588+
return ""
589+
q_col = quote_identifier(column)
590+
frequent = self.execute(
591+
f"SELECT quote({q_col}), COUNT(*) AS n FROM {source}"
592+
f" WHERE {q_col} IS NOT NULL GROUP BY {q_col} ORDER BY n DESC, 1 LIMIT ?",
593+
(top,),
594+
).rows
595+
return ", ".join(f"{value} ({count})" for value, count in frequent)
511596

512597
def search(
513598
self, text: str, *, tables: Sequence[str] | None = None, limit: int | None = None
@@ -608,3 +693,8 @@ def import_rows(
608693

609694

610695
_SEARCH_COLUMNS = ("table", "column", "rowid", "value")
696+
_STATS_COLUMNS = ("column", "type", "nulls", "distinct", "min", "max", "top")
697+
# Columns per aggregate query: five expressions each, well below SQLITE_MAX_COLUMN.
698+
_STATS_CHUNK = 100
699+
_SAMPLE_NAME = '"sqlitexplorer_sample"'
700+
_SAMPLE_TABLE = f"temp.{_SAMPLE_NAME}"

‎sqlitexplorer/shell.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ def _dot_command(
213213
elif command == ".indexes":
214214
emit(db.indexes(argument or None), options, empty="(no indexes)")
215215
elif command == ".stats":
216-
emit(db.stats(_required(argument, command)), options)
216+
emit(db.stats(_required(argument, command)).result, options)
217217
elif command == ".format":
218218
options.format = _parse_format(argument)
219219
elif command == ".null":

‎tests/test_cli.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,25 @@ def test_stats_reports_per_column_summary(invoke: Callable, database: Path) -> N
319319
assert "age,INTEGER,1,2,12,30,12 (1)" in result.output
320320

321321

322+
def test_stats_columns_sample_and_no_top(invoke: Callable, database: Path) -> None:
323+
result = invoke(
324+
"stats", database, "users", "-c", "name,age", "--top", "0", "--sample", "2", "-f", "csv"
325+
)
326+
assert result.exit_code == 0
327+
lines = result.stdout.splitlines()
328+
assert lines[0] == "column,type,nulls,distinct,min,max,top"
329+
assert [line.split(",")[0] for line in lines[1:]] == ["name", "age"]
330+
assert all(line.endswith(",") for line in lines[1:]) # no top values
331+
assert "computed on a random sample of 2 rows" in result.stderr
332+
333+
334+
def test_tables_no_count(invoke: Callable, database: Path) -> None:
335+
result = invoke("tables", database, "--no-count", "-f", "csv")
336+
assert result.exit_code == 0
337+
assert result.stdout.splitlines()[0] == "type,name"
338+
assert "table,users" in result.stdout
339+
340+
322341
def test_stats_unknown_table_fails(invoke: Callable, database: Path) -> None:
323342
result = invoke("stats", database, "nope")
324343
assert result.exit_code == 1

‎tests/test_core.py‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,10 @@ def test_rows_rejects_unknown_column(database: Path) -> None:
209209

210210
def test_stats_reports_nulls_distinct_min_max_top(database: Path) -> None:
211211
with open_database(database) as db:
212-
result = db.stats("users", top=2)
212+
report = db.stats("users", top=2)
213+
assert report.rows == 3
214+
assert not report.sampled
215+
result = report.result
213216
assert result.columns == ("column", "type", "nulls", "distinct", "min", "max", "top")
214217
by_column = {row[0]: row for row in result.rows}
215218
assert by_column["age"][1:6] == ("INTEGER", 1, 2, 12, 30)
@@ -222,12 +225,51 @@ def test_stats_on_view_falls_back_to_typeof_for_expressions(database: Path) -> N
222225
connection.execute("CREATE VIEW doubled AS SELECT name, age * 2 AS twice FROM users")
223226
connection.close()
224227
with open_database(database) as db:
225-
adults = db.stats("adults")
226-
doubled = db.stats("doubled")
228+
adults = db.stats("adults").result
229+
doubled = db.stats("doubled").result
227230
assert [(row[0], row[1]) for row in adults.rows] == [("name", "TEXT"), ("age", "INTEGER")]
228231
assert [(row[0], row[1]) for row in doubled.rows] == [("name", "TEXT"), ("twice", "integer")]
229232

230233

234+
def test_stats_columns_top_zero_and_sample(database: Path) -> None:
235+
with open_database(database) as db:
236+
subset = db.stats("users", columns=["AGE"], top=0)
237+
assert [row[0] for row in subset.result.rows] == ["age"]
238+
assert subset.result.rows[0][1:] == ("INTEGER", 1, 2, 12, 30, "")
239+
sampled = db.stats("users", sample=2)
240+
assert sampled.sampled
241+
assert sampled.rows == 2
242+
assert all(row[2] + row[3] <= 2 for row in sampled.result.rows) # nulls + distinct
243+
assert db.execute(
244+
"SELECT COUNT(*) FROM sqlite_temp_master WHERE name = 'sqlitexplorer_sample'"
245+
).rows == [(0,)] # the sample is dropped afterwards
246+
with pytest.raises(ExplorerError, match="no such column in users: nope"):
247+
db.stats("users", columns=["nope"])
248+
assert db.stats("empty").rows == 0
249+
250+
251+
def test_stats_handles_hundreds_of_columns_in_one_pass(tmp_path: Path) -> None:
252+
definition = ", ".join(f"c{index} INTEGER" for index in range(250))
253+
path = make_database(
254+
tmp_path / "wide.db",
255+
f"CREATE TABLE wide ({definition}); INSERT INTO wide (c0, c249) VALUES (1, 2);",
256+
)
257+
with open_database(path) as db:
258+
report = db.stats("wide", top=1)
259+
assert len(report.result.rows) == 250
260+
by_column = {row[0]: row for row in report.result.rows}
261+
assert by_column["c0"][2:] == (0, 1, 1, 1, "1 (1)")
262+
assert by_column["c1"][2:] == (1, 0, None, None, "")
263+
assert by_column["c249"][5] == 2
264+
265+
266+
def test_tables_without_counts(database: Path) -> None:
267+
with open_database(database) as db:
268+
result = db.tables(count=False)
269+
assert result.columns == ("type", "name")
270+
assert ("table", "users") in result.rows
271+
272+
231273
def test_search_finds_text_in_any_table(relational_database: Path) -> None:
232274
with open_database(relational_database) as db:
233275
result = db.search("world")

0 commit comments

Comments
 (0)