|
23 | 23 | "ReadOnlyError", |
24 | 24 | "ResultSet", |
25 | 25 | "RowStream", |
| 26 | + "StatsReport", |
26 | 27 | "open_database", |
27 | 28 | "plan_tree", |
28 | 29 | "quote_identifier", |
@@ -90,6 +91,14 @@ def collect(self) -> ResultSet: |
90 | 91 | return ResultSet(columns=self.columns, rows=list(self.rows), rowcount=self.rowcount) |
91 | 92 |
|
92 | 93 |
|
| 94 | +class StatsReport(NamedTuple): |
| 95 | + """Per-column statistics, plus how many rows they were computed on.""" |
| 96 | + |
| 97 | + result: ResultSet |
| 98 | + rows: int |
| 99 | + sampled: bool |
| 100 | + |
| 101 | + |
93 | 102 | def _rows_of(cursor: sqlite3.Cursor) -> Iterator[tuple]: |
94 | 103 | try: |
95 | 104 | yield from cursor |
@@ -271,13 +280,16 @@ def objects(self, *, include_internal: bool = False) -> list[tuple[str, str, str |
271 | 280 | sql += " ORDER BY rowid" |
272 | 281 | return [(kind, name, statement) for kind, name, statement in self.execute(sql).rows] |
273 | 282 |
|
274 | | - def tables(self, *, include_internal: bool = False) -> ResultSet: |
275 | | - """List tables and views together with their row counts.""" |
| 283 | + def tables(self, *, include_internal: bool = False, count: bool = True) -> ResultSet: |
| 284 | + """List tables and views, with their row counts unless *count* is false.""" |
276 | 285 | sql = "SELECT type, name FROM sqlite_master WHERE type IN ('table', 'view')" |
277 | 286 | if not include_internal: |
278 | 287 | sql += " AND name NOT LIKE 'sqlite_%'" |
279 | 288 | sql += " ORDER BY type, name" |
280 | | - rows = [(kind, name, self._count_rows(name)) for kind, name in self.execute(sql).rows] |
| 289 | + objects = self.execute(sql).rows |
| 290 | + if not count: |
| 291 | + return ResultSet(columns=("type", "name"), rows=[tuple(row) for row in objects]) |
| 292 | + rows = [(kind, name, self._count_rows(name)) for kind, name in objects] |
281 | 293 | return ResultSet(columns=("type", "name", "rows"), rows=rows) |
282 | 294 |
|
283 | 295 | def _count_rows(self, name: str) -> int | str: |
@@ -488,26 +500,99 @@ def stream_rows( |
488 | 500 | query = f"SELECT {selection} {source}{order} LIMIT ? OFFSET ?" |
489 | 501 | return self.stream(query, (-1 if limit is None else limit, offset)) |
490 | 502 |
|
491 | | - def stats(self, table: str, *, top: int = 3) -> ResultSet: |
492 | | - """Per-column summary of *table*: nulls, distinct values, min, max, top values.""" |
493 | | - q_table = quote_identifier(table) |
494 | | - rows = [] |
495 | | - for _cid, name, declared, *_ in self.columns(table).rows: |
496 | | - q_col = quote_identifier(name) |
497 | | - nulls, distinct, minimum, maximum, min_type = self.execute( |
498 | | - f"SELECT COUNT(*) - COUNT({q_col}), COUNT(DISTINCT {q_col})," |
499 | | - f" MIN({q_col}), MAX({q_col}), typeof(MIN({q_col})) FROM {q_table}" |
500 | | - ).rows[0] |
501 | | - frequent = self.execute( |
502 | | - f"SELECT quote({q_col}), COUNT(*) AS n FROM {q_table}" |
503 | | - f" WHERE {q_col} IS NOT NULL GROUP BY {q_col} ORDER BY n DESC, 1 LIMIT ?", |
504 | | - (top,), |
505 | | - ).rows |
506 | | - summary = ", ".join(f"{value} ({count})" for value, count in frequent) |
507 | | - rows.append((name, declared or min_type, nulls, distinct, minimum, maximum, summary)) |
508 | | - return ResultSet( |
509 | | - columns=("column", "type", "nulls", "distinct", "min", "max", "top"), rows=rows |
| 503 | + def stats( |
| 504 | + self, |
| 505 | + table: str, |
| 506 | + *, |
| 507 | + top: int = 3, |
| 508 | + columns: Sequence[str] | None = None, |
| 509 | + sample: int | None = None, |
| 510 | + ) -> StatsReport: |
| 511 | + """Per-column summary of *table*: nulls, distinct values, min, max, top values. |
| 512 | +
|
| 513 | + Nulls, distinct values, minimum and maximum of every column come from a |
| 514 | + single pass over the table; the *top* most frequent values need one |
| 515 | + ``GROUP BY`` per column, so ``top=0`` is much cheaper on big tables. |
| 516 | + With *sample*, everything is computed on a random sample of that many |
| 517 | + rows copied to a temporary table. |
| 518 | + """ |
| 519 | + info = self.columns(table).rows |
| 520 | + known = [row[1] for row in info] |
| 521 | + declared = {row[1]: row[2] for row in info} |
| 522 | + if columns is None: |
| 523 | + selected = known |
| 524 | + else: |
| 525 | + selected = [self._resolve_column(table, name, known) for name in columns] |
| 526 | + if sample is None: |
| 527 | + return self._collect_stats(quote_identifier(table), selected, declared, top, False) |
| 528 | + source = self._sample(table, selected, sample) |
| 529 | + try: |
| 530 | + return self._collect_stats(source, selected, declared, top, True) |
| 531 | + finally: |
| 532 | + self._connection.execute(f"DROP TABLE IF EXISTS {source}") |
| 533 | + |
| 534 | + def _sample(self, table: str, columns: Sequence[str], size: int) -> str: |
| 535 | + """Copy a random sample of *table* into a temporary table; return its name.""" |
| 536 | + selection = ", ".join(quote_identifier(name) for name in columns) |
| 537 | + self._connection.execute(f"DROP TABLE IF EXISTS {_SAMPLE_TABLE}") |
| 538 | + self._connection.execute( |
| 539 | + f"CREATE TEMP TABLE {_SAMPLE_NAME} AS SELECT {selection}" |
| 540 | + f" FROM {quote_identifier(table)} ORDER BY random() LIMIT ?", |
| 541 | + (size,), |
510 | 542 | ) |
| 543 | + return _SAMPLE_TABLE |
| 544 | + |
| 545 | + def _collect_stats( |
| 546 | + self, |
| 547 | + source: str, |
| 548 | + columns: Sequence[str], |
| 549 | + declared: Mapping[str, str], |
| 550 | + top: int, |
| 551 | + sampled: bool, |
| 552 | + ) -> StatsReport: |
| 553 | + rows = [] |
| 554 | + examined = 0 |
| 555 | + for start in range(0, len(columns), _STATS_CHUNK): |
| 556 | + chunk = columns[start : start + _STATS_CHUNK] |
| 557 | + expressions = ["COUNT(*)"] |
| 558 | + for name in chunk: |
| 559 | + q_col = quote_identifier(name) |
| 560 | + expressions += [ |
| 561 | + f"COUNT({q_col})", |
| 562 | + f"COUNT(DISTINCT {q_col})", |
| 563 | + f"MIN({q_col})", |
| 564 | + f"MAX({q_col})", |
| 565 | + f"typeof(MIN({q_col}))", |
| 566 | + ] |
| 567 | + values = self.execute(f"SELECT {', '.join(expressions)} FROM {source}").rows[0] |
| 568 | + examined = values[0] |
| 569 | + for index, name in enumerate(chunk): |
| 570 | + filled, distinct, minimum, maximum, min_type = values[1 + index * 5 : 6 + index * 5] |
| 571 | + rows.append( |
| 572 | + ( |
| 573 | + name, |
| 574 | + declared.get(name) or min_type, |
| 575 | + examined - filled, |
| 576 | + distinct, |
| 577 | + minimum, |
| 578 | + maximum, |
| 579 | + self._top_values(source, name, top), |
| 580 | + ) |
| 581 | + ) |
| 582 | + if not columns: |
| 583 | + examined = self.execute(f"SELECT COUNT(*) FROM {source}").rows[0][0] |
| 584 | + return StatsReport(ResultSet(columns=_STATS_COLUMNS, rows=rows), examined, sampled) |
| 585 | + |
| 586 | + def _top_values(self, source: str, column: str, top: int) -> str: |
| 587 | + if top <= 0: |
| 588 | + return "" |
| 589 | + q_col = quote_identifier(column) |
| 590 | + frequent = self.execute( |
| 591 | + f"SELECT quote({q_col}), COUNT(*) AS n FROM {source}" |
| 592 | + f" WHERE {q_col} IS NOT NULL GROUP BY {q_col} ORDER BY n DESC, 1 LIMIT ?", |
| 593 | + (top,), |
| 594 | + ).rows |
| 595 | + return ", ".join(f"{value} ({count})" for value, count in frequent) |
511 | 596 |
|
512 | 597 | def search( |
513 | 598 | self, text: str, *, tables: Sequence[str] | None = None, limit: int | None = None |
@@ -608,3 +693,8 @@ def import_rows( |
608 | 693 |
|
609 | 694 |
|
610 | 695 | _SEARCH_COLUMNS = ("table", "column", "rowid", "value") |
| 696 | +_STATS_COLUMNS = ("column", "type", "nulls", "distinct", "min", "max", "top") |
| 697 | +# Columns per aggregate query: five expressions each, well below SQLITE_MAX_COLUMN. |
| 698 | +_STATS_CHUNK = 100 |
| 699 | +_SAMPLE_NAME = '"sqlitexplorer_sample"' |
| 700 | +_SAMPLE_TABLE = f"temp.{_SAMPLE_NAME}" |
0 commit comments