Skip to content

Clarify DuckLake metadata query intent #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bill-ph wants to merge 2 commits into
base: v1.5-variegata
Choose a base branch
from
Draft

Clarify DuckLake metadata query intent #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
40a06fc
Route Postgres global stats through passthrough
bill-ph May 13, 2026
014dea2
Clarify DuckLake metadata query intent
bill-ph May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/functions/ducklake_add_data_files.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ struct DuckLakeFileProcessor {
};

void DuckLakeFileProcessor::ReadParquetFullMetadata(const string &glob, vector<DuckLakeDataFile> &written_files) {
auto result = transaction.Query(StringUtil::Format(R"(
auto result = transaction.RawQuery(StringUtil::Format(R"(
SELECT
list_transform(parquet_file_metadata, lambda x: struct_pack(
file_name := x.file_name,
Expand Down Expand Up @@ -196,7 +196,7 @@ SELECT
)) AS parquet_schema
FROM parquet_full_metadata(%s)
)",
SQLString(glob)));
SQLString(glob)));
if (result->HasError()) {
result->GetErrorObject().Throw("Failed to add data files to DuckLake: ");
}
Expand Down
12 changes: 6 additions & 6 deletions src/functions/ducklake_flush_inlined_data.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ SinkFinalizeType DuckLakeFlushData::Finalize(Pipeline &pipeline, Event &event, C
order_by = sort_order_sql + ", row_id, begin_snapshot";
}
auto deleted_rows_result =
transaction.Query(snapshot, StringUtil::Format(R"(
transaction.SnapshotQuery(snapshot, StringUtil::Format(R"(
WITH all_rows AS (
SELECT end_snapshot, ROW_NUMBER() OVER (ORDER BY %s) - 1 AS output_position
FROM {METADATA_CATALOG}.%s
Expand All @@ -156,8 +156,8 @@ SinkFinalizeType DuckLakeFlushData::Finalize(Pipeline &pipeline, Event &event, C
FROM all_rows
WHERE end_snapshot IS NOT NULL
AND output_position >= %d AND output_position < %d;)",
order_by, inlined_table.table_name, extra_filter,
file_offset, file_offset + file.row_count));
order_by, inlined_table.table_name, extra_filter,
file_offset, file_offset + file.row_count));

for (auto &row : *deleted_rows_result) {
auto end_snap = row.GetValue<int64_t>(0);
Expand Down Expand Up @@ -427,7 +427,7 @@ static void FlushInlinedFileDeletions(ClientContext &context, DuckLakeCatalog &c
}

// Query the inlined deletions with file paths and existing delete file info
auto deletions_result = transaction.Query(snapshot, StringUtil::Format(R"(
auto deletions_result = transaction.SnapshotQuery(snapshot, StringUtil::Format(R"(
SELECT del.file_id, data.path, data.path_is_relative, del.row_id, del.begin_snapshot,
existing_del.delete_file_id, existing_del.path as del_path, existing_del.path_is_relative as del_path_is_relative,
existing_del.begin_snapshot as del_begin_snapshot, existing_del.encryption_key as del_encryption_key,
Expand All @@ -440,7 +440,7 @@ LEFT JOIN (
AND ({SNAPSHOT_ID} < end_snapshot OR end_snapshot IS NULL)
) existing_del ON del.file_id = existing_del.data_file_id
)",
inlined_table_name, table_id.index));
inlined_table_name, table_id.index));
if (deletions_result->HasError()) {
deletions_result->GetErrorObject().Throw("Failed to query inlined file deletions for flush: ");
}
Expand Down Expand Up @@ -572,7 +572,7 @@ LEFT JOIN (

// Delete the flushed inlined deletions
auto delete_result =
transaction.Query(snapshot, StringUtil::Format("DELETE FROM {METADATA_CATALOG}.%s", inlined_table_name));
transaction.Execute(StringUtil::Format("DELETE FROM {METADATA_CATALOG}.%s", inlined_table_name));
if (delete_result->HasError()) {
delete_result->GetErrorObject().Throw("Failed to delete inlined file deletions after flush: ");
}
Expand Down
5 changes: 3 additions & 2 deletions src/include/metadata_manager/postgres_metadata_manager.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ class PostgresMetadataManager : public DuckLakeMetadataManager {
unique_ptr<QueryResult> Execute(DuckLakeSnapshot snapshot, string &query) override;
unique_ptr<QueryResult> Execute(string &query) override;

unique_ptr<QueryResult> Query(DuckLakeSnapshot snapshot, string &query) override;
unique_ptr<QueryResult> Query(string &query) override;
unique_ptr<QueryResult> SnapshotQuery(DuckLakeSnapshot snapshot, string &query) override;
unique_ptr<QueryResult> CurrentQuery(DuckLakeSnapshot snapshot, string &query) override;
unique_ptr<QueryResult> CurrentQuery(string &query) override;

protected:
string GetLatestSnapshotQuery() const override;
Expand Down
8 changes: 6 additions & 2 deletions src/include/storage/ducklake_metadata_manager.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,15 @@ class DuckLakeMetadataManager {
virtual void InitializeDuckLake(bool has_explicit_schema, DuckLakeEncryption encryption);
virtual DuckLakeMetadata LoadDuckLake();

//! Execute metadata DDL/DML.
virtual unique_ptr<QueryResult> Execute(DuckLakeSnapshot snapshot, string &query);
virtual unique_ptr<QueryResult> Execute(string &query);

virtual unique_ptr<QueryResult> Query(DuckLakeSnapshot snapshot, string &query);
virtual unique_ptr<QueryResult> Query(string &query);
//! Read metadata rows visible at the supplied DuckLake snapshot.
virtual unique_ptr<QueryResult> SnapshotQuery(DuckLakeSnapshot snapshot, string &query);
//! Read current metadata state.
virtual unique_ptr<QueryResult> CurrentQuery(DuckLakeSnapshot snapshot, string &query);
virtual unique_ptr<QueryResult> CurrentQuery(string &query);
//! Get the catalog information for a specific snapshot
virtual DuckLakeCatalogInfo GetCatalogForSnapshot(DuckLakeSnapshot snapshot);
virtual vector<DuckLakeGlobalStatsInfo> GetGlobalTableStats(DuckLakeSnapshot snapshot);
Expand Down
15 changes: 13 additions & 2 deletions src/include/storage/ducklake_transaction.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,16 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this<D
DuckLakeSnapshotCommit &GetCommitInfo() {
return commit_info;
}
unique_ptr<QueryResult> Query(DuckLakeSnapshot snapshot, string query);
unique_ptr<QueryResult> Query(string query);
//! Execute metadata DDL/DML.
unique_ptr<QueryResult> Execute(DuckLakeSnapshot snapshot, string query);
unique_ptr<QueryResult> Execute(string query);
//! Read metadata rows visible at the supplied DuckLake snapshot.
unique_ptr<QueryResult> SnapshotQuery(DuckLakeSnapshot snapshot, string query);
//! Read current metadata state.
unique_ptr<QueryResult> CurrentQuery(DuckLakeSnapshot snapshot, string query);
unique_ptr<QueryResult> CurrentQuery(string query);
//! Execute a local DuckDB connection utility query, not a semantic metadata read/write.
unique_ptr<QueryResult> RawQuery(string query);
Connection &GetConnection();

DuckLakeSnapshot GetSnapshot();
Expand Down Expand Up @@ -338,6 +346,9 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this<D
case_insensitive_map_t<unique_ptr<DuckLakeCatalogSet>> &GetNewMacroMap(CatalogType type);

private:
unique_ptr<QueryResult> RunQuery(DuckLakeSnapshot snapshot, string query);
unique_ptr<QueryResult> RunQuery(string query);

DuckLakeCatalog &ducklake_catalog;
DuckLakeSnapshotCommit commit_info;
DatabaseInstance &db;
Expand Down
12 changes: 8 additions & 4 deletions src/metadata_manager/postgres_metadata_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ unique_ptr<QueryResult> PostgresMetadataManager::ExecuteQuery(DuckLakeSnapshot s
query = StringUtil::Replace(query, "{DATA_PATH}", data_path);

auto passthrough_query = StringUtil::Format("CALL %s(%s, %s)", command, catalog_literal, SQLString(query));
auto result = transaction.Query(passthrough_query);
auto result = transaction.RawQuery(passthrough_query);
if (command == "postgres_execute" && !result->HasError()) {
while (result->Fetch()) {
}
Expand All @@ -297,11 +297,15 @@ unique_ptr<QueryResult> PostgresMetadataManager::Execute(string &query) {
return ExecuteQuery(query, "postgres_execute");
}

unique_ptr<QueryResult> PostgresMetadataManager::Query(DuckLakeSnapshot snapshot, string &query) {
unique_ptr<QueryResult> PostgresMetadataManager::SnapshotQuery(DuckLakeSnapshot snapshot, string &query) {
return ExecuteQuery(snapshot, query, "postgres_query");
}

unique_ptr<QueryResult> PostgresMetadataManager::Query(string &query) {
unique_ptr<QueryResult> PostgresMetadataManager::CurrentQuery(DuckLakeSnapshot snapshot, string &query) {
return ExecuteQuery(snapshot, query, "postgres_query");
}

unique_ptr<QueryResult> PostgresMetadataManager::CurrentQuery(string &query) {
return ExecuteQuery(query, "postgres_query");
}

Expand All @@ -325,7 +329,7 @@ SELECT EXISTS (
AND table_name = %s
))",
DuckLakeUtil::SQLLiteralToString(table_name));
auto result = Query(snapshot, query);
auto result = SnapshotQuery(snapshot, query);
if (result->HasError()) {
return false;
}
Expand Down
8 changes: 4 additions & 4 deletions src/storage/ducklake_initializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,15 @@ string DuckLakeInitializer::GetAttachOptions() {
void DuckLakeInitializer::Initialize() {
auto &transaction = DuckLakeTransaction::Get(context, catalog);
// attach the metadata database
auto result = transaction.Query("ATTACH OR REPLACE {METADATA_PATH} AS {METADATA_CATALOG_NAME_IDENTIFIER}" +
GetAttachOptions());
auto result = transaction.RawQuery("ATTACH OR REPLACE {METADATA_PATH} AS {METADATA_CATALOG_NAME_IDENTIFIER}" +
GetAttachOptions());
if (result->HasError()) {
auto &error_obj = result->GetErrorObject();
error_obj.Throw("Failed to attach DuckLake MetaData \"" + catalog.MetadataDatabaseName() + "\" at path + \"" +
catalog.MetadataPath() + "\"");
}
// explicitly load all secrets - work-around to secret initialization bug
transaction.Query("FROM duckdb_secrets()");
transaction.RawQuery("FROM duckdb_secrets()");

bool has_explicit_schema = !options.metadata_schema.empty();
if (options.metadata_schema.empty()) {
Expand All @@ -77,7 +77,7 @@ void DuckLakeInitializer::Initialize() {
// directly query a known ducklake metadata table to avoid scanning all attached catalogs via duckdb_tables()
// this prevents a corrupted ducklake catalog from blocking initialization of unrelated ducklake databases
// FIXME: verify that all ducklake tables are in the correct format
result = transaction.Query("SELECT NULL FROM {METADATA_CATALOG}.ducklake_metadata LIMIT 1");
result = transaction.CurrentQuery("SELECT NULL FROM {METADATA_CATALOG}.ducklake_metadata LIMIT 1");
if (result->HasError()) {
auto &error_obj = result->GetErrorObject();
if (error_obj.Type() == ExceptionType::CATALOG) {
Expand Down
Loading
Loading