|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! See `main.rs` for how to run it. |
| 19 | +//! |
| 20 | +//! The same problem as `composed_extension_codec.rs`, solved without composing |
| 21 | +//! codecs: two plans from two independent crates in one tree. |
| 22 | +//! |
| 23 | +//! ```text |
| 24 | +//! ParentExec (from crate A) |
| 25 | +//! ChildExec (from crate B) |
| 26 | +//! ``` |
| 27 | +//! |
| 28 | +//! With `ComposedPhysicalExtensionCodec` the decoding side has to register the |
| 29 | +//! same codecs in the same *order* the encoder used, because the composed codec |
| 30 | +//! writes the position of the encoding codec into the payload. Here each plan |
| 31 | +//! names itself on the wire instead, so order does not matter and a name two |
| 32 | +//! crates both claim is an error at registration rather than a wrong decode. |
| 33 | +
|
| 34 | +use std::fmt::Formatter; |
| 35 | +use std::sync::Arc; |
| 36 | + |
| 37 | +use datafusion::common::tree_node::TreeNodeRecursion; |
| 38 | +use datafusion::common::{Result, internal_err}; |
| 39 | +use datafusion::execution::TaskContext; |
| 40 | +use datafusion::physical_expr::PhysicalExpr; |
| 41 | +use datafusion::physical_plan::proto::{ |
| 42 | + ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionPlanFromProto, |
| 43 | + ProtoDecoderRegistry, register_execution_plan, |
| 44 | +}; |
| 45 | +use datafusion::physical_plan::{ |
| 46 | + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, |
| 47 | + ReplaceChildrenOptions, SendableRecordBatchStream, |
| 48 | +}; |
| 49 | +use datafusion::prelude::{SessionConfig, SessionContext}; |
| 50 | +use datafusion_proto::physical_plan::{AsExecutionPlan, DefaultPhysicalExtensionCodec}; |
| 51 | +use datafusion_proto::protobuf::PhysicalPlanNode; |
| 52 | +use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType; |
| 53 | + |
| 54 | +pub fn extension_plan_registry() -> Result<()> { |
| 55 | + let plan: Arc<dyn ExecutionPlan> = Arc::new(ParentExec { |
| 56 | + input: Arc::new(ChildExec {}), |
| 57 | + }); |
| 58 | + |
| 59 | + // Each crate exposes a function that fills a registry it is handed; the |
| 60 | + // application that owns the session composes them. The two crates need to |
| 61 | + // know nothing about each other, and the call order does not matter. |
| 62 | + let mut registry = ProtoDecoderRegistry::new(); |
| 63 | + crate_a::register(&mut registry)?; |
| 64 | + crate_b::register(&mut registry)?; |
| 65 | + |
| 66 | + let ctx = SessionContext::new_with_config( |
| 67 | + SessionConfig::new().with_extension(Arc::new(registry)), |
| 68 | + ); |
| 69 | + |
| 70 | + // `DefaultPhysicalExtensionCodec` decodes nothing at all, so whatever |
| 71 | + // survives the round trip came from the registry. |
| 72 | + let codec = DefaultPhysicalExtensionCodec {}; |
| 73 | + let node = PhysicalPlanNode::try_from_physical_plan(Arc::clone(&plan), &codec)?; |
| 74 | + let decoded = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?; |
| 75 | + |
| 76 | + assert_eq!(format!("{plan:?}"), format!("{decoded:?}")); |
| 77 | + |
| 78 | + // A session that does not register the plans cannot decode them, and the |
| 79 | + // error names what is missing rather than only what the codec said. Children |
| 80 | + // are decoded before their parent falls back to the codec, so the innermost |
| 81 | + // unregistered plan is the one reported: |
| 82 | + // |
| 83 | + // No decoder is registered for the extension ExecutionPlan |
| 84 | + // 'example-crate-b.ChildExec'. Register the plan in the |
| 85 | + // ProtoDecoderRegistry attached to the decoding session's |
| 86 | + // SessionConfig, ... Registered extension plans: none |
| 87 | + let bare = SessionContext::new(); |
| 88 | + let err = node |
| 89 | + .try_into_physical_plan(bare.task_ctx().as_ref(), &codec) |
| 90 | + .expect_err("an unregistered plan must not decode"); |
| 91 | + assert!(err.to_string().contains("No decoder is registered")); |
| 92 | + assert!(err.to_string().contains(ChildExec::NAME)); |
| 93 | + |
| 94 | + Ok(()) |
| 95 | +} |
| 96 | + |
| 97 | +/// What the crate that owns `ParentExec` would export. |
| 98 | +mod crate_a { |
| 99 | + use super::*; |
| 100 | + |
| 101 | + pub fn register(registry: &mut ProtoDecoderRegistry) -> Result<()> { |
| 102 | + register_execution_plan::<ParentExec>(registry) |
| 103 | + } |
| 104 | +} |
| 105 | + |
| 106 | +/// What the crate that owns `ChildExec` would export. |
| 107 | +mod crate_b { |
| 108 | + use super::*; |
| 109 | + |
| 110 | + pub fn register(registry: &mut ProtoDecoderRegistry) -> Result<()> { |
| 111 | + register_execution_plan::<ChildExec>(registry) |
| 112 | + } |
| 113 | +} |
| 114 | + |
| 115 | +#[derive(Debug)] |
| 116 | +struct ParentExec { |
| 117 | + input: Arc<dyn ExecutionPlan>, |
| 118 | +} |
| 119 | + |
| 120 | +impl ExtensionPlanFromProto for ParentExec { |
| 121 | + // Namespaced with the owning crate, so a collision with another crate is |
| 122 | + // an error at registration rather than a wrong decode. |
| 123 | + const NAME: &'static str = "example-crate-a.ParentExec"; |
| 124 | + |
| 125 | + fn try_from_proto( |
| 126 | + node: &PhysicalPlanNode, |
| 127 | + ctx: &ExecutionPlanDecodeCtx<'_>, |
| 128 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 129 | + let Some(PhysicalPlanType::Extension(extension)) = &node.physical_plan_type |
| 130 | + else { |
| 131 | + return internal_err!("PhysicalPlanNode is not an Extension"); |
| 132 | + }; |
| 133 | + let mut children = ctx.decode_children(&extension.inputs)?; |
| 134 | + if children.len() != 1 { |
| 135 | + return internal_err!("ParentExec expects exactly one child"); |
| 136 | + } |
| 137 | + Ok(Arc::new(ParentExec { |
| 138 | + input: children.remove(0), |
| 139 | + })) |
| 140 | + } |
| 141 | +} |
| 142 | + |
| 143 | +#[derive(Debug)] |
| 144 | +struct ChildExec {} |
| 145 | + |
| 146 | +impl ExtensionPlanFromProto for ChildExec { |
| 147 | + const NAME: &'static str = "example-crate-b.ChildExec"; |
| 148 | + |
| 149 | + fn try_from_proto( |
| 150 | + _node: &PhysicalPlanNode, |
| 151 | + _ctx: &ExecutionPlanDecodeCtx<'_>, |
| 152 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 153 | + Ok(Arc::new(ChildExec {})) |
| 154 | + } |
| 155 | +} |
| 156 | + |
| 157 | +/// Both plans are serde-only: this example never executes them. |
| 158 | +macro_rules! serde_only_body { |
| 159 | + ($name:ident) => { |
| 160 | + fn name(&self) -> &str { |
| 161 | + stringify!($name) |
| 162 | + } |
| 163 | + |
| 164 | + fn properties(&self) -> &Arc<PlanProperties> { |
| 165 | + unreachable!("this example only serializes") |
| 166 | + } |
| 167 | + |
| 168 | + fn apply_expressions( |
| 169 | + &self, |
| 170 | + _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>, |
| 171 | + ) -> Result<TreeNodeRecursion> { |
| 172 | + Ok(TreeNodeRecursion::Continue) |
| 173 | + } |
| 174 | + |
| 175 | + fn with_new_children( |
| 176 | + self: Arc<Self>, |
| 177 | + children: Vec<Arc<dyn ExecutionPlan>>, |
| 178 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 179 | + self.replace_children( |
| 180 | + children, |
| 181 | + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), |
| 182 | + ) |
| 183 | + } |
| 184 | + |
| 185 | + fn execute( |
| 186 | + &self, |
| 187 | + _partition: usize, |
| 188 | + _context: Arc<TaskContext>, |
| 189 | + ) -> Result<SendableRecordBatchStream> { |
| 190 | + internal_err!("{} is a serde-only example plan", stringify!($name)) |
| 191 | + } |
| 192 | + |
| 193 | + /// `extension_node` stamps `Self::NAME`, so the name on the wire and |
| 194 | + /// the registry key cannot drift apart. Neither plan has state of its |
| 195 | + /// own, hence the empty payload. |
| 196 | + fn try_to_proto( |
| 197 | + &self, |
| 198 | + ctx: &ExecutionPlanEncodeCtx<'_>, |
| 199 | + ) -> Result<Option<PhysicalPlanNode>> { |
| 200 | + Ok(Some(ctx.extension_node::<Self>(vec![], self.children())?)) |
| 201 | + } |
| 202 | + }; |
| 203 | +} |
| 204 | + |
| 205 | +impl DisplayAs for ParentExec { |
| 206 | + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { |
| 207 | + write!(f, "ParentExec") |
| 208 | + } |
| 209 | +} |
| 210 | + |
| 211 | +impl ExecutionPlan for ParentExec { |
| 212 | + serde_only_body!(ParentExec); |
| 213 | + |
| 214 | + fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> { |
| 215 | + vec![&self.input] |
| 216 | + } |
| 217 | + |
| 218 | + fn replace_children( |
| 219 | + self: Arc<Self>, |
| 220 | + mut children: Vec<Arc<dyn ExecutionPlan>>, |
| 221 | + _: ReplaceChildrenOptions, |
| 222 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 223 | + if children.len() != 1 { |
| 224 | + return internal_err!("ParentExec expects exactly one child"); |
| 225 | + } |
| 226 | + Ok(Arc::new(ParentExec { |
| 227 | + input: children.remove(0), |
| 228 | + })) |
| 229 | + } |
| 230 | +} |
| 231 | + |
| 232 | +impl DisplayAs for ChildExec { |
| 233 | + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { |
| 234 | + write!(f, "ChildExec") |
| 235 | + } |
| 236 | +} |
| 237 | + |
| 238 | +impl ExecutionPlan for ChildExec { |
| 239 | + serde_only_body!(ChildExec); |
| 240 | + |
| 241 | + fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> { |
| 242 | + vec![] |
| 243 | + } |
| 244 | + |
| 245 | + fn replace_children( |
| 246 | + self: Arc<Self>, |
| 247 | + _children: Vec<Arc<dyn ExecutionPlan>>, |
| 248 | + _: ReplaceChildrenOptions, |
| 249 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 250 | + Ok(self) |
| 251 | + } |
| 252 | +} |
0 commit comments