Skip to content

Commit a1f0616

Browse files
robert3005robert3005
committed
more
Signed-off-by: Robert Kruszewski <github@robertk.io>
1 parent d95b769 commit a1f0616

65 files changed

Lines changed: 1422 additions & 1430 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/specs/editions.md‎

Lines changed: 115 additions & 99 deletions
Large diffs are not rendered by default.

‎encodings/alp/src/alp/plugin.rs‎

Lines changed: 56 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,18 @@
77
//! This enables zero-cost backward compatibility with previously written datasets.
88
99
use vortex_array::Array;
10+
use vortex_array::ArrayContext;
11+
use vortex_array::ArrayDeserialization;
1012
use vortex_array::ArrayId;
1113
use vortex_array::ArrayPlugin;
1214
use vortex_array::ArrayRef;
15+
use vortex_array::ArraySerialization;
1316
use vortex_array::ArrayVTable;
1417
use vortex_array::IntoArray;
1518
use vortex_array::VortexSessionExecute;
1619
use vortex_array::arrays::Patched;
17-
use vortex_array::buffer::BufferHandle;
18-
use vortex_array::dtype::DType;
19-
use vortex_array::serde::ArrayChildren;
2020
use vortex_error::VortexResult;
21+
use vortex_error::vortex_ensure;
2122
use vortex_error::vortex_err;
2223
use vortex_session::VortexSession;
2324

@@ -41,23 +42,31 @@ impl ArrayPlugin for ALPPatchedPlugin {
4142
fn serialize(
4243
&self,
4344
array: &ArrayRef,
45+
ctx: &ArrayContext,
4446
session: &VortexSession,
45-
) -> VortexResult<Option<Vec<u8>>> {
47+
) -> VortexResult<Option<ArraySerialization>> {
4648
// Delegate to ALP's metadata serde
47-
ALP.serialize(array, session)
49+
ArrayPlugin::serialize(&ALP, array, ctx, session)
4850
}
4951

5052
fn deserialize(
5153
&self,
52-
dtype: &DType,
53-
len: usize,
54-
metadata: &[u8],
55-
buffers: &[BufferHandle],
56-
children: &dyn ArrayChildren,
54+
parts: ArrayDeserialization<'_>,
5755
session: &VortexSession,
5856
) -> VortexResult<ArrayRef> {
57+
vortex_ensure!(
58+
parts.serialized_id == self.id(),
59+
"ALP plugin does not recognize serialized ID {}",
60+
parts.serialized_id,
61+
);
5962
let alp_array = Array::<ALP>::try_from_parts(ArrayVTable::deserialize(
60-
&ALP, dtype, len, metadata, buffers, children, session,
63+
&ALP,
64+
parts.dtype,
65+
parts.len,
66+
parts.metadata,
67+
parts.buffers,
68+
parts.children,
69+
session,
6170
)?)
6271
.map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?;
6372

@@ -91,6 +100,8 @@ mod tests {
91100
use std::f64::consts::PI;
92101
use std::sync::LazyLock;
93102

103+
use vortex_array::ArrayContext;
104+
use vortex_array::ArrayDeserialization;
94105
use vortex_array::ArrayPlugin;
95106
use vortex_array::IntoArray;
96107
use vortex_array::VortexSessionExecute;
@@ -133,7 +144,9 @@ mod tests {
133144

134145
let array = alp_encoded.as_array();
135146

136-
let metadata = SESSION.array_serialize(array)?.unwrap();
147+
let serialization = SESSION
148+
.array_serialize(array, &ArrayContext::empty())?
149+
.unwrap();
137150
let children = array.children();
138151
let buffers = array
139152
.buffers()
@@ -142,11 +155,14 @@ mod tests {
142155
.collect::<Vec<_>>();
143156

144157
let deserialized = ALPPatchedPlugin.deserialize(
145-
array.dtype(),
146-
array.len(),
147-
&metadata,
148-
&buffers,
149-
&children,
158+
ArrayDeserialization::new(
159+
ALPPatchedPlugin.id(),
160+
array.dtype(),
161+
array.len(),
162+
&serialization.metadata,
163+
&buffers,
164+
&children,
165+
),
150166
&SESSION,
151167
)?;
152168

@@ -182,7 +198,9 @@ mod tests {
182198

183199
let array = alp_encoded.as_array();
184200

185-
let metadata = SESSION.array_serialize(array)?.unwrap();
201+
let serialization = SESSION
202+
.array_serialize(array, &ArrayContext::empty())?
203+
.unwrap();
186204
let children = array.children();
187205
let buffers = array
188206
.buffers()
@@ -191,11 +209,14 @@ mod tests {
191209
.collect::<Vec<_>>();
192210

193211
let deserialized = ALPPatchedPlugin.deserialize(
194-
array.dtype(),
195-
array.len(),
196-
&metadata,
197-
&buffers,
198-
&children,
212+
ArrayDeserialization::new(
213+
ALPPatchedPlugin.id(),
214+
array.dtype(),
215+
array.len(),
216+
&serialization.metadata,
217+
&buffers,
218+
&children,
219+
),
199220
&SESSION,
200221
)?;
201222

@@ -213,7 +234,10 @@ mod tests {
213234
fn primitive_array_returns_error() {
214235
let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array();
215236

216-
let metadata = SESSION.array_serialize(&array).unwrap().unwrap();
237+
let serialization = SESSION
238+
.array_serialize(&array, &ArrayContext::empty())
239+
.unwrap()
240+
.unwrap();
217241
let children = array.children();
218242
let buffers = array
219243
.buffers()
@@ -223,11 +247,14 @@ mod tests {
223247

224248
// This panics because PrimitiveArray has no children and ALP requires encoded child.
225249
let _result = ALPPatchedPlugin.deserialize(
226-
array.dtype(),
227-
array.len(),
228-
&metadata,
229-
&buffers,
230-
&children,
250+
ArrayDeserialization::new(
251+
ALPPatchedPlugin.id(),
252+
array.dtype(),
253+
array.len(),
254+
&serialization.metadata,
255+
&buffers,
256+
&children,
257+
),
231258
&SESSION,
232259
);
233260
}

‎encodings/fastlanes/src/bitpacking/plugin.rs‎

Lines changed: 55 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,18 @@
77
//! This enables zero-cost backward compatibility with previously written datasets.
88
99
use vortex_array::Array;
10+
use vortex_array::ArrayContext;
11+
use vortex_array::ArrayDeserialization;
1012
use vortex_array::ArrayId;
1113
use vortex_array::ArrayPlugin;
1214
use vortex_array::ArrayRef;
15+
use vortex_array::ArraySerialization;
1316
use vortex_array::ArrayVTable;
1417
use vortex_array::IntoArray;
1518
use vortex_array::VortexSessionExecute;
1619
use vortex_array::arrays::Patched;
17-
use vortex_array::buffer::BufferHandle;
18-
use vortex_array::dtype::DType;
19-
use vortex_array::serde::ArrayChildren;
2020
use vortex_error::VortexResult;
21+
use vortex_error::vortex_ensure;
2122
use vortex_error::vortex_err;
2223
use vortex_session::VortexSession;
2324

@@ -40,23 +41,31 @@ impl ArrayPlugin for BitPackedPatchedPlugin {
4041
fn serialize(
4142
&self,
4243
array: &ArrayRef,
44+
ctx: &ArrayContext,
4345
session: &VortexSession,
44-
) -> VortexResult<Option<Vec<u8>>> {
46+
) -> VortexResult<Option<ArraySerialization>> {
4547
// delegate to BitPacked VTable for serialization
46-
BitPacked.serialize(array, session)
48+
ArrayPlugin::serialize(&BitPacked, array, ctx, session)
4749
}
4850

4951
fn deserialize(
5052
&self,
51-
dtype: &DType,
52-
len: usize,
53-
metadata: &[u8],
54-
buffers: &[BufferHandle],
55-
children: &dyn ArrayChildren,
53+
parts: ArrayDeserialization<'_>,
5654
session: &VortexSession,
5755
) -> VortexResult<ArrayRef> {
56+
vortex_ensure!(
57+
parts.serialized_id == self.id(),
58+
"BitPacked plugin does not recognize serialized ID {}",
59+
parts.serialized_id,
60+
);
5861
let bitpacked = Array::<BitPacked>::try_from_parts(ArrayVTable::deserialize(
59-
&BitPacked, dtype, len, metadata, buffers, children, session,
62+
&BitPacked,
63+
parts.dtype,
64+
parts.len,
65+
parts.metadata,
66+
parts.buffers,
67+
parts.children,
68+
session,
6069
)?)
6170
.map_err(|_| vortex_err!("BitPacked plugin should only deserialize fastlanes.bitpacked"))?;
6271

@@ -93,6 +102,8 @@ impl ArrayPlugin for BitPackedPatchedPlugin {
93102
mod tests {
94103
use std::sync::LazyLock;
95104

105+
use vortex_array::ArrayContext;
106+
use vortex_array::ArrayDeserialization;
96107
use vortex_array::ArrayPlugin;
97108
use vortex_array::IntoArray;
98109
use vortex_array::VortexSessionExecute;
@@ -134,7 +145,9 @@ mod tests {
134145

135146
let array = bitpacked.as_array();
136147

137-
let metadata = SESSION.array_serialize(array)?.unwrap();
148+
let serialization = SESSION
149+
.array_serialize(array, &ArrayContext::empty())?
150+
.unwrap();
138151
let children = array.children();
139152
let buffers = array
140153
.buffers()
@@ -143,11 +156,14 @@ mod tests {
143156
.collect::<Vec<_>>();
144157

145158
let deserialized = BitPackedPatchedPlugin.deserialize(
146-
array.dtype(),
147-
array.len(),
148-
&metadata,
149-
&buffers,
150-
&children,
159+
ArrayDeserialization::new(
160+
BitPackedPatchedPlugin.id(),
161+
array.dtype(),
162+
array.len(),
163+
&serialization.metadata,
164+
&buffers,
165+
&children,
166+
),
151167
&SESSION,
152168
)?;
153169

@@ -184,7 +200,9 @@ mod tests {
184200

185201
let array = bitpacked.as_array();
186202

187-
let metadata = SESSION.array_serialize(array)?.unwrap();
203+
let serialization = SESSION
204+
.array_serialize(array, &ArrayContext::empty())?
205+
.unwrap();
188206
let children = array.children();
189207
let buffers = array
190208
.buffers()
@@ -193,11 +211,14 @@ mod tests {
193211
.collect::<Vec<_>>();
194212

195213
let deserialized = BitPackedPatchedPlugin.deserialize(
196-
array.dtype(),
197-
array.len(),
198-
&metadata,
199-
&buffers,
200-
&children,
214+
ArrayDeserialization::new(
215+
BitPackedPatchedPlugin.id(),
216+
array.dtype(),
217+
array.len(),
218+
&serialization.metadata,
219+
&buffers,
220+
&children,
221+
),
201222
&SESSION,
202223
)?;
203224

@@ -214,7 +235,9 @@ mod tests {
214235
fn primitive_array_returns_error() -> VortexResult<()> {
215236
let array = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();
216237

217-
let metadata = SESSION.array_serialize(&array)?.unwrap();
238+
let serialization = SESSION
239+
.array_serialize(&array, &ArrayContext::empty())?
240+
.unwrap();
218241
let children = array.children();
219242
let buffers = array
220243
.buffers()
@@ -223,11 +246,14 @@ mod tests {
223246
.collect::<Vec<_>>();
224247

225248
let result = BitPackedPatchedPlugin.deserialize(
226-
array.dtype(),
227-
array.len(),
228-
&metadata,
229-
&buffers,
230-
&children,
249+
ArrayDeserialization::new(
250+
BitPackedPatchedPlugin.id(),
251+
array.dtype(),
252+
array.len(),
253+
&serialization.metadata,
254+
&buffers,
255+
&children,
256+
),
231257
&SESSION,
232258
);
233259

0 commit comments

Comments
 (0)