Skip to content

Commit 33928df

Browse files
feat(python): support array and tensor conversions for DenseVector
1 parent 6ddcda4 commit 33928df

4 files changed

Lines changed: 265 additions & 15 deletions

File tree

‎client/python/USAGE.md‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,12 +228,29 @@ validation and protobuf conversion internally
228228
### `DenseVector`
229229

230230
```
231-
DenseVector(values: list[float] | tuple[float, ...])
231+
DenseVector(values: list[float] | tuple[float, ...] | array | tensor)
232232
```
233233
- Validates numeric input
234+
- Accepts 1D NumPy arrays and PyTorch, TensorFlow, or JAX tensors
235+
- Rejects arrays and tensors that are not 1D
234236
- Normalizes values to `float`
235237
- Immutable (`frozen=True`)
236238

239+
Input conversion uses the capabilities exposed by the value rather than requiring
240+
NumPy, PyTorch, TensorFlow, or JAX as client dependencies.
241+
242+
```python
243+
vector = DenseVector(numpy_array)
244+
245+
vector.to_list()
246+
vector.to_numpy()
247+
vector.to_torch()
248+
vector.to_tensorflow()
249+
vector.to_jax()
250+
```
251+
252+
Each reverse conversion helper requires its corresponding library to be installed.
253+
237254
---
238255

239256
### `Payload`
@@ -374,4 +391,4 @@ python -m grpc_tools.protoc \
374391

375392
After running this:
376393
- `vector_db_pb2_grpc.py` and `vector_db_pb2.py` will be updated
377-
- No other client code should need changes
394+
- No other client code should need changes

‎client/python/tests/test_models.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,82 @@ def test_dense_vector_valid():
1717
a = [1, 2.5, 3]
1818
v = DenseVector(a)
1919
assert v.values == [1.0, 2.5, 3.0]
20+
assert all(isinstance(value, float) for value in v.values)
2021

2122

2223
def test_dense_vector_accepts_tuple():
2324
v = DenseVector((1, 2, 3))
2425
assert v.values == [1.0, 2.0, 3.0]
2526

2627

28+
def test_dense_vector_accepts_numpy_array():
29+
numpy = pytest.importorskip("numpy")
30+
v = DenseVector(numpy.array([1, 2.5, 3], dtype=numpy.float32))
31+
assert v.values == [1.0, 2.5, 3.0]
32+
33+
34+
def test_dense_vector_accepts_pytorch_tensor():
35+
torch = pytest.importorskip("torch")
36+
v = DenseVector(torch.tensor([1, 2.5, 3]))
37+
assert v.values == [1.0, 2.5, 3.0]
38+
39+
40+
def test_dense_vector_accepts_tensorflow_tensor():
41+
tensorflow = pytest.importorskip("tensorflow")
42+
v = DenseVector(tensorflow.constant([1.0, 2.5, 3.0]))
43+
assert v.values == [1.0, 2.5, 3.0]
44+
45+
46+
def test_dense_vector_accepts_jax_array():
47+
jax_numpy = pytest.importorskip("jax.numpy")
48+
v = DenseVector(jax_numpy.array([1, 2.5, 3]))
49+
assert v.values == [1.0, 2.5, 3.0]
50+
51+
52+
class _TensorLike:
53+
def tolist(self):
54+
return [1, 2.5, 3]
55+
56+
57+
def test_dense_vector_accepts_tensor_with_tolist():
58+
v = DenseVector(_TensorLike())
59+
assert v.values == [1.0, 2.5, 3.0]
60+
61+
62+
class _TensorShape:
63+
rank = 1
64+
65+
66+
class _MaterializedTensor:
67+
def tolist(self):
68+
return [1, 2.5, 3]
69+
70+
71+
class _TensorFlowLike:
72+
shape = _TensorShape()
73+
74+
def numpy(self):
75+
return _MaterializedTensor()
76+
77+
78+
def test_dense_vector_accepts_tensor_with_numpy_conversion():
79+
v = DenseVector(_TensorFlowLike())
80+
assert v.values == [1.0, 2.5, 3.0]
81+
82+
83+
class _JaxLike:
84+
ndim = 1
85+
86+
def __array__(self):
87+
numpy = pytest.importorskip("numpy")
88+
return numpy.array([1, 2.5, 3])
89+
90+
91+
def test_dense_vector_accepts_tensor_with_array_protocol():
92+
v = DenseVector(_JaxLike())
93+
assert v.values == [1.0, 2.5, 3.0]
94+
95+
2796
def test_dense_vector_rejects_empty():
2897
with pytest.raises(ValueError):
2998
DenseVector([])
@@ -34,6 +103,36 @@ def test_dense_vector_rejects_non_numeric():
34103
DenseVector([1, "a", 3])
35104

36105

106+
@pytest.mark.parametrize("values", [[[1, 2], [3, 4]], [[1, 2, 3]]])
107+
def test_dense_vector_rejects_nested_sequences(values):
108+
with pytest.raises(ValueError, match="one-dimensional"):
109+
DenseVector(values)
110+
111+
112+
def test_dense_vector_rejects_multidimensional_array():
113+
numpy = pytest.importorskip("numpy")
114+
with pytest.raises(ValueError, match="one-dimensional"):
115+
DenseVector(numpy.array([[1, 2], [3, 4]]))
116+
117+
118+
def test_dense_vector_rejects_scalar_array():
119+
numpy = pytest.importorskip("numpy")
120+
with pytest.raises(ValueError, match="one-dimensional"):
121+
DenseVector(numpy.array(1.0))
122+
123+
124+
class _MatrixLike:
125+
ndim = 2
126+
127+
def tolist(self):
128+
return [[1, 2], [3, 4]]
129+
130+
131+
def test_dense_vector_rejects_multidimensional_tensor_like_value():
132+
with pytest.raises(ValueError, match="one-dimensional"):
133+
DenseVector(_MatrixLike())
134+
135+
37136
def test_dense_vector_is_frozen():
38137
v = DenseVector([1, 2, 3])
39138
with pytest.raises(Exception):
@@ -46,6 +145,41 @@ def test_dense_vector_to_proto():
46145
assert list(proto.values) == [1.0, 2.0, 3.0]
47146

48147

148+
def test_dense_vector_to_numpy():
149+
numpy = pytest.importorskip("numpy")
150+
result = DenseVector([1, 2.5, 3]).to_numpy()
151+
assert isinstance(result, numpy.ndarray)
152+
assert result.ndim == 1
153+
assert numpy.issubdtype(result.dtype, numpy.floating)
154+
assert result.tolist() == [1.0, 2.5, 3.0]
155+
156+
157+
def test_dense_vector_to_torch():
158+
torch = pytest.importorskip("torch")
159+
result = DenseVector([1, 2.5, 3]).to_torch()
160+
assert isinstance(result, torch.Tensor)
161+
assert result.ndim == 1
162+
assert result.dtype.is_floating_point
163+
assert result.tolist() == [1.0, 2.5, 3.0]
164+
165+
166+
def test_dense_vector_to_tensorflow():
167+
tensorflow = pytest.importorskip("tensorflow")
168+
result = DenseVector([1, 2.5, 3]).to_tensorflow()
169+
assert tensorflow.is_tensor(result)
170+
assert result.shape.rank == 1
171+
assert result.dtype.is_floating
172+
assert result.numpy().tolist() == [1.0, 2.5, 3.0]
173+
174+
175+
def test_dense_vector_to_jax():
176+
jax_numpy = pytest.importorskip("jax.numpy")
177+
result = DenseVector([1, 2.5, 3]).to_jax()
178+
assert result.ndim == 1
179+
assert jax_numpy.issubdtype(result.dtype, jax_numpy.floating)
180+
assert result.tolist() == [1.0, 2.5, 3.0]
181+
182+
49183
# Similarity Test
50184

51185

‎client/python/vortexdb/models.py‎

Lines changed: 86 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from dataclasses import dataclass
22
from enum import Enum
3-
from typing import List
3+
from typing import Any, List
44
from vortexdb.grpc import vector_db_pb2
55

66

@@ -43,32 +43,109 @@ def from_proto(value: int) -> "ContentType":
4343
}[value]
4444

4545

46-
# TODO Extend support to other data types than lists or tuples (numpy arrays probably)
47-
# TODO Further compatibility to allow conversions directly to numpy arrays (similar to .to_list())
4846
@dataclass(frozen=True)
4947
class DenseVector:
5048
values: List[float]
5149

5250
def __post_init__(self):
53-
if not isinstance(self.values, (list, tuple)):
54-
raise TypeError("DenseVector expects a list or tuple of floats")
51+
if isinstance(self.values, (list, tuple)):
52+
normalized_values = list(self.values)
53+
else:
54+
normalized_values = self._array_like_to_list(self.values)
5555

56-
if not self.values:
56+
if not normalized_values:
5757
raise ValueError("DenseVector cannot be empty")
5858

59-
for v in self.values:
59+
if any(isinstance(value, (list, tuple)) for value in normalized_values):
60+
raise ValueError("DenseVector expects a one-dimensional vector")
61+
62+
for v in normalized_values:
6063
if not isinstance(v, (int, float)):
6164
raise TypeError("DenseVector values must be numeric (int or float)")
6265

63-
# force float normalization
64-
object.__setattr__(self, "values", [float(v) for v in self.values])
66+
object.__setattr__(self, "values", [float(v) for v in normalized_values])
67+
68+
@staticmethod
69+
def _array_like_to_list(values: Any) -> list[Any]:
70+
shape = getattr(values, "shape", None)
71+
rank = getattr(values, "ndim", None)
72+
73+
if rank is None and shape is not None:
74+
rank = getattr(shape, "rank", None)
75+
if rank is None:
76+
try:
77+
rank = len(shape)
78+
except (TypeError, ValueError):
79+
pass
80+
81+
if rank is not None and rank != 1:
82+
raise ValueError("DenseVector expects a one-dimensional vector")
83+
84+
to_list = getattr(values, "tolist", None)
85+
if callable(to_list):
86+
converted = to_list()
87+
else:
88+
to_numpy = getattr(values, "numpy", None)
89+
if callable(to_numpy):
90+
converted = to_numpy()
91+
else:
92+
to_array = getattr(values, "__array__", None)
93+
if not callable(to_array):
94+
raise TypeError("DenseVector could not convert the array or tensor")
95+
converted = to_array()
96+
97+
converted_to_list = getattr(converted, "tolist", None)
98+
if not callable(converted_to_list):
99+
raise TypeError("DenseVector could not convert the array or tensor")
100+
converted = converted_to_list()
101+
102+
if not isinstance(converted, list):
103+
raise TypeError("DenseVector could not convert the array or tensor")
104+
return converted
65105

66106
def to_proto(self) -> vector_db_pb2.DenseVector:
67107
return vector_db_pb2.DenseVector(values=self.values)
68108

69109
def to_list(self) -> list[float]:
70110
return list(self.values)
71111

112+
# Keep framework dependencies optional by importing only when conversion is requested.
113+
def to_numpy(self) -> Any:
114+
try:
115+
import numpy
116+
except ImportError as error:
117+
raise ImportError(
118+
"NumPy is required to convert DenseVector to an array"
119+
) from error
120+
return numpy.asarray(self.values, dtype=float)
121+
122+
def to_torch(self) -> Any:
123+
try:
124+
import torch
125+
except ImportError as error:
126+
raise ImportError(
127+
"PyTorch is required to convert DenseVector to a tensor"
128+
) from error
129+
return torch.tensor(self.values)
130+
131+
def to_tensorflow(self) -> Any:
132+
try:
133+
import tensorflow
134+
except ImportError as error:
135+
raise ImportError(
136+
"TensorFlow is required to convert DenseVector to a tensor"
137+
) from error
138+
return tensorflow.convert_to_tensor(self.values)
139+
140+
def to_jax(self) -> Any:
141+
try:
142+
import jax.numpy
143+
except ImportError as error:
144+
raise ImportError(
145+
"JAX is required to convert DenseVector to an array"
146+
) from error
147+
return jax.numpy.asarray(self.values)
148+
72149

73150
# & Helper Function for Batch of DenseVectors
74151
def to_dense_vectors(arr):

‎docs/sdk/reference.mdx‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -291,15 +291,17 @@ from vortexdb import DenseVector
291291
### Constructor
292292

293293
```python
294-
DenseVector(values: List[float] | Tuple[float, ...])
294+
DenseVector(values: List[float] | Tuple[float, ...] | array | tensor)
295295
```
296296

297-
<ParamField body="values" type="List[float] | Tuple[float, ...]" required>
298-
The vector components. Must be non-empty and contain numeric values.
297+
<ParamField body="values" type="List[float] | Tuple[float, ...] | array | tensor" required>
298+
A list, tuple, 1D NumPy array, or 1D PyTorch, TensorFlow, or JAX tensor.
299+
Must be non-empty and contain numeric values.
299300
</ParamField>
300301

301302
**Raises:**
302-
- `TypeError`: If values is not a list or tuple
303+
- `TypeError`: If values is not a supported sequence, array, or tensor
304+
- `ValueError`: If values is not one-dimensional
303305
- `ValueError`: If values is empty
304306
- `TypeError`: If any value is not numeric
305307

@@ -314,6 +316,12 @@ vec = DenseVector((0.1, 0.2, 0.3, 0.4))
314316

315317
# Integers are converted to floats
316318
vec = DenseVector([1, 2, 3, 4]) # -> [1.0, 2.0, 3.0, 4.0]
319+
320+
# NumPy arrays and tensors can be passed directly
321+
vec = DenseVector(numpy_array)
322+
vec = DenseVector(torch_tensor)
323+
vec = DenseVector(tensorflow_tensor)
324+
vec = DenseVector(jax_array)
317325
```
318326

319327
### Methods
@@ -333,6 +341,20 @@ vec = DenseVector([0.1, 0.2, 0.3])
333341
values = vec.to_list() # [0.1, 0.2, 0.3]
334342
```
335343

344+
#### Framework conversions
345+
346+
Convert the vector to an array or tensor from an installed framework.
347+
348+
```python
349+
numpy_array = vec.to_numpy()
350+
torch_tensor = vec.to_torch()
351+
tensorflow_tensor = vec.to_tensorflow()
352+
jax_array = vec.to_jax()
353+
```
354+
355+
The framework libraries are optional and are imported only when their conversion
356+
methods are called.
357+
336358
### Properties
337359

338360
#### values

0 commit comments

Comments
 (0)