Skip to content

Commit b2fe932

Browse files
committed
Add DocumentClient to RoboSystemsExtensions class
## Summary This commit enhances the `RoboSystemsExtensions` class by integrating the `DocumentClient`, allowing for improved document management capabilities within the client library. ## Key Accomplishments - **New Client Integration**: Added `DocumentClient` to the `RoboSystemsExtensions` class, enabling access to document-related functionalities. - **Resource Management**: Updated the `close` method to ensure proper resource management for the new `documents` attribute. ## Changes Breakdown - Modified `extensions.py` to include the `DocumentClient` and updated the initialization and cleanup logic accordingly. ## Testing Notes - Verify that the `DocumentClient` is correctly instantiated and that resources are properly released during the `close` operation. ## Infrastructure Considerations - No breaking changes introduced; existing functionality remains intact.
1 parent 443a615 commit b2fe932

2 files changed

Lines changed: 368 additions & 0 deletions

File tree

Lines changed: 364 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,364 @@
1+
"""Document Client for RoboSystems API
2+
3+
Upload, search, list, and delete text documents indexed in OpenSearch.
4+
Documents are sectioned on markdown headings, embedded for semantic search,
5+
and searchable alongside structured graph data.
6+
"""
7+
8+
from dataclasses import dataclass, field
9+
from pathlib import Path
10+
from typing import Any, Dict, List, Optional
11+
12+
import httpx
13+
import logging
14+
15+
logger = logging.getLogger(__name__)
16+
17+
18+
@dataclass
19+
class DocumentUploadResult:
20+
"""Result from document upload."""
21+
22+
document_id: str
23+
sections_indexed: int
24+
total_content_length: int
25+
section_ids: List[str]
26+
success: bool = True
27+
error: Optional[str] = None
28+
29+
30+
@dataclass
31+
class DocumentSearchHit:
32+
"""A single search result."""
33+
34+
document_id: str
35+
score: float
36+
source_type: str
37+
snippet: str
38+
section_label: Optional[str] = None
39+
document_title: Optional[str] = None
40+
entity_ticker: Optional[str] = None
41+
form_type: Optional[str] = None
42+
filing_date: Optional[str] = None
43+
tags: Optional[List[str]] = None
44+
folder: Optional[str] = None
45+
46+
47+
@dataclass
48+
class DocumentSearchResult:
49+
"""Result from document search."""
50+
51+
total: int
52+
hits: List[DocumentSearchHit]
53+
query: str
54+
graph_id: str
55+
56+
57+
@dataclass
58+
class DocumentInfo:
59+
"""Information about an indexed document."""
60+
61+
document_title: str
62+
section_count: int
63+
source_type: str
64+
folder: Optional[str] = None
65+
tags: Optional[List[str]] = None
66+
last_indexed: Optional[str] = None
67+
68+
69+
@dataclass
70+
class DocumentListResult:
71+
"""Result from listing documents."""
72+
73+
total: int
74+
documents: List[DocumentInfo]
75+
graph_id: str
76+
77+
78+
class DocumentClient:
79+
"""Client for document upload, search, and management."""
80+
81+
def __init__(self, config: Dict[str, Any]):
82+
self.config = config
83+
self.base_url = config["base_url"]
84+
self.headers = config.get("headers", {})
85+
self.timeout = config.get("timeout", 60)
86+
self._http_client = httpx.Client(
87+
timeout=self.timeout,
88+
headers=self.headers,
89+
)
90+
91+
def _url(self, graph_id: str, path: str = "") -> str:
92+
return f"{self.base_url}/v1/graphs/{graph_id}/documents{path}"
93+
94+
def _search_url(self, graph_id: str, path: str = "") -> str:
95+
return f"{self.base_url}/v1/graphs/{graph_id}/search{path}"
96+
97+
def upload(
98+
self,
99+
graph_id: str,
100+
title: str,
101+
content: str,
102+
tags: Optional[List[str]] = None,
103+
folder: Optional[str] = None,
104+
external_id: Optional[str] = None,
105+
) -> DocumentUploadResult:
106+
"""Upload a markdown document for text indexing.
107+
108+
The document is sectioned on headings, embedded, and indexed
109+
into OpenSearch for full-text and semantic search.
110+
111+
Args:
112+
graph_id: Target graph ID.
113+
title: Document title.
114+
content: Markdown content (may include YAML frontmatter).
115+
tags: Optional tags for filtering.
116+
folder: Optional folder/category.
117+
external_id: Optional external ID for upsert behavior.
118+
119+
Returns:
120+
DocumentUploadResult with section IDs and counts.
121+
"""
122+
payload: Dict[str, Any] = {"title": title, "content": content}
123+
if tags is not None:
124+
payload["tags"] = tags
125+
if folder is not None:
126+
payload["folder"] = folder
127+
if external_id is not None:
128+
payload["external_id"] = external_id
129+
130+
response = self._http_client.post(self._url(graph_id), json=payload)
131+
132+
if response.status_code == 200:
133+
data = response.json()
134+
return DocumentUploadResult(
135+
document_id=data["document_id"],
136+
sections_indexed=data["sections_indexed"],
137+
total_content_length=data["total_content_length"],
138+
section_ids=data["section_ids"],
139+
)
140+
else:
141+
error_detail = response.text
142+
try:
143+
error_detail = response.json().get("detail", response.text)
144+
except Exception:
145+
pass
146+
return DocumentUploadResult(
147+
document_id="",
148+
sections_indexed=0,
149+
total_content_length=0,
150+
section_ids=[],
151+
success=False,
152+
error=f"{response.status_code}: {error_detail}",
153+
)
154+
155+
def upload_file(
156+
self,
157+
graph_id: str,
158+
file_path: str | Path,
159+
title: Optional[str] = None,
160+
tags: Optional[List[str]] = None,
161+
folder: Optional[str] = None,
162+
external_id: Optional[str] = None,
163+
) -> DocumentUploadResult:
164+
"""Upload a markdown file by path.
165+
166+
Reads the file, optionally extracts title from filename,
167+
and uploads via the document API.
168+
169+
Args:
170+
graph_id: Target graph ID.
171+
file_path: Path to markdown file.
172+
title: Document title (defaults to filename).
173+
tags: Optional tags.
174+
folder: Optional folder.
175+
external_id: Optional external ID.
176+
"""
177+
path = Path(file_path)
178+
content = path.read_text()
179+
180+
if title is None:
181+
# Try frontmatter title, fall back to filename
182+
title = path.stem.replace("-", " ").replace("_", " ").title()
183+
184+
return self.upload(
185+
graph_id=graph_id,
186+
title=title,
187+
content=content,
188+
tags=tags,
189+
folder=folder,
190+
external_id=external_id,
191+
)
192+
193+
def upload_directory(
194+
self,
195+
graph_id: str,
196+
directory: str | Path,
197+
pattern: str = "*.md",
198+
folder: Optional[str] = None,
199+
) -> List[DocumentUploadResult]:
200+
"""Upload all markdown files from a directory.
201+
202+
Args:
203+
graph_id: Target graph ID.
204+
directory: Path to directory containing markdown files.
205+
pattern: Glob pattern for file matching (default: *.md).
206+
folder: Optional folder to apply to all documents.
207+
208+
Returns:
209+
List of upload results, one per file.
210+
"""
211+
dir_path = Path(directory)
212+
results = []
213+
214+
for md_file in sorted(dir_path.glob(pattern)):
215+
result = self.upload_file(
216+
graph_id=graph_id,
217+
file_path=md_file,
218+
folder=folder,
219+
)
220+
results.append(result)
221+
222+
return results
223+
224+
def search(
225+
self,
226+
graph_id: str,
227+
query: str,
228+
source_type: Optional[str] = None,
229+
entity: Optional[str] = None,
230+
form_type: Optional[str] = None,
231+
section: Optional[str] = None,
232+
semantic: bool = False,
233+
size: int = 10,
234+
) -> DocumentSearchResult:
235+
"""Search documents with full-text or semantic search.
236+
237+
Args:
238+
graph_id: Target graph ID.
239+
query: Search query string.
240+
source_type: Filter by source type (uploaded_doc, memory, etc.).
241+
entity: Filter by entity ticker/CIK/name.
242+
form_type: Filter by SEC form type.
243+
section: Filter by section ID.
244+
semantic: Enable semantic (vector) search.
245+
size: Max results to return.
246+
247+
Returns:
248+
DocumentSearchResult with ranked hits.
249+
"""
250+
payload: Dict[str, Any] = {"query": query, "size": size}
251+
if source_type is not None:
252+
payload["source_type"] = source_type
253+
if entity is not None:
254+
payload["entity"] = entity
255+
if form_type is not None:
256+
payload["form_type"] = form_type
257+
if section is not None:
258+
payload["section"] = section
259+
if semantic:
260+
payload["semantic"] = True
261+
262+
response = self._http_client.post(self._search_url(graph_id), json=payload)
263+
response.raise_for_status()
264+
data = response.json()
265+
266+
hits = [
267+
DocumentSearchHit(
268+
document_id=h["document_id"],
269+
score=h["score"],
270+
source_type=h["source_type"],
271+
snippet=h["snippet"],
272+
section_label=h.get("section_label"),
273+
document_title=h.get("document_title"),
274+
entity_ticker=h.get("entity_ticker"),
275+
form_type=h.get("form_type"),
276+
filing_date=h.get("filing_date"),
277+
tags=h.get("tags"),
278+
folder=h.get("folder"),
279+
)
280+
for h in data.get("hits", [])
281+
]
282+
283+
return DocumentSearchResult(
284+
total=data["total"],
285+
hits=hits,
286+
query=data["query"],
287+
graph_id=data["graph_id"],
288+
)
289+
290+
def get_section(self, graph_id: str, document_id: str) -> Optional[Dict[str, Any]]:
291+
"""Retrieve the full text of a document section by ID.
292+
293+
Args:
294+
graph_id: Target graph ID.
295+
document_id: Document section ID.
296+
297+
Returns:
298+
Dict with full section content and metadata, or None if not found.
299+
"""
300+
response = self._http_client.get(self._search_url(graph_id, f"/{document_id}"))
301+
if response.status_code == 404:
302+
return None
303+
response.raise_for_status()
304+
return response.json()
305+
306+
def list(
307+
self, graph_id: str, source_type: Optional[str] = None
308+
) -> DocumentListResult:
309+
"""List indexed documents for a graph.
310+
311+
Args:
312+
graph_id: Target graph ID.
313+
source_type: Optional filter by source type.
314+
315+
Returns:
316+
DocumentListResult with document inventory.
317+
"""
318+
params = {}
319+
if source_type is not None:
320+
params["source_type"] = source_type
321+
322+
response = self._http_client.get(self._url(graph_id), params=params)
323+
response.raise_for_status()
324+
data = response.json()
325+
326+
documents = [
327+
DocumentInfo(
328+
document_title=d["document_title"],
329+
section_count=d["section_count"],
330+
source_type=d["source_type"],
331+
folder=d.get("folder"),
332+
tags=d.get("tags"),
333+
last_indexed=d.get("last_indexed"),
334+
)
335+
for d in data.get("documents", [])
336+
]
337+
338+
return DocumentListResult(
339+
total=data["total"],
340+
documents=documents,
341+
graph_id=data["graph_id"],
342+
)
343+
344+
def delete(self, graph_id: str, document_id: str) -> bool:
345+
"""Delete a document and all its sections.
346+
347+
Args:
348+
graph_id: Target graph ID.
349+
document_id: Document ID to delete.
350+
351+
Returns:
352+
True if deleted, False if not found.
353+
"""
354+
response = self._http_client.delete(self._url(graph_id, f"/{document_id}"))
355+
if response.status_code == 204:
356+
return True
357+
if response.status_code == 404:
358+
return False
359+
response.raise_for_status()
360+
return False
361+
362+
def close(self):
363+
"""Close the HTTP client."""
364+
self._http_client.close()

‎robosystems_client/extensions/extensions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from .agent_client import AgentClient
1111
from .operation_client import OperationClient
1212
from .file_client import FileClient
13+
from .document_client import DocumentClient
1314
from .materialization_client import MaterializationClient
1415
from .table_client import TableClient
1516
from .graph_client import GraphClient
@@ -68,6 +69,7 @@ def __init__(self, config: RoboSystemsExtensionConfig = None):
6869
self.files = FileClient(self.config)
6970
self.materialization = MaterializationClient(self.config)
7071
self.tables = TableClient(self.config)
72+
self.documents = DocumentClient(self.config)
7173
self.graphs = GraphClient(self.config)
7274

7375
def monitor_operation(
@@ -104,6 +106,8 @@ def close(self):
104106
self.materialization.close()
105107
if hasattr(self.tables, "close"):
106108
self.tables.close()
109+
if hasattr(self.documents, "close"):
110+
self.documents.close()
107111
self.graphs.close()
108112

109113
# Convenience methods that delegate to the appropriate clients

0 commit comments

Comments
 (0)