|
| 1 | +"""Document Client for RoboSystems API |
| 2 | +
|
| 3 | +Upload, search, list, and delete text documents indexed in OpenSearch. |
| 4 | +Documents are sectioned on markdown headings, embedded for semantic search, |
| 5 | +and searchable alongside structured graph data. |
| 6 | +""" |
| 7 | + |
| 8 | +from dataclasses import dataclass, field |
| 9 | +from pathlib import Path |
| 10 | +from typing import Any, Dict, List, Optional |
| 11 | + |
| 12 | +import httpx |
| 13 | +import logging |
| 14 | + |
| 15 | +logger = logging.getLogger(__name__) |
| 16 | + |
| 17 | + |
| 18 | +@dataclass |
| 19 | +class DocumentUploadResult: |
| 20 | + """Result from document upload.""" |
| 21 | + |
| 22 | + document_id: str |
| 23 | + sections_indexed: int |
| 24 | + total_content_length: int |
| 25 | + section_ids: List[str] |
| 26 | + success: bool = True |
| 27 | + error: Optional[str] = None |
| 28 | + |
| 29 | + |
| 30 | +@dataclass |
| 31 | +class DocumentSearchHit: |
| 32 | + """A single search result.""" |
| 33 | + |
| 34 | + document_id: str |
| 35 | + score: float |
| 36 | + source_type: str |
| 37 | + snippet: str |
| 38 | + section_label: Optional[str] = None |
| 39 | + document_title: Optional[str] = None |
| 40 | + entity_ticker: Optional[str] = None |
| 41 | + form_type: Optional[str] = None |
| 42 | + filing_date: Optional[str] = None |
| 43 | + tags: Optional[List[str]] = None |
| 44 | + folder: Optional[str] = None |
| 45 | + |
| 46 | + |
| 47 | +@dataclass |
| 48 | +class DocumentSearchResult: |
| 49 | + """Result from document search.""" |
| 50 | + |
| 51 | + total: int |
| 52 | + hits: List[DocumentSearchHit] |
| 53 | + query: str |
| 54 | + graph_id: str |
| 55 | + |
| 56 | + |
| 57 | +@dataclass |
| 58 | +class DocumentInfo: |
| 59 | + """Information about an indexed document.""" |
| 60 | + |
| 61 | + document_title: str |
| 62 | + section_count: int |
| 63 | + source_type: str |
| 64 | + folder: Optional[str] = None |
| 65 | + tags: Optional[List[str]] = None |
| 66 | + last_indexed: Optional[str] = None |
| 67 | + |
| 68 | + |
| 69 | +@dataclass |
| 70 | +class DocumentListResult: |
| 71 | + """Result from listing documents.""" |
| 72 | + |
| 73 | + total: int |
| 74 | + documents: List[DocumentInfo] |
| 75 | + graph_id: str |
| 76 | + |
| 77 | + |
| 78 | +class DocumentClient: |
| 79 | + """Client for document upload, search, and management.""" |
| 80 | + |
| 81 | + def __init__(self, config: Dict[str, Any]): |
| 82 | + self.config = config |
| 83 | + self.base_url = config["base_url"] |
| 84 | + self.headers = config.get("headers", {}) |
| 85 | + self.timeout = config.get("timeout", 60) |
| 86 | + self._http_client = httpx.Client( |
| 87 | + timeout=self.timeout, |
| 88 | + headers=self.headers, |
| 89 | + ) |
| 90 | + |
| 91 | + def _url(self, graph_id: str, path: str = "") -> str: |
| 92 | + return f"{self.base_url}/v1/graphs/{graph_id}/documents{path}" |
| 93 | + |
| 94 | + def _search_url(self, graph_id: str, path: str = "") -> str: |
| 95 | + return f"{self.base_url}/v1/graphs/{graph_id}/search{path}" |
| 96 | + |
| 97 | + def upload( |
| 98 | + self, |
| 99 | + graph_id: str, |
| 100 | + title: str, |
| 101 | + content: str, |
| 102 | + tags: Optional[List[str]] = None, |
| 103 | + folder: Optional[str] = None, |
| 104 | + external_id: Optional[str] = None, |
| 105 | + ) -> DocumentUploadResult: |
| 106 | + """Upload a markdown document for text indexing. |
| 107 | +
|
| 108 | + The document is sectioned on headings, embedded, and indexed |
| 109 | + into OpenSearch for full-text and semantic search. |
| 110 | +
|
| 111 | + Args: |
| 112 | + graph_id: Target graph ID. |
| 113 | + title: Document title. |
| 114 | + content: Markdown content (may include YAML frontmatter). |
| 115 | + tags: Optional tags for filtering. |
| 116 | + folder: Optional folder/category. |
| 117 | + external_id: Optional external ID for upsert behavior. |
| 118 | +
|
| 119 | + Returns: |
| 120 | + DocumentUploadResult with section IDs and counts. |
| 121 | + """ |
| 122 | + payload: Dict[str, Any] = {"title": title, "content": content} |
| 123 | + if tags is not None: |
| 124 | + payload["tags"] = tags |
| 125 | + if folder is not None: |
| 126 | + payload["folder"] = folder |
| 127 | + if external_id is not None: |
| 128 | + payload["external_id"] = external_id |
| 129 | + |
| 130 | + response = self._http_client.post(self._url(graph_id), json=payload) |
| 131 | + |
| 132 | + if response.status_code == 200: |
| 133 | + data = response.json() |
| 134 | + return DocumentUploadResult( |
| 135 | + document_id=data["document_id"], |
| 136 | + sections_indexed=data["sections_indexed"], |
| 137 | + total_content_length=data["total_content_length"], |
| 138 | + section_ids=data["section_ids"], |
| 139 | + ) |
| 140 | + else: |
| 141 | + error_detail = response.text |
| 142 | + try: |
| 143 | + error_detail = response.json().get("detail", response.text) |
| 144 | + except Exception: |
| 145 | + pass |
| 146 | + return DocumentUploadResult( |
| 147 | + document_id="", |
| 148 | + sections_indexed=0, |
| 149 | + total_content_length=0, |
| 150 | + section_ids=[], |
| 151 | + success=False, |
| 152 | + error=f"{response.status_code}: {error_detail}", |
| 153 | + ) |
| 154 | + |
| 155 | + def upload_file( |
| 156 | + self, |
| 157 | + graph_id: str, |
| 158 | + file_path: str | Path, |
| 159 | + title: Optional[str] = None, |
| 160 | + tags: Optional[List[str]] = None, |
| 161 | + folder: Optional[str] = None, |
| 162 | + external_id: Optional[str] = None, |
| 163 | + ) -> DocumentUploadResult: |
| 164 | + """Upload a markdown file by path. |
| 165 | +
|
| 166 | + Reads the file, optionally extracts title from filename, |
| 167 | + and uploads via the document API. |
| 168 | +
|
| 169 | + Args: |
| 170 | + graph_id: Target graph ID. |
| 171 | + file_path: Path to markdown file. |
| 172 | + title: Document title (defaults to filename). |
| 173 | + tags: Optional tags. |
| 174 | + folder: Optional folder. |
| 175 | + external_id: Optional external ID. |
| 176 | + """ |
| 177 | + path = Path(file_path) |
| 178 | + content = path.read_text() |
| 179 | + |
| 180 | + if title is None: |
| 181 | + # Try frontmatter title, fall back to filename |
| 182 | + title = path.stem.replace("-", " ").replace("_", " ").title() |
| 183 | + |
| 184 | + return self.upload( |
| 185 | + graph_id=graph_id, |
| 186 | + title=title, |
| 187 | + content=content, |
| 188 | + tags=tags, |
| 189 | + folder=folder, |
| 190 | + external_id=external_id, |
| 191 | + ) |
| 192 | + |
| 193 | + def upload_directory( |
| 194 | + self, |
| 195 | + graph_id: str, |
| 196 | + directory: str | Path, |
| 197 | + pattern: str = "*.md", |
| 198 | + folder: Optional[str] = None, |
| 199 | + ) -> List[DocumentUploadResult]: |
| 200 | + """Upload all markdown files from a directory. |
| 201 | +
|
| 202 | + Args: |
| 203 | + graph_id: Target graph ID. |
| 204 | + directory: Path to directory containing markdown files. |
| 205 | + pattern: Glob pattern for file matching (default: *.md). |
| 206 | + folder: Optional folder to apply to all documents. |
| 207 | +
|
| 208 | + Returns: |
| 209 | + List of upload results, one per file. |
| 210 | + """ |
| 211 | + dir_path = Path(directory) |
| 212 | + results = [] |
| 213 | + |
| 214 | + for md_file in sorted(dir_path.glob(pattern)): |
| 215 | + result = self.upload_file( |
| 216 | + graph_id=graph_id, |
| 217 | + file_path=md_file, |
| 218 | + folder=folder, |
| 219 | + ) |
| 220 | + results.append(result) |
| 221 | + |
| 222 | + return results |
| 223 | + |
| 224 | + def search( |
| 225 | + self, |
| 226 | + graph_id: str, |
| 227 | + query: str, |
| 228 | + source_type: Optional[str] = None, |
| 229 | + entity: Optional[str] = None, |
| 230 | + form_type: Optional[str] = None, |
| 231 | + section: Optional[str] = None, |
| 232 | + semantic: bool = False, |
| 233 | + size: int = 10, |
| 234 | + ) -> DocumentSearchResult: |
| 235 | + """Search documents with full-text or semantic search. |
| 236 | +
|
| 237 | + Args: |
| 238 | + graph_id: Target graph ID. |
| 239 | + query: Search query string. |
| 240 | + source_type: Filter by source type (uploaded_doc, memory, etc.). |
| 241 | + entity: Filter by entity ticker/CIK/name. |
| 242 | + form_type: Filter by SEC form type. |
| 243 | + section: Filter by section ID. |
| 244 | + semantic: Enable semantic (vector) search. |
| 245 | + size: Max results to return. |
| 246 | +
|
| 247 | + Returns: |
| 248 | + DocumentSearchResult with ranked hits. |
| 249 | + """ |
| 250 | + payload: Dict[str, Any] = {"query": query, "size": size} |
| 251 | + if source_type is not None: |
| 252 | + payload["source_type"] = source_type |
| 253 | + if entity is not None: |
| 254 | + payload["entity"] = entity |
| 255 | + if form_type is not None: |
| 256 | + payload["form_type"] = form_type |
| 257 | + if section is not None: |
| 258 | + payload["section"] = section |
| 259 | + if semantic: |
| 260 | + payload["semantic"] = True |
| 261 | + |
| 262 | + response = self._http_client.post(self._search_url(graph_id), json=payload) |
| 263 | + response.raise_for_status() |
| 264 | + data = response.json() |
| 265 | + |
| 266 | + hits = [ |
| 267 | + DocumentSearchHit( |
| 268 | + document_id=h["document_id"], |
| 269 | + score=h["score"], |
| 270 | + source_type=h["source_type"], |
| 271 | + snippet=h["snippet"], |
| 272 | + section_label=h.get("section_label"), |
| 273 | + document_title=h.get("document_title"), |
| 274 | + entity_ticker=h.get("entity_ticker"), |
| 275 | + form_type=h.get("form_type"), |
| 276 | + filing_date=h.get("filing_date"), |
| 277 | + tags=h.get("tags"), |
| 278 | + folder=h.get("folder"), |
| 279 | + ) |
| 280 | + for h in data.get("hits", []) |
| 281 | + ] |
| 282 | + |
| 283 | + return DocumentSearchResult( |
| 284 | + total=data["total"], |
| 285 | + hits=hits, |
| 286 | + query=data["query"], |
| 287 | + graph_id=data["graph_id"], |
| 288 | + ) |
| 289 | + |
| 290 | + def get_section(self, graph_id: str, document_id: str) -> Optional[Dict[str, Any]]: |
| 291 | + """Retrieve the full text of a document section by ID. |
| 292 | +
|
| 293 | + Args: |
| 294 | + graph_id: Target graph ID. |
| 295 | + document_id: Document section ID. |
| 296 | +
|
| 297 | + Returns: |
| 298 | + Dict with full section content and metadata, or None if not found. |
| 299 | + """ |
| 300 | + response = self._http_client.get(self._search_url(graph_id, f"/{document_id}")) |
| 301 | + if response.status_code == 404: |
| 302 | + return None |
| 303 | + response.raise_for_status() |
| 304 | + return response.json() |
| 305 | + |
| 306 | + def list( |
| 307 | + self, graph_id: str, source_type: Optional[str] = None |
| 308 | + ) -> DocumentListResult: |
| 309 | + """List indexed documents for a graph. |
| 310 | +
|
| 311 | + Args: |
| 312 | + graph_id: Target graph ID. |
| 313 | + source_type: Optional filter by source type. |
| 314 | +
|
| 315 | + Returns: |
| 316 | + DocumentListResult with document inventory. |
| 317 | + """ |
| 318 | + params = {} |
| 319 | + if source_type is not None: |
| 320 | + params["source_type"] = source_type |
| 321 | + |
| 322 | + response = self._http_client.get(self._url(graph_id), params=params) |
| 323 | + response.raise_for_status() |
| 324 | + data = response.json() |
| 325 | + |
| 326 | + documents = [ |
| 327 | + DocumentInfo( |
| 328 | + document_title=d["document_title"], |
| 329 | + section_count=d["section_count"], |
| 330 | + source_type=d["source_type"], |
| 331 | + folder=d.get("folder"), |
| 332 | + tags=d.get("tags"), |
| 333 | + last_indexed=d.get("last_indexed"), |
| 334 | + ) |
| 335 | + for d in data.get("documents", []) |
| 336 | + ] |
| 337 | + |
| 338 | + return DocumentListResult( |
| 339 | + total=data["total"], |
| 340 | + documents=documents, |
| 341 | + graph_id=data["graph_id"], |
| 342 | + ) |
| 343 | + |
| 344 | + def delete(self, graph_id: str, document_id: str) -> bool: |
| 345 | + """Delete a document and all its sections. |
| 346 | +
|
| 347 | + Args: |
| 348 | + graph_id: Target graph ID. |
| 349 | + document_id: Document ID to delete. |
| 350 | +
|
| 351 | + Returns: |
| 352 | + True if deleted, False if not found. |
| 353 | + """ |
| 354 | + response = self._http_client.delete(self._url(graph_id, f"/{document_id}")) |
| 355 | + if response.status_code == 204: |
| 356 | + return True |
| 357 | + if response.status_code == 404: |
| 358 | + return False |
| 359 | + response.raise_for_status() |
| 360 | + return False |
| 361 | + |
| 362 | + def close(self): |
| 363 | + """Close the HTTP client.""" |
| 364 | + self._http_client.close() |
0 commit comments