|
5 | 5 |
|
6 | 6 | #![allow(clippy::result_large_err)] |
7 | 7 |
|
| 8 | +pub mod otel_tracing; |
| 9 | + |
8 | 10 | use bollard::Docker; |
9 | 11 | use bollard::errors::Error as BollardError; |
10 | 12 | use bollard::models::{ |
@@ -54,10 +56,12 @@ use openshell_core::proto_struct::{ |
54 | 56 | }; |
55 | 57 | use openshell_core::{Error, Result as CoreResult}; |
56 | 58 | use std::collections::{HashMap, HashSet}; |
| 59 | +use std::future::Future; |
57 | 60 | use std::net::{IpAddr, Ipv4Addr, SocketAddr}; |
58 | 61 | use std::path::{Path, PathBuf}; |
59 | 62 | use std::pin::Pin; |
60 | 63 | use std::sync::Arc; |
| 64 | +use std::task::{Context, Poll}; |
61 | 65 | use std::time::Duration; |
62 | 66 | use tokio::sync::{Mutex, broadcast, mpsc}; |
63 | 67 | use tokio::task::JoinHandle; |
@@ -385,6 +389,119 @@ fn default_true() -> bool { |
385 | 389 | type WatchStream = |
386 | 390 | Pin<Box<dyn Stream<Item = Result<WatchSandboxesEvent, Status>> + Send + 'static>>; |
387 | 391 |
|
| 392 | +struct TracedWatchStream { |
| 393 | + inner: WatchStream, |
| 394 | + span: tracing::Span, |
| 395 | + finished: bool, |
| 396 | +} |
| 397 | + |
| 398 | +impl Stream for TracedWatchStream { |
| 399 | + type Item = Result<WatchSandboxesEvent, Status>; |
| 400 | + |
| 401 | + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> { |
| 402 | + let span = self.span.clone(); |
| 403 | + let _entered = span.enter(); |
| 404 | + let result = self.inner.as_mut().poll_next(cx); |
| 405 | + if !self.finished { |
| 406 | + match &result { |
| 407 | + Poll::Ready(Some(Err(status))) => { |
| 408 | + openshell_otel::mark_error(&self.span); |
| 409 | + self.span |
| 410 | + .record("rpc.grpc.status_code", status.code() as i32); |
| 411 | + self.finished = true; |
| 412 | + } |
| 413 | + Poll::Ready(None) => { |
| 414 | + self.span |
| 415 | + .record("rpc.grpc.status_code", tonic::Code::Ok as i32); |
| 416 | + self.finished = true; |
| 417 | + } |
| 418 | + Poll::Pending | Poll::Ready(Some(Ok(_))) => {} |
| 419 | + } |
| 420 | + } |
| 421 | + result |
| 422 | + } |
| 423 | +} |
| 424 | + |
| 425 | +impl Drop for TracedWatchStream { |
| 426 | + fn drop(&mut self) { |
| 427 | + if !self.finished { |
| 428 | + openshell_otel::mark_error(&self.span); |
| 429 | + self.span |
| 430 | + .record("rpc.grpc.status_code", tonic::Code::Cancelled as i32); |
| 431 | + } |
| 432 | + } |
| 433 | +} |
| 434 | + |
| 435 | +/// Compute-driver service wrapper that preserves the standalone RPC trace |
| 436 | +/// boundary while Docker runs in the gateway process. |
| 437 | +#[derive(Clone)] |
| 438 | +pub struct ComputeDriverService { |
| 439 | + driver: DockerComputeDriver, |
| 440 | + trace_in_process_rpc: bool, |
| 441 | +} |
| 442 | + |
| 443 | +impl ComputeDriverService { |
| 444 | + #[must_use] |
| 445 | + pub fn new(driver: DockerComputeDriver) -> Self { |
| 446 | + Self { |
| 447 | + driver, |
| 448 | + trace_in_process_rpc: false, |
| 449 | + } |
| 450 | + } |
| 451 | + |
| 452 | + #[must_use] |
| 453 | + pub fn new_in_process(driver: DockerComputeDriver) -> Self { |
| 454 | + Self { |
| 455 | + driver, |
| 456 | + trace_in_process_rpc: true, |
| 457 | + } |
| 458 | + } |
| 459 | + |
| 460 | + fn in_process_rpc_span( |
| 461 | + &self, |
| 462 | + operation: &'static str, |
| 463 | + method: &'static str, |
| 464 | + ) -> Option<tracing::Span> { |
| 465 | + self.trace_in_process_rpc.then(|| { |
| 466 | + tracing::info_span!( |
| 467 | + target: "openshell_driver_docker::otel_tracing", |
| 468 | + "driver_rpc", |
| 469 | + otel.name = operation, |
| 470 | + otel.kind = "server", |
| 471 | + otel.status_code = tracing::field::Empty, |
| 472 | + rpc.system = "grpc", |
| 473 | + rpc.service = "openshell.compute.v1.ComputeDriver", |
| 474 | + rpc.method = method, |
| 475 | + rpc.grpc.status_code = tracing::field::Empty, |
| 476 | + ) |
| 477 | + }) |
| 478 | + } |
| 479 | + |
| 480 | + async fn trace_rpc<T>( |
| 481 | + &self, |
| 482 | + operation: &'static str, |
| 483 | + method: &'static str, |
| 484 | + future: impl Future<Output = Result<T, Status>>, |
| 485 | + ) -> Result<T, Status> { |
| 486 | + use tracing::Instrument as _; |
| 487 | + |
| 488 | + let Some(span) = self.in_process_rpc_span(operation, method) else { |
| 489 | + return future.await; |
| 490 | + }; |
| 491 | + let result = future.instrument(span.clone()).await; |
| 492 | + match &result { |
| 493 | + Ok(_) => { |
| 494 | + span.record("rpc.grpc.status_code", tonic::Code::Ok as i32); |
| 495 | + } |
| 496 | + Err(status) => { |
| 497 | + openshell_otel::mark_error(&span); |
| 498 | + span.record("rpc.grpc.status_code", status.code() as i32); |
| 499 | + } |
| 500 | + } |
| 501 | + result |
| 502 | + } |
| 503 | +} |
| 504 | + |
388 | 505 | /// Return the first responsive local Docker API socket. |
389 | 506 | #[must_use] |
390 | 507 | pub fn detect_socket() -> Option<PathBuf> { |
@@ -1536,12 +1653,188 @@ impl DockerComputeDriver { |
1536 | 1653 | } |
1537 | 1654 | } |
1538 | 1655 |
|
| 1656 | +#[tonic::async_trait] |
| 1657 | +impl ComputeDriver for ComputeDriverService { |
| 1658 | + type WatchSandboxesStream = WatchStream; |
| 1659 | + |
| 1660 | + async fn authenticate_sandbox( |
| 1661 | + &self, |
| 1662 | + request: Request<openshell_core::proto::compute::v1::AuthenticateSandboxRequest>, |
| 1663 | + ) -> Result<Response<openshell_core::proto::compute::v1::AuthenticateSandboxResponse>, Status> |
| 1664 | + { |
| 1665 | + self.trace_rpc( |
| 1666 | + "driver.authenticate_sandbox", |
| 1667 | + "authenticate_sandbox", |
| 1668 | + ComputeDriver::authenticate_sandbox(&self.driver, request), |
| 1669 | + ) |
| 1670 | + .await |
| 1671 | + } |
| 1672 | + |
| 1673 | + async fn get_capabilities( |
| 1674 | + &self, |
| 1675 | + request: Request<GetCapabilitiesRequest>, |
| 1676 | + ) -> Result<Response<GetCapabilitiesResponse>, Status> { |
| 1677 | + self.trace_rpc( |
| 1678 | + "driver.get_capabilities", |
| 1679 | + "get_capabilities", |
| 1680 | + ComputeDriver::get_capabilities(&self.driver, request), |
| 1681 | + ) |
| 1682 | + .await |
| 1683 | + } |
| 1684 | + |
| 1685 | + async fn get_gateway_listener_requirements( |
| 1686 | + &self, |
| 1687 | + request: Request<GetGatewayListenerRequirementsRequest>, |
| 1688 | + ) -> Result<Response<GetGatewayListenerRequirementsResponse>, Status> { |
| 1689 | + self.trace_rpc( |
| 1690 | + "driver.get_gateway_listener_requirements", |
| 1691 | + "get_gateway_listener_requirements", |
| 1692 | + ComputeDriver::get_gateway_listener_requirements(&self.driver, request), |
| 1693 | + ) |
| 1694 | + .await |
| 1695 | + } |
| 1696 | + |
| 1697 | + async fn validate_sandbox_create( |
| 1698 | + &self, |
| 1699 | + request: Request<ValidateSandboxCreateRequest>, |
| 1700 | + ) -> Result<Response<ValidateSandboxCreateResponse>, Status> { |
| 1701 | + self.trace_rpc( |
| 1702 | + "driver.validate_sandbox_create", |
| 1703 | + "validate_sandbox_create", |
| 1704 | + ComputeDriver::validate_sandbox_create(&self.driver, request), |
| 1705 | + ) |
| 1706 | + .await |
| 1707 | + } |
| 1708 | + |
| 1709 | + async fn get_sandbox( |
| 1710 | + &self, |
| 1711 | + request: Request<GetSandboxRequest>, |
| 1712 | + ) -> Result<Response<GetSandboxResponse>, Status> { |
| 1713 | + self.trace_rpc( |
| 1714 | + "driver.get_sandbox", |
| 1715 | + "get_sandbox", |
| 1716 | + ComputeDriver::get_sandbox(&self.driver, request), |
| 1717 | + ) |
| 1718 | + .await |
| 1719 | + } |
| 1720 | + |
| 1721 | + async fn list_sandboxes( |
| 1722 | + &self, |
| 1723 | + request: Request<ListSandboxesRequest>, |
| 1724 | + ) -> Result<Response<ListSandboxesResponse>, Status> { |
| 1725 | + self.trace_rpc( |
| 1726 | + "driver.list_sandboxes", |
| 1727 | + "list_sandboxes", |
| 1728 | + ComputeDriver::list_sandboxes(&self.driver, request), |
| 1729 | + ) |
| 1730 | + .await |
| 1731 | + } |
| 1732 | + |
| 1733 | + async fn create_sandbox( |
| 1734 | + &self, |
| 1735 | + request: Request<CreateSandboxRequest>, |
| 1736 | + ) -> Result<Response<CreateSandboxResponse>, Status> { |
| 1737 | + self.trace_rpc( |
| 1738 | + "driver.create_sandbox", |
| 1739 | + "create_sandbox", |
| 1740 | + ComputeDriver::create_sandbox(&self.driver, request), |
| 1741 | + ) |
| 1742 | + .await |
| 1743 | + } |
| 1744 | + |
| 1745 | + async fn stop_sandbox( |
| 1746 | + &self, |
| 1747 | + request: Request<StopSandboxRequest>, |
| 1748 | + ) -> Result<Response<StopSandboxResponse>, Status> { |
| 1749 | + self.trace_rpc( |
| 1750 | + "driver.stop_sandbox", |
| 1751 | + "stop_sandbox", |
| 1752 | + ComputeDriver::stop_sandbox(&self.driver, request), |
| 1753 | + ) |
| 1754 | + .await |
| 1755 | + } |
| 1756 | + |
| 1757 | + async fn start_sandbox( |
| 1758 | + &self, |
| 1759 | + request: Request<StartSandboxRequest>, |
| 1760 | + ) -> Result<Response<StartSandboxResponse>, Status> { |
| 1761 | + self.trace_rpc( |
| 1762 | + "driver.start_sandbox", |
| 1763 | + "start_sandbox", |
| 1764 | + ComputeDriver::start_sandbox(&self.driver, request), |
| 1765 | + ) |
| 1766 | + .await |
| 1767 | + } |
| 1768 | + |
| 1769 | + async fn delete_sandbox( |
| 1770 | + &self, |
| 1771 | + request: Request<DeleteSandboxRequest>, |
| 1772 | + ) -> Result<Response<DeleteSandboxResponse>, Status> { |
| 1773 | + self.trace_rpc( |
| 1774 | + "driver.delete_sandbox", |
| 1775 | + "delete_sandbox", |
| 1776 | + ComputeDriver::delete_sandbox(&self.driver, request), |
| 1777 | + ) |
| 1778 | + .await |
| 1779 | + } |
| 1780 | + |
| 1781 | + async fn watch_sandboxes( |
| 1782 | + &self, |
| 1783 | + request: Request<WatchSandboxesRequest>, |
| 1784 | + ) -> Result<Response<Self::WatchSandboxesStream>, Status> { |
| 1785 | + use tracing::Instrument as _; |
| 1786 | + |
| 1787 | + let create_stream = ComputeDriver::watch_sandboxes(&self.driver, request); |
| 1788 | + let Some(span) = self.in_process_rpc_span("driver.watch_sandboxes", "watch_sandboxes") |
| 1789 | + else { |
| 1790 | + return create_stream.await; |
| 1791 | + }; |
| 1792 | + match create_stream.instrument(span.clone()).await { |
| 1793 | + Ok(response) => Ok(Response::new(Box::pin(TracedWatchStream { |
| 1794 | + inner: response.into_inner(), |
| 1795 | + span, |
| 1796 | + finished: false, |
| 1797 | + }))), |
| 1798 | + Err(status) => { |
| 1799 | + openshell_otel::mark_error(&span); |
| 1800 | + span.record("rpc.grpc.status_code", status.code() as i32); |
| 1801 | + Err(status) |
| 1802 | + } |
| 1803 | + } |
| 1804 | + } |
| 1805 | + |
| 1806 | + async fn ensure_workspace( |
| 1807 | + &self, |
| 1808 | + request: Request<EnsureWorkspaceRequest>, |
| 1809 | + ) -> Result<Response<EnsureWorkspaceResponse>, Status> { |
| 1810 | + self.trace_rpc( |
| 1811 | + "driver.ensure_workspace", |
| 1812 | + "ensure_workspace", |
| 1813 | + ComputeDriver::ensure_workspace(&self.driver, request), |
| 1814 | + ) |
| 1815 | + .await |
| 1816 | + } |
| 1817 | + |
| 1818 | + async fn delete_workspace( |
| 1819 | + &self, |
| 1820 | + request: Request<DeleteWorkspaceRequest>, |
| 1821 | + ) -> Result<Response<DeleteWorkspaceResponse>, Status> { |
| 1822 | + self.trace_rpc( |
| 1823 | + "driver.delete_workspace", |
| 1824 | + "delete_workspace", |
| 1825 | + ComputeDriver::delete_workspace(&self.driver, request), |
| 1826 | + ) |
| 1827 | + .await |
| 1828 | + } |
| 1829 | +} |
| 1830 | + |
1539 | 1831 | #[tonic::async_trait] |
1540 | 1832 | impl ComputeDriver for DockerComputeDriver { |
1541 | 1833 | async fn authenticate_sandbox( |
1542 | 1834 | &self, |
1543 | 1835 | _request: Request<openshell_core::proto::compute::v1::AuthenticateSandboxRequest>, |
1544 | | - ) -> Result<Response<openshell_core::proto::compute::v1::AuthenticateSandboxResponse>, Status> { |
| 1836 | + ) -> Result<Response<openshell_core::proto::compute::v1::AuthenticateSandboxResponse>, Status> |
| 1837 | + { |
1545 | 1838 | Err(Status::unimplemented( |
1546 | 1839 | "docker does not authenticate sandbox credentials", |
1547 | 1840 | )) |
|
0 commit comments