fix per-route services binding to the same port

Author: mvdschee

src/core/handlers/filesystem.rs

@@ -20,6 +20,13 @@ pub fn write_file(file_path: SafePath, content: &[u8]) -> Result<()> {
 	Ok(())
 }
 
+pub fn read_file(file_path: &SafePath) -> Result<Vec<u8>> {
+	let file = fs::read(file_path.as_str())
+		.map_err(|e| Error::FileSystem(format!("Failed to read {}: {}", file_path, e)))?;
+
+	Ok(file)
+}
+
 pub fn check_file_exists(file_path: &SafePath) -> bool {
 	// foxguard: ignore[rs/no-path-traversal]
 	// validated with SafePath

src/core/handlers/proxy.rs

@@ -1,21 +1,30 @@
 use crate::{
 	Error, Result,
-	core::models::proxy::{ProxyConfig, ProxyRoute},
+	core::{
+		handlers::filesystem::read_file,
+		models::{
+			certs::{TlsCerts, TlsMaterial},
+			proxy::{ProxyConfig, ProxyRoute, ProxyRouteMap},
+			routes::Host,
+		},
+	},
+	error,
 };
 use async_trait::async_trait;
 use http::{Response, header};
 use pingora::{
+	ErrorType,
 	apps::http_app::ServeHttp,
-	http::StatusCode,
-	prelude::{HttpPeer, Result as PingoraResult},
+	http::{ResponseHeader, StatusCode},
+	listeners::{TlsAccept, tls::TlsSettings},
+	prelude::{Error as PingoraError, HttpPeer, Result as PingoraResult},
 	protocols::http::ServerSession,
 	proxy::{ProxyHttp, Session, http_proxy_service},
 	server::{Server, configuration::ServerConf},
 	services::{Service, listening::Service as ListeningService},
+	tls::{self, ssl},
 };
-use std::sync::Arc;
-
-static BODY: &[u8] = "<html><body>301 Moved Permanently</body></html>".as_bytes();
+use std::{collections::HashMap, sync::Arc};
 
 pub fn run_proxy(proxy_config: ProxyConfig, routes: Vec<ProxyRoute>) -> Result<()> {
 	let mut server = Server::new(None).map_err(|e| Error::Proxy(e.to_string()))?;
@@ -26,89 +35,92 @@ pub fn run_proxy(proxy_config: ProxyConfig, routes: Vec<ProxyRoute>) -> Result<(
 	let http_addr = format!("{}:{}", proxy_config.input_address, *proxy_config.http_port);
 	let https_addr = format!("{}:{}", proxy_config.input_address, *proxy_config.https_port);
 
-	// tls proxies
-	let tls_services =
-		tls_routes_services(server_conf.clone(), https_addr.clone(), routes.clone())?;
+	let mut routes_map = HashMap::new();
+	let mut tls_certs = HashMap::new();
+
+	for route in routes {
+		if *route.tls {
+			let cert_bytes = read_file(&route.cert_path)?;
+			let cert = tls::x509::X509::from_pem(&cert_bytes)
+				.map_err(|e| Error::Certificate(format!("Failed to parse certificate: {}", e)))?;
+
+			let key_bytes = read_file(&route.key_path)?;
+			let key = tls::pkey::PKey::private_key_from_pem(&key_bytes)
+				.map_err(|e| Error::Certificate(format!("Failed to parse private key: {}", e)))?;
+
+			tls_certs.insert(
+				route.host.clone(),
+				TlsMaterial {
+					cert,
+					key,
+				},
+			);
+		}
 
-	for service in tls_services {
-		server.add_service(service);
+		routes_map.insert(route.host.clone(), route);
 	}
 
-	// redirect to tls proxies
-	let redirect_service = http_redirect_service(http_addr.clone());
+	let routes_map = Arc::new(routes_map);
+	let tls_certs = Arc::new(tls_certs);
 
-	server.add_service(redirect_service);
-	// plain proxies
-	let plain_services = plain_routes_services(server_conf, http_addr.clone(), routes.clone())?;
+	// plain proxies with redirect
+	let plain_service =
+		plain_routes_service(server_conf.clone(), http_addr.clone(), routes_map.clone())?;
+	server.add_service(plain_service);
 
-	for service in plain_services {
-		server.add_service(service);
-	}
+	// tls proxies
+	let tls_service =
+		tls_routes_service(server_conf, https_addr.clone(), routes_map.clone(), tls_certs)?;
+	server.add_service(tls_service);
 
 	server.run_forever();
 
 	Err(Error::Proxy("proxy stopped".to_string()))
 }
 
-pub fn tls_routes_services(
+pub fn tls_routes_service(
 	server_conf: Arc<ServerConf>,
 	listen_addr: String,
-	routes: Vec<ProxyRoute>,
-) -> Result<Vec<impl Service>> {
-	let mut services = Vec::new();
-
-	let tls_routes = routes.into_iter().filter(|route| *route.tls == true).collect::<Vec<_>>();
+	routes_map: ProxyRouteMap,
+	tls_certs: TlsCerts,
+) -> Result<impl Service> {
+	let proxy_app = ProxyToUpstream::new(routes_map.clone(), false);
 
-	for route in tls_routes {
-		let proxy_app = ProxyToUpstream::new(route.clone());
-
-		let mut service = http_proxy_service(&server_conf, proxy_app);
-		service
-			.add_tls(&listen_addr, &route.cert_path, &route.key_path)
-			.map_err(|e| Error::Proxy(e.to_string()));
+	let mut service = http_proxy_service(&server_conf, proxy_app);
+	service.add_tcp(&listen_addr);
 
-		services.push(service);
-	}
+	let sni_resolver = SniResolver::new(tls_certs);
+	let callback = Box::new(sni_resolver);
+	let tls_settings =
+		TlsSettings::with_callbacks(callback).map_err(|e| Error::Proxy(e.to_string()))?;
+	service.add_tls_with_settings(&listen_addr, None, tls_settings);
 
-	Ok(services)
+	Ok(service)
 }
 
-pub fn plain_routes_services(
+pub fn plain_routes_service(
 	server_conf: Arc<ServerConf>,
 	listen_addr: String,
-	routes: Vec<ProxyRoute>,
-) -> Result<Vec<impl Service>> {
-	let mut services = Vec::new();
-
-	let plain_routes = routes.into_iter().filter(|route| *route.tls == false).collect::<Vec<_>>();
-
-	for route in plain_routes {
-		let proxy_app = ProxyToUpstream::new(route.clone());
-
-		let mut service = http_proxy_service(&server_conf, proxy_app);
-		service.add_tcp(&listen_addr);
-
-		services.push(service);
-	}
-
-	Ok(services)
-}
+	routes_map: ProxyRouteMap,
+) -> Result<impl Service> {
+	let proxy_app = ProxyToUpstream::new(routes_map, true);
 
-fn http_redirect_service(listen_addr: String) -> impl Service {
-	let mut service = ListeningService::new("HTTPS Redirect".to_string(), RedirectToHttps {});
+	let mut service = http_proxy_service(&server_conf, proxy_app);
 	service.add_tcp(&listen_addr);
 
-	service
+	Ok(service)
 }
 
 pub struct ProxyToUpstream {
-	route: ProxyRoute,
+	routes_map: ProxyRouteMap,
+	upgrade_to_https: bool,
 }
 
 impl ProxyToUpstream {
-	pub fn new(route: ProxyRoute) -> Self {
+	pub fn new(routes_map: ProxyRouteMap, upgrade_to_https: bool) -> Self {
 		Self {
-			route,
+			routes_map,
+			upgrade_to_https,
 		}
 	}
 }
@@ -118,36 +130,106 @@ impl ProxyHttp for ProxyToUpstream {
 	type CTX = ();
 	fn new_ctx(&self) {}
 
+	// ProxyToUpstream is shared with both HTTP and HTTPS
+	// here we do a quick check to upgrade to HTTPS if its a TLS route
+	async fn request_filter(&self, session: &mut Session, _ctx: &mut ()) -> PingoraResult<bool> {
+		if self.upgrade_to_https {
+			let host = host_from_session(session)
+				// 400 bad request no host
+				.ok_or_else(|| PingoraError::new(ErrorType::HTTPStatus(400)))?;
+
+			match self.routes_map.get(host) {
+				// 421 Misdirected Request doesnt match any hosts
+				None => {
+					error!("no route for host: {}", host);
+					let header = ResponseHeader::build(421, None)?;
+					session.write_response_header(Box::new(header), true).await?;
+					return Ok(true);
+				},
+				// 301 Moved Permanently redirect to https
+				Some(route) if *route.tls => {
+					let mut header = ResponseHeader::build(301, None)?;
+					header.insert_header(header::LOCATION, format!("https://{}", route.host))?;
+					header.insert_header(header::CONTENT_LENGTH, 0)?;
+
+					session.write_response_header(Box::new(header), true).await?;
+
+					return Ok(true);
+				},
+				// All good let it pass.
+				Some(_) => return Ok(false),
+			}
+		}
+
+		Ok(false)
+	}
+
 	async fn upstream_peer(
 		&self,
 		session: &mut Session,
 		_ctx: &mut (),
 	) -> PingoraResult<Box<HttpPeer>> {
-		// foxguard: ignore[rs/no-unwrap-in-lib]
-		let host_header = session.get_header(header::HOST).unwrap().to_str().unwrap();
+		let host = host_from_session(session)
+			// 400 bad request no host
+			.ok_or_else(|| PingoraError::new(ErrorType::HTTPStatus(400)))?;
 
-		let proxy_to =
-			HttpPeer::new(self.route.upstream.as_str(), false, self.route.host.to_string());
+		let route = self
+			.routes_map
+			.get(host)
+			// 421 Misdirected Request doesnt match any hosts
+			.ok_or_else(|| PingoraError::new(ErrorType::HTTPStatus(421)))?;
+
+		let proxy_to = HttpPeer::new(route.upstream.as_str(), false, route.host.to_string());
 		let peer = Box::new(proxy_to);
 		Ok(peer)
 	}
 }
 
-pub struct RedirectToHttps;
+// note to self we are not stripping port from the host header
+// so Host: example.com:443 will be rejected with a 421
+// this is a strict design decision not a bug
+fn host_from_session(session: &Session) -> Option<&str> {
+	session.get_header(header::HOST).and_then(|h| h.to_str().ok())
+}
+
+struct SniResolver {
+	certs: TlsCerts,
+}
+
+impl SniResolver {
+	fn new(tls_certs: TlsCerts) -> Self {
+		Self {
+			certs: tls_certs,
+		}
+	}
+}
 
 #[async_trait]
-impl ServeHttp for RedirectToHttps {
-	async fn response(&self, http_stream: &mut ServerSession) -> Response<Vec<u8>> {
-		// foxguard: ignore[rs/no-unwrap-in-lib]
-		let host_header = http_stream.get_header(header::HOST).unwrap().to_str().unwrap();
-
-		Response::builder()
-			.status(StatusCode::MOVED_PERMANENTLY)
-			.header(header::CONTENT_TYPE, "text/html")
-			.header(header::CONTENT_LENGTH, BODY.len())
-			.header(header::LOCATION, format!("https://{host_header}"))
-			.body(BODY.to_owned())
-			// foxguard: ignore[rs/no-unwrap-in-lib]
-			.unwrap()
+impl TlsAccept for SniResolver {
+	async fn certificate_callback(&self, ssl: &mut ssl::SslRef) -> () {
+		let sni_provided = ssl.servername(ssl::NameType::HOST_NAME).map(str::to_owned);
+
+		let Some(sni_provided) = sni_provided else {
+			error!("No SNI provided");
+			return;
+		};
+
+		let Some(TlsMaterial {
+			cert,
+			key,
+		}) = self.certs.get(sni_provided.as_str())
+		else {
+			error!("No certificate found for SNI: {}", sni_provided);
+			return;
+		};
+
+		if let Err(e) = tls::ext::ssl_use_certificate(ssl, cert) {
+			error!("Failed to use certificate for SNI {}: {}", sni_provided, e);
+			return;
+		}
+		if let Err(e) = tls::ext::ssl_use_private_key(ssl, key) {
+			error!("Failed to use private key for SNI {}: {}", sni_provided, e);
+			return;
+		}
 	}
 }

src/core/models/certs.rs

@@ -1,7 +1,13 @@
 use crate::core::models::routes::Host;
+use pingora::tls::{
+	pkey::{PKey, Private},
+	x509::X509,
+};
 use serde::Deserialize;
+use std::collections::HashMap;
 use std::fmt;
 use std::ops::Deref;
+use std::sync::Arc;
 
 #[derive(Debug, Clone, Deserialize, Default, PartialEq)]
 #[serde(rename_all = "snake_case")]
@@ -20,6 +26,13 @@ pub struct Certificate {
 	pub cert_type: CertificateType,
 }
 
+pub struct TlsMaterial {
+	pub cert: X509,
+	pub key: PKey<Private>,
+}
+
+pub type TlsCerts = Arc<HashMap<Host, TlsMaterial>>;
+
 // --- EMAIL ---
 #[derive(Debug, Clone, Deserialize)]
 pub struct Email(String);

src/core/models/proxy.rs

@@ -4,7 +4,9 @@ use crate::core::models::{
 	routes::{Host, Upstream},
 };
 use http::{Response, StatusCode, header};
-use std::{fmt, ops::Deref};
+use std::{collections::HashMap, fmt, ops::Deref, sync::Arc};
+
+pub type ProxyRouteMap = Arc<HashMap<Host, ProxyRoute>>;
 
 #[derive(Debug, Clone)]
 pub struct ProxyRoute {

src/core/models/routes.rs

@@ -11,7 +11,7 @@ pub struct Route {
 }
 
 // --- HOST ---
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Hash, Eq, PartialEq)]
 pub struct Host(String);
 
 impl Deref for Host {
@@ -22,6 +22,12 @@ impl Deref for Host {
 	}
 }
 
+impl std::borrow::Borrow<str> for Host {
+	fn borrow(&self) -> &str {
+		&self.0
+	}
+}
+
 impl fmt::Display for Host {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 		write!(f, "{}", self.0)