Merge pull request #16 from adab-tech/claude/globalopportunities-deploy-yukhn1

Harden scraper against SSRF, add search autocomplete and email copy button

Author: adab-tech

backend/app/routes/opportunities.py

@@ -7,7 +7,12 @@
 
 from app.database import get_db
 from app.models import Opportunity
-from app.schemas import OpportunityResponse, PaginatedOpportunities, StatsResponse
+from app.schemas import (
+    OpportunityResponse,
+    PaginatedOpportunities,
+    StatsResponse,
+    SuggestionsResponse,
+)
 
 router = APIRouter(prefix="/opportunities", tags=["Opportunities"])
 
@@ -88,6 +93,28 @@ def list_opportunities(
     )
 
 
+@router.get("/suggest", response_model=SuggestionsResponse)
+def suggest(
+    q: str = Query(..., min_length=2, max_length=100),
+    limit: int = Query(8, ge=1, le=20),
+    db: Session = Depends(get_db),
+):
+    """Distinct titles matching the query, for header search autocomplete."""
+    term = f"%{q}%"
+    # Postgres requires ORDER BY expressions to appear in the SELECT list
+    # for SELECT DISTINCT, so this orders alphabetically rather than by
+    # recency (fine for an autocomplete list).
+    titles = (
+        _public_visible(db.query(Opportunity.title).filter(Opportunity.is_active.is_(True)))
+        .filter(Opportunity.title.ilike(term))
+        .distinct()
+        .order_by(Opportunity.title.asc())
+        .limit(limit)
+        .all()
+    )
+    return SuggestionsResponse(suggestions=[t[0] for t in titles])
+
+
 @router.get("/stats", response_model=StatsResponse)
 def get_stats(db: Session = Depends(get_db)):
     counts = dict(

backend/app/schemas.py

@@ -38,6 +38,10 @@ class PaginatedOpportunities(BaseModel):
     data: list[OpportunityResponse]
 
 
+class SuggestionsResponse(BaseModel):
+    suggestions: list[str]
+
+
 class ScrapeRequest(BaseModel):
     opportunity_types: list[str] | None = Field(
         default_factory=lambda: ["scholarship", "fellowship", "grant", "job"]

backend/app/scrapers/base_scraper.py

@@ -3,10 +3,12 @@
 rate limiting, and common text-extraction helpers.
 """
 
+import ipaddress
 import logging
 import random
+import socket
 import time
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
 from urllib.robotparser import RobotFileParser
 
 import requests
@@ -19,6 +21,40 @@
 
 logger = logging.getLogger(__name__)
 
+_MAX_REDIRECTS = 5
+
+
+def resolves_to_public_address(url: str) -> bool:
+    """False if the URL's host is missing, unresolvable, or resolves to
+    any private/loopback/link-local/reserved/multicast address.
+
+    Discovery URLs come from search results and RSS feeds — content
+    this app doesn't control. Without this check, a malicious or
+    compromised third-party page could redirect the scraper's request
+    into internal infrastructure (e.g. a cloud metadata endpoint) since
+    fetch_page previously followed redirects with no destination check
+    at all. Applied both to the initial URL and to every redirect hop.
+    """
+    host = urlparse(url).hostname
+    if not host:
+        return False
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except socket.gaierror:
+        return False
+    for info in infos:
+        ip = ipaddress.ip_address(info[4][0])
+        if (
+            not ip.is_global
+            or ip.is_private
+            or ip.is_loopback
+            or ip.is_link_local
+            or ip.is_multicast
+            or ip.is_reserved
+        ):
+            return False
+    return True
+
 
 class BaseScraper:
     def __init__(self):
@@ -62,6 +98,10 @@ def _can_fetch(self, url: str) -> bool:
 
     def fetch_page(self, url: str, delay: bool = True) -> BeautifulSoup | None:
         """Fetch a URL and return a parsed BeautifulSoup tree, or None on failure."""
+        if not resolves_to_public_address(url):
+            logger.warning(f"Skipping {url} — does not resolve to a public address")
+            return None
+
         if not self._can_fetch(url):
             logger.info(f"Skipping {url} — blocked by robots.txt")
             return None
@@ -71,18 +111,35 @@ def fetch_page(self, url: str, delay: bool = True) -> BeautifulSoup | None:
 
         for attempt in range(settings.MAX_RETRIES):
             try:
-                response = self.session.get(
-                    url,
-                    headers=self._get_headers(),
-                    timeout=settings.REQUEST_TIMEOUT,
-                    allow_redirects=True,
-                )
-                response.raise_for_status()
-                content_type = response.headers.get("Content-Type", "")
-                if content_type and "html" not in content_type and "xml" not in content_type:
-                    logger.info(f"Skipping {url} — non-HTML content ({content_type})")
+                current_url = url
+                for _ in range(_MAX_REDIRECTS + 1):
+                    response = self.session.get(
+                        current_url,
+                        headers=self._get_headers(),
+                        timeout=settings.REQUEST_TIMEOUT,
+                        allow_redirects=False,
+                    )
+                    if response.is_redirect or response.is_permanent_redirect:
+                        location = response.headers.get("Location")
+                        if not location:
+                            return None
+                        next_url = urljoin(current_url, location)
+                        if not resolves_to_public_address(next_url):
+                            logger.warning(
+                                f"Skipping redirect from {current_url} to {next_url} — not a public address"
+                            )
+                            return None
+                        current_url = next_url
+                        continue
+                    response.raise_for_status()
+                    content_type = response.headers.get("Content-Type", "")
+                    if content_type and "html" not in content_type and "xml" not in content_type:
+                        logger.info(f"Skipping {url} — non-HTML content ({content_type})")
+                        return None
+                    return BeautifulSoup(response.text, "lxml")
+                else:
+                    logger.warning(f"Skipping {url} — too many redirects")
                     return None
-                return BeautifulSoup(response.text, "lxml")
             except requests.RequestException as exc:
                 logger.warning(f"Attempt {attempt + 1}/{settings.MAX_RETRIES} failed for {url}: {exc}")
                 if attempt < settings.MAX_RETRIES - 1:

backend/tests/test_base_scraper.py

@@ -0,0 +1,107 @@
+import socket
+from unittest.mock import MagicMock, patch
+
+from app.scrapers.base_scraper import BaseScraper, resolves_to_public_address
+
+
+class TestResolvesToPublicAddress:
+    def test_no_hostname_is_rejected(self):
+        assert resolves_to_public_address("not-a-url") is False
+
+    def test_unresolvable_host_is_rejected(self):
+        with patch("app.scrapers.base_scraper.socket.getaddrinfo", side_effect=socket.gaierror):
+            assert resolves_to_public_address("https://nonexistent.invalid/") is False
+
+    def test_loopback_address_is_rejected(self):
+        with patch(
+            "app.scrapers.base_scraper.socket.getaddrinfo",
+            return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 443))],
+        ):
+            assert resolves_to_public_address("https://sneaky.example/") is False
+
+    def test_private_address_is_rejected(self):
+        with patch(
+            "app.scrapers.base_scraper.socket.getaddrinfo",
+            return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 443))],
+        ):
+            assert resolves_to_public_address("https://sneaky.example/") is False
+
+    def test_link_local_metadata_address_is_rejected(self):
+        # 169.254.169.254 is the AWS/GCP/Azure cloud metadata endpoint —
+        # the canonical SSRF target this check exists to block.
+        with patch(
+            "app.scrapers.base_scraper.socket.getaddrinfo",
+            return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("169.254.169.254", 80))],
+        ):
+            assert resolves_to_public_address("http://sneaky.example/") is False
+
+    def test_public_address_is_accepted(self):
+        with patch(
+            "app.scrapers.base_scraper.socket.getaddrinfo",
+            return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))],
+        ):
+            assert resolves_to_public_address("https://example.org/") is True
+
+
+def _redirect_response(location):
+    resp = MagicMock()
+    resp.is_redirect = True
+    resp.is_permanent_redirect = False
+    resp.headers = {"Location": location}
+    return resp
+
+
+def _ok_response(text="<html><body>ok</body></html>", content_type="text/html"):
+    resp = MagicMock()
+    resp.is_redirect = False
+    resp.is_permanent_redirect = False
+    resp.raise_for_status.return_value = None
+    resp.headers = {"Content-Type": content_type}
+    resp.text = text
+    return resp
+
+
+class TestFetchPageSSRF:
+    def test_initial_url_not_public_is_skipped(self):
+        scraper = BaseScraper()
+        with patch("app.scrapers.base_scraper.resolves_to_public_address", return_value=False):
+            with patch.object(scraper.session, "get") as mock_get:
+                result = scraper.fetch_page("http://169.254.169.254/latest/meta-data/", delay=False)
+        assert result is None
+        mock_get.assert_not_called()
+
+    def test_redirect_to_private_address_is_rejected(self):
+        scraper = BaseScraper()
+        with patch.object(scraper, "_can_fetch", return_value=True):
+            with patch(
+                "app.scrapers.base_scraper.resolves_to_public_address",
+                side_effect=[True, False],
+            ):
+                with patch.object(
+                    scraper.session,
+                    "get",
+                    return_value=_redirect_response("http://169.254.169.254/"),
+                ):
+                    result = scraper.fetch_page("https://example.org/redirector", delay=False)
+        assert result is None
+
+    def test_normal_page_is_fetched(self):
+        scraper = BaseScraper()
+        with patch("app.scrapers.base_scraper.resolves_to_public_address", return_value=True):
+            with patch.object(scraper, "_can_fetch", return_value=True):
+                with patch.object(scraper.session, "get", return_value=_ok_response()):
+                    result = scraper.fetch_page("https://example.org/page", delay=False)
+        assert result is not None
+        assert result.body.get_text() == "ok"
+
+    def test_too_many_redirects_gives_up(self):
+        scraper = BaseScraper()
+        with patch.object(scraper, "_can_fetch", return_value=True):
+            with patch("app.scrapers.base_scraper.resolves_to_public_address", return_value=True):
+                with patch.object(
+                    scraper.session,
+                    "get",
+                    return_value=_redirect_response("https://example.org/next"),
+                ):
+                    result = scraper.fetch_page("https://example.org/loop", delay=False)
+        assert result is None

backend/tests/test_suggest.py

@@ -0,0 +1,90 @@
+"""Regression tests for the /opportunities/suggest autocomplete endpoint."""
+
+from datetime import date, timedelta
+
+from fastapi.testclient import TestClient
+
+from app.database import SessionLocal
+from app.main import app
+from app.models import Opportunity
+
+client = TestClient(app)
+
+_TEST_URL_PREFIX = "https://example.org/suggest-test-"
+
+
+def _cleanup(db):
+    db.query(Opportunity).filter(Opportunity.url.like(f"{_TEST_URL_PREFIX}%")).delete(
+        synchronize_session=False
+    )
+    db.commit()
+
+
+def _make(db, suffix: str, **overrides) -> Opportunity:
+    defaults = dict(
+        title=f"Suggest Fellowship {suffix}",
+        opportunity_type="fellowship",
+        url=f"{_TEST_URL_PREFIX}{suffix}",
+        source_name="Test Source",
+        is_active=True,
+        review_status="approved",
+    )
+    defaults.update(overrides)
+    opp = Opportunity(**defaults)
+    db.add(opp)
+    db.commit()
+    db.refresh(opp)
+    return opp
+
+
+class TestSuggest:
+    def setup_method(self):
+        db = SessionLocal()
+        _cleanup(db)
+        db.close()
+
+    def teardown_method(self):
+        db = SessionLocal()
+        _cleanup(db)
+        db.close()
+
+    def test_query_too_short_is_rejected(self):
+        response = client.get("/api/v1/opportunities/suggest", params={"q": "a"})
+        assert response.status_code == 422
+
+    def test_matches_are_returned(self):
+        db = SessionLocal()
+        _make(db, "1")
+        db.close()
+
+        response = client.get("/api/v1/opportunities/suggest", params={"q": "Suggest Fellowship"})
+        assert response.status_code == 200
+        assert "Suggest Fellowship 1" in response.json()["suggestions"]
+
+    def test_pending_opportunities_are_excluded(self):
+        db = SessionLocal()
+        _make(db, "2", review_status="pending")
+        db.close()
+
+        response = client.get("/api/v1/opportunities/suggest", params={"q": "Suggest Fellowship"})
+        assert response.json()["suggestions"] == []
+
+    def test_expired_opportunities_are_excluded(self):
+        db = SessionLocal()
+        _make(db, "3", deadline_at=date.today() - timedelta(days=1))
+        db.close()
+
+        response = client.get("/api/v1/opportunities/suggest", params={"q": "Suggest Fellowship"})
+        assert response.json()["suggestions"] == []
+
+    def test_limit_is_respected(self):
+        db = SessionLocal()
+        for i in range(5):
+            _make(db, f"limit-{i}")
+        db.close()
+
+        response = client.get(
+            "/api/v1/opportunities/suggest",
+            params={"q": "Suggest Fellowship", "limit": 2},
+        )
+        assert len(response.json()["suggestions"]) <= 2