|
| 1 | +# |
| 2 | +# Copyright (c) nexB Inc. and others. All rights reserved. |
| 3 | +# VulnerableCode is a trademark of nexB Inc. |
| 4 | +# SPDX-License-Identifier: Apache-2.0 |
| 5 | +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. |
| 6 | +# See https://github.com/aboutcode-org/vulnerablecode for support or download. |
| 7 | +# See https://aboutcode.org for more information about nexB OSS projects. |
| 8 | +# |
| 9 | + |
| 10 | +import json |
| 11 | +from pathlib import Path |
| 12 | + |
| 13 | +from fetchcode.vcs import fetch_via_vcs |
| 14 | + |
| 15 | +from vulnerabilities.models import AdvisoryAlias |
| 16 | +from vulnerabilities.models import AdvisoryV2 |
| 17 | +from vulnerabilities.models import DetectionRule |
| 18 | +from vulnerabilities.models import DetectionRuleTypes |
| 19 | +from vulnerabilities.pipelines import VulnerableCodePipeline |
| 20 | + |
| 21 | + |
| 22 | +class DetectionRulesPipeline(VulnerableCodePipeline): |
| 23 | + """ |
| 24 | + Pipeline to collect vulnerability scanner rules (Sigma, YARA, Suricata, ClamAV entries) |
| 25 | + """ |
| 26 | + |
| 27 | + pipeline_id = "detection_rules" |
| 28 | + license_url = "https://github.com/ziadhany/detection-rules-collector/blob/master/LICENSE" |
| 29 | + precedence = 200 |
| 30 | + |
| 31 | + @classmethod |
| 32 | + def steps(cls): |
| 33 | + return ( |
| 34 | + cls.clone, |
| 35 | + cls.collect_detection_rules, |
| 36 | + cls.clean_downloads, |
| 37 | + ) |
| 38 | + |
| 39 | + def clone(self): |
| 40 | + self.repo_url = "git+https://github.com/aboutcode-data/detection-rules-collector" |
| 41 | + self.log(f"Cloning `{self.repo_url}`") |
| 42 | + self.vcs_response = fetch_via_vcs(self.repo_url) |
| 43 | + |
| 44 | + def advisories_count(self): |
| 45 | + root = Path(self.vcs_response.dest_dir) |
| 46 | + return sum(1 for _ in root.rglob("*.json")) |
| 47 | + |
| 48 | + def collect_detection_rules(self): |
| 49 | + base_path = Path(self.vcs_response.dest_dir) / "data" |
| 50 | + rule_type_mapping = { |
| 51 | + DetectionRuleTypes.YARA: "yara/**/*.json", |
| 52 | + DetectionRuleTypes.SURICATA: "suricata/**/*.json", |
| 53 | + DetectionRuleTypes.SIGMA: "sigma/**/*.json", |
| 54 | + DetectionRuleTypes.CLAMAV: "clamav/**/*.json", |
| 55 | + } |
| 56 | + |
| 57 | + for rule_type, glob_pattern in rule_type_mapping.items(): |
| 58 | + for file_path in base_path.glob(glob_pattern): |
| 59 | + with open(file_path, "r") as f: |
| 60 | + try: |
| 61 | + json_data = json.load(f) |
| 62 | + except json.JSONDecodeError: |
| 63 | + self.log(f"Failed to parse JSON in {file_path}") |
| 64 | + continue |
| 65 | + |
| 66 | + source_url = json_data.get("source_url") |
| 67 | + for rule in json_data.get("rules", []): |
| 68 | + advisories = set() |
| 69 | + for vulnerability_id in rule.get("vulnerabilities", []): |
| 70 | + try: |
| 71 | + if alias := AdvisoryAlias.objects.get(alias=vulnerability_id): |
| 72 | + for adv in alias.advisories.all(): |
| 73 | + advisories.add(adv) |
| 74 | + else: |
| 75 | + advs = AdvisoryV2.objects.filter( |
| 76 | + advisory_id=vulnerability_id |
| 77 | + ).latest_per_avid() |
| 78 | + for adv in advs: |
| 79 | + advisories.add(adv) |
| 80 | + except AdvisoryAlias.DoesNotExist: |
| 81 | + self.log(f"No advisory found for aliases {vulnerability_id}") |
| 82 | + |
| 83 | + raw_text = rule.get("rule_text") |
| 84 | + detection_rule, _ = DetectionRule.objects.get_or_create( |
| 85 | + rule_text=raw_text, |
| 86 | + rule_type=rule_type, |
| 87 | + defaults={ |
| 88 | + "source_url": source_url, |
| 89 | + }, |
| 90 | + ) |
| 91 | + if advisories: |
| 92 | + detection_rule.related_advisories.add(*advisories) |
| 93 | + |
| 94 | + def clean_downloads(self): |
| 95 | + """Cleanup any temporary repository data.""" |
| 96 | + if self.vcs_response: |
| 97 | + self.log(f"Removing cloned repository") |
| 98 | + self.vcs_response.delete() |
| 99 | + |
| 100 | + def on_failure(self): |
| 101 | + """Ensure cleanup is always performed on failure.""" |
| 102 | + self.clean_downloads() |
0 commit comments