palos/paloalto_scraper_exceptions.yaml at main · dr4gon123/palos · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
# Palo Alto PAN-OS Log Scraper - Exceptions and Corrections
#
# This file encodes corrections needed because:
#   - PA documentation is inconsistent across log types (same concept, different spelling)
#   - PA field tables sometimes have wrong/truncated variable names (typos)
#   - The scraper's name-mapping logic misses some long-name → variable-name lookups
#   - One PA doc page has a literal bug (period instead of comma as separator)

# per_log_corrections: raw format string token corrections applied at extraction time.
# These operate on the raw comma-split tokens BEFORE any variable name lookup.
# Two selector types: match (value-based, preferred) or position (0-indexed, fragile).
# Two correction types: new (replace token) or split_into (expand token into multiple).
per_log_corrections:
  Correlated_Events_Log:
    # PA docs bug: period instead of comma produces single token "Source Address. Source User"
    - match: "Source Address. Source User"
      split_into: ["src", "srcuser"]
  GlobalProtect_Log:
    # Format has "Serial Number" at two positions:
    #   position 2  (firewall serial) → should map to Serial # (serial)   → serial
    #   position 20 (machine serial)  → should map to Serial Number (serialnumber) → serialnumber
    # Rename the first occurrence at extraction time so lookup finds the correct field table row.
    - match: "Serial Number"
      new: "Serial #"

# field_name_lookup_corrections: normalizes "Field Name lookup" column values to match
# format string tokens. Applied before _lookup_variable_names.
# Keys are values as they appear in the field table's "Field Name lookup" column.
# Values are the corresponding token as it appears in the format string.
#
# IMPORTANT: Only add a global entry when the table key is NEVER the correct format token
# for any log type. If ANY log type uses the table key as its format token, the global
# correction would break that log type. Use per_log_type instead.
field_name_lookup_corrections:
  global:
    "Threat/Content Name": "Threat ID" # Data/Threat/URL: table name differs from format token
    "Security Rule UUID": "Rule UUID" # Data/Threat/Traffic/Tunnel_Inspection/URL: table name differs from format token
    "Generate Time": "Generated Time" # most logs: table "Generate Time", format "Generated Time" (extra 'd')
    "Parent Session Start Time": "Parent Start Time" # Data/Threat/URL: table "Parent Session Start Time", format "Parent Start Time"

    # DG Hierarchy variants handled by regex — no config entries needed
    # "Server Name Indication" and "Tunnel Inspection Rule" handled by relaxed \s*\( regex
    #
    # Entries NOT here because they are used as format tokens in some log types and would
    # break those log types if renamed globally. Handled per_log_type below instead:
    #   "Source Country" / "Destination Country" — Traffic/URL/Data use these as format tokens (→ srcloc/dstloc)
    #   "IP Protocol" — Decryption uses "IP Protocol" as format token; Traffic/GTP/Tunnel_Inspection use "Protocol"
    #   "High Resolution Timestamp" — most logs use this as format token; Decryption uses "High Res Timestamp" (per_log_type); GlobalProtect also uses "High Res Timestamp" but has no table row for it (caught by variable_name_corrections)
    #   "Threat/Content Type" — Traffic/URL/Data/Audit use this as format token; Config/Tunnel_Inspection use "Subtype"
    #   "Tunnel Type" — Traffic/GlobalProtect use this as format token; Decryption/Tunnel_Inspection use "Tunnel"
    #   "Dynamic User Group Name" — most logs use this; Tunnel_Inspection uses "Dynamic User Group"
    #   "PCAP ID" — GTP/Tunnel_Inspection use "PCAP ID" as format token; Threat/URL/Data use "PCAP_ID"
  per_log_type:
    HIP_Match_Log:
      "IPv6 System Address": "IPv6 Source Address" # HIP_Match: table name differs from format token
    IP_Tag_Log:
      "Serial Number": "Serial" # IP_Tag: format uses abbreviated token "Serial", table has "Serial Number"
      "Generated Time": "Generate Time" # IP_Tag: format "Generate Time", table "Generated Time" — per_log_type rename makes lookup succeed; parenthetical gives time_generated directly
    Audit_Log:
      "Generate Time": "Generate Time" # Audit: identity mapping prevents global from renaming "Generate Time"→"Generated Time"; table key stays "Generate Time" so format "Generate Time" finds the (empty) row → placeholder pass-through → variable_name_corrections["Generate Time"] catches it
    Threat_Log:
      "Source address": "Source Address" # Threat: field table lowercase 'a', format uppercase 'A'
      "Destination address": "Destination Address" # Threat: same case mismatch
      "Source Country": "Source Location" # Threat: format "Source Location", table "Source Country"
      "Destination Country": "Destination Location" # Threat: format "Destination Location", table "Destination Country"
      "PCAP ID": "PCAP_ID" # Threat: format "PCAP_ID" (underscore), table "PCAP ID" (space)
    URL_Filtering_Log:
      "Source address": "Source Address" # URL: same case mismatch as Threat
      "Destination address": "Destination Address" # URL: same case mismatch
      "PCAP ID": "PCAP_ID" # URL: format "PCAP_ID" (underscore), table "PCAP ID" (space)
    Data_Filtering_Log:
      "Source address": "Source Address" # Data: same case mismatch as Threat
      "Destination address": "Destination Address" # Data: same case mismatch
      "PCAP ID": "PCAP_ID" # Data: same as Threat
    Config_Log:
      "Threat/Content Type": "Subtype" # Config: format "Subtype", table "Threat/Content Type"
    Traffic_Log:
      "IP Protocol": "Protocol" # Traffic: format "Protocol", table "IP Protocol"
    GTP_Log:
      "Source Country": "Source Location" # GTP: format "Source Location", table "Source Country"
      "Destination Country": "Destination Location" # GTP: format "Destination Location", table "Destination Country"
      "IP Protocol": "Protocol" # GTP: format "Protocol", table "IP Protocol"
      "End IP Address": "End User IP Address" # GTP: table name differs from format token
      "Serving Network MCC": "Serving Country MCC" # GTP: table name differs from format token
    Decryption_Log:
      "High Resolution Timestamp": "High Res Timestamp" # Decryption: format "High Res Timestamp", table "High Resolution Timestamp"
      "Tunnel Type": "Tunnel" # Decryption: format "Tunnel", table "Tunnel Type"
      "Certificate Fingerprint": "Fingerprint" # Decryption: table name differs from format token
      "Threat/ContentType": "Threat/Content Type" # Decryption: malformed key (missing space before parenthetical)
      "Issuer Common Name": "Issuer Subject Common Name" # Decryption: table name differs from format token
      "Root Common Name": "Root Subject Common Name" # Decryption: table name differs from format token
    GlobalProtect_Log:
      "Gateway Priority": "Priority" # GlobalProtect: table name differs from format token
      "Gateway Name": "Gateway" # GlobalProtect: table name differs from format token
      "Gateway Selection Method": "Selection Type" # GlobalProtect: table name differs from format token
      "SSL Response Time": "Response Time" # GlobalProtect: table name differs from format token
    # GlobalProtect_Log: GlobalProtect field table does not include a "High Resolution Timestamp" row;
    # "High Res Timestamp" format token passes through and is caught by variable_name_corrections below.
    Tunnel_Inspection_Log:
      "Source Country": "Source Location" # Tunnel_Inspection: format "Source Location", table "Source Country"
      "Destination Country": "Destination Location" # Tunnel_Inspection: format "Destination Location", table "Destination Country"
      "IP Protocol": "Protocol" # Tunnel_Inspection: format "Protocol", table "IP Protocol"
      "Threat/Content Type": "Subtype" # Tunnel_Inspection: format "Subtype", table "Threat/Content Type"
      "Tunnel Type": "Tunnel" # Tunnel_Inspection: format "Tunnel", table "Tunnel Type"
      "Dynamic User Group Name": "Dynamic User Group" # Tunnel_Inspection: format "Dynamic User Group", table "Dynamic User Group Name"
      "Strict Checking": "Strict Check" # Tunnel_Inspection: table name differs from format token

# variable_name_corrections: applied to variable names after _lookup_variable_names.
# Corrects both the format token list AND the Variable Name column of the field table.
# Applied in order: global first, then per_log_type for the current log type.
variable_name_corrections:
  global:
    # Lookup failures: raw long names that pass through _lookup_variable_names unchanged.
    "Parent Start Time": "parent_start_time" # safety fallback for logs where field is absent from field table

    "High Res Timestamp": "high_res_timestamp" # GlobalProtect: field table has no "High Resolution Timestamp" row → pass-through

    # MAC address capitalization: format string is inconsistent across log types.
    # Threat/URL/Data use "Source MAC Address" (all-caps) → table match → parenthetical gives src_mac directly.
    # Traffic/Auth/Decryption use "Source Mac Address" (mixed case) → table "Source MAC Address" → lookup fails → caught here.
    "Source Mac Address": "src_mac" # Traffic/Auth/Decryption: format "Mac" (mixed), table "MAC" (all-caps) → lookup fails
    "Destination Mac Address": "dst_mac" # Traffic/Decryption: same capitalization mismatch

    # Audit_Log: fields with no parenthetical in PA docs → pass through as raw long names
    "Generate Time": "time_generated" # Audit_Log: "Generate Time" format token → placeholder pass-through → caught here
    "Serial Number": "serial" # Audit_Log: no parenthetical; found + empty Variable Name → placeholder pass-through
    "Event ID": "eventid" # Audit_Log: no parenthetical
    "Object": "object" # Audit_Log: no parenthetical
    "CLI Command": "cmd" # Audit_Log: no parenthetical
    "Severity": "severity" # Audit_Log: no parenthetical

    # Field table typos / multi-word parentheticals: incorrect variable names produced by parenthetical extraction
    "FUTURE_USER": "FUTURE_USE" # User-ID log: typo in PA docs field table
    "high_res timestamp": "high_res_timestamp" # IP-Tag, User-ID, URL, Threat, etc.: space in PA field table parenthetical
    "high_res _timestamp": "high_res_timestamp" # Authentication: PA docs typo — space before underscore in parenthetical
    "receive_time or cef-formatted-receive_time": "receive_time" # multi-word parenthetical
    "time_generated or cef-formatted-time_generated": "time_generated" # multi-word parenthetical
    "event_id": "eventid" # IP-Tag: underscore wrong in PA field table
    "tunnelid/imsi": "tunnel_id/imsi" # Traffic: PA field table lists without underscore before slash
    "Tunnel ID/IMSI": "tunnel_id/imsi" # Tunnel_Inspection_Log: format "Tunnel ID/IMSI", table "Tunnel ID" → lookup fails
    "Monitor Tag/IMEI": "monitortag/imei" # Tunnel_Inspection_Log: format "Monitor Tag/IMEI", table "Monitor Tag" → lookup fails
    #"tunnelid": "tunnel_id/imsi" # Tunnel_Inspection_Log: parenthetical produces "tunnelid" without /imsi suffix
    #"monitortag": "monitortag/imei" # Tunnel_Inspection_Log: parenthetical produces "monitortag" without /imei suffix
    "nsdsai_sst": "nssai_sst" # GTP: "nsdsai" → "nssai" typo in PA docs
    "nsdsai_sd": "nssai_sd" # GTP: same typo
  per_log_type: {}