spark agent #917 Add Column Level Lineage support into HttpOpenLineag… (#919)

* spark agent #917 Add Column Level Lineage support into HttpOpenLineageDispatcher

- SchemaDatasetFacet
- ColumnLineageDatasetFacet

Author: cerveada

.gitignore

@@ -77,3 +77,6 @@ metastore_db/
 
 #JEnv
 .java-version
+
+# Project specific
+local/

core/pom.xml

@@ -237,17 +237,6 @@
                             <language />
                         </configuration>
                     </execution>
-                    <execution>
-                        <id>swagger.openlineage.api.v0_3_1</id>
-                        <goals>
-                            <goal>generate</goal>
-                        </goals>
-                        <configuration>
-                            <inputSpec>${project.basedir}/src/main/resources/api/openlineage/OpenLineage.yml</inputSpec>
-                            <modelPackage>za.co.absa.spline.producer.model.openlineage.v0_3_1</modelPackage>
-                            <language />
-                        </configuration>
-                    </execution>
                 </executions>
             </plugin>
             <plugin>

core/src/main/resources/api/openlineage/ColumnLineageDatasetFacet.json

@@ -0,0 +1,107 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json",
+  "$defs": {
+    "ColumnLineageDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "fields": {
+              "description": "Column level lineage that maps output fields into input fields used to evaluate them.",
+              "type": "object",
+              "additionalProperties": {
+                "type": "object",
+                "properties": {
+                  "inputFields": {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/$defs/InputField"
+                    }
+                  },
+                  "transformationDescription": {
+                    "type": "string",
+                    "description": "a string representation of the transformation applied",
+                    "deprecated": true
+                  },
+                  "transformationType": {
+                    "type": "string",
+                    "description": "IDENTITY|MASKED reflects a clearly defined behavior. IDENTITY: exact same as input; MASKED: no original data available (like a hash of PII for example)",
+                    "deprecated": true
+                  }
+                },
+                "additionalProperties": true,
+                "required": ["inputFields"]
+              }
+            },
+            "dataset": {
+              "description": "Column level lineage that affects the whole dataset. This includes filtering, sorting, grouping (aggregates), joining, window functions, etc.",
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/InputField"
+              }
+            }
+          },
+          "additionalProperties": true,
+          "required": ["fields"]
+        }
+      ],
+      "type": "object"
+    },
+    "InputField": {
+      "description": "Represents a single dependency on some field (column).",
+      "type": "object",
+      "properties": {
+        "namespace": {
+          "type": "string",
+          "description": "The input dataset namespace"
+        },
+        "name": {
+          "type": "string",
+          "description": "The input dataset name"
+        },
+        "field": {
+          "type": "string",
+          "description": "The input field"
+        },
+        "transformations": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "description": "The type of the transformation. Allowed values are: DIRECT, INDIRECT",
+                "type": "string"
+              },
+              "subtype": {
+                "type": "string",
+                "description": "The subtype of the transformation"
+              },
+              "description": {
+                "type": "string",
+                "description": "a string representation of the transformation applied"
+              },
+              "masking": {
+                "type": "boolean",
+                "description": "is transformation masking the data or not"
+              }
+            },
+            "required": ["type"],
+            "additionalProperties": true
+          }
+        }
+      },
+      "additionalProperties": true,
+      "required": ["namespace", "name", "field"]
+    }
+  },
+  "type": "object",
+  "properties": {
+    "columnLineage": {
+      "$ref": "#/$defs/ColumnLineageDatasetFacet"
+    }
+  }
+}

…sources/api/openlineage/OpenLineage.json …/api/openlineage/OpenLineage.v1.0.2.jsoncore/src/main/resources/api/openlineage/OpenLineage.json renamed to core/src/main/resources/api/openlineage/OpenLineage.v1.0.2.json core/src/main/resources/api/openlineage/OpenLineage.json renamed to core/src/main/resources/api/openlineage/OpenLineage.v1.0.2.json

@@ -18,8 +18,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: 'OpenLineage.json#/$defs/RunEvent'
+              $ref: 'OpenLineage.v1.0.2.json#/$defs/RunEvent'
       responses:
         '200':
           description: OK
-