Repository navigation
Expand file tree
/
Copy pathout.state_after_migration.json
More file actions
143 lines (143 loc) · 3.49 KB
/
Copy pathout.state_after_migration.json
File metadata and controls
143 lines (143 loc) · 3.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
{
"state_version": 3,
"cli_version": "[CLI_VERSION]",
"lineage": "[UUID]",
"serial": 5,
"state": {
"resources.jobs.sample_job": {
"__id__": "[SAMPLE_JOB_ID]",
"state": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "job_cluster",
"new_cluster": {
"autoscale": {
"max_workers": 4,
"min_workers": 1
},
"data_security_mode": "DATA_SECURITY_MODE_AUTO",
"node_type_id": "[NODE_TYPE_ID]",
"spark_version": "16.4.x-scala2.12"
}
}
],
"max_concurrent_runs": 4,
"name": "[dev [USERNAME]] sample_job",
"parameters": [
{
"default": "hive_metastore",
"name": "catalog"
},
{
"default": "[USERNAME]",
"name": "schema"
}
],
"queue": {
"enabled": true
},
"tags": {
"dev": "[USERNAME]"
},
"tasks": [
{
"job_cluster_key": "job_cluster",
"libraries": [
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl"
}
],
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook"
},
"task_key": "notebook_task"
},
{
"depends_on": [
{
"task_key": "notebook_task"
}
],
"job_cluster_key": "job_cluster",
"libraries": [
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl"
}
],
"python_wheel_task": {
"entry_point": "main",
"package_name": "my_default_python",
"parameters": [
"--catalog",
"hive_metastore",
"--schema",
"[USERNAME]"
]
},
"task_key": "python_wheel_task"
},
{
"depends_on": [
{
"task_key": "notebook_task"
}
],
"pipeline_task": {
"pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]"
},
"task_key": "refresh_pipeline"
}
],
"trigger": {
"pause_status": "PAUSED",
"periodic": {
"interval": 1,
"unit": "DAYS"
}
}
},
"depends_on": [
{
"node": "resources.pipelines.my_default_python_etl",
"label": "${resources.pipelines.my_default_python_etl.id}"
}
]
},
"resources.pipelines.my_default_python_etl": {
"__id__": "[MY_DEFAULT_PYTHON_ETL_ID]",
"state": {
"channel": "CURRENT",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json"
},
"development": true,
"edition": "ADVANCED",
"environment": {
"dependencies": [
"--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files"
]
},
"libraries": [
{
"glob": {
"include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**"
}
}
],
"name": "[dev [USERNAME]] my_default_python_etl",
"root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl",
"schema": "[USERNAME]",
"tags": {
"dev": "[USERNAME]"
}
}
}
}
}