-
Notifications
You must be signed in to change notification settings - Fork 78
/
Copy pathAI_complex.py
181 lines (153 loc) · 9.06 KB
/
AI_complex.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
import kfp
import kfp.dsl as dsl
# import kfp.notebook # jupter
from kubernetes import client as k8s_client
print('Start client!!!!2019-11-24')
client = kfp.Client()
# EXPERIMENT_NAME = 'liu1234'
print('start exp!!!!')
# exp = client.create_experiment(name=EXPERIMENT_NAME)
class DataCollect(dsl.ContainerOp):
def __init__(self, data_path, data_job_id, user_name):
super(DataCollect, self).__init__(
name="data_collect",
image="10.18.101.90:80/library/dataset_base:v1",
command=["bash", "-c"],
arguments=[
"bash /root/classify_data.sh %s %s %s && date > /root/data.txt" % (data_path, data_job_id, user_name)],
file_outputs={"data": "/root/data.txt"}
)
class AIClassify(dsl.ContainerOp):
def __init__(self, pre_input, var_job_id, user_name):
super(AIClassify, self).__init__(
name="ai_classify",
image="10.18.101.90:80/library/camb_arm_execute:v5",
command=["bash", "-c"],
arguments=["bash /root/camb/run_camb_eg.sh /home/newnfs/%s/jobs/%s resnet && date > /root/data.txt" % (
user_name, var_job_id), "echo %s" % pre_input],
file_outputs={"data": "/root/data.txt"}
)
class Data_Adjust(dsl.ContainerOp):
def __init__(self, pre_input, data_path, data_job_id, user_name):
super(Data_Adjust, self).__init__(
name="data_adjust",
image="10.18.101.90:80/library/dataset_base:v1",
command=["bash", "-c"],
arguments=[
"bash /root/detection_data.sh %s %s %s && date > /root/data.txt" % (data_path, data_job_id, user_name),"echo %s" % pre_input],
file_outputs={"data": "/root/data.txt"}
)
class AIDetection(dsl.ContainerOp):
def __init__(self, pre_input, var_job_id, user_name):
super(AIDetection, self).__init__(
name="ai_detection",
image="10.18.101.90:80/library/camb_arm_execute:v5",
command=["bash", "-c"],
arguments=["bash /root/camb/run_camb_eg.sh /home/newnfs/%s/jobs/%s aircraft && date > /root/data.txt" % (
user_name, var_job_id), "echo %s" % pre_input],
file_outputs={"data": "/root/data.txt"}
)
class Picture_Modify(dsl.ContainerOp):
def __init__(self, pre_input, var_job_id, user_name,log_id):
super(Picture_Modify, self).__init__(
name="picture_modify",
image="10.18.101.90:80/library/camb_arm_execute:v5",
command=["bash", "-c"],
arguments=["python3 /root/camb/copy_ret.py --job_id=%s --user_name=%s --log_id=%s && date > /root/data.txt" % (var_job_id, user_name,log_id), "echo %s" % pre_input],
file_outputs={"data": "/root/data.txt"},
)
@dsl.pipeline(
name='ldg_pipeline_test_01',
description='one demo test'
)
def demo():
classify_data_path = "/root/AID"
detection_data_path = "/root/aircraft"
user_name = "admin"
# 检测的
detection_job_id = "20200514-detection"
# 分类的job_id
classification_job_id = "20200514-classify"
classify_camb_id = "20200514-camb-classify"
detection_camb_id = "20200514-camb-detection"
picture_modify_id = "20200514-camb-picture_modify"
device_name = "dev-cambricon"
data_collect = DataCollect(classify_data_path, classification_job_id, user_name).add_volume(k8s_client.V1Volume(
name='nfs-storage',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/newnfs", name='nfs-storage')).add_node_selector_constraint(
'beta.kubernetes.io/arch', 'amd64')
ai_classify = AIClassify(data_collect.output, classify_camb_id, user_name).add_volume(k8s_client.V1Volume(
name='nfs-storage',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/newnfs", name='nfs-storage')).add_volume(k8s_client.V1Volume(
name='aaa',
host_path=k8s_client.V1LocalVolumeSource(path="/sys/kernel/debug"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/sys/kernel/debug", name='aaa')).add_volume(k8s_client.V1Volume(
name='bbb',
host_path=k8s_client.V1LocalVolumeSource(path="/tmp/.X11-unix"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/tmp/.X11-unix", name='bbb')).add_volume(k8s_client.V1Volume(
name='ccc',
host_path=k8s_client.V1LocalVolumeSource(path="/mnt/xfs/project/camb/v8.2_arm"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/Cambricon-Test-v8.2_arm", name='ccc')).add_volume(
k8s_client.V1Volume(
name='ddd',
host_path=k8s_client.V1LocalVolumeSource(path="/mnt/xfs/project/camb/arm_v8.0/v8.0_arm/ARM64-v8.0/arm64/congcan"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/congcan", name='ddd')).add_volume(
k8s_client.V1Volume(name=device_name, host_path=k8s_client.V1HostPathVolumeSource(
path="/dev/cambricon_c10Dev0"))).add_volume_mount(
k8s_client.V1VolumeMount(name=device_name, mount_path="/dev/cambricon_c10Dev0")).add_node_selector_constraint(
'beta.kubernetes.io/arch', 'arm64')
ai_classify.add_resource_limit("cambricon.com/mlu", "1")
data_adjust = Data_Adjust(ai_classify.output,detection_data_path,detection_job_id,user_name).add_volume(k8s_client.V1Volume(
name='nfs-storage',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/newnfs", name='nfs-storage')).add_node_selector_constraint(
'beta.kubernetes.io/arch', 'amd64')
ai_detection = AIDetection(data_adjust.output, detection_camb_id, user_name).add_volume(k8s_client.V1Volume(
name='nfs-storage',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/newnfs", name='nfs-storage')).add_volume(k8s_client.V1Volume(
name='aaa',
host_path=k8s_client.V1LocalVolumeSource(path="/sys/kernel/debug"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/sys/kernel/debug", name='aaa')).add_volume(k8s_client.V1Volume(
name='bbb',
host_path=k8s_client.V1LocalVolumeSource(path="/tmp/.X11-unix"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/tmp/.X11-unix", name='bbb')).add_volume(k8s_client.V1Volume(
name='ccc',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs/newldg/Cambricon-Test-v8.2_arm"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/Cambricon-Test-v8.2_arm", name='ccc')).add_volume(
k8s_client.V1Volume(
name='ddd',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs/newldg/congcan"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/congcan", name='ddd')).add_volume(
k8s_client.V1Volume(name=device_name, host_path=k8s_client.V1HostPathVolumeSource(
path="/dev/cambricon_c10Dev0"))).add_volume_mount(
k8s_client.V1VolumeMount(name=device_name, mount_path="/dev/cambricon_c10Dev0")).add_node_selector_constraint(
'beta.kubernetes.io/arch', 'arm64')
ai_detection.add_resource_limit("cambricon.com/mlu", "1")
picture_modify = Picture_Modify(ai_detection.output, picture_modify_id, user_name,detection_camb_id).add_volume(k8s_client.V1Volume(
name='nfs-storage',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/newnfs", name='nfs-storage')).add_volume(k8s_client.V1Volume(
name='aaa',
host_path=k8s_client.V1LocalVolumeSource(path="/sys/kernel/debug"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/sys/kernel/debug", name='aaa')).add_volume(k8s_client.V1Volume(
name='bbb',
host_path=k8s_client.V1LocalVolumeSource(path="/tmp/.X11-unix"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/tmp/.X11-unix", name='bbb')).add_volume(k8s_client.V1Volume(
name='ccc',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs/newldg/Cambricon-Test-v8.2_arm"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/Cambricon-Test-v8.2_arm", name='ccc')).add_volume(
k8s_client.V1Volume(
name='ddd',
host_path=k8s_client.V1LocalVolumeSource(path="/home/newnfs/newldg/congcan"))).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/home/congcan", name='ddd')).add_volume(
k8s_client.V1Volume(name=device_name, host_path=k8s_client.V1HostPathVolumeSource(
path="/dev/cambricon_c10Dev0"))).add_volume_mount(
k8s_client.V1VolumeMount(name=device_name, mount_path="/dev/cambricon_c10Dev0")).add_node_selector_constraint(
'beta.kubernetes.io/arch', 'arm64')
picture_modify.add_resource_limit("cambricon.com/mlu", "1")
if __name__ == '__main__':
import kfp.compiler as compiler
compiler.Compiler().compile(demo, "demo.tar.gz")