Skip to content

Commit 4dbc381

Browse files
lukeseawalkerlukeseawalker
committed
Regionalize links and set retry for curl and remote_file (chef resource)
For curl - retry 3 times before giving up (--retry) For remote_file - 3 times to catch exceptions and retry the resource (retries) - wait 5 seconds before next retry (retry_delay) Signed-off-by: Luca Carrogu <[email protected]>
1 parent 7dd5f94 commit 4dbc381

13 files changed

+61
-16
lines changed

‎amis/packer_alinux.json

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@
160160
{
161161
"type" : "shell",
162162
"inline" : [
163-
"curl -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
163+
"curl --retry 3 -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
164164
]
165165
},
166166
{
@@ -229,7 +229,10 @@
229229
{
230230
"type" : "shell",
231231
"inline" : [
232-
"sudo curl https://s3.amazonaws.com/us-east-1-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
232+
"region=\"{{user `region`}}\"",
233+
"bucket=\"s3.${region}.amazonaws.com\"",
234+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
235+
"sudo curl --retry 3 https://${bucket}/${region}-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
233236
]
234237
},
235238
{

‎amis/packer_centos6.json

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@
167167
{
168168
"type" : "shell",
169169
"inline" : [
170-
"curl -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
170+
"curl --retry 3 -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
171171
]
172172
},
173173
{
@@ -238,14 +238,20 @@
238238
{
239239
"type" : "shell",
240240
"inline" : [
241-
"wget -O /tmp/aws-cfn-bootstrap-latest.zip https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
241+
"region=\"{{user `region`}}\"",
242+
"bucket=\"s3.${region}.amazonaws.com\"",
243+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
244+
"curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.zip https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
242245
"sudo pip install /tmp/aws-cfn-bootstrap-latest.zip"
243246
]
244247
},
245248
{
246249
"type" : "shell",
247250
"inline" : [
248-
"sudo curl https://s3.amazonaws.com/us-east-1-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
251+
"region=\"{{user `region`}}\"",
252+
"bucket=\"s3.${region}.amazonaws.com\"",
253+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
254+
"sudo curl --retry 3 https://${bucket}/${region}-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
249255
]
250256
},
251257
{

‎amis/packer_centos7.json

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@
166166
{
167167
"type" : "shell",
168168
"inline" : [
169-
"curl -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
169+
"curl --retry 3 -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
170170
]
171171
},
172172
{
@@ -235,15 +235,21 @@
235235
{
236236
"type" : "shell",
237237
"inline" : [
238-
"wget -O /tmp/aws-cfn-bootstrap-latest.zip https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
238+
"region=\"{{user `region`}}\"",
239+
"bucket=\"s3.${region}.amazonaws.com\"",
240+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
241+
"curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.zip https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
239242
"which pip2",
240243
"if [ $? -eq 0 ]; then sudo pip2 install /tmp/aws-cfn-bootstrap-latest.zip; else sudo pip install /tmp/aws-cfn-bootstrap-latest.zip; fi"
241244
]
242245
},
243246
{
244247
"type" : "shell",
245248
"inline" : [
246-
"sudo curl https://s3.amazonaws.com/us-east-1-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
249+
"region=\"{{user `region`}}\"",
250+
"bucket=\"s3.${region}.amazonaws.com\"",
251+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
252+
"sudo curl --retry 3 https://${bucket}/${region}-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
247253
]
248254
},
249255
{

‎amis/packer_ubuntu1404.json

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@
174174
{
175175
"type" : "shell",
176176
"inline" : [
177-
"curl -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
177+
"curl --retry 3 -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
178178
]
179179
},
180180
{
@@ -243,14 +243,20 @@
243243
{
244244
"type" : "shell",
245245
"inline" : [
246-
"wget -O /tmp/aws-cfn-bootstrap-latest.zip https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
246+
"region=\"{{user `region`}}\"",
247+
"bucket=\"s3.${region}.amazonaws.com\"",
248+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
249+
"curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.zip https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
247250
"sudo pip install /tmp/aws-cfn-bootstrap-latest.zip"
248251
]
249252
},
250253
{
251254
"type" : "shell",
252255
"inline" : [
253-
"sudo curl https://s3.amazonaws.com/us-east-1-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
256+
"region=\"{{user `region`}}\"",
257+
"bucket=\"s3.${region}.amazonaws.com\"",
258+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
259+
"sudo curl --retry 3 https://${bucket}/${region}-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
254260
]
255261
},
256262
{

‎amis/packer_ubuntu1604.json

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@
177177
{
178178
"type" : "shell",
179179
"inline" : [
180-
"curl -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
180+
"curl --retry 3 -L https://www.chef.io/chef/install.sh | sudo bash -s -- -v {{user `chef_version`}}"
181181
]
182182
},
183183
{
@@ -246,14 +246,20 @@
246246
{
247247
"type" : "shell",
248248
"inline" : [
249-
"wget -O /tmp/aws-cfn-bootstrap-latest.zip https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
249+
"region=\"{{user `region`}}\"",
250+
"bucket=\"s3.${region}.amazonaws.com\"",
251+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
252+
"curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.zip https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.zip",
250253
"sudo pip install /tmp/aws-cfn-bootstrap-latest.zip"
251254
]
252255
},
253256
{
254257
"type" : "shell",
255258
"inline" : [
256-
"sudo curl https://s3.amazonaws.com/us-east-1-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
259+
"region=\"{{user `region`}}\"",
260+
"bucket=\"s3.${region}.amazonaws.com\"",
261+
"[[ ${region} =~ ^cn- ]] && bucket=\"s3.${region}.amazonaws.com.cn\"",
262+
"sudo curl --retry 3 https://${bucket}/${region}-aws-parallelcluster/cookbooks/aws-parallelcluster-cookbook-{{user `parallelcluster_cookbook_version`}}.tgz --silent --location -o /etc/chef/aws-parallelcluster-cookbook.tgz"
257263
]
258264
},
259265
{

‎recipes/_ganglia_install.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
remote_file ganglia_tarball do
3232
source node['cfncluster']['ganglia']['url']
3333
mode '0644'
34+
retries 3
35+
retry_delay 5
3436
# TODO: Add version or checksum checks
3537
not_if { ::File.exist?(ganglia_tarball) }
3638
end
@@ -84,6 +86,8 @@
8486
remote_file ganglia_web_tarball do
8587
source node['cfncluster']['ganglia']['web_url']
8688
mode '0644'
89+
retries 3
90+
retry_delay 5
8791
# TODO: Add version or checksum checks
8892
not_if { ::File.exist?(ganglia_web_tarball) }
8993
end

‎recipes/_nvidia_install.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
remote_file nvidia_tmp_runfile do
3232
source node['cfncluster']['nvidia']['driver_url']
3333
mode '0755'
34+
retries 3
35+
retry_delay 5
3436
not_if { ::File.exist?(nvidia_tmp_runfile) }
3537
end
3638

@@ -50,6 +52,8 @@
5052
remote_file cuda_tmp_runfile do
5153
source node['cfncluster']['nvidia']['cuda_url']
5254
mode '0755'
55+
retries 3
56+
retry_delay 5
5357
not_if { ::File.exist?(cuda_tmp_runfile) }
5458
end
5559

‎recipes/awsbatch_config.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
cwd Chef::Config[:file_cache_path]
4141
code <<-CLI
4242
source /tmp/proxy.sh
43-
curl -v -L -o aws-parallelcluster.tgz #{node['cfncluster']['custom_awsbatchcli_package']}
43+
curl --retry 3 -v -L -o aws-parallelcluster.tgz #{node['cfncluster']['custom_awsbatchcli_package']}
4444
tar -xzf aws-parallelcluster.tgz
4545
cd aws-parallelcluster-*
4646
pip install cli/

‎recipes/base_install.rb

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@
121121
user 'root'
122122
group 'root'
123123
mode '0755'
124+
retries 3
125+
retry_delay 5
124126
end
125127

126128
# Fix dependencies for CentOS 6 (Python 2.6)
@@ -138,7 +140,7 @@
138140
code <<-NODE
139141
source /tmp/proxy.sh
140142
pip uninstall --yes aws-parallelcluster-node
141-
curl -v -L -o aws-parallelcluster-node.tgz #{node['cfncluster']['custom_node_package']}
143+
curl --retry 3 -v -L -o aws-parallelcluster-node.tgz #{node['cfncluster']['custom_node_package']}
142144
tar -xzf aws-parallelcluster-node.tgz
143145
cd aws-parallelcluster-node-*
144146
/usr/bin/python setup.py install

‎recipes/munge_install.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
remote_file munge_tarball do
2222
source node['cfncluster']['munge']['munge_url']
2323
mode '0644'
24+
retries 3
25+
retry_delay 5
2426
# TODO: Add version or checksum checks
2527
not_if { ::File.exist?(munge_tarball) }
2628
end

0 commit comments

Comments
 (0)