engine-storage: control download redirection

Add a global setting to control whether redirection is allowed while
downloading templates and volumes

Signed-off-by: Abhishek Kumar <[email protected]>
(cherry picked from commit 52dc0da)
Signed-off-by: Rohit Yadav <[email protected]>

Author: shwstppr

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -26,6 +26,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Date;
+import java.util.List;
 
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
@@ -81,6 +82,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
     private long maxTemplateSizeInBytes;
     private ResourceType resourceType = ResourceType.TEMPLATE;
     private final HttpMethodRetryHandler myretryhandler;
+    private boolean followRedirects = false;
 
     public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
             String user, String password, Proxy proxy, ResourceType resourceType) {
@@ -112,7 +114,7 @@ public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, Str
     private GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(followRedirects);
         return request;
     }
 
@@ -336,6 +338,12 @@ private boolean checkServerResponse(long localFileSize) throws IOException {
         } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
             status = Status.UNRECOVERABLE_ERROR;
             errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
+            if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
+                    && !followRedirects) {
+                errorString = String.format("Failed to download %s due to redirection, response code: %d",
+                        downloadUrl, responseCode);
+                s_logger.error(errorString);
+            }
             return true; //FIXME: retry?
         }
         return false;
@@ -537,4 +545,12 @@ public VerifyFormat invoke() {
             return this;
         }
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+        if (this.request != null) {
+            this.request.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java

@@ -60,7 +60,7 @@ public MetalinkTemplateDownloader(StorageLayer storageLayer, String downloadUrl,
     protected GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
-        request.setFollowRedirects(true);
+        request.setFollowRedirects(followRedirects);
         if (!toFileSet) {
             String[] parts = downloadUrl.split("/");
             String filename = parts[parts.length - 1];
@@ -173,4 +173,12 @@ public Status getStatus() {
     public void setStatus(Status status) {
         this.status = status;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        super.setFollowRedirects(followRedirects);
+        if (this.request != null) {
+            this.request.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java

@@ -34,6 +34,7 @@
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.params.HttpMethodParams;
@@ -44,6 +45,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Date;
+import java.util.List;
 
 import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
 import static java.util.Arrays.asList;
@@ -72,8 +74,8 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
     private long downloadTime;
     private long totalBytes;
     private long maxTemplateSizeInByte;
-
     private boolean resume = false;
+    private boolean followRedirects = false;
 
     public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
             long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
@@ -91,7 +93,7 @@ public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, D
         this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
 
         // Follow redirects
-        this.getMethod.setFollowRedirects(true);
+        this.getMethod.setFollowRedirects(followRedirects);
 
         // Set file extension.
         this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
@@ -124,10 +126,11 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
             return 0;
         }
 
-        if (!HTTPUtils.verifyResponseCode(responseCode)) {
+        boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY,
+                HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects;
+        if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) {
             errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
             LOGGER.warn(errorString);
-
             status = Status.UNRECOVERABLE_ERROR;
             return 0;
         }
@@ -373,4 +376,12 @@ public long getTotalBytes() {
     public String getFileExtension() {
         return fileExtension;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+        if (this.getMethod != null) {
+            this.getMethod.setFollowRedirects(followRedirects);
+        }
+    }
 }

core/src/main/java/com/cloud/storage/template/TemplateDownloader.java

@@ -92,4 +92,6 @@ enum Status {
     boolean isInited();
 
     long getMaxTemplateSizeInBytes();
+
+    void setFollowRedirects(boolean followRedirects);
 }

core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java

@@ -43,6 +43,7 @@ public abstract class TemplateDownloaderBase extends ManagedContextRunnable impl
     protected long _start;
     protected StorageLayer _storage;
     protected boolean _inited = false;
+    protected boolean followRedirects = false;
     private long maxTemplateSizeInBytes;
 
     public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) {
@@ -149,4 +150,9 @@ public void setResume(boolean resume) {
     public boolean isInited() {
         return _inited;
     }
+
+    @Override
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java

@@ -25,6 +25,7 @@ public class CheckUrlCommand extends Command {
 
     private String format;
     private String url;
+    private boolean followRedirects;
 
     public String getFormat() {
         return format;
@@ -34,10 +35,15 @@ public String getUrl() {
         return url;
     }
 
-    public CheckUrlCommand(final String format,final String url) {
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public CheckUrlCommand(final String format, final String url, final boolean followRedirects) {
         super();
         this.format = format;
         this.url = url;
+        this.followRedirects = followRedirects;
     }
 
     @Override

core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java

@@ -45,7 +45,11 @@ public enum DownloadProtocol {
     private Long templateSize;
     private Storage.ImageFormat format;
 
-    protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
+    private boolean followRedirects;
+
+    protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool,
+             final String checksum, final Map<String, String> headers, final Integer connectTimeout,
+             final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) {
         this.url = url;
         this.templateId = templateId;
         this.destData = destData;
@@ -55,6 +59,7 @@ protected DirectDownloadCommand (final String url, final Long templateId, final
         this.connectTimeout = connectTimeout;
         this.soTimeout = soTimeout;
         this.connectionRequestTimeout = connectionRequestTimeout;
+        this.followRedirects = followRedirects;
     }
 
     public String getUrl() {
@@ -137,4 +142,12 @@ public boolean executeInSequence() {
     public int getWaitInMillSeconds() {
         return getWait() * 1000;
     }
+
+    public boolean isFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        this.followRedirects = followRedirects;
+    }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java

@@ -24,8 +24,10 @@
 
 public class HttpDirectDownloadCommand extends DirectDownloadCommand {
 
-    public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+    public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+             Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+                null, followRedirects);
     }
 
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java

@@ -25,7 +25,10 @@
 
 public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
 
-    public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
+    public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+              Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout,
+              boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+                connectionRequestTimeout, followRedirects);
     }
 }

core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java

@@ -24,8 +24,9 @@
 
 public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
 
-    public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
-        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+    public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+                 Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+        super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects);
     }
 
 }