Skip to content

Commit

Permalink
curl: Retry operations on potentially transient errors
Browse files Browse the repository at this point in the history 
Resolves: #557
  • Loading branch information
@ximion
ximion committed Dec 14, 2023
1 parent f19debc commit 51b97d0
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 10 deletions.
105 changes: 95 additions & 10 deletions src/as-curl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ struct _AsCurl {
typedef struct {
CURL *curl;
const gchar *user_agent;
guint n_retries;

curl_off_t bytes_downloaded;
} AsCurlPrivate;
Expand Down Expand Up @@ -81,6 +82,9 @@ as_curl_init (AsCurl *acurl)
AsCurlPrivate *priv = GET_PRIVATE (acurl);

priv->user_agent = "appstream/" PACKAGE_VERSION;

/* retry downloads 3 times by default */
priv->n_retries = 3;
}

static void
Expand Down Expand Up @@ -110,7 +114,11 @@ as_curl_download_write_bytearray_cb (char *ptr, size_t size, size_t nmemb, void
}

static gboolean
as_curl_perform_download (AsCurl *acurl, gboolean abort_is_error, GError **error)
as_curl_perform_download_once (AsCurl *acurl,
gboolean abort_is_error,
CURLcode *curl_status,
glong *response_code,
GError **error)
{
AsCurlPrivate *priv = GET_PRIVATE (acurl);
CURLcode res;
Expand All @@ -121,6 +129,9 @@ as_curl_perform_download (AsCurl *acurl, gboolean abort_is_error, GError **error

res = curl_easy_perform (priv->curl);
curl_easy_getinfo (priv->curl, CURLINFO_RESPONSE_CODE, &status_code);

if (curl_status != NULL)
*curl_status = res;
if (res != CURLE_OK) {
/* check if this issue was an intentional abort */
if (!abort_is_error && res == CURLE_ABORTED_BY_CALLBACK)
Expand Down Expand Up @@ -151,6 +162,9 @@ as_curl_perform_download (AsCurl *acurl, gboolean abort_is_error, GError **error
}

verify_and_return:
if (response_code != NULL)
*response_code = status_code;

if (status_code == 404) {
g_set_error (
error,
Expand All @@ -159,9 +173,6 @@ as_curl_perform_download (AsCurl *acurl, gboolean abort_is_error, GError **error
/* TRANSLATORS: We tried to download an URL, but received a 404 error code */
_("URL was not found on the server."));
return FALSE;
} else if (status_code == 302) {
/* redirects are fine, we ignore them until we reach a different code */
return TRUE;
} else if (status_code != 200) {
g_set_error (
error,
Expand All @@ -175,6 +186,54 @@ as_curl_perform_download (AsCurl *acurl, gboolean abort_is_error, GError **error
return TRUE;
}

static gboolean
as_curl_perform_download (AsCurl *acurl, gboolean abort_is_error, const gchar *url, GError **error)
{
AsCurlPrivate *priv = GET_PRIVATE (acurl);
guint n_retries_remaining = priv->n_retries;
gboolean success;

curl_easy_setopt (priv->curl, CURLOPT_URL, url);
do {
g_autoptr(GError) tmp_error = NULL;
glong response_code;
CURLcode curl_status;

success = as_curl_perform_download_once (acurl,
abort_is_error,
&curl_status,
&response_code,
&tmp_error);
if (n_retries_remaining == 0) {
if (tmp_error != NULL)
g_propagate_error (error, g_steal_pointer (&tmp_error));
return success;
}
n_retries_remaining--;

/* if any of these matched, we attempt a retry */
if (curl_status == CURLE_OPERATION_TIMEDOUT ||
curl_status == CURLE_COULDNT_RESOLVE_HOST ||
curl_status == CURLE_COULDNT_RESOLVE_PROXY ||
curl_status == CURLE_COULDNT_CONNECT

|| response_code >= 405) {
g_debug ("Retrying failed download of %s (attempt: %d/%d)",
url,
priv->n_retries - n_retries_remaining,
priv->n_retries);
continue;
}

/* if we are here, we failed with an error that a retry can not fix */
if (tmp_error != NULL)
g_propagate_error (error, g_steal_pointer (&tmp_error));
break;
} while (TRUE);

return success;
}

/**
* as_curl_set_cainfo:
* @acurl: an #AsCurl instance.
Expand All @@ -189,6 +248,35 @@ as_curl_set_cainfo (AsCurl *acurl, const gchar *cainfo)
curl_easy_setopt (priv->curl, CURLOPT_CAINFO, cainfo);
}

/**
* as_curl_get_retry_count:
* @acurl: an #AsCurl instance.
*
* Get the number of download retries before giving up.
**/
guint
as_curl_get_retry_count (AsCurl *acurl)
{
AsCurlPrivate *priv = GET_PRIVATE (acurl);
return priv->n_retries;
}

/**
* as_curl_set_retry_count:
* @acurl: an #AsCurl instance.
* @count: number of download retries
*
* Set the number of download retries before giving up.
**/
void
as_curl_set_retry_count (AsCurl *acurl, guint count)
{
AsCurlPrivate *priv = GET_PRIVATE (acurl);
if (count > 24)
count = 24;
priv->n_retries = count;
}

/**
* as_curl_download_bytes:
* @acurl: an #AsCurl instance.
Expand All @@ -203,13 +291,12 @@ as_curl_download_bytes (AsCurl *acurl, const gchar *url, GError **error)
AsCurlPrivate *priv = GET_PRIVATE (acurl);
g_autoptr(GByteArray) buf = g_byte_array_new ();

curl_easy_setopt (priv->curl, CURLOPT_URL, url);
curl_easy_setopt (priv->curl, CURLOPT_WRITEFUNCTION, as_curl_download_write_bytearray_cb);
curl_easy_setopt (priv->curl, CURLOPT_WRITEDATA, buf);
curl_easy_setopt (priv->curl, CURLOPT_XFERINFOFUNCTION, as_curl_progress_dummy_cb);
curl_easy_setopt (priv->curl, CURLOPT_XFERINFODATA, acurl);

if (!as_curl_perform_download (acurl, TRUE, error))
if (!as_curl_perform_download (acurl, TRUE, url, error))
return NULL;

return g_byte_array_free_to_bytes (g_steal_pointer (&buf));
Expand Down Expand Up @@ -262,13 +349,12 @@ as_curl_download_to_filename (AsCurl *acurl, const gchar *url, const gchar *fnam

dos = g_data_output_stream_new (G_OUTPUT_STREAM (fos));

curl_easy_setopt (priv->curl, CURLOPT_URL, url);
curl_easy_setopt (priv->curl, CURLOPT_WRITEFUNCTION, as_curl_download_write_data_stream_cb);
curl_easy_setopt (priv->curl, CURLOPT_WRITEDATA, dos);
curl_easy_setopt (priv->curl, CURLOPT_XFERINFOFUNCTION, as_curl_progress_dummy_cb);
curl_easy_setopt (priv->curl, CURLOPT_XFERINFODATA, acurl);

if (!as_curl_perform_download (acurl, TRUE, error))
if (!as_curl_perform_download (acurl, TRUE, url, error))
return FALSE;

return TRUE;
Expand Down Expand Up @@ -307,14 +393,13 @@ as_curl_check_url_exists (AsCurl *acurl, const gchar *url, GError **error)
AsCurlPrivate *priv = GET_PRIVATE (acurl);
g_autoptr(GByteArray) buf = g_byte_array_new ();

curl_easy_setopt (priv->curl, CURLOPT_URL, url);
curl_easy_setopt (priv->curl, CURLOPT_WRITEFUNCTION, as_curl_download_write_bytearray_cb);
curl_easy_setopt (priv->curl, CURLOPT_WRITEDATA, buf);
curl_easy_setopt (priv->curl, CURLOPT_XFERINFOFUNCTION, as_curl_progress_check_url_cb);
curl_easy_setopt (priv->curl, CURLOPT_XFERINFODATA, acurl);

priv->bytes_downloaded = 0;
if (!as_curl_perform_download (acurl, FALSE, error))
if (!as_curl_perform_download (acurl, FALSE, url, error))
return FALSE;

/* check if it's a zero sized file */
Expand Down
3 changes: 3 additions & 0 deletions src/as-curl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ AsCurl *as_curl_new (GError **error);

void as_curl_set_cainfo (AsCurl *acurl, const gchar *cainfo);

guint as_curl_get_retry_count (AsCurl *acurl);
void as_curl_set_retry_count (AsCurl *acurl, guint count);

GBytes *as_curl_download_bytes (AsCurl *acurl, const gchar *url, GError **error);
gboolean as_curl_download_to_filename (AsCurl *acurl,
const gchar *url,
Expand Down

0 comments on commit 51b97d0

Please sign in to comment.