Specify a metadata format for external release descriptions

compose/asc-compose.c

@@ -1409,6 +1409,7 @@ asc_compose_process_task_cb (AscComposeTask *ctask, AscCompose *compose)
 	/* process metadata */
 	for (guint i = 0; i < mi_fnames->len; i++) {
 		g_autoptr(GBytes) mi_bytes = NULL;
+		g_autoptr(GBytes) rel_bytes = NULL;
 		g_autoptr(GError) local_error = NULL;
 		g_autoptr(AsComponent) cpt = NULL;
 		g_autofree gchar *mi_basename = NULL;
@@ -1455,13 +1456,23 @@ asc_compose_process_task_cb (AscComposeTask *ctask, AscCompose *compose)
 					  g_strdup (as_component_get_id (cpt)));
 		}
 
+		/* process any release information of this component and download release data if needed */
+		asc_process_metainfo_releases (ctask->result,
+					       ctask->unit,
+					       cpt,
+					       mi_fname,
+					       as_flags_contains (priv->flags, ASC_COMPOSE_FLAG_ALLOW_NET),
+					       acurl,
+					       &rel_bytes);
+
 		/* validate the data */
 		if (as_flags_contains (priv->flags, ASC_COMPOSE_FLAG_VALIDATE)) {
 			asc_validate_metainfo_data_for_component (ctask->result,
-									validator,
-									cpt,
-									mi_bytes,
-									mi_basename);
+								  validator,
+								  cpt,
+								  mi_bytes,
+								  mi_basename,
+								  rel_bytes);
 		}
 
 		/* legacy support: Synthesize launchable entry if none was set,

compose/asc-hint-tags.c

@@ -103,6 +103,16 @@ AscHintTagStatic asc_hint_tag_list[] =  {
 	  "<code>type=</code> property of the component root-node in the MetaInfo XML file does not contain a spelling mistake."
 	},
 
+	{ "metainfo-releases-download-failed",
+	  AS_ISSUE_SEVERITY_WARNING,
+	  "Unable to download release information from <code>{{url}}</code>. The error message was: {{msg}}."
+	},
+
+	{ "metainfo-releases-read-failed",
+	  AS_ISSUE_SEVERITY_ERROR,
+	  "Unable to read release information from <code>{{path}}</code>. The error message was: {{msg}}."
+	},
+
 	{ "file-read-error",
 	  AS_ISSUE_SEVERITY_ERROR,
 	  "Unable to read data from file <code>{{fname}}</code>: {{msg}}",

compose/asc-utils-metainfo.c

@@ -95,15 +95,6 @@ asc_parse_metainfo_data (AscResult *cres, AsMetadata *mdata, GBytes *bytes, cons
 		return NULL;
 	}
 
-	/* limit the amount of releases that we add to the output metadata.
-	 * since releases are sorted with the newest one at the top, we will only
-	 * remove the older ones. */
-	if (as_component_get_kind (cpt) != AS_COMPONENT_KIND_OPERATING_SYSTEM) {
-		GPtrArray *releases = as_component_get_releases (cpt);
-		if (releases->len > MAX_RELEASE_INFO_COUNT)
-			g_ptr_array_set_size (releases, MAX_RELEASE_INFO_COUNT);
-	}
-
 	return g_object_ref (cpt);
 }
 
@@ -132,71 +123,191 @@ asc_parse_metainfo_data_simple (AscResult *cres, GBytes *bytes, const gchar *mi_
 					mi_basename);
 }
 
+/**
+ * asc_process_metainfo_releases:
+ * @cres: an #AscResult instance.
+ * @unit: an #AscUnit where release information could be read from.
+ * @cpt: the #AsComponent to read release data for.
+ * @mi_filename: the metadata filename for the component, as found in the passed unit.
+ * @allow_net: set to %TRUE if network access should be allowed.
+ * @acurl: the #AsCurl to use for network access.
+ * @used_reldata: (out) (optional): Receive the release data that was used, if set.
+ *
+ * Reads an external release description file, either from a network location or from
+ * the given unit. Also performs further processing on the release information, like sorting
+ * releases or pruning old ones.
+ **/
+void
+asc_process_metainfo_releases (AscResult *cres,
+			       AscUnit *unit,
+			       AsComponent *cpt,
+			       const gchar *mi_filename,
+			       gboolean allow_net,
+			       AsCurl *acurl,
+			       GBytes **used_reldata)
+{
+	g_autoptr(GError) local_error = NULL;
+	g_autoptr(GBytes) relmd_bytes = NULL;
+
+	/* download external release metadata or fetch local release data */
+	if (as_component_get_releases_kind (cpt) == AS_RELEASES_KIND_EXTERNAL) {
+		g_autofree gchar *relmd_uri = NULL;
+
+		g_ptr_array_set_size (as_component_get_releases (cpt), 0);
+		if (allow_net && as_component_get_releases_url (cpt) != NULL) {
+			/* get the release data from a network location */
+			const gchar *releases_url = as_component_get_releases_url (cpt);
+
+			relmd_bytes = as_curl_download_bytes (acurl, releases_url, &local_error);
+			if (relmd_bytes == NULL) {
+				asc_result_add_hint (cres, NULL,
+						     "metainfo-releases-download-failed",
+						     "url", releases_url,
+						     "msg", local_error->message,
+						     NULL);
+				goto out;
+			}
+			relmd_uri = g_strdup (releases_url);
+		} else {
+			/* we need to find local release information */
+			g_autofree gchar *relfile_path = NULL;
+			g_autofree gchar *relfile_name = NULL;
+			g_autofree gchar *tmp = NULL;
+
+			relfile_name = g_strconcat (as_component_get_id (cpt), ".releases.xml", NULL);
+			tmp = g_path_get_dirname (mi_filename);
+			relfile_path = g_build_filename (tmp, "releases", relfile_name, NULL);
+
+			relmd_bytes = asc_unit_read_data (unit, relfile_path, &local_error);
+			if (relmd_bytes == NULL) {
+				asc_result_add_hint (cres, NULL,
+						     "file-read-error",
+						     "fname", relfile_path,
+						     "msg", local_error->message,
+						     NULL);
+				goto out;
+			}
+			relmd_uri = g_steal_pointer (&relfile_path);
+		}
+
+		if (!as_component_load_releases_from_bytes (cpt, relmd_bytes, &local_error)) {
+			asc_result_add_hint (cres, NULL,
+						"metainfo-releases-read-failed",
+						"path", relmd_uri,
+						"msg", local_error->message,
+						NULL);
+			goto out;
+		}
+	}
+
+	/* limit the amount of releases that we add to the output metadata.
+	 * since releases are sorted with the newest one at the top, we will only
+	 * remove the older ones. */
+	if (as_component_get_kind (cpt) != AS_COMPONENT_KIND_OPERATING_SYSTEM) {
+		GPtrArray *releases = as_component_get_releases (cpt);
+		if (releases->len > MAX_RELEASE_INFO_COUNT)
+			g_ptr_array_set_size (releases, MAX_RELEASE_INFO_COUNT);
+	}
+
+out:
+	if (used_reldata != NULL)
+		*used_reldata = g_steal_pointer (&relmd_bytes);
+}
+
 /**
  * asc_validate_metainfo_data_for_component:
  * @cres: an #AscResult instance.
  * @validator: an #AsValidator to validate with.
  * @cpt: the loaded #AsComponent which we are validating
  * @bytes: the data @cpt was constructed from.
  * @mi_basename: the basename of the MetaInfo file we are analyzing.
+ * @relmd_bytes: (nullable): the release metadata for this component.
  *
  * Validates MetaInfo data for the given component and stores the validation result as issue hints
  * in the given #AscResult.
  * Both the result as well as the validator's state may be modified by this function.
  **/
 void
-asc_validate_metainfo_data_for_component (AscResult *cres, AsValidator *validator,
-					  AsComponent *cpt, GBytes *bytes, const gchar *mi_basename)
+asc_validate_metainfo_data_for_component (AscResult *cres,
+					  AsValidator *validator,
+					  AsComponent *cpt,
+					  GBytes *bytes,
+					  const gchar *mi_basename,
+					  GBytes *relmd_bytes)
 {
-	g_autoptr(GList) issues = NULL;
+	GHashTable *issues_files;
+	GHashTableIter hiter;
+	gpointer hkey, hvalue;
 
 	/* don't check web URLs for validity, we catch those issues differently */
 	as_validator_set_check_urls (validator, FALSE);
 
 	/* remove issues from a potential previous use of this validator */
 	as_validator_clear_issues (validator);
+	as_validator_clear_release_data (validator);
+
+	/* add release data if we have any */
+	if (relmd_bytes != NULL) {
+		g_autoptr(GError) tmp_error = NULL;
+		g_autofree gchar *release_name = g_strconcat (as_component_get_id (cpt), ".releases.xml", NULL);
+		if (!as_validator_add_release_bytes (validator,
+						     release_name,
+						     relmd_bytes,
+						     &tmp_error))
+			g_warning ("Failed to add release metadata for %s: %s",
+				   as_component_get_id (cpt), tmp_error->message);
+	}
 
 	/* validate */
 	as_validator_validate_bytes (validator, bytes);
 
 	/* convert & register found issues */
-	issues = as_validator_get_issues (validator);
-	for (GList *l = issues; l != NULL; l = l->next) {
-		g_autofree gchar *asv_tag = NULL;
-		g_autofree gchar *location = NULL;
-		glong line;
-		const gchar *issue_hint;
-		AsValidatorIssue *issue = AS_VALIDATOR_ISSUE (l->data);
-
-		/* we have a special hint tag for legacy metadata,
-		 * with its proper "error" priority */
-		if (g_strcmp0 (as_validator_issue_get_tag (issue), "metainfo-ancient") == 0) {
-			asc_result_add_hint_simple (cres, cpt, "ancient-metadata");
-			continue;
+	issues_files = as_validator_get_issues_per_file (validator);
+	g_hash_table_iter_init (&hiter, issues_files);
+	while (g_hash_table_iter_next (&hiter, &hkey, &hvalue)) {
+		const gchar *filename = (const gchar*) hkey;
+		const GPtrArray *issues = (const GPtrArray*) hvalue;
+
+		if (filename == NULL)
+			filename = mi_basename;
+
+		for (guint i = 0; i < issues->len; i++) {
+			g_autofree gchar *asv_tag = NULL;
+			g_autofree gchar *location = NULL;
+			glong line;
+			const gchar *issue_hint;
+			AsValidatorIssue *issue = AS_VALIDATOR_ISSUE (g_ptr_array_index (issues, i));
+
+			/* we have a special hint tag for legacy metadata,
+			* with its proper "error" priority */
+			if (g_strcmp0 (as_validator_issue_get_tag (issue), "metainfo-ancient") == 0) {
+				asc_result_add_hint_simple (cres, cpt, "ancient-metadata");
+				continue;
+			}
+
+			/* create a tag for asgen out of the AppStream validator tag by prefixing it */
+			asv_tag = g_strconcat ("asv-",
+					as_validator_issue_get_tag (issue),
+					NULL);
+
+			line = as_validator_issue_get_line (issue);
+			if (line >= 0)
+				location = g_strdup_printf ("%s:%ld", filename, line);
+			else
+				location = g_strdup (filename);
+
+			/* we don't need to do much here, with the tag generated here,
+			* the hint registry will automatically assign the right explanation
+			* text and severity to the issue. */
+			issue_hint = as_validator_issue_get_hint (issue);
+			if (issue_hint == NULL)
+				issue_hint = "";
+			asc_result_add_hint (cres, cpt,
+					     asv_tag,
+					     "location", location,
+					     "hint", issue_hint,
+					     NULL);
 		}
-
-		/* create a tag for asgen out of the AppStream validator tag by prefixing it */
-		asv_tag = g_strconcat ("asv-",
-				       as_validator_issue_get_tag (issue),
-				       NULL);
-
-		line = as_validator_issue_get_line (issue);
-		if (line >= 0)
-			location = g_strdup_printf ("%s:%ld", mi_basename, line);
-		else
-			location = g_strdup (mi_basename);
-
-		/* we don't need to do much here, with the tag generated here,
-		 * the hint registry will automatically assign the right explanation
-		 * text and severity to the issue. */
-		issue_hint = as_validator_issue_get_hint (issue);
-		if (issue_hint == NULL)
-			issue_hint = "";
-		asc_result_add_hint (cres, cpt,
-				     asv_tag,
-				     "location", location,
-				     "hint", issue_hint,
-				     NULL);
 	}
 }
 

compose/asc-utils-metainfo.h

@@ -24,6 +24,7 @@
 #include <appstream.h>
 
 #include "as-settings-private.h"
+#include "as-curl.h"
 #include "asc-result.h"
 #include "asc-compose.h"
 
@@ -37,12 +38,20 @@ AsComponent		*asc_parse_metainfo_data (AscResult *cres,
 AsComponent		*asc_parse_metainfo_data_simple (AscResult *cres,
 							 GBytes *bytes,
 							 const gchar *mi_basename);
+void			asc_process_metainfo_releases (AscResult *cres,
+						       AscUnit *unit,
+						       AsComponent *cpt,
+						       const gchar *mi_filename,
+						       gboolean allow_net,
+						       AsCurl *acurl,
+						       GBytes **used_reldata);
 
 void			asc_validate_metainfo_data_for_component (AscResult *cres,
 								  AsValidator *validator,
 								  AsComponent *cpt,
 								  GBytes *bytes,
-								  const gchar *mi_basename);
+								  const gchar *mi_basename,
+								  GBytes *relmd_bytes);
 
 AS_INTERNAL_VISIBLE
 AsComponent		*asc_parse_desktop_entry_data (AscResult *cres,

data/its/metainfo.its

@@ -35,6 +35,13 @@
   <its:translateRule selector="/component/releases/release/description[@translate = 'no']"
                      translate="no"/>
 
+  <!-- Release metadata -->
+  <its:translateRule selector="/releases" translate="no"/>
+  <its:translateRule selector="/releases/release/description"
+                     translate="yes"/>
+  <its:translateRule selector="/releases/release/description[@translate = 'no']"
+                     translate="no"/>
+
   <!-- DEPRECATED -->
   <its:translateRule selector="/component/name[@translatable = 'no']"
                      translate="no"/>

data/its/metainfo.loc

@@ -5,10 +5,14 @@
   SPDX-License-Identifier: FSFAP
 -->
 <locatingRules>
-  <locatingRule name="MetaInfo" pattern="*.appdata.xml">
+  <locatingRule name="MetaInfo" pattern="*.metainfo.xml">
     <documentRule localName="component" target="metainfo.its"/>
   </locatingRule>
-  <locatingRule name="MetaInfo" pattern="*.metainfo.xml">
+  <locatingRule name="MetaInfo" pattern="*.appdata.xml">
     <documentRule localName="component" target="metainfo.its"/>
   </locatingRule>
+
+  <locatingRule name="ReleaseData" pattern="*.releases.xml">
+    <documentRule localName="releases" target="metainfo.its"/>
+  </locatingRule>
 </locatingRules>

docs/xml/collection-xmldata.xml

@@ -505,8 +505,7 @@
 				<term>&lt;releases/&gt;</term>
 				<listitem>
 					<para>
-						The <literal>releases</literal> tag and its <literal>release</literal> children are structured as described in <xref linkend="tag-releases"/>, with the
-						additional requirement that releases must be sorted in a latest-to-oldest order.
+						The <literal>releases</literal> tag and its <literal>release</literal> children are structured as described in <xref linkend="sect-Metadata-Releases"/>.
 					</para>
 					<para>
 						Each <literal>release</literal> tag may have a <literal>description</literal> tag as child, containing a brief description of what is new in the release.
@@ -517,6 +516,11 @@
 						It may also convert ISO 8601 <literal>date</literal> properties of the metainfo file into an UNIX timestamp <literal>timestamp</literal> property.
 						It should avoid generating metadata containing both properties on a <literal>release</literal> tag.
 					</para>
+					<para>
+						If a <xref linkend="tag-releases"/> tag in a metainfo file references an <literal>external</literal> release description, the release description should
+						be read either from the release file provided locally, or, if possible and provided, be downloaded from the URL referenced in the component's <literal>releases</literal>
+						tag.
+					</para>
 					<para>
 						Example for a valid releases tag:
 					</para>