Skip to content

Commit 6a44948

Browse files
marticlimentmarticliment
committed
Better resiliency if a source fails when loading NuGet packages (fix #4172)
1 parent 9c39721 commit 6a44948

2 files changed

Lines changed: 101 additions & 87 deletions

File tree

  • src
    • UniGetUI.PackageEngine.Managers.Generic.NuGet
    • UniGetUI.PackageEngine.Managers.Scoop

‎src/UniGetUI.PackageEngine.Managers.Generic.NuGet/BaseNuGet.cs‎

Lines changed: 100 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -62,81 +62,87 @@ protected sealed override IReadOnlyList<Package> FindPackages_UnSafe(string quer
6262

6363
foreach(IManagerSource source in sources)
6464
{
65-
Uri? SearchUrl = new($"{source.Url}/Search()" +
66-
$"?$filter=IsLatestVersion" +
67-
$"&$orderby=Id&searchTerm='{HttpUtility.UrlEncode(query)}'" +
68-
$"&targetFramework=''" +
69-
$"&includePrerelease={(canPrerelease? "true": "false")}" +
70-
$"&$skip=0" +
71-
$"&$top=50" +
72-
$"&semVerLevel=2.0.0");
73-
// Uri SearchUrl = new($"{source.Url}/Search()?$filter=IsLatestVersion&searchTerm=%27{HttpUtility.UrlEncode(query)}%27&targetFramework=%27%27&includePrerelease=false");
74-
logger.Log($"Begin package search with url={SearchUrl} on manager {Name}");
75-
Dictionary<string, SearchResult> AlreadyProcessedPackages = [];
76-
77-
78-
using HttpClient client = new(CoreTools.GenericHttpClientParameters);
79-
client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString);
80-
81-
while (SearchUrl is not null)
65+
try
8266
{
83-
HttpResponseMessage response = client.GetAsync(SearchUrl).GetAwaiter().GetResult();
84-
85-
if (!response.IsSuccessStatusCode)
67+
Uri? SearchUrl = new($"{source.Url}/Search()" +
68+
$"?$filter=IsLatestVersion" +
69+
$"&$orderby=Id&searchTerm='{HttpUtility.UrlEncode(query)}'" +
70+
$"&targetFramework=''" +
71+
$"&includePrerelease={(canPrerelease ? "true" : "false")}" +
72+
$"&$skip=0" +
73+
$"&$top=50" +
74+
$"&semVerLevel=2.0.0");
75+
// Uri SearchUrl = new($"{source.Url}/Search()?$filter=IsLatestVersion&searchTerm=%27{HttpUtility.UrlEncode(query)}%27&targetFramework=%27%27&includePrerelease=false");
76+
logger.Log($"Begin package search with url={SearchUrl} on manager {Name}");
77+
Dictionary<string, SearchResult> AlreadyProcessedPackages = [];
78+
79+
80+
using HttpClient client = new(CoreTools.GenericHttpClientParameters);
81+
client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString);
82+
83+
while (SearchUrl is not null)
8684
{
87-
logger.Error($"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}");
88-
SearchUrl = null;
89-
continue;
90-
}
91-
92-
string SearchResults = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
93-
MatchCollection matches = Regex.Matches(SearchResults, "<entry>([\\s\\S]*?)<\\/entry>");
85+
HttpResponseMessage response = client.GetAsync(SearchUrl).GetAwaiter().GetResult();
9486

95-
foreach (Match match in matches)
96-
{
97-
if (!match.Success)
87+
if (!response.IsSuccessStatusCode)
9888
{
89+
logger.Error($"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}");
90+
SearchUrl = null;
9991
continue;
10092
}
10193

102-
string id = Regex.Match(match.Value, "Id='([^<>']+)'").Groups[1].Value;
103-
string version = Regex.Match(match.Value, "Version='([^<>']+)'").Groups[1].Value;
104-
var float_version = CoreTools.VersionStringToStruct(version);
105-
// Match title = Regex.Match(match.Value, "<title[ \\\"\\=A-Za-z0-9]+>([^<>]+)<\\/title>");
94+
string SearchResults = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
95+
MatchCollection matches = Regex.Matches(SearchResults, "<entry>([\\s\\S]*?)<\\/entry>");
10696

107-
if (AlreadyProcessedPackages.TryGetValue(id, out var value) && value.version_float >= float_version)
97+
foreach (Match match in matches)
10898
{
109-
continue;
99+
if (!match.Success)
100+
{
101+
continue;
102+
}
103+
104+
string id = Regex.Match(match.Value, "Id='([^<>']+)'").Groups[1].Value;
105+
string version = Regex.Match(match.Value, "Version='([^<>']+)'").Groups[1].Value;
106+
var float_version = CoreTools.VersionStringToStruct(version);
107+
// Match title = Regex.Match(match.Value, "<title[ \\\"\\=A-Za-z0-9]+>([^<>]+)<\\/title>");
108+
109+
if (AlreadyProcessedPackages.TryGetValue(id, out var value) && value.version_float >= float_version)
110+
{
111+
continue;
112+
}
113+
114+
AlreadyProcessedPackages[id] =
115+
new SearchResult { id = id, version = version, version_float = float_version, manifest = match.Value };
110116
}
111117

112-
AlreadyProcessedPackages[id] =
113-
new SearchResult { id = id, version = version, version_float = float_version, manifest = match.Value };
118+
SearchUrl = null;
119+
Match next = Regex.Match(SearchResults, "<link rel=\"next\" href=\"([^\"]+)\" ?\\/>");
120+
if (next.Success)
121+
{
122+
SearchUrl = new Uri(next.Groups[1].Value.Replace("&amp;", "&"));
123+
logger.Log($"Adding extra info from URL={SearchUrl}");
124+
}
114125
}
115126

116-
SearchUrl = null;
117-
Match next = Regex.Match(SearchResults, "<link rel=\"next\" href=\"([^\"]+)\" ?\\/>");
118-
if (next.Success)
127+
foreach (SearchResult package in AlreadyProcessedPackages.Values)
119128
{
120-
SearchUrl = new Uri(next.Groups[1].Value.Replace("&amp;", "&"));
121-
logger.Log($"Adding extra info from URL={SearchUrl}");
129+
logger.Log($"Found package {package.id} version {package.version} on source {source.Name}");
130+
var nativePackage = new Package(CoreTools.FormatAsName(package.id), package.id, package.version, source, this);
131+
Packages.Add(nativePackage);
132+
Manifests[nativePackage.GetHash()] = package.manifest;
122133
}
123134
}
124-
125-
foreach (SearchResult package in AlreadyProcessedPackages.Values)
135+
catch (Exception ex)
126136
{
127-
logger.Log($"Found package {package.id} version {package.version} on source {source.Name}");
128-
var nativePackage = new Package(CoreTools.FormatAsName(package.id), package.id, package.version, source, this);
129-
Packages.Add(nativePackage);
130-
Manifests[nativePackage.GetHash()] = package.manifest;
137+
logger.Error($"Source {source.Name} on manager {source.Manager.Name} failed to find package data");
138+
logger.Error(ex);
131139
}
132140
}
133141

134142
logger.Close(0);
135-
136143
return Packages;
137144
}
138145

139-
140146
protected override IReadOnlyList<Package> GetAvailableUpdates_UnSafe()
141147
{
142148
int errors = 0;
@@ -157,51 +163,59 @@ protected override IReadOnlyList<Package> GetAvailableUpdates_UnSafe()
157163

158164
foreach (var pair in sourceMapping)
159165
{
160-
var packageIds = new StringBuilder();
161-
var packageVers = new StringBuilder();
162-
var packageIdVersion = new Dictionary<string, string>();
163-
foreach (var package in pair.Value)
166+
try
164167
{
165-
packageIds.Append(package.Id + "|");
166-
packageVers.Append(package.VersionString + "|");
167-
packageIdVersion[package.Id.ToLower()] = package.VersionString;
168-
}
169-
170-
var SearchUrl = $"{pair.Key.Url.ToString().Trim('/')}/GetUpdates()" +
171-
$"?packageIds=%27{HttpUtility.UrlEncode(packageIds.ToString().Trim('|'))}%27" +
172-
$"&versions=%27{HttpUtility.UrlEncode(packageVers.ToString().Trim('|'))}%27" +
173-
$"&includePrerelease={(canPrerelease ? "true" : "false")}" +
174-
$"&includeAllVersions=0";
168+
var packageIds = new StringBuilder();
169+
var packageVers = new StringBuilder();
170+
var packageIdVersion = new Dictionary<string, string>();
171+
foreach (var package in pair.Value)
172+
{
173+
packageIds.Append(package.Id + "|");
174+
packageVers.Append(package.VersionString + "|");
175+
packageIdVersion[package.Id.ToLower()] = package.VersionString;
176+
}
175177

176-
using HttpClient client = new(CoreTools.GenericHttpClientParameters);
177-
client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString);
178-
HttpResponseMessage response = client.GetAsync(SearchUrl).GetAwaiter().GetResult();
178+
var SearchUrl = $"{pair.Key.Url.ToString().Trim('/')}/GetUpdates()" +
179+
$"?packageIds=%27{HttpUtility.UrlEncode(packageIds.ToString().Trim('|'))}%27" +
180+
$"&versions=%27{HttpUtility.UrlEncode(packageVers.ToString().Trim('|'))}%27" +
181+
$"&includePrerelease={(canPrerelease ? "true" : "false")}" +
182+
$"&includeAllVersions=0";
179183

180-
if (!response.IsSuccessStatusCode)
181-
{
182-
logger.Error($"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}");
183-
errors++;
184-
}
185-
else
186-
{
187-
string SearchResults = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
188-
MatchCollection matches = Regex.Matches(SearchResults, "<entry>([\\s\\S]*?)<\\/entry>");
184+
using HttpClient client = new(CoreTools.GenericHttpClientParameters);
185+
client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString);
186+
HttpResponseMessage response = client.GetAsync(SearchUrl).GetAwaiter().GetResult();
189187

190-
foreach (Match match in matches)
188+
if (!response.IsSuccessStatusCode)
191189
{
192-
if (!match.Success) continue;
190+
logger.Error($"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}");
191+
errors++;
192+
}
193+
else
194+
{
195+
string SearchResults = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
196+
MatchCollection matches = Regex.Matches(SearchResults, "<entry>([\\s\\S]*?)<\\/entry>");
193197

194-
string id = Regex.Match(match.Value, "<d:Id>([^<]+)</d:Id>").Groups[1].Value;
195-
string new_version = Regex.Match(match.Value, "<d:Version>([^<]+)</d:Version>").Groups[1].Value;
196-
// Match title = Regex.Match(match.Value, "<title[ \\\"\\=A-Za-z0-9]+>([^<>]+)<\\/title>");
198+
foreach (Match match in matches)
199+
{
200+
if (!match.Success) continue;
197201

198-
logger.Log($"Found package {id} version {new_version} on source {pair.Key.Name}");
202+
string id = Regex.Match(match.Value, "<d:Id>([^<]+)</d:Id>").Groups[1].Value;
203+
string new_version = Regex.Match(match.Value, "<d:Version>([^<]+)</d:Version>").Groups[1].Value;
204+
// Match title = Regex.Match(match.Value, "<title[ \\\"\\=A-Za-z0-9]+>([^<>]+)<\\/title>");
199205

200-
var nativePackage = new Package(CoreTools.FormatAsName(id), id, packageIdVersion[id.ToLower()], new_version, pair.Key, this);
201-
Packages.Add(nativePackage);
202-
Manifests[nativePackage.GetHash()] = match.Value;
206+
logger.Log($"Found package {id} version {new_version} on source {pair.Key.Name}");
207+
208+
var nativePackage = new Package(CoreTools.FormatAsName(id), id, packageIdVersion[id.ToLower()], new_version, pair.Key, this);
209+
Packages.Add(nativePackage);
210+
Manifests[nativePackage.GetHash()] = match.Value;
211+
}
203212
}
204213
}
214+
catch (Exception ex)
215+
{
216+
logger.Error($"Source {pair.Key.Name} on manager {pair.Key.Manager.Name} failed to load updates info with exception");
217+
logger.Error(ex);
218+
}
205219
}
206220

207221
var maxVersions = new Dictionary<string, CoreTools.Version?>();

‎src/UniGetUI.PackageEngine.Managers.Scoop/Scoop.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ namespace UniGetUI.PackageEngine.Managers.ScoopManager
2222

2323
public class Scoop : PackageManager
2424
{
25-
public static new string[] FALSE_PACKAGE_IDS = ["No"];
25+
public static new string[] FALSE_PACKAGE_IDS = ["No", "WARN"];
2626
public static new string[] FALSE_PACKAGE_VERSIONS = ["Matches", "Install", "failed", "failed,", "Manifest", "removed", "removed,"];
2727

2828
private long LastScoopSourceUpdateTime;

0 commit comments

Comments
 (0)