Skip to content

Commit

Permalink
Check NuGet.org for SiteExtensions (projectkudu#2568)
Browse files Browse the repository at this point in the history 
Check NuGet.org for SiteExtensions
  • Loading branch information
@EricSten-MSFT @davidebbo
EricSten-MSFT authored and davidebbo committed Sep 27, 2017
1 parent 150dab5 commit 75a3839
Show file tree
Hide file tree
Showing 5 changed files with 179 additions and 59 deletions.
1 change: 1 addition & 0 deletions Kudu.Contracts/Settings/DeploymentSettingsExtension.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public static class DeploymentSettingsExtension
public const int DefaultMaxJobRunsHistoryCount = 50;

public static readonly string DefaultSiteExtensionFeedUrl = "https://www.siteextensions.net/api/v2/";
public static readonly string NuGetSiteExtensionFeedUrl = "https://www.nuget.org/api/v2/";

public static string GetValue(this IDeploymentSettingsManager settings, string key)
{
Expand Down
41 changes: 33 additions & 8 deletions Kudu.Core/SiteExtensions/FeedExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public static async Task<IEnumerable<UIPackageMetadata>> Search(this SourceRepos

// When using nuget.org, we only look at packages that have our tag. Later, we should switch to filtering
// by PackageType once nuget.org starts supporting that
if (srcRepo.PackageSource.Source.StartsWith("https://www.nuget.org/", StringComparison.InvariantCultureIgnoreCase))
if (IsNuGetRepo(srcRepo.PackageSource.Source))
{
if (String.IsNullOrWhiteSpace(searchTerm))
{
Expand All @@ -52,9 +52,9 @@ public static async Task<IEnumerable<UIPackageMetadata>> Search(this SourceRepos
}
else
{
// User provided simple string: treat it as a title. This is not ideal behavior, but
// User provided simple string: treat it as a id. This is not ideal behavior, but
// is the best we can do based on how nuget.org works
searchTerm = $"tags:AzureSiteExtension title:\"{searchTerm}\"";
searchTerm = $"tags:AzureSiteExtension id:\"{searchTerm}\"";
}
}

Expand All @@ -68,14 +68,22 @@ internal static async Task<UIPackageMetadata> GetLatestPackageByIdFromSrcRepo(th
{
// 7 references none of which uses bool includePrerelease = true, bool includeUnlisted = false
var metadataResource = await srcRepo.GetResourceAndValidateAsync<UIMetadataResource>();
return await metadataResource.GetLatestPackageByIdFromMetaRes(packageId);
return await metadataResource.GetLatestPackageByIdFromMetaRes(packageId,
explicitTag: IsNuGetRepo(srcRepo.PackageSource.Source));
}

// can be called concurrently if metaDataResource is provided
internal static async Task<UIPackageMetadata> GetLatestPackageByIdFromMetaRes(this UIMetadataResource metadataResource, string packageId, bool includePrerelease = true, bool includeUnlisted = false)
internal static async Task<UIPackageMetadata> GetLatestPackageByIdFromMetaRes(this UIMetadataResource metadataResource, string packageId, bool includePrerelease = true, bool includeUnlisted = false, bool explicitTag = false)
{
UIPackageMetadata latestPackage = null;
IEnumerable<UIPackageMetadata> packages = await metadataResource.GetMetadata(packageId, includePrerelease, includeUnlisted, token: CancellationToken.None);

// When using nuget.org, we only look at packages that have our tag.
if (explicitTag)
{
packages = packages.Where(item => item.Tags.IndexOf("azuresiteextension", StringComparison.OrdinalIgnoreCase) >= 0);
}

foreach (var p in packages)
{
if (latestPackage == null ||
Expand All @@ -90,7 +98,7 @@ internal static async Task<UIPackageMetadata> GetLatestPackageByIdFromMetaRes(th
// explicit id, even without specifying a version
if (latestPackage == null && !includeUnlisted)
{
latestPackage = await GetLatestPackageByIdFromMetaRes(metadataResource, packageId, includePrerelease, includeUnlisted: true);
latestPackage = await GetLatestPackageByIdFromMetaRes(metadataResource, packageId, includePrerelease, includeUnlisted: true, explicitTag: explicitTag);
}

return latestPackage;
Expand All @@ -108,7 +116,16 @@ public static async Task<UIPackageMetadata> GetPackageByIdentity(this SourceRepo
var metadataResource = await srcRepo.GetResourceAndValidateAsync<UIMetadataResource>();
NuGetVersion expectedVersion = NuGetVersion.Parse(version);
var identity = new PackageIdentity(packageId, expectedVersion);
return await metadataResource.GetMetadata(identity, CancellationToken.None);
UIPackageMetadata ret = await metadataResource.GetMetadata(identity, CancellationToken.None);

// When using nuget.org, we only look at packages that have our tag.
if (ret != null &&
IsNuGetRepo(srcRepo.PackageSource.Source) &&
(ret.Tags.IndexOf("azuresiteextension", StringComparison.OrdinalIgnoreCase) < 0))
{
ret = null;
}
return ret;
}

/// <summary>
Expand Down Expand Up @@ -385,5 +402,13 @@ private static async Task<Stream> GetPackageStream(this SourceRepository srcRepo
}
}
}

internal static bool IsNuGetRepo(string repoUrl)
{
return System.Text.RegularExpressions.Regex.IsMatch(
repoUrl,
@"https://.*\.nuget\.org/.*",
System.Text.RegularExpressions.RegexOptions.IgnoreCase);
}
}
}
145 changes: 106 additions & 39 deletions Kudu.Core/SiteExtensions/SiteExtensionManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Collections.Generic;
using System.ComponentModel.Composition.Hosting;
using System.Configuration;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
Expand Down Expand Up @@ -75,17 +76,24 @@ public async Task<IEnumerable<SiteExtensionInfo>> GetRemoteExtensions(string fil
{
ITracer tracer = _traceFactory.GetTracer();
var extensions = new List<SiteExtensionInfo>();
SourceRepository remoteRepo = GetRemoteRepository(feedUrl);
IEnumerable<SourceRepository> remoteRepos = GetRemoteRepositories(feedUrl);

SearchFilter filterOptions = new SearchFilter();
filterOptions.IncludePrerelease = allowPrereleaseVersions;

IEnumerable<UIPackageMetadata> packages = null;
IEnumerable<UIPackageMetadata> packages = new List<UIPackageMetadata>();

using (tracer.Step("Search site extensions by filter: {0}", filter))
{
packages = (await remoteRepo.Search(string.IsNullOrWhiteSpace(filter) ? string.Empty : filter, filterOptions: filterOptions))
.OrderByDescending(p => p.DownloadCount);
foreach (SourceRepository remoteRepo in remoteRepos)
{
var foundUIPackages = await remoteRepo.Search(string.IsNullOrWhiteSpace(filter) ? string.Empty : filter, filterOptions: filterOptions);
if (null != foundUIPackages && foundUIPackages.Count() > 0)
{
packages = packages.Concat(foundUIPackages);
}
}
packages = packages.OrderByDescending(p => p.DownloadCount);
}

using (tracer.Step("Convert search result to SiteExtensionInfos with max concurrent requests: {0}", System.Environment.ProcessorCount))
Expand All @@ -110,21 +118,35 @@ public async Task<SiteExtensionInfo> GetRemoteExtension(string id, string versio
{
ITracer tracer = _traceFactory.GetTracer();

SourceRepository remoteRepo = GetRemoteRepository(feedUrl);
IEnumerable<SourceRepository> remoteRepos = GetRemoteRepositories(feedUrl);
UIPackageMetadata package = null;

if (string.IsNullOrWhiteSpace(version))
{
using (tracer.Step("Version is null, search latest package by id: {0}", id))
{
package = await remoteRepo.GetLatestPackageByIdFromSrcRepo(id);
foreach (SourceRepository remoteRepo in remoteRepos)
{
package = await remoteRepo.GetLatestPackageByIdFromSrcRepo(id);
if (package != null)
{
break;
}
}
}
}
else
{
using (tracer.Step("Search package by id: {0} and version: {1}", id, version))
{
package = await remoteRepo.GetPackageByIdentity(id, version);
foreach (SourceRepository remoteRepo in remoteRepos)
{
package = await remoteRepo.GetPackageByIdentity(id, version);
if (package != null)
{
break;
}
}
}
}

Expand Down Expand Up @@ -249,9 +271,10 @@ private async Task<SiteExtensionInfo> TryInstallExtension(string id, string vers
{
JsonSettings siteExtensionSettings = GetSettingManager(id);
feedUrl = (string.IsNullOrEmpty(feedUrl) ? siteExtensionSettings.GetValue(_feedUrlSetting) : feedUrl);
SourceRepository remoteRepo = GetRemoteRepository(feedUrl);
IEnumerable<SourceRepository> remoteRepos = GetRemoteRepositories(feedUrl);
UIPackageMetadata localPackage = null;
UIPackageMetadata repoPackage = null;
SourceRepository remoteRepo = null;

if (this.IsInstalledToWebRoot(id))
{
Expand All @@ -264,23 +287,40 @@ private async Task<SiteExtensionInfo> TryInstallExtension(string id, string vers
{
using (tracer.Step("Version is null, search latest package by id: {0}, will not search for unlisted package.", id))
{
repoPackage = await remoteRepo.GetLatestPackageByIdFromSrcRepo(id);
foreach (SourceRepository rr in remoteRepos)
{
repoPackage = await rr.GetLatestPackageByIdFromSrcRepo(id);
if (repoPackage != null)
{
remoteRepo = rr;
break;
}
}
}
}
else
{
using (tracer.Step("Search package by id: {0} and version: {1}, will also search for unlisted package.", id, version))
{
repoPackage = await remoteRepo.GetPackageByIdentity(id, version);
foreach (SourceRepository rr in remoteRepos)
{
repoPackage = await rr.GetPackageByIdentity(id, version);
if (repoPackage != null)
{
remoteRepo = rr;
break;
}
}
}
}

if (repoPackage != null)
{
Debug.Assert(remoteRepo != null, "remote SourceRepository should not be null!");
using (tracer.Step("Install package: {0}.", id))
{
string installationDirectory = GetInstallationDirectory(id);
localPackage = await InstallExtension(repoPackage, installationDirectory, feedUrl, type, tracer, installationArgs);
localPackage = await InstallExtension(repoPackage, installationDirectory, remoteRepo, type, tracer, installationArgs);
siteExtensionSettings.SetValues(new KeyValuePair<string, JToken>[] {
new KeyValuePair<string, JToken>(_versionSetting, localPackage.Identity.Version.ToNormalizedString()),
new KeyValuePair<string, JToken>(_feedUrlSetting, feedUrl),
Expand Down Expand Up @@ -396,15 +436,14 @@ private async Task<SiteExtensionInfo> TryInstallExtension(string id, string vers
/// <para>3. Deploy site extension job</para>
/// <para>4. Execute install.cmd if exist</para>
/// </summary>
private async Task<UIPackageMetadata> InstallExtension(UIPackageMetadata package, string installationDirectory, string feedUrl, SiteExtensionInfo.SiteExtensionType type, ITracer tracer, string installationArgs)
private async Task<UIPackageMetadata> InstallExtension(UIPackageMetadata package, string installationDirectory, SourceRepository remoteRepo, SiteExtensionInfo.SiteExtensionType type, ITracer tracer, string installationArgs)
{
try
{
EnsureInstallationEnviroment(installationDirectory, tracer);

string packageLocalFilePath = GetNuGetPackageFile(package.Identity.Id, package.Identity.Version.ToNormalizedString());
bool packageExisted = FileSystemHelpers.DirectoryExists(installationDirectory);
SourceRepository remoteRepo = GetRemoteRepository(feedUrl);

using (tracer.Step("Download site extension: {0}", package.Identity))
{
Expand Down Expand Up @@ -571,23 +610,30 @@ private async Task<bool> IsSiteExtensionInstalled(string id, string version, str

// Try to use given feed
// If given feed is null, try with feed that from local package
// And GetRemoteRepository will fallback to use default feed if pass in feed param is null
SourceRepository remoteRepo = GetRemoteRepository(feedUrl ?? localPackageFeedUrl);
// And GetRemoteRepositories will fallback to use default feed if pass in feed param is null
IEnumerable<SourceRepository> remoteRepos = GetRemoteRepositories(feedUrl ?? localPackageFeedUrl);

// case 1 and 2
if (!string.IsNullOrWhiteSpace(version)
&& version.Equals(localPackageVersion, StringComparison.OrdinalIgnoreCase)
&& remoteRepo.PackageSource.Source.Equals(localPackageFeedUrl, StringComparison.OrdinalIgnoreCase))
{
isInstalled = true;
}
else if (string.IsNullOrWhiteSpace(version) && string.IsNullOrWhiteSpace(feedUrl))
foreach (SourceRepository rr in remoteRepos)
{
// case 3
UIPackageMetadata remotePackage = await remoteRepo.GetLatestPackageByIdFromSrcRepo(id);
if (remotePackage != null)
// case 1 and 2
if (!string.IsNullOrWhiteSpace(version)
&& version.Equals(localPackageVersion, StringComparison.OrdinalIgnoreCase)
&& rr.PackageSource.Source.Equals(localPackageFeedUrl, StringComparison.OrdinalIgnoreCase))
{
isInstalled = true;
}
else if (string.IsNullOrWhiteSpace(version) && string.IsNullOrWhiteSpace(feedUrl))
{
// case 3
UIPackageMetadata remotePackage = await rr.GetLatestPackageByIdFromSrcRepo(id);
if (remotePackage != null)
{
isInstalled = remotePackage.Identity.Version.ToNormalizedString().Equals(localPackageVersion, StringComparison.OrdinalIgnoreCase) && string.Equals(localPackageInstallationArgs, installationArgs);
}
}
if (isInstalled)
{
isInstalled = remotePackage.Identity.Version.ToNormalizedString().Equals(localPackageVersion, StringComparison.OrdinalIgnoreCase) && string.Equals(localPackageInstallationArgs, installationArgs);
break;
}
}

Expand Down Expand Up @@ -673,11 +719,20 @@ public async Task<bool> UninstallExtension(string id)
return await GetLocalExtension(id, checkLatest: false) == null;
}

private SourceRepository GetRemoteRepository(string feedUrl)
private IEnumerable<SourceRepository> GetRemoteRepositories(string feedUrl)
{
return string.IsNullOrWhiteSpace(feedUrl) ?
GetSourceRepository(_settings.GetSiteExtensionRemoteUrl()) :
GetSourceRepository(feedUrl);
List<SourceRepository> repos = new List<SourceRepository>();

if (!string.IsNullOrWhiteSpace(feedUrl))
{
repos.Add(GetSourceRepository(feedUrl));
}
else
{
repos.Add(GetSourceRepository(DeploymentSettingsExtension.NuGetSiteExtensionFeedUrl));
repos.Add(GetSourceRepository(_settings.GetSiteExtensionRemoteUrl()));
}
return repos;
}

private string GetInstallationDirectory(string id)
Expand Down Expand Up @@ -801,7 +856,15 @@ private async Task<SiteExtensionInfo> ConvertRemotePackageToSiteExtensionInfo(UI

private async Task<SiteExtensionInfo> CheckRemotePackageLatestVersion(SiteExtensionInfo info, UIMetadataResource metadataResource)
{
UIPackageMetadata localPackage = await metadataResource.GetLatestPackageByIdFromMetaRes(info.Id);
bool isNuGetPackage = false;

if (!string.IsNullOrEmpty(info.FeedUrl))
{
isNuGetPackage = FeedExtensions.IsNuGetRepo(info.FeedUrl);
}

UIPackageMetadata localPackage = await metadataResource.GetLatestPackageByIdFromMetaRes(info.Id,
explicitTag: isNuGetPackage);

if (localPackage != null)
{
Expand Down Expand Up @@ -838,14 +901,18 @@ private async Task TryCheckLocalPackageLatestVersionFromRemote(SiteExtensionInfo
try
{
// FindPackage gets back the latest version.
SourceRepository remoteRepo = GetRemoteRepository(info.FeedUrl);
UIPackageMetadata latestPackage = await remoteRepo.GetLatestPackageByIdFromSrcRepo(info.Id);
if (latestPackage != null)
IEnumerable<SourceRepository> remoteRepos = GetRemoteRepositories(info.FeedUrl);
foreach (SourceRepository remoteRepo in remoteRepos)
{
NuGetVersion currentVersion = NuGetVersion.Parse(info.Version);
info.LocalIsLatestVersion = NuGetVersion.Parse(info.Version).Equals(latestPackage.Identity.Version);
info.DownloadCount = latestPackage.DownloadCount;
info.PublishedDateTime = latestPackage.Published;
UIPackageMetadata latestPackage = await remoteRepo.GetLatestPackageByIdFromSrcRepo(info.Id);
if (latestPackage != null)
{
NuGetVersion currentVersion = NuGetVersion.Parse(info.Version);
info.LocalIsLatestVersion = NuGetVersion.Parse(info.Version).Equals(latestPackage.Identity.Version);
info.DownloadCount = latestPackage.DownloadCount;
info.PublishedDateTime = latestPackage.Published;
break;
}
}
}
catch (Exception ex)
Expand Down
Loading

0 comments on commit 75a3839

Please sign in to comment.