Support webjobs in secondary location site/jobs

Kudu.Contracts/IEnvironment.cs

@@ -24,6 +24,7 @@ public interface IEnvironment
         string DataPath { get; }                // e.g. /data
         string JobsDataPath { get; }            // e.g. /data/jobs
         string JobsBinariesPath { get; }        // e.g. /site/wwwroot/app_data/jobs
+        string SecondaryJobsBinariesPath { get; } // e.g. /site/jobs
         string FunctionsPath { get; }           // e.g. /site/wwwroot
         string AppBaseUrlPrefix { get; }        // e.g. siteName.azurewebsites.net
         string RequestId { get; }               // e.g. x-arr-log-id or x-ms-request-id header value

Kudu.Contracts/Jobs/IJobsManager.cs

@@ -8,6 +8,8 @@ namespace Kudu.Contracts.Jobs
     {
         IEnumerable<TJob> ListJobs(bool forceRefreshCache);
 
+        bool HasJob(string jobName);
+
         TJob GetJob(string jobName);
 
         TJob CreateOrReplaceJobFromZipStream(Stream zipStream, string jobName);

Kudu.Contracts/Jobs/ITriggeredJobsManager.cs

@@ -9,7 +9,5 @@ public interface ITriggeredJobsManager : IJobsManager<TriggeredJob>
         TriggeredJobRun GetJobRun(string jobName, string runId);
 
         TriggeredJobRun GetLatestJobRun(string jobName);
-
-        string JobsBinariesPath { get; }
     }
 }

Kudu.Contracts/Settings/DeploymentSettingsExtension.cs

@@ -261,5 +261,8 @@ public static bool RunFromRemoteZip(this IDeploymentSettingsManager settings)
 
             return value != null && value.StartsWith("http", StringComparison.OrdinalIgnoreCase);
         }
+
+        public static bool RunFromZip(this IDeploymentSettingsManager settings)
+            => settings.RunFromLocalZip() || settings.RunFromRemoteZip();
     }
 }

Kudu.Core.Test/Jobs/ContinuousJobRunnerFacts.cs

@@ -50,7 +50,7 @@ public ContinuousJobRunnerFacts()
             mockTracer.Setup(p => p.Trace(It.IsAny<string>(), It.IsAny<IDictionary<string, string>>()));
             mockTraceFactory.Setup(p => p.GetTracer()).Returns(mockTracer.Object);
 
-            _runner = new ContinuousJobRunner(_job, _environment, mockSettingsManager, mockTraceFactory.Object, mockAnalytics.Object);
+            _runner = new ContinuousJobRunner(_job, _environment.JobsBinariesPath, _environment, mockSettingsManager, mockTraceFactory.Object, mockAnalytics.Object);
 
             FileSystemHelpers.DeleteFileSafe(_logFilePath);
         }

Kudu.Core/Environment.cs

@@ -35,6 +35,7 @@ public class Environment : IEnvironment
         private readonly string _jobsDataPath;
         private readonly string _jobsBinariesPath;
         private readonly string _sitePackagesPath;
+        private readonly string _secondaryJobsBinariesPath;
 
         // This ctor is used only in unit tests
         public Environment(
@@ -80,6 +81,7 @@ public Environment(
 
             _jobsDataPath = Path.Combine(_dataPath, Constants.JobsPath);
             _jobsBinariesPath = _jobsDataPath;
+            _secondaryJobsBinariesPath = _jobsDataPath;
 
             _logFilesPath = Path.Combine(rootPath, Constants.LogFilesPath);
             _applicationLogFilesPath = Path.Combine(_logFilesPath, Constants.ApplicationLogFilesDirectory);
@@ -112,7 +114,7 @@ public Environment(
             _siteExtensionSettingsPath = Path.Combine(SiteRootPath, Constants.SiteExtensionsCachePath);
             _diagnosticsPath = Path.Combine(SiteRootPath, Constants.DiagnosticsPath);
             _locksPath = Path.Combine(SiteRootPath, Constants.LocksPath);
-            
+
             if (OSDetector.IsOnWindows())
             {
                 _sshKeyPath = Path.Combine(rootPath, Constants.SSHKeyPath);
@@ -132,6 +134,7 @@ public Environment(
             _dataPath = Path.Combine(rootPath, Constants.DataPath);
             _jobsDataPath = Path.Combine(_dataPath, Constants.JobsPath);
             _jobsBinariesPath = Path.Combine(_webRootPath, Constants.AppDataPath, Constants.JobsPath);
+            _secondaryJobsBinariesPath = Path.Combine(SiteRootPath, Constants.JobsPath);
             string userDefinedWebJobRoot = System.Environment.GetEnvironmentVariable(SettingsKeys.WebJobsRootPath);
             if (!String.IsNullOrEmpty(userDefinedWebJobRoot))
             {
@@ -314,6 +317,11 @@ public string JobsBinariesPath
             get { return _jobsBinariesPath; }
         }
 
+        public string SecondaryJobsBinariesPath
+        {
+            get { return _secondaryJobsBinariesPath; }
+        }
+
         public string SiteExtensionSettingsPath
         {
             get { return _siteExtensionSettingsPath; }

Kudu.Core/Jobs/AggregrateContinuousJobsManager.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Kudu.Contracts.Jobs;
+using Kudu.Contracts.Settings;
+using Kudu.Core.Tracing;
+
+namespace Kudu.Core.Jobs
+{
+    public class AggregrateContinuousJobsManager : AggregrateJobsManagerBase<ContinuousJob>, IContinuousJobsManager
+    {
+        public AggregrateContinuousJobsManager(ITraceFactory traceFactory, IEnvironment environment, IDeploymentSettingsManager settings, IAnalytics analytics)
+            : base(new ContinuousJobsManager(environment.JobsBinariesPath, traceFactory, environment, settings, analytics),
+                  excludedList => new ContinuousJobsManager(environment.SecondaryJobsBinariesPath, traceFactory, environment, settings, analytics, excludedList),
+                  settings)
+        {
+        }
+
+        public void DisableJob(string jobName)
+            => GetContinuousWriteJobManager(jobName).DisableJob(jobName);
+
+        public void EnableJob(string jobName)
+            => GetContinuousWriteJobManager(jobName).EnableJob(jobName);
+
+        public Task<HttpResponseMessage> HandleRequest(string jobName, string path, HttpRequestMessage request)
+            => PrimaryJobManager.HasJob(jobName)
+            ? (PrimaryJobManager as IContinuousJobsManager).HandleRequest(jobName, path, request)
+            : (SecondaryJobManager as IContinuousJobsManager).HandleRequest(jobName, path, request);
+
+        IContinuousJobsManager GetContinuousWriteJobManager(string jobName) => GetWriteJobManagerForJob(jobName) as IContinuousJobsManager;
+    }
+}

Kudu.Core/Jobs/AggregrateJobsManagerBase.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Kudu.Contracts.Jobs;
+using Kudu.Contracts.Settings;
+
+namespace Kudu.Core.Jobs
+{
+    public abstract class AggregrateJobsManagerBase<TJob> where TJob : JobBase, new()
+    {
+        protected JobsManagerBase<TJob> PrimaryJobManager { get; private set; }
+        protected JobsManagerBase<TJob> SecondaryJobManager { get; private set; }
+        private readonly IDeploymentSettingsManager _settings;
+
+        protected AggregrateJobsManagerBase(JobsManagerBase<TJob> primaryManager, Func<IEnumerable<string>, JobsManagerBase<TJob>> secondaryManagerFactory, IDeploymentSettingsManager settings)
+        {
+            PrimaryJobManager = primaryManager;
+            // pass the list of primary job names so the second manager can excluded them
+            SecondaryJobManager = secondaryManagerFactory(PrimaryJobManager.ListJobs(forceRefreshCache: false).Select(j => j.Name));
+            _settings = settings;
+        }
+
+        public void DeleteJob(string jobName)
+        {
+            if (_settings.RunFromZip())
+            {
+                SecondaryJobManager.DeleteJob(jobName);
+            }
+            else
+            {
+                // Make sure to delete the job from both managers if it exists.
+                // Calling DeleteJob on a non-existing job is a no-op.
+                PrimaryJobManager.DeleteJob(jobName);
+                SecondaryJobManager.DeleteJob(jobName);
+            }
+        }
+
+        public void CleanupDeletedJobs()
+        {
+            PrimaryJobManager.CleanupDeletedJobs();
+            SecondaryJobManager.CleanupDeletedJobs();
+        }
+
+        public TJob CreateOrReplaceJobFromFileStream(Stream scriptFileStream, string jobName, string scriptFileName)
+            => GetWriteJobManagerForJob(jobName).CreateOrReplaceJobFromFileStream(scriptFileStream, jobName, scriptFileName);
+
+        public TJob CreateOrReplaceJobFromZipStream(Stream zipStream, string jobName)
+            => GetWriteJobManagerForJob(jobName).CreateOrReplaceJobFromZipStream(zipStream, jobName);
+
+        public TJob GetJob(string jobName) 
+            => PrimaryJobManager.HasJob(jobName) ? PrimaryJobManager.GetJob(jobName) : SecondaryJobManager.GetJob(jobName);
+
+        public JobSettings GetJobSettings(string jobName)
+            => PrimaryJobManager.HasJob(jobName) ? PrimaryJobManager.GetJobSettings(jobName) : SecondaryJobManager.GetJobSettings(jobName);
+
+        public bool HasJob(string jobName)
+            => PrimaryJobManager.HasJob(jobName) || SecondaryJobManager.HasJob(jobName);
+
+        public IEnumerable<TJob> ListJobs(bool forceRefreshCache)
+            => PrimaryJobManager.ListJobs(forceRefreshCache).Concat(SecondaryJobManager.ListJobs(forceRefreshCache));
+
+        public void RegisterExtraEventHandlerForFileChange(Action<string> action)
+        {
+            PrimaryJobManager.RegisterExtraEventHandlerForFileChange(action);
+            SecondaryJobManager.RegisterExtraEventHandlerForFileChange(action);
+        }
+
+        public void SetJobSettings(string jobName, JobSettings jobSettings)
+            => GetWriteJobManagerForJob(jobName).SetJobSettings(jobName, jobSettings);
+
+        // Always sync webjobs brought in by site extensions
+        // into the writable location.
+        public void SyncExternalJobs(string sourcePath, string sourceName)
+            => WritableJobManager.SyncExternalJobs(sourcePath, sourceName);
+
+        // Always sync webjobs brought in by site extensions
+        // into the writable location.
+        public void CleanupExternalJobs(string sourceName)
+            => WritableJobManager.CleanupExternalJobs(sourceName);
+
+        // Writable manager is secondary if run from zip, and primary otherwise.
+        private JobsManagerBase<TJob> WritableJobManager => _settings.RunFromZip()
+            ? SecondaryJobManager
+            : PrimaryJobManager;
+
+        // This checks both run from zip, and Primary.HasJob()
+        // The point is that if we are in run from zip, we always use the secondary.
+        // Otherwise, we use the manager where the job exists. This can be either the primary or the secondary.
+        protected JobsManagerBase<TJob> GetWriteJobManagerForJob(string jobName)
+        {
+            if (_settings.RunFromZip())
+            {
+                return SecondaryJobManager;
+            }
+            else if (PrimaryJobManager.HasJob(jobName))
+            {
+                return PrimaryJobManager;
+            }
+            else if (SecondaryJobManager.HasJob(jobName))
+            {
+                return SecondaryJobManager;
+            }
+            else
+            {
+                return PrimaryJobManager;
+            }
+        }
+    }
+}

Kudu.Core/Jobs/AggregrateTriggeredJobsManager.cs

@@ -0,0 +1,35 @@
+using System;
+using Kudu.Contracts.Jobs;
+using Kudu.Contracts.Settings;
+using Kudu.Core.Hooks;
+using Kudu.Core.Tracing;
+
+namespace Kudu.Core.Jobs
+{
+    public class AggregrateTriggeredJobsManager : AggregrateJobsManagerBase<TriggeredJob>, ITriggeredJobsManager
+    {
+        public AggregrateTriggeredJobsManager(ITraceFactory traceFactory, IEnvironment environment, IDeploymentSettingsManager settings, IAnalytics analytics, IWebHooksManager hooksManager)
+            : base(new TriggeredJobsManager(environment.JobsBinariesPath, traceFactory, environment, settings, analytics, hooksManager),
+                  excludedList => new TriggeredJobsManager(environment.SecondaryJobsBinariesPath, traceFactory, environment, settings, analytics, hooksManager, excludedList),
+                  settings)
+        {
+        }
+
+        public TriggeredJobHistory GetJobHistory(string jobName, string etag, out string currentETag)
+            => GetManagerForJob(jobName).GetJobHistory(jobName, etag, out currentETag);
+
+        public TriggeredJobRun GetJobRun(string jobName, string runId)
+            => GetManagerForJob(jobName).GetJobRun(jobName, runId);
+
+        public TriggeredJobRun GetLatestJobRun(string jobName)
+            => GetManagerForJob(jobName).GetLatestJobRun(jobName);
+
+        public Uri InvokeTriggeredJob(string jobName, string arguments, string trigger)
+            => GetManagerForJob(jobName).InvokeTriggeredJob(jobName, arguments, trigger);
+
+        private ITriggeredJobsManager GetManagerForJob(string jobName)
+            => PrimaryJobManager.HasJob(jobName)
+            ? PrimaryJobManager as ITriggeredJobsManager
+            : SecondaryJobManager as ITriggeredJobsManager;
+    }
+}

Kudu.Core/Jobs/BaseJobRunner.cs

@@ -29,7 +29,7 @@ public abstract class BaseJobRunner
         private string _inPlaceWorkingDirectory;
         private Dictionary<string, FileInfoBase> _cachedSourceDirectoryFileMap;
 
-        protected BaseJobRunner(string jobName, string jobsTypePath, IEnvironment environment,
+        protected BaseJobRunner(string jobName, string jobsTypePath, string basePath, IEnvironment environment,
             IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics)
         {
             TraceFactory = traceFactory;
@@ -38,7 +38,8 @@ protected BaseJobRunner(string jobName, string jobsTypePath, IEnvironment enviro
             JobName = jobName;
             _analytics = analytics;
 
-            JobBinariesPath = Path.Combine(Environment.JobsBinariesPath, jobsTypePath, jobName);
+            JobBinariesPath = Path.Combine(basePath, jobName);
+
             JobTempPath = Path.Combine(Environment.TempPath, Constants.JobsPath, jobsTypePath, jobName);
             JobDataPath = Path.Combine(Environment.DataPath, Constants.JobsPath, jobsTypePath, jobName);
 

Kudu.Core/Jobs/ContinuousJobRunner.cs

@@ -28,8 +28,8 @@ public class ContinuousJobRunner : BaseJobRunner, IDisposable
         private bool _alwaysOnWarningLogged;
         private bool? _isSingleton;
 
-        public ContinuousJobRunner(ContinuousJob continuousJob, IEnvironment environment, IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics)
-            : base(continuousJob.Name, Constants.ContinuousPath, environment, settings, traceFactory, analytics)
+        public ContinuousJobRunner(ContinuousJob continuousJob, string basePath, IEnvironment environment, IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics)
+            : base(continuousJob.Name, Constants.ContinuousPath, basePath, environment, settings, traceFactory, analytics)
         {
             _analytics = analytics;
             _continuousJobLogger = new ContinuousJobLogger(continuousJob.Name, Environment, TraceFactory);

Kudu.Core/Jobs/ContinuousJobsManager.cs

@@ -22,8 +22,8 @@ public class ContinuousJobsManager : JobsManagerBase<ContinuousJob>, IContinuous
 
         private readonly Dictionary<string, ContinuousJobRunner> _continuousJobRunners = new Dictionary<string, ContinuousJobRunner>(StringComparer.OrdinalIgnoreCase);
 
-        public ContinuousJobsManager(ITraceFactory traceFactory, IEnvironment environment, IDeploymentSettingsManager settings, IAnalytics analytics)
-            : base(traceFactory, environment, settings, analytics, Constants.ContinuousPath)
+        public ContinuousJobsManager(string basePath, ITraceFactory traceFactory, IEnvironment environment, IDeploymentSettingsManager settings, IAnalytics analytics, IEnumerable<string> excludedJobsNames = null)
+            : base(traceFactory, environment, settings, analytics, Constants.ContinuousPath, basePath, excludedJobsNames)
         {
             RegisterExtraEventHandlerForFileChange(OnJobChanged);
         }
@@ -216,7 +216,7 @@ private void RefreshJob(ContinuousJob continuousJob, bool logRefresh)
             ContinuousJobRunner continuousJobRunner;
             if (!_continuousJobRunners.TryGetValue(continuousJob.Name, out continuousJobRunner))
             {
-                continuousJobRunner = new ContinuousJobRunner(continuousJob, Environment, Settings, TraceFactory, Analytics);
+                continuousJobRunner = new ContinuousJobRunner(continuousJob, JobsBinariesPath, Environment, Settings, TraceFactory, Analytics);
                 _continuousJobRunners.Add(continuousJob.Name, continuousJobRunner);
             }
 

Kudu.Core/Jobs/JobsManagerBase.cs

@@ -64,25 +64,28 @@ public static bool IsUsingSdk(string specificJobDataPath)
 
         protected IAnalytics Analytics { get; private set; }
 
+        private readonly IEnumerable<string> _excludedJobsNames;
+
         protected JobsFileWatcher JobsWatcher { get; set; }
-        internal static IEnumerable<TJob> JobListCache { get; set; }
+
+        internal static IDictionary<string, IEnumerable<TJob>> CacheMap { get; } = new Dictionary<string, IEnumerable<TJob>>();
 
         private DateTime _jobListCacheExpiryDate;
 
         private const int _jobListCacheTimeOutInMinutes = 10;
 
         List<Action<string>> FileWatcherExtraEventHandlers;
 
-        protected JobsManagerBase(ITraceFactory traceFactory, IEnvironment environment, IDeploymentSettingsManager settings, IAnalytics analytics, string jobsTypePath)
+        protected JobsManagerBase(ITraceFactory traceFactory, IEnvironment environment, IDeploymentSettingsManager settings, IAnalytics analytics, string jobsTypePath, string basePath, IEnumerable<string> excludedJobsNames)
         {
             TraceFactory = traceFactory;
             Environment = environment;
             Settings = settings;
             Analytics = analytics;
 
+            _excludedJobsNames = excludedJobsNames ?? Enumerable.Empty<string>();
             _jobsTypePath = jobsTypePath;
-
-            JobsBinariesPath = Path.Combine(Environment.JobsBinariesPath, jobsTypePath);
+            JobsBinariesPath = Path.Combine(basePath, jobsTypePath);
             JobsDataPath = Path.Combine(Environment.JobsDataPath, jobsTypePath);
             JobsWatcher = new JobsFileWatcher(JobsBinariesPath, OnJobChanged, null, ListJobNames, traceFactory, analytics, jobsTypePath);
             HostingEnvironment.RegisterObject(this);
@@ -130,13 +133,30 @@ public IEnumerable<TJob> ListJobs(bool forceRefreshCache)
             return jobList;
         }
 
+        internal IEnumerable<TJob> JobListCache
+        {
+            get
+            {
+                return CacheMap.ContainsKey(JobsBinariesPath)
+                    ? CacheMap[JobsBinariesPath]
+                    : null;
+            }
+
+            set
+            {
+                CacheMap[JobsBinariesPath] = value;
+            }
+        }
+
         internal static void ClearJobListCache()
         {
-            JobListCache = null;
+            CacheMap.Clear();
         }
 
         public abstract TJob GetJob(string jobName);
 
+        public bool HasJob(string jobName) => FileSystemHelpers.DirectoryExists(Path.Combine(JobsBinariesPath, jobName));
+
         public TJob CreateOrReplaceJobFromZipStream(Stream zipStream, string jobName)
         {
             return CreateOrReplaceJob(jobName,
@@ -245,7 +265,8 @@ protected IEnumerable<TJob> ListJobsInternal()
         {
             var jobs = new List<TJob>();
 
-            IEnumerable<DirectoryInfoBase> jobDirectories = ListJobDirectories(JobsBinariesPath);
+            IEnumerable<DirectoryInfoBase> jobDirectories = ListJobDirectories(JobsBinariesPath)
+                .Where(d => !_excludedJobsNames.Any(e => d.Name.Equals(e, StringComparison.OrdinalIgnoreCase)));
             foreach (DirectoryInfoBase jobDirectory in jobDirectories)
             {
                 TJob job = BuildJob(jobDirectory);