diff --git a/src/Agni/Agni.csproj b/src/Agni/Agni.csproj new file mode 100644 index 0000000..a9847a7 --- /dev/null +++ b/src/Agni/Agni.csproj @@ -0,0 +1,115 @@ + + + + netstandard2.0 + Agni OS Main Assembly + + + + + ..\..\out\Debug\ + ..\..\out\Debug\Agni.xml + true + + + + ..\..\out\Release\ + ..\..\out\Release\Agni.xml + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ..\lib\nfx\NFX.dll + + + ..\lib\nfx\NFX.Wave.dll + + + ..\lib\nfx\NFX.Web.dll + + + + + + + + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + + diff --git a/src/Agni/AgniException.cs b/src/Agni/AgniException.cs new file mode 100644 index 0000000..3558f3f --- /dev/null +++ b/src/Agni/AgniException.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni +{ + /// + /// Marker interfaces for all Agni exceptions + /// + public interface IAgniException + { + } + + /// + /// Base exception thrown by the framework + /// + [Serializable] + public class AgniException : NFXException, IAgniException + { + public const string SENDER_FLD_NAME = "AE-S"; + public const string TOPIC_FLD_NAME = "AE-T"; + + public static string DefaultSender; + public static string DefaultTopic; + + public readonly string Sender; + public readonly string Topic; + + + public AgniException() + { + Sender = DefaultSender; + Topic = DefaultTopic; + } + + public AgniException(int code) + { + Code = code; + Sender = DefaultSender; + Topic = DefaultTopic; + } + + public AgniException(int code, string message) : this(message, null, code, null, null) {} + public AgniException(string message) : this(message, null, 0, null, null) { } + public AgniException(string message, Exception inner) : this(message, inner, 0, null, null) { } + + public AgniException(string message, Exception inner, int code, string sender, string topic) : base(message, inner) + { + Code = code; + Sender = sender ?? DefaultSender; + Topic = topic ?? DefaultTopic; + } + + protected AgniException(SerializationInfo info, StreamingContext context) : base(info, context) + { + Sender = info.GetString(SENDER_FLD_NAME); + Topic = info.GetString(TOPIC_FLD_NAME); + } + + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + if (info == null) + throw new NFXException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".GetObjectData(info=null)"); + info.AddValue(SENDER_FLD_NAME, Sender); + info.AddValue(TOPIC_FLD_NAME, Topic); + base.GetObjectData(info, context); + } + } +} diff --git a/src/Agni/AgniExtensions.cs b/src/Agni/AgniExtensions.cs new file mode 100644 index 0000000..324c077 --- /dev/null +++ b/src/Agni/AgniExtensions.cs @@ -0,0 +1,199 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Glue; +using NFX.Environment; + +namespace Agni +{ + public static class AgniExtensions + { + /// + /// Tries to resolve mnemonic name of the Agni service into port, i.e. "hgov" into int port number + /// + public static Node ToResolvedServiceNode(this string connectString, bool appTerminal = false) + { + return ToResolvedServiceNode(new Node(connectString), appTerminal); + } + + /// + /// Tries to resolve mnemonic name of the Agni service into port, i.e. "hgov" into int port number + /// + public static Node ToResolvedServiceNode(this Node node, bool appTerminal = false) + { + if (!node.Assigned ) return node; + + if (node.Binding.IsNullOrWhiteSpace()) + node = new Node("{0}://{1}:{2}".Args(SysConsts.DEFAULT_BINDING, node.Host, node.Service)); + + int p; + if (int.TryParse(node.Service, out p)) return node; + + var sync = node.Binding.Trim().EqualsIgnoreCase(SysConsts.SYNC_BINDING); + var port = ServiceNameToPort(node.Service, sync, appTerminal); + return new Node("{0}://{1}:{2}".Args(node.Binding, node.Host, port)); + } + + /// + /// Translates mnemonic name of the major service (i.e. "hgov") into its default port + /// + public static int ServiceNameToPort(string service, bool sync, bool appTerminal) + { + var result = SysConsts.DEFAULT_ZONE_GOV_SVC_SYNC_PORT; + if (service.IsNotNullOrWhiteSpace()) + { + service = service.Trim().ToLowerInvariant(); + if (service=="ahgov" || + service=="hgov" || + service=="hgv" || + service=="h") result = SysConsts.DEFAULT_HOST_GOV_SVC_SYNC_PORT; + + else if (service=="azgov" || + service=="zgov" || + service=="zgv" || + service=="z") result = SysConsts.DEFAULT_ZONE_GOV_SVC_SYNC_PORT; + + else if (service=="agdida" || + service == "gdida" || + service == "gdid" || + service=="id" || + service=="g") result = SysConsts.DEFAULT_GDID_AUTH_SVC_SYNC_PORT; + + else if (service=="aws" || + service=="www" || + service=="web" || + service=="http") result = SysConsts.DEFAULT_AWS_SVC_SYNC_PORT; + + else if (service=="aph" || + service=="proc" || + service=="ph") result = SysConsts.DEFAULT_PH_SVC_SYNC_PORT; + + else if (service=="log") result = SysConsts.DEFAULT_LOG_SVC_SYNC_PORT; + + else if (service=="telem" || + service=="telemetry" || + service=="tlm") result = SysConsts.DEFAULT_TELEMETRY_SVC_SYNC_PORT; + + else if (service=="wm" || + service=="msg" || + service=="wmsg") result = SysConsts.DEFAULT_WEB_MESSAGE_SYSTEM_SVC_APPTERM_PORT; + + else if (service=="ash" || service=="sh") + { + if (appTerminal) return SysConsts.DEFAULT_ASH_APPTERM_PORT; + else throw new AgniException("Not supported ASH service"); + } + else throw new AgniException("Not supported service: `{0}`".Args(service)); + } + + return appTerminal ? result + SysConsts.APP_TERMINAL_PORT_OFFSET : sync ? result : result+1; + } + + + /// + /// Checks two strings for region paths that reference the same regional entity disregarding entity extensions (such as '.r' or '.noc'). + /// Note: this method DOES NOT check whether this path resolves to actual catalog entity as it only compares names. + /// This function should be used in conjunction with GetRegionPathHashCode() while implementing Equals/GetHashCode. + /// Ignores dynamic host name suffixes + /// + public static bool IsSameRegionPath(this string path1, string path2) + { + if (path1==null || path2==null) return false; + + var segs1 = path1.Split('/').Where(s=>s.IsNotNullOrWhiteSpace()).ToList(); + var segs2 = path2.Split('/').Where(s=>s.IsNotNullOrWhiteSpace()).ToList(); + + if (segs1.Count!=segs2.Count) return false; + + for(var i=0; i0) seg1 = seg1.Substring(0, si).Trim(); + si = seg2.LastIndexOf(Metabase.Metabank.HOST_DYNAMIC_SUFFIX_SEPARATOR); if (si>0) seg2 = seg2.Substring(0, si).Trim(); + } + + var di = seg1.LastIndexOf('.'); if (di>0) seg1 = seg1.Substring(0, di); + di = seg2.LastIndexOf('.'); if (di>0) seg2 = seg2.Substring(0, di); + + if (!seg1.EqualsIgnoreCase( seg2 )) return false; + } + + return true; + } + + /// + /// Deletes region extensions (such as '.r' or '.noc') from path + /// Ignores dynamic host name suffixes + /// + public static string StripPathOfRegionExtensions(this string path) + { + if (path==null) return null; + + var segs = path.Split('/').Where(s=>s.IsNotNullOrWhiteSpace()).ToList(); + var result = new StringBuilder(); + for(var i=0; i0) seg = seg.Substring(0, si).Trim(); + } + + var di = seg.LastIndexOf('.'); + if (di>0) seg = seg.Substring(0, di); + + if (i>0) result.Append('/'); + result.Append(seg.Trim()); + } + + return result.ToString(); + } + + /// + /// Computes has code for region path string disregarding case and extensions (such as '.r' or '.noc') + /// Note: This function should be used in conjunction with IsSameRegionPath() while implementing Equals/GetHashCode + /// Ignores dynamic host name suffixes + /// + public static int GetRegionPathHashCode(this string path) + { + var stripped = path.StripPathOfRegionExtensions(); + if (stripped==null) return 0; + return stripped.ToLowerInvariant().GetHashCode(); + } + + + /// + /// Returns true if the supplied string is a valid name + /// a name that does not contain SysConsts.NAME_INVALID_CHARS + /// + public static bool IsValidName(this string name) + { + if (name==null) return false; + + var started = false; + var ws = false; + + for(var i=0; i + /// Provides a shortcut access to app-global Agni context + /// + public static class AgniSystem + { + private static BuildInformation s_CoreBuildInfo; + + /// + /// Returns BuildInformation object for the core agni assembly + /// + public static BuildInformation CoreBuildInfo + { + get + { + //multithreading: 2nd copy is ok + if (s_CoreBuildInfo == null) + s_CoreBuildInfo = new BuildInformation(typeof(AgniSystem).Assembly); + + return s_CoreBuildInfo; + } + } + + private static string s_MetabaseApplicationName; + + /// + /// Every agni application MUST ASSIGN THIS property at its entry point ONCE. Example: void Main(string[]args){ AgniSystem.MetabaseApplicationName = "MyApp1";... + /// + public static string MetabaseApplicationName + { + get { return s_MetabaseApplicationName; } + set + { + if (s_MetabaseApplicationName != null || value.IsNullOrWhiteSpace()) + throw new AgniException(StringConsts.METABASE_APP_NAME_ASSIGNMENT_ERROR); + s_MetabaseApplicationName = value; + } + } + + + /// + /// Returns instance of agni application container that this AgniSystem services + /// + public static IAgniApplication Application + { + get { return (App.Instance as IAgniApplication) ?? (IAgniApplication)AgniNOPApplication.Instance; } + } + + /// + /// Denotes system application/process type that this app container has, i.e.: HostGovernor, WebServer, etc. + /// + public static SystemApplicationType SystemApplicationType { get { return Application.SystemApplicationType; } } + + /// + /// Returns current instance + /// + public static IAgniSystem Instance { get { return Application.TheSystem; } } + + /// + /// Returns true when AgniSystem is active non-NOP instance + /// + public static bool Available { get { return Instance != null && Instance.Available; } } + + /// + /// References application configuration root used to boot this application instance + /// + public static IConfigSectionNode BootConfigRoot { get { return Application.BootConfigRoot; } } + + /// + /// Host name of this machine as determined at boot. This is a shortcut to Agni.AppModel.BootConfLoader.HostName + /// + public static string HostName { get { return BootConfLoader.HostName; } } + + + /// + /// True if this host is dynamic + /// + public static bool DynamicHost { get { return BootConfLoader.DynamicHost; } } + + + /// + /// Returns parent zone governor host name or null if this is the top-level host in Agni. + /// This is a shortcut to Agni.AppModel.BootConfLoader.ParentZoneGovernorPrimaryHostName + /// + public static string ParentZoneGovernorPrimaryHostName { get { return BootConfLoader.ParentZoneGovernorPrimaryHostName; } } + + + /// + /// NOC name for this host as determined at boot + /// + public static string NOCName { get { return NOCMetabaseSection.Name; } } + + + /// + /// True when metabase is mounted!=null + /// + public static bool IsMetabase { get { return BootConfLoader.Metabase != null; } } + + + /// + /// Returns metabank instance that interfaces the metabase as determined at application boot. + /// If metabase is null then exception is thrown. Use IsMetabase to test for null instead + /// + public static Metabank Metabase + { + get + { + var result = BootConfLoader.Metabase; + + if (result == null) + { + var trace = new System.Diagnostics.StackTrace(false); + throw new AgniException(StringConsts.METABASE_NOT_AVAILABLE_ERROR.Args(trace.ToString())); + } + + return result; + } + } + + /// + /// Returns Metabank.SectionHost (metabase's information about this host) + /// + public static Metabank.SectionHost HostMetabaseSection { get { return Metabase.CatalogReg.NavigateHost(HostName); } } + + /// + /// Returns Metabank.SectionNOC (metabase's information about the NOC this host is in) + /// + public static Metabank.SectionNOC NOCMetabaseSection { get { return HostMetabaseSection.NOC; } } + + + /// + /// Returns Agni distributed lock manager + /// + public static Locking.ILockManager LockManager { get { return Application.LockManager; } } + + /// + /// References distributed GDID provider + /// + public static IGDIDProvider GDIDProvider { get { return Application.GDIDProvider; } } + + /// + /// Returns Agni distributed process manager + /// + public static Workers.IProcessManager ProcessManager { get { return Application.ProcessManager; } } + + /// + /// Returns Agni distributed dynamic host manager + /// + public static Dynamic.IHostManager DynamicHostManager { get { return Application.DynamicHostManager; } } + } +} diff --git a/src/Agni/AppModel/AgniNOPApplication.cs b/src/Agni/AppModel/AgniNOPApplication.cs new file mode 100644 index 0000000..d6a368e --- /dev/null +++ b/src/Agni/AppModel/AgniNOPApplication.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Agni.Dynamic; +using Agni.Workers; +using NFX.ApplicationModel; +using NFX.DataAccess; +using NFX.Environment; + +namespace Agni.AppModel +{ + /// + /// Represents an application that consists of pure-nop providers, consequently + /// this application does not log, does not store data and does not do anything else + /// still satisfying its contract + /// + public class AgniNOPApplication : NOPApplication, IAgniApplication + { + private static AgniNOPApplication s_Instance = new AgniNOPApplication(); + + protected AgniNOPApplication() : base() {} + + /// + /// Returns a singlelton instance of the AgniNOPApplication + /// + public static new AgniNOPApplication Instance { get { return s_Instance; } } + + public string MetabaseApplicationName { get { return string.Empty; } } + + public IAgniSystem TheSystem { get { return NOPAgniSystem.Instance; } } + + public IConfigSectionNode BootConfigRoot { get { return m_Configuration.Root; } } + + public bool ConfiguredFromLocalBootConfig { get { return false; } } + + public SystemApplicationType SystemApplicationType { get { return SystemApplicationType.Unspecified; } } + + public Locking.ILockManager LockManager { get { return Locking.NOPLockManager.Instance; } } + + public IGDIDProvider GDIDProvider { get { throw new NotSupportedException("NOPApp.GDIDProvider"); } } + + public IProcessManager ProcessManager { get { throw new NotSupportedException("NOPApp.ProcessManager"); } } + + public IHostManager DynamicHostManager { get { throw new NotSupportedException("NOPApp.HostManager"); } } + } +} diff --git a/src/Agni/AppModel/AgniServiceApplication.cs b/src/Agni/AppModel/AgniServiceApplication.cs new file mode 100644 index 0000000..80990be --- /dev/null +++ b/src/Agni/AppModel/AgniServiceApplication.cs @@ -0,0 +1,358 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Log; +using NFX.ApplicationModel; +using NFX.ServiceModel; +using NFX.Environment; +using NFX.DataAccess; +using NFX.Wave; + +using Agni.Identification; + +namespace Agni.AppModel +{ + /// + /// Provides base implementation of IAgniApplication for applications like services and console apps. + /// This class IS thread safe + /// + public class AgniServiceApplication : ServiceBaseApplication, IAgniApplication + { + #region CONSTS + + public const string CONFIG_WEB_MANAGER_SECTION = "web-manager"; + + public const string CONFIG_LOCK_MANAGER_SECTION = "lock-manager"; + + public const string CONFIG_PROCESS_MANAGER_SECTION = "process-manager"; + + public const string CONFIG_HOST_MANAGER_SECTION = "host-manager"; + + #endregion + + #region .ctor + + public AgniServiceApplication(SystemApplicationType sysAppType, string[] args, ConfigSectionNode rootConfig) + : base(BootConfLoader.SetSystemApplicationType(sysAppType, args), rootConfig) + {} + + protected override void Destructor() + { + BootConfLoader.Unload(); + base.Destructor(); + } + + #endregion + + #region Fields + + protected IAgniSystem m_TheSystem; + private ConfigSectionNode m_BootConfigRoot; + + private WaveServer m_WebManagerServer; + + private Locking.ILockManagerImplementation m_LockManager; + private GDIDGenerator m_GDIDProvider; + private Workers.IProcessManagerImplementation m_ProcessManager; + private Dynamic.IHostManagerImplementation m_DynamicHostManager; + + #endregion + + #region Properties + + /// + /// References a singleton instance of AgniServiceApplication + /// + public static AgniServiceApplication Instance { get{ return App.Instance as AgniServiceApplication; } } + + /// + /// Denotes system application/process type that this app container has, i.e.: HostGovernor, WebServer, etc. + /// The value is set in .ctor and kept in BootConfLoader.SystemApplicationType + /// + public SystemApplicationType SystemApplicationType { get {return BootConfLoader.SystemApplicationType; } } + + + public string MetabaseApplicationName { get{ return Agni.AgniSystem.MetabaseApplicationName; } } + + + public IConfigSectionNode BootConfigRoot { get { return m_BootConfigRoot; } } + public IAgniSystem TheSystem { get{ return m_TheSystem;} } + + internal WaveServer WebManagerServer{ get{return m_WebManagerServer;}} + + public Locking.ILockManager LockManager { get{ return m_LockManager ?? Locking.NOPLockManager.Instance; } } + + public IGDIDProvider GDIDProvider { get { return m_GDIDProvider; } } + + public Workers.IProcessManager ProcessManager { get { return m_ProcessManager; } } + + public Dynamic.IHostManager DynamicHostManager { get { return m_DynamicHostManager; } } + #endregion + + #region Protected + + protected override Configuration GetConfiguration() + { + var localConfig = base.GetConfiguration(); + + BootConfLoader.ProcessAllExistingIncludes(localConfig.Root, null, "boot"); + + m_BootConfigRoot = localConfig.Root; + + var cmdArgs = new string[]{}; + + if (CommandArgs.Configuration is CommandArgsConfiguration) + cmdArgs = ((CommandArgsConfiguration)this.CommandArgs.Configuration).Arguments; + + return BootConfLoader.Load(cmdArgs, localConfig); + } + + protected override void DoInitApplication() + { + base.DoInitApplication(); + + var FROM = GetType().FullName+".DoInitApplication()"; + + var csvc = new AgniSystemBase(this); + csvc.Start(); + + m_TheSystem = csvc; + var metabase = BootConfLoader.Metabase; + + try + { + m_GDIDProvider = new GDIDGenerator("Agni", this); + + foreach(var ah in metabase.GDIDAuthorities) + { + m_GDIDProvider.AuthorityHosts.Register(ah); + WriteLog(MessageType.Info, FROM+"{GDIDProvider init}", "Registered GDID authority host: "+ah.ToString()); + } + + WriteLog(MessageType.Info, FROM, "GDIProvider made"); + } + catch(Exception error) + { + WriteLog(MessageType.CatastrophicError, FROM+"{GDIDProvider init}", error.ToMessageWithType()); + try + { + m_GDIDProvider.Dispose(); + } + catch{ } + + m_GDIDProvider = null; + } + + var wmSection = ConfigRoot[CONFIG_WEB_MANAGER_SECTION]; + if (wmSection.Exists && wmSection.AttrByName(CONFIG_ENABLED_ATTR).ValueAsBool(false)) + try + { + m_WebManagerServer = new WaveServer(); + m_WebManagerServer.Configure(wmSection); + m_WebManagerServer.Start(); + } + catch(Exception error) + { + WriteLog(MessageType.CatastrophicError, FROM+"{WebManagerServer start}", error.ToMessageWithType()); + try + { + m_WebManagerServer.Dispose(); + } + catch{} + + m_WebManagerServer = null; + } + + var lockSection = ConfigRoot[CONFIG_LOCK_MANAGER_SECTION]; + try + { + m_LockManager = FactoryUtils.MakeAndConfigure(lockSection, typeof(Locking.LockManager)); + + WriteLog(MessageType.Info, FROM, "Lock Manager made"); + + if (m_LockManager is Service) + { + ((Service)m_LockManager).Start(); + WriteLog(MessageType.Info, FROM, "Lock Manager STARTED"); + } + } + catch(Exception error) + { + WriteLog(MessageType.CatastrophicError, FROM+"{LockManager start}", error.ToMessageWithType()); + try + { + m_LockManager.Dispose(); + } + catch{} + + m_LockManager = null; + } + + var procSection = ConfigRoot[CONFIG_PROCESS_MANAGER_SECTION]; + try + { + m_ProcessManager = FactoryUtils.MakeAndConfigure(procSection, typeof(Workers.ProcessManager), new object[] { this }); + + WriteLog(MessageType.Info, FROM, "Process Manager made"); + + if (m_ProcessManager is Service) + { + ((Service)m_ProcessManager).Start(); + WriteLog(MessageType.Info, FROM, "Process Manager STARTED"); + } + } + catch (Exception error) + { + WriteLog(MessageType.CatastrophicError, FROM+"{ProcessManager start}", error.ToMessageWithType()); + try + { + m_ProcessManager.Dispose(); + } + catch{} + + m_ProcessManager = null; + } + + var hostSection = ConfigRoot[CONFIG_HOST_MANAGER_SECTION]; + try + { + m_DynamicHostManager = FactoryUtils.MakeAndConfigure(procSection, typeof(Dynamic.HostManager), new object[] { this }); + + WriteLog(MessageType.Info, FROM, "Dynamic Host Manager made"); + + if (m_DynamicHostManager is Service) + { + ((Service)m_DynamicHostManager).Start(); + WriteLog(MessageType.Info, FROM, "Dynamic Host Manager STARTED"); + } + } + catch (Exception error) + { + WriteLog(MessageType.CatastrophicError, FROM+ "{HostManager start}", error.ToMessageWithType()); + try + { + m_DynamicHostManager.Dispose(); + } + catch{} + + m_DynamicHostManager = null; + } + } + + protected override void DoCleanupApplication() + { + var FROM = GetType().FullName+".DoCleanupApplication()"; + + if (m_DynamicHostManager != null) + { + WriteLog(MessageType.Info, FROM, "Finalizing Dynamic Host Manager"); + try + { + if (m_DynamicHostManager is Service) + { + ((Service)m_DynamicHostManager).SignalStop(); + ((Service)m_DynamicHostManager).WaitForCompleteStop(); + WriteLog(MessageType.Info, FROM, "Dynamic Host Manager STOPPED"); + } + + DisposableObject.DisposeAndNull(ref m_DynamicHostManager); + WriteLog(MessageType.Info, FROM, "Dynamic Host Manager DISPOSED"); + } + catch(Exception error) + { + WriteLog(MessageType.Error, FROM, "ERROR finalizing Dynamic Host Manager: " + error.ToMessageWithType()); + } + } + + if (m_ProcessManager!=null) + { + WriteLog(MessageType.Info, FROM, "Finalizing Process Manager"); + try + { + if (m_ProcessManager is Service) + { + ((Service)m_ProcessManager).SignalStop(); + ((Service)m_ProcessManager).WaitForCompleteStop(); + WriteLog(MessageType.Info, FROM, "Process Manager STOPPED"); + } + + DisposableObject.DisposeAndNull(ref m_ProcessManager); + WriteLog(MessageType.Info, FROM, "Process Manager DISPOSED"); + } + catch(Exception error) + { + WriteLog(MessageType.Error, FROM, "ERROR finalizing Process Manager: " + error.ToMessageWithType()); + } + } + + if (m_LockManager!=null) + { + WriteLog(MessageType.Info, FROM, "Finalizing Lock Manager"); + try + { + if (m_LockManager is Service) + { + ((Service)m_LockManager).SignalStop(); + ((Service)m_LockManager).WaitForCompleteStop(); + WriteLog(MessageType.Info, FROM, "Lock Manager STOPPED"); + } + + DisposableObject.DisposeAndNull(ref m_LockManager); + WriteLog(MessageType.Info, FROM, "lock manager DISPOSED"); + } + catch(Exception error) + { + WriteLog(MessageType.Error, FROM, "ERROR finalizing Lock Manager: " + error.ToMessageWithType()); + } + } + + if (m_WebManagerServer!=null) + { + WriteLog(MessageType.Info, FROM, "Finalizing Web Manager Server"); + try + { + DisposableObject.DisposeAndNull(ref m_WebManagerServer); + WriteLog(MessageType.Info, FROM, "Web Manager Server DISPOSED"); + } + catch (Exception error) + { + WriteLog(MessageType.CatastrophicError, FROM, "ERROR finalizing Web Manager Server: " + error.ToMessageWithType()); + } + } + + if (m_GDIDProvider!=null) + { + WriteLog(MessageType.Info, FROM, "Finalizing GDIDProvider"); + try + { + DisposableObject.DisposeAndNull(ref m_GDIDProvider); + WriteLog(MessageType.Info, FROM, "GDIDProvider DISPOSED"); + } + catch(Exception error) + { + WriteLog(MessageType.Error, FROM, "ERROR finalizing GDIDProvider: " + error.ToMessageWithType()); + } + } + + //Turn off Node - must be right before shutdown + WriteLog(MessageType.Info, FROM, "Finalizing TheSystem"); + try + { + DisposableObject.DisposeAndNull(ref m_TheSystem); + WriteLog(MessageType.Info, FROM, "TheSystem DISPOSED"); + } + catch(Exception error) + { + WriteLog(MessageType.CatastrophicError, FROM, "ERROR finalizing TheSystem: " + error.ToMessageWithType()); + } + + // Shutdown - must be last + base.DoCleanupApplication(); + } + + #endregion + } +} diff --git a/src/Agni/AppModel/AgniSystemBase.cs b/src/Agni/AppModel/AgniSystemBase.cs new file mode 100644 index 0000000..399a0e3 --- /dev/null +++ b/src/Agni/AppModel/AgniSystemBase.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ServiceModel; + +namespace Agni.AppModel +{ + /// + /// Provides base AgniSystem implementation + /// + public class AgniSystemBase : Service, IAgniSystem + { + public const string AGNI_COMPONENT_NAME = "agni"; + + #region .ctor + internal AgniSystemBase(IAgniApplication app) : base(app) + { + + } + #endregion + + public override string ComponentCommonName { get { return AGNI_COMPONENT_NAME; } } + + public bool Available { get { return false; } } + + IOperationalStatus IAgniSystem.Status + { + get + { + throw new NotImplementedException(); + } + } + + #region Protected + protected override void DoStart() + { + base.DoStart(); + } + + protected override void DoWaitForCompleteStop() + { + base.DoWaitForCompleteStop(); + } + #endregion + } +} diff --git a/src/Agni/AppModel/BootConfLoader.cs b/src/Agni/AppModel/BootConfLoader.cs new file mode 100644 index 0000000..5a53b72 --- /dev/null +++ b/src/Agni/AppModel/BootConfLoader.cs @@ -0,0 +1,475 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Security; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX.ApplicationModel; + +using Agni.Identification; +using Agni.Metabase; + +namespace Agni.AppModel +{ + /// + /// Gets the boot configuration for app container. + /// Reads the local process config to determine where the metabase is and what file system to use to connect to it. + /// Once metabase connection is established get all information form there identifying the host by agni/Host/$name. + /// If the name of the host is not set in config, then take it from AGNI_HOST_NAME environment var. If that name is blank then + /// take host name from: DEFAULT_WORLD_GLOBAL_ZONE_PATH+LOCAL_COMPUTER_NAME (NetBIOSName) + /// + public static class BootConfLoader + { + #region CONSTS + public const string ENV_VAR_METABASE_FS_ROOT = "AGNI_METABASE_FS_ROOT"; + public const string ENV_VAR_METABASE_FS_TYPE = "AGNI_METABASE_FS_TYPE"; + public const string ENV_VAR_METABASE_FS_CSTRING = "AGNI_METABASE_FS_CSTRING"; + + public const string ENV_VAR_HOST_NAME = "AGNI_HOST_NAME"; + + /// + /// Used to append to local machine name if ENV_VAR_HOST_NAME is not set, then this value is concatenated with local machine name + /// + public const string DEFAULT_HOST_ZONE_PATH = SysConsts.DEFAULT_WORLD_GLOBAL_ZONE_PATH; + + public const string CONFIG_AGNI_SECTION = "agni"; + public const string CONFIG_HOST_SECTION = "host"; + public const string CONFIG_METABASE_SECTION = "metabase"; + public const string CONFIG_FS_SECTION = "file-system"; + public const string CONFIG_SESSION_CONNECT_PARAMS_SECTION = "session-connect-params"; + public const string CONFIG_APPLICATION_NAME_ATTR = "app-name"; + + + /// + /// Switch for agni options + /// + public const string CMD_ARG_AGNI_SWITCH = "agni"; + + /// + /// Used to specify application name from command line + /// + public const string CMD_ARG_APP_NAME = "app-name"; + + /// + /// Specifies host name in metabase, i.e. "/USA/East/Cle/A/I/wmed0001" + /// + public const string CONFIG_HOST_NAME_ATTR = "name"; + + /// + /// Root of file system + /// + public const string CONFIG_ROOT_ATTR = "root"; + + + public const string LOG_FROM_BOOTLOADER = "AppBootLoader"; + + + #endregion + + private static SystemApplicationType s_SystemApplicationType; + private static bool s_Loaded; + private static Exception s_LoadException; + private static string s_HostName; + private static string s_DynamicHostNameSuffix; + private static Metabank s_Metabase; + private static string s_ParentZoneGovernorPrimaryHostName; + + /// + /// Internal hack to compensate for c# inability to call .ctor within .ctor body + /// + internal static string[] SetSystemApplicationType(SystemApplicationType appType, string[] args) + { + s_SystemApplicationType = appType; + return args; + } + + /// + /// Application container system type + /// + public static SystemApplicationType SystemApplicationType { get { return s_SystemApplicationType; } } + + /// + /// Returns true after configuration has loaded + /// + public static bool Loaded { get{ return s_Loaded;} } + + /// + /// Returns exception (if any) has occured during application config loading process + /// + public static Exception LoadException { get{ return s_LoadException;} } + + + /// + /// Host name as determined at boot + /// + public static string HostName { get { return s_HostName ?? string.Empty;} } + + + /// + /// For dynamic hosts, host name suffix as determined at boot. It is the last part of HostName for dynamic hosts + /// including the separation character + /// + public static string DynamicHostNameSuffix { get { return s_DynamicHostNameSuffix ?? string.Empty;} } + + + /// + /// True when metabase section host declares this host as dynamic and HostName ends with DynamicHostNameSuffix + /// + public static bool DynamicHost {get { return s_DynamicHostNameSuffix.IsNotNullOrWhiteSpace();}} + + + /// + /// Returns primary zone governer parent host as determined at boot or null if this is the top-level host + /// + public static string ParentZoneGovernorPrimaryHostName { get { return s_ParentZoneGovernorPrimaryHostName;}} + + + + /// + /// Metabase as determined at boot or null in case of failure + /// + public static Metabank Metabase { get { return s_Metabase;} } + + + + private class TestDisposer : IDisposable { public void Dispose(){ BootConfLoader.Unload();} } + + + internal static void SetDomainInvariantCulture() + { + System.Globalization.CultureInfo.DefaultThreadCurrentCulture = + System.Globalization.CultureInfo.InvariantCulture; + } + + public static IDisposable LoadForTest(SystemApplicationType appType, Metabank mbase, string host, string dynamicHostNameSuffix = null) + { + SetDomainInvariantCulture(); + s_Loaded = true; + s_SystemApplicationType = appType; + s_Metabase = mbase; + s_HostName = host; + + if (dynamicHostNameSuffix.IsNotNullOrWhiteSpace()) + s_DynamicHostNameSuffix = Metabank.HOST_DYNAMIC_SUFFIX_SEPARATOR + dynamicHostNameSuffix; + else + { + var sh = mbase.CatalogReg.NavigateHost(host); + if (sh.Dynamic) + s_DynamicHostNameSuffix = thisMachineDynamicNameSuffix(); + } + + if (s_DynamicHostNameSuffix.IsNotNullOrWhiteSpace()) + s_HostName = s_HostName + s_DynamicHostNameSuffix; + + SystemVarResolver.Bind(); + + Configuration.ProcesswideConfigNodeProviderType = typeof(Metabase.MetabankFileConfigNodeProvider); + + return new TestDisposer(); + } + + private static string thisMachineDynamicNameSuffix() + { + //no spaces between + return "{0}{1}".Args(Metabank.HOST_DYNAMIC_SUFFIX_SEPARATOR, NFX.OS.Computer.UniqueNetworkSignature.Trim());//trim() as a safeguard + } + + + private static void writeLog(this ServiceBaseApplication app, NFX.Log.MessageType type, string text) + { + app.Log.Write( new NFX.Log.Message{ + Type = type, + From = LOG_FROM_BOOTLOADER, + Text = "Entering agni app bootloader..." + }); + + } + + + /// + /// Loads initial application container configuration (app container may re-read it in future using metabase) per supplied local one also connecting the metabase + /// + public static Configuration Load(string[] cmdArgs, Configuration bootConfig) + { + if (s_Loaded) + throw new AgniException(StringConsts.APP_LOADER_ALREADY_LOADED_ERROR); + + SetDomainInvariantCulture(); + + SystemVarResolver.Bind(); + + Configuration.ProcesswideConfigNodeProviderType = typeof(Metabase.MetabankFileConfigNodeProvider); + + try + { + Configuration result = null; + + //init Boot app container + using(var bootApp = new ServiceBaseApplication(cmdArgs, bootConfig.Root)) + { + bootApp.writeLog(NFX.Log.MessageType.Info, "Entering agni app bootloader..."); + + determineHostName(bootApp); + + NFX.Log.Message.DefaultHostName = s_HostName; + + mountMetabank(bootApp); + + Metabank.SectionHost zoneGov; + bool isDynamicHost; + result = getEffectiveAppConfigAndZoneGovernor(bootApp, out zoneGov, out isDynamicHost); + + if (zoneGov!=null) s_ParentZoneGovernorPrimaryHostName = zoneGov.RegionPath; + + if (isDynamicHost) + { + bootApp.writeLog(NFX.Log.MessageType.Info, "The meatabase host '{0}' is dynamic".Args(s_HostName)); + s_DynamicHostNameSuffix = thisMachineDynamicNameSuffix(); + bootApp.writeLog(NFX.Log.MessageType.Info, "Obtained actual host dynamic suffix: '{0}' ".Args(s_DynamicHostNameSuffix)); + s_HostName = s_HostName + s_DynamicHostNameSuffix;//no spaces between + bootApp.writeLog(NFX.Log.MessageType.Info, "The actual dynamic instance host name is: '{0}'".Args(s_HostName)); + + NFX.Log.Message.DefaultHostName = s_HostName; + } + + + bootApp.writeLog(NFX.Log.MessageType.Info, "...exiting agni app bootloader"); + } + + return result; + } + catch(Exception error) + { + s_LoadException = error; + throw new AgniException(StringConsts.APP_LOADER_ERROR + error.ToMessageWithType(), error); + } + finally + { + s_Loaded = true; + } + } + + public static void ProcessAllExistingIncludes(ConfigSectionNode node, string includePragma, string level) + { + const int CONST_MAX_INCLUDE_DEPTH = 7; + try + { + for (int count = 0; node.ProcessIncludePragmas(true, includePragma); count++) + if (count >= CONST_MAX_INCLUDE_DEPTH) + throw new ConfigException(StringConsts.CONFIGURATION_INCLUDE_PRAGMA_DEPTH_ERROR.Args(CONST_MAX_INCLUDE_DEPTH)); + } + catch (Exception error) + { + throw new ConfigException(StringConsts.CONFIGURATION_INCLUDE_PRAGMA_ERROR.Args(level, error.ToMessageWithType()), error); + } + } + + + internal static void Unload() + { + if (!s_Loaded) return; + s_Loaded = false; + s_SystemApplicationType = AppModel.SystemApplicationType.Unspecified; + s_HostName = null; + s_DynamicHostNameSuffix = null; + s_ParentZoneGovernorPrimaryHostName = null; + try + { + if (s_Metabase!=null) + { + var mb = s_Metabase; + var fs = mb.FileSystem; + s_Metabase = null; + + try + { + mb.Dispose(); + } + finally + { + if (fs!=null) + { + fs.Dispose(); + } + }//finally + } + } + catch + { + //nowhere to log anymore as all loggers have stopped + } + } + + #region .pvt .impl + + private static void determineHostName(ServiceBaseApplication bootApp) + { + var hNode = bootApp.ConfigRoot[CONFIG_AGNI_SECTION][CONFIG_HOST_SECTION]; + + s_HostName = hNode.AttrByName(CONFIG_HOST_NAME_ATTR).Value; + + if (s_HostName.IsNullOrWhiteSpace()) + { + bootApp.writeLog(NFX.Log.MessageType.Warning, "Host name was not specified in config, trying to take from machine env var {0}".Args(ENV_VAR_HOST_NAME)); + s_HostName = System.Environment.GetEnvironmentVariable(ENV_VAR_HOST_NAME); + } + if (s_HostName.IsNullOrWhiteSpace()) + { + bootApp.writeLog(NFX.Log.MessageType.Warning, "Host name was not specified in neither config nor env var, taking from local computer name"); + s_HostName = "{0}/{1}".Args(DEFAULT_HOST_ZONE_PATH, System.Environment.MachineName); + } + + bootApp.writeLog(NFX.Log.MessageType.Info, "Host name: " + s_HostName); + } + + + private static void mountMetabank(ServiceBaseApplication bootApp) + { + var mNode = bootApp.ConfigRoot[CONFIG_AGNI_SECTION][CONFIG_METABASE_SECTION]; + + ensureMetabaseAppName(bootApp, mNode); + + + FileSystemSessionConnectParams fsSessionConnectParams; + var fs = getFileSystem(bootApp, mNode, out fsSessionConnectParams); + + var fsRoot = mNode[CONFIG_FS_SECTION].AttrByName(CONFIG_ROOT_ATTR).Value; + if (fsRoot.IsNullOrWhiteSpace()) + { + bootApp.writeLog(NFX.Log.MessageType.Info, + "Metabase fs root is null in config, trying to take from machine env var {0}".Args(ENV_VAR_METABASE_FS_ROOT)); + fsRoot = System.Environment.GetEnvironmentVariable(ENV_VAR_METABASE_FS_ROOT); + } + + + bootApp.writeLog(NFX.Log.MessageType.Info, "Metabase FS root: " + fsRoot); + + + bootApp.writeLog(NFX.Log.MessageType.Info, "Mounting metabank..."); + try + { + s_Metabase = new Metabank(fs, fsSessionConnectParams, fsRoot ); + } + catch(Exception error) + { + bootApp.writeLog(NFX.Log.MessageType.CatastrophicError, error.ToMessageWithType()); + throw error; + } + bootApp.writeLog(NFX.Log.MessageType.Info, "...Metabank mounted"); + } + + + private static Configuration getEffectiveAppConfigAndZoneGovernor(ServiceBaseApplication bootApp, + out Metabank.SectionHost zoneGovernorSection, + out bool isDynamicHost) + { + Configuration result = null; + + bootApp.writeLog(NFX.Log.MessageType.Info, "Getting effective app config for '{0}'...".Args(AgniSystem.MetabaseApplicationName)); + try + { + var host = s_Metabase.CatalogReg.NavigateHost(s_HostName); + + result = host.GetEffectiveAppConfig(AgniSystem.MetabaseApplicationName).Configuration; + + zoneGovernorSection = host.ParentZoneGovernorPrimaryHost();//Looking in the same NOC only + isDynamicHost = host.Dynamic; + } + catch(Exception error) + { + bootApp.writeLog(NFX.Log.MessageType.CatastrophicError, error.ToMessageWithType()); + throw error; + } + bootApp.writeLog(NFX.Log.MessageType.Info, "...config obtained"); + + return result; + } + + + private static void ensureMetabaseAppName(ServiceBaseApplication bootApp, IConfigSectionNode mNode) + { + if (AgniSystem.MetabaseApplicationName==null) + { + var appName = bootApp.CommandArgs[CMD_ARG_AGNI_SWITCH].AttrByName(CMD_ARG_APP_NAME).Value; + if (appName.IsNotNullOrWhiteSpace()) + bootApp.writeLog(NFX.Log.MessageType.Info, + "Metabase application name was not defined in code, but is injected from cmd arg '-{0} {1}='".Args(CMD_ARG_AGNI_SWITCH, CMD_ARG_APP_NAME)); + else + { + bootApp.writeLog(NFX.Log.MessageType.Warning, + "Metabase application name was not defined in code or cmd switch, reading from '{0}/${1}'".Args(CONFIG_METABASE_SECTION, CONFIG_APPLICATION_NAME_ATTR)); + appName = mNode.AttrByName(CONFIG_APPLICATION_NAME_ATTR).Value; + } + + try + { + AgniSystem.MetabaseApplicationName = appName; + } + catch(Exception error) + { + bootApp.writeLog(NFX.Log.MessageType.CatastrophicError, error.ToMessageWithType()); + throw error; + } + bootApp.writeLog(NFX.Log.MessageType.Info, + "Metabase application name from config set to: "+AgniSystem.MetabaseApplicationName); + } + else + { + bootApp.writeLog(NFX.Log.MessageType.Info, + "Metabase application name defined in code: "+AgniSystem.MetabaseApplicationName); + + if (mNode.AttrByName(CONFIG_APPLICATION_NAME_ATTR).Exists) + bootApp.writeLog(NFX.Log.MessageType.Warning, + "Metabase application name defined in code but the boot config also defines the name which was ignored"); + } + } + + + private static IFileSystem getFileSystem(ServiceBaseApplication bootApp, + IConfigSectionNode mNode, + out FileSystemSessionConnectParams cParams) + { + IFileSystem result = null; + + bootApp.writeLog(NFX.Log.MessageType.Info, "Making metabase FS instance..."); + + var fsNode = mNode[CONFIG_FS_SECTION]; + + var fsFallbackTypeName = Environment.GetEnvironmentVariable(ENV_VAR_METABASE_FS_TYPE); + var fsFallbackType = typeof(NFX.IO.FileSystem.Local.LocalFileSystem); + + if (fsFallbackTypeName.IsNotNullOrWhiteSpace()) + fsFallbackType = Type.GetType(fsFallbackTypeName, true); + + result = FactoryUtils.MakeAndConfigure(fsNode, + fsFallbackType, + args: new object[]{CONFIG_METABASE_SECTION, fsNode}); + + var paramsNode = fsNode[CONFIG_SESSION_CONNECT_PARAMS_SECTION]; + if (paramsNode.Exists) + { + cParams = FileSystemSessionConnectParams.Make(paramsNode); + } + else + { + var fsFallbackCString = Environment.GetEnvironmentVariable(ENV_VAR_METABASE_FS_CSTRING); + if (fsFallbackCString.IsNotNullOrWhiteSpace()) + cParams = FileSystemSessionConnectParams.Make(fsFallbackCString); + else + cParams = new FileSystemSessionConnectParams(){ User = User.Fake}; + } + + bootApp.writeLog(NFX.Log.MessageType.Info, "...Metabase FS FileSystemSessionConnectParams instance of '{0}' made".Args(cParams.GetType().FullName)); + bootApp.writeLog(NFX.Log.MessageType.Info, "...Metabase FS instance of '{0}' made".Args(result.GetType().FullName)); + + return result; + } + + + #endregion + + } +} diff --git a/src/Agni/AppModel/HostGovernor/Cmdlets/Install.cs b/src/Agni/AppModel/HostGovernor/Cmdlets/Install.cs new file mode 100644 index 0000000..4e0c233 --- /dev/null +++ b/src/Agni/AppModel/HostGovernor/Cmdlets/Install.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; +using Agni.AppModel.Terminal; + +namespace Agni.AppModel.HostGovernor.Cmdlets +{ + public class Install : Cmdlet + { + public const string CONFIG_FORCE_ATTR = "force"; + + public Install(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var list = new List(); + var force = m_Args.AttrByName(CONFIG_FORCE_ATTR).ValueAsBool(false); + + if (force) + App.Log.Write( new NFX.Log.Message + { + Type = NFX.Log.MessageType.Warning, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = "{0}.Force".Args(GetType().FullName), + Text = "Installation with force=true initiated" + }); + + var anew = HostGovernorService.Instance.CheckAndPerformLocalSoftwareInstallation(list, force); + + var progress = list.Aggregate(new StringBuilder(), (sb, s) => sb.AppendLine(s)).ToString(); + + App.Log.Write( new NFX.Log.Message + { + Type = NFX.Log.MessageType.Warning, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = "{0}.Force".Args(GetType().FullName), + Text = "Installation finished. Installed anew: " + anew, + Parameters = progress + }); + + return progress; + } + + public override string GetHelp() + { + return +@"Initiates check and installation of local software. + Parameters: + force=bool - force reinstall +"; + } + } +} diff --git a/src/Agni/AppModel/HostGovernor/Cmdlets/Run.cs b/src/Agni/AppModel/HostGovernor/Cmdlets/Run.cs new file mode 100644 index 0000000..6b0bdae --- /dev/null +++ b/src/Agni/AppModel/HostGovernor/Cmdlets/Run.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; +using Agni.AppModel.Terminal; + +namespace Agni.AppModel.HostGovernor.Cmdlets +{ + public class Run : Cmdlet + { + public const string CONFIG_CMD_ATTR = "cmd"; + public const string CONFIG_ARGS_ATTR = "args"; + public const string CONFIG_TIMEOUT_ATTR = "timeout"; + + + public Run(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + + public override string Execute() + { + var cmd = m_Args.AttrByName(CONFIG_CMD_ATTR).Value; + if (cmd.IsNullOrWhiteSpace()) return "Missing 'cmd' - command to run"; + + var args = m_Args.AttrByName(CONFIG_ARGS_ATTR).ValueAsString(string.Empty); + + var timeout = m_Args.AttrByName(CONFIG_TIMEOUT_ATTR).ValueAsInt(5000); + + if (timeout<0) return "Timeout must be > 0 or blank"; + + + bool timedOut; + var result = NFX.OS.ProcessRunner.Run(cmd, args, out timedOut, timeout); + + if (timedOut) + result += "\n ....Process TIMED OUT...."; + + return result; + } + + public override string GetHelp() + { + return +@"Runs process blocking until it either exits or timeout expires. + Parameters: + cmd=string - command to run + args=int - arguments + timeout=int_ms - for how long to wait for process exit +"; + } + } +} diff --git a/src/Agni/AppModel/HostGovernor/Exceptions.cs b/src/Agni/AppModel/HostGovernor/Exceptions.cs new file mode 100644 index 0000000..128de6e --- /dev/null +++ b/src/Agni/AppModel/HostGovernor/Exceptions.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +namespace Agni.AppModel.HostGovernor +{ + /// + /// Thrown to indicate AHGOV related problems + /// + [Serializable] + public class AHGOVException : AgniException + { + public AHGOVException() : base() { } + public AHGOVException(string message) : base(message) { } + public AHGOVException(string message, Exception inner) : base(message, inner) { } + protected AHGOVException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } + + /// + /// Thrown to indicate AHGOV ManagedApp-related problems + /// + [Serializable] + public class ManagedAppException : AHGOVException + { + public ManagedAppException() : base() { } + public ManagedAppException(string message) : base(message) { } + public ManagedAppException(string message, Exception inner) : base(message, inner) { } + protected ManagedAppException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/AppModel/HostGovernor/HostGovernorServer.cs b/src/Agni/AppModel/HostGovernor/HostGovernorServer.cs new file mode 100644 index 0000000..454b896 --- /dev/null +++ b/src/Agni/AppModel/HostGovernor/HostGovernorServer.cs @@ -0,0 +1,22 @@ +using System; + +namespace Agni.AppModel.HostGovernor +{ + /// + /// Implements contracts trampoline that uses a singleton instance of HostGovernorService + /// + public class HostGovernorServer + : Agni.Contracts.IHostGovernor, + Agni.Contracts.IPinger + { + public Contracts.HostInfo GetHostInfo() + { + return HostGovernorService.Instance.GetHostInfo(); + } + + public void Ping() + { + HostGovernorService.Instance.Ping(); + } + } +} diff --git a/src/Agni/AppModel/HostGovernor/HostGovernorService.cs b/src/Agni/AppModel/HostGovernor/HostGovernorService.cs new file mode 100644 index 0000000..d08d686 --- /dev/null +++ b/src/Agni/AppModel/HostGovernor/HostGovernorService.cs @@ -0,0 +1,513 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Threading; + +using NFX; +using NFX.Log; +using NFX.Environment; +using NFX.Collections; +using NFX.ApplicationModel; +using NFX.ServiceModel; +using NFX.IO.FileSystem; +using NFX.IO.FileSystem.Packaging; + +using Agni.Metabase; + +namespace Agni.AppModel.HostGovernor +{ + + /// + /// Provides Host Governor Services - this is a singleton class + /// + public sealed class HostGovernorService : Service, Contracts.IPinger + { + #region CONSTS + public const string THREAD_NAME = "HostGovernorService"; + public const int THREAD_GRANULARITY_MS = 3000; + + public const string CONFIG_HOST_GOVERNOR_SECTION = "host-governor"; + + public const string CONFIG_DYNAMIC_HOST_ID_ATTR = "dynamic-host-id"; + public const string CONFIG_STARTUP_INSTALL_CHECK_ATTR = "startup-install-check"; + #endregion + + #region Static + private static object s_InstanceLock = new object(); + private static volatile HostGovernorService s_Instance; + + /// + /// Returns true to indicate that this process has host governor instance + /// + public static bool IsHostGovernor + { + get { return s_Instance!=null; } + } + + /// + /// Returns singleton instance or throws if service has not been allocated yet + /// + public static HostGovernorService Instance + { + get + { + var instance = s_Instance; + if (instance==null) + throw new AHGOVException(StringConsts.AHGOV_INSTANCE_NOT_ALLOCATED_ERROR); + + return instance; + } + } + #endregion + + #region .ctor/.dctor + /// + /// Creates a singleton instance or throws if instance is already created + /// + public HostGovernorService(bool launchedByARD, bool ardUpdateProblem) : base(null) + { + if (!AgniSystem.IsMetabase) + throw new AHGOVException(StringConsts.METABASE_NOT_AVAILABLE_ERROR.Args(GetType().FullName+".ctor()")); + + lock(s_InstanceLock) + { + if (s_Instance!=null) + throw new AHGOVException(StringConsts.AHGOV_INSTANCE_ALREADY_ALLOCATED_ERROR); + + m_LaunchedByARD = launchedByARD; + m_ARDUpdateProblem = ardUpdateProblem; + + var exeName = System.Reflection.Assembly.GetEntryAssembly().Location; + m_RootPath = Directory.GetParent(Path.GetDirectoryName(exeName)).FullName; + + s_Instance = this; + } + } + + protected override void Destructor() + { + lock(s_InstanceLock) + { + base.Destructor(); + s_Instance = null; + } + } + #endregion + + #region Fields + + + private string m_RootPath; + + private bool m_LaunchedByARD; + private bool m_ARDUpdateProblem; + private bool m_NeedsProcessRestart; + private bool m_StartupInstallCheck = true; + private Thread m_Thread; + private AutoResetEvent m_WaitEvent; + + private List m_Apps = new List(); + + private int m_AppStartOrder; + + private string m_DynamicHostID; + #endregion + + #region Properties + + public override string ComponentCommonName { get { return "hgov"; }} + + /// + /// Returns true when this process was launched by Agni Root Daemon as opposed to being launched from console/script + /// + public bool LaunchedByARD {get{ return m_LaunchedByARD;}} + + /// + /// Returns true when this process was launched by Agni Root Daemon that could not perform update properly - + /// most likely UPD and RUN folder slocked by some other process + /// + public bool ARDUpdateProblem {get{ return m_ARDUpdateProblem;}} + + + /// + /// Indicates whether install version check is done on service start (vs being invoked by command) + /// + public bool StartupInstallCheck {get{ return m_StartupInstallCheck;}} + + /// + /// Returns the very root path under which "ard","/run-netf","/run-core","/upd" are + /// + public string RootPath { get{return m_RootPath;}} + + /// + /// Returns the full path to "/run" directory out of which the current AHGOV process was launched + /// + public string RunPath + { + get + { + var isCore = NFX.PAL.PlatformAbstractionLayer.IsNetCore; + return Path.Combine(m_RootPath, isCore ? SysConsts.HGOV_RUN_CORE_DIR : SysConsts.HGOV_RUN_NETF_DIR); + } + } + + /// + /// Returns the full path to "/upd" directory where AHGOV will place newer files + /// + public string UpdatePath { get{return Path.Combine(m_RootPath, SysConsts.HGOV_UPDATE_DIR);}} + + + /// + /// Returns a thread-safe copy of ManagedApps in the instance - applications managed by Agni Host Governor + /// + public IEnumerable ManagedApps + { + get + { + lock(m_Apps) + return m_Apps.ToList(); + } + } + + /// + /// Becomes true to indocate that the app;ication process should be restarted (by ARD) + /// + public bool NeedsProcessRestart + { + get { return m_NeedsProcessRestart;} + } + + /// + /// Gets all unique Metabank.SectionApplication.AppPackage(s) for all applications on this host + /// + public IEnumerable AllPackages + { + get + { + var result = new List(); + foreach(var application in m_Apps) + foreach(var package in application.Packages) + if (!result.Any(ap => ap.Name.EqualsIgnoreCase( package.Name )&& + ap.MatchedPackage.Equals(package.MatchedPackage) && + ap.Path.EqualsIgnoreCase( package.Path ))) result.Add(package); + return result; + } + } + + /// + /// Returns current application start order, the sequence # of the next app start + /// + public int AppStartOrder + { + get { return m_AppStartOrder;} + } + + public Contracts.DynamicHostID? DynamicHostID + { + get + { + if (m_DynamicHostID.IsNullOrWhiteSpace()) return null; + return new Contracts.DynamicHostID(m_DynamicHostID, AgniSystem.HostMetabaseSection.ParentZone.RegionPath); + } + } + + #endregion + + #region Public + + public Contracts.HostInfo GetHostInfo() + { + return Contracts.HostInfo.ForThisHost(); + } + + public void Ping() + { + //does nothing. just comes back + } + + /// + /// Initiates check of locally installed packages and if they are different than install set, reinstalls in UpdatePath + /// + /// True if physical install was performed and AHGOV needs to restart so ARD may respawn it + public bool CheckAndPerformLocalSoftwareInstallation(IList progress, bool force = false) + { + IOMiscUtils.EnsureDirectoryDeleted(UpdatePath); + + var anew = AgniSystem.Metabase.CatalogBin.CheckAndPerformLocalSoftwareInstallation(progress, force); + if (!anew) return false; + //Flag the end of successfull installation + File.WriteAllText(Path.Combine(UpdatePath, SysConsts.HGOV_UPDATE_FINISHED_FILE), SysConsts.HGOV_UPDATE_FINISHED_FILE_OK_CONTENT); + + m_NeedsProcessRestart = true; + + return true; + } + + + + #endregion + + #region Protected + + protected override void DoConfigure(IConfigSectionNode node) + { + if (node==null) + node = App.ConfigRoot[CONFIG_HOST_GOVERNOR_SECTION]; + + + m_StartupInstallCheck = node.AttrByName(CONFIG_STARTUP_INSTALL_CHECK_ATTR).ValueAsBool(true); + m_DynamicHostID = node.AttrByName(CONFIG_DYNAMIC_HOST_ID_ATTR).ValueAsString(); + + base.DoConfigure(node); + } + + protected override void DoStart() + { + try + { + lock(m_Apps) + { + m_Apps.Clear(); + + //add managed apps as specified by host's role + foreach(var appInfo in AgniSystem.HostMetabaseSection.Role.Applications) + { + var app = new ManagedApp(this, appInfo); + m_Apps.Add(app);//the app is started later when it is ready + } + } + + + m_WaitEvent = new AutoResetEvent(false); + + m_Thread = new Thread(threadSpin); + m_Thread.Name = THREAD_NAME; + m_Thread.Start(); + + + } + catch + { + AbortStart(); + throw; + } + } + + protected override void DoSignalStop() + { + stopAllApps(false); + } + + protected override void DoWaitForCompleteStop() + { + m_WaitEvent.Set(); + + m_Thread.Join(); + m_Thread = null; + + m_WaitEvent.Close(); + m_WaitEvent = null; + + base.DoWaitForCompleteStop(); + } + + #endregion + + + #region .pvt .impl + private void threadSpin() + { + const string FROM = "threadSpin()"; + try + { + if (m_ARDUpdateProblem) + log(MessageType.CatastrophicError, FROM, StringConsts.AHGOV_ARD_UPDATE_PROBLEM_ERROR); + else + { + if (StartupInstallCheck) + if (checkInstall()) return; + } + } + catch(Exception error) + { + log(MessageType.CatastrophicError, FROM, "checkInstall() leaked: " + error.ToMessageWithType(), error); + } + + try + { + autoStartApps(); + } + catch(Exception error) + { + log(MessageType.CatastrophicError, FROM, "autoStartApps() leaked: " + error.ToMessageWithType(), error); + } + + try + { + var first = true; + while (Running) + { + try + { + if (!first) m_WaitEvent.WaitOne(THREAD_GRANULARITY_MS); + else first = false; + + var now = App.TimeSource.UTCNow; + //check for stopped processes that supposed to be auto-started + + //Notify Zone governor about this host(Host Registry service) + registerWithZGov(now); + } + catch(Exception error) + { + log(MessageType.CatastrophicError, FROM, "while(Running){} leaked: " + error.ToMessageWithType(), error); + } + } + } + finally + { + stopAllApps(true);//block + } + } + + private DateTime m_ScheduledZGovRegistration; + private int m_ConsecutiveZGovRegFailures; + private const int CONSECUTIVE_ZGOV_REG_FAIL_LONG_RETRY_THRESHOLD = 7; + + private void registerWithZGov(DateTime now) + { + const string FROM = "registerWithZGov()"; + + if (now(zgov.RegionPath)) + { + cl.RegisterSubordinateHost(Contracts.HostInfo.ForThisHost(), DynamicHostID); + ok = true; + break; + } + } + catch (Exception error) + { + log(MessageType.Error, FROM, "RegisterSubordinateHost('{0}') svc call threw: {1} ".Args(zgov.RegionPath, error.ToMessageWithType()), error, related: logid); + } + } + + if (!ok && !thisHostHasZGov) + log(MessageType.Error, FROM, "Could not send this host registration to any of the ZGovs tried", related: logid); + + //20151016 DKh added IF so if there is no ZGov it does not get bombarded + if (ok) + { + m_ScheduledZGovRegistration = now.AddMilliseconds(5000 + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 25000)); + m_ConsecutiveZGovRegFailures = 0; + } + else + {//20151016 DKh added IF so if there is no ZGov it does not get bombarded + if (any) + { + m_ConsecutiveZGovRegFailures++; + if (m_ConsecutiveZGovRegFailures(); + list.GetReadOnlyEvent = (c)=>false; + list.ChangeEvent = delegate(EventedList cl, EventedList.ChangeType change, EventPhase phase, int idx, string item) + { + if (phase==EventPhase.After) + log(MessageType.Trace, "checkInstall()", " * " + item, related: id); + }; + var anew = CheckAndPerformLocalSoftwareInstallation(list, false); + if (anew) + { + log(MessageType.Info, "checkInstall()", "CheckAndPerformLocalSoftwareInstallation() returned true. Will be restarting", related: id); + return true; //needs restart + } + } + catch(Exception error) + { + log(MessageType.CatastrophicError, "checkInstall()", "CheckAndPerformLocalSoftwareInstallation() leaked: " + error.ToMessageWithType(), error, related: id); + } + return false; + } + + private void autoStartApps() + { + var autoApps = m_Apps.Where(a => !a.AppInfo.Name.EqualsIgnoreCase(SysConsts.APP_NAME_HGOV) && + a.AppInfo.AutoRun.HasValue); + foreach(var app in autoApps.OrderBy(a=>a.AppInfo.AutoRun.Value)) + try + { + app.Start(); + app.m_StartOrder = m_AppStartOrder; + m_AppStartOrder++; + } + catch(Exception error) + { + log(MessageType.CatastrophicError, "autoStartApps()", "App '{0}' Leaked: {1}".Args(app.Name, error.ToMessageWithType()), error); + } + } + + private void stopAllApps(bool block) + { + foreach(var app in m_Apps.OrderBy(p=>-p.StartOrder))//in reverse order of start + try + { + if (block) + app.WaitForCompleteStop(); + else + app.SignalStop(); + } + catch(Exception error) + { + log(MessageType.CatastrophicError, "stopAllApps(block:{0})".Args(block), "Svc stop leaked: " + error.ToMessageWithType(), error); + } + } + + + internal void log(MessageType type, string from, string text, Exception error = null, Guid? related = null) + { + var msg = new NFX.Log.Message + { + Type = type, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = "{0}.{1}".Args(GetType().FullName, from), + Text = text, + Exception = error + }; + + if (related.HasValue) msg.RelatedTo = related.Value; + + App.Log.Write( msg ); + } + #endregion + } +} diff --git a/src/Agni/AppModel/HostGovernor/ManagedApp.cs b/src/Agni/AppModel/HostGovernor/ManagedApp.cs new file mode 100644 index 0000000..185a254 --- /dev/null +++ b/src/Agni/AppModel/HostGovernor/ManagedApp.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using System.Threading; + +using NFX; +using NFX.Log; +using NFX.ServiceModel; + +using Agni.Metabase; + +namespace Agni.AppModel.HostGovernor +{ + /// + /// Represents an application managed by the HostGovernorService instance - the agni application that gets installed/updated/executed by + /// the Agni Governor Process. The standard service's Start/Stop commands launch the actual application process + /// + public sealed class ManagedApp : Service + { + #region CONSTS + + public const int APP_PROCESS_LAUNCH_TIMEOUT_MS = 20000; + + #endregion + + + #region .ctor + internal ManagedApp(HostGovernorService director, Metabank.SectionRole.AppInfo appInfo) : base(director) + { + Name = appInfo.ToString(); + m_AppInfo = appInfo; + m_Packages = AgniSystem.HostMetabaseSection.GetAppPackages(appInfo.Name).ToList(); + } + #endregion + + #region Fields + private Metabank.SectionRole.AppInfo m_AppInfo; + internal int m_StartOrder; + private List m_Packages; + + private Process m_Process; + + + #endregion + + #region Properties + + /// + /// Returns the AppInfo as feteched from the metabase + /// + public Metabank.SectionRole.AppInfo AppInfo { get{ return m_AppInfo;}} + + + /// + /// Returns packages that this application have + /// + public IEnumerable Packages { get{return m_Packages;}} + + /// + /// Returns the start order of this app - when it was launched relative to others + /// + public int StartOrder { get{ return m_StartOrder;}} + + + /// + /// Returns executable launch command obtained from the role, or if it is blank from app itself + /// + public string ExeFile + { + get + { + var result = AppInfo.ExeFile; + if (result.IsNotNullOrWhiteSpace()) return result; + result = AgniSystem.Metabase.CatalogApp.Applications[AppInfo.Name].ExeFile; + return result; + } + } + + /// + /// Returns executable launch command arguments obtained from the role, or if it is blank from app itself + /// + public string ExeArgs + { + get + { + var result = AppInfo.ExeArgs; + if (result.IsNotNullOrWhiteSpace()) return result; + result = AgniSystem.Metabase.CatalogApp.Applications[AppInfo.Name].ExeArgs; + return result; + } + } + + + #endregion + + #region Public + + + #endregion + + #region Protected + + protected override void DoStart() + { + try + { + startProcess(); + } + catch(Exception error) + { + AbortStart(); + log(MessageType.CatastrophicError, "DoStart()", "Svc start leaked: " + error.ToMessageWithType(), error); + throw error; + } + } + + + + protected override void DoSignalStop() + { + try + { + if (m_Process!=null) + { + if (!m_Process.HasExited) + { + log(MessageType.Info, "DoSignalStop()", "Sending application process a line to gracefully exit"); + m_Process.StandardInput.WriteLine("");//Gracefully tells Application to exit + } + } + } + catch(Exception error) + { + log(MessageType.CatastrophicError, "DoSignalStop()", "Svc signal stop leaked: " + error.ToMessageWithType(), error); + throw error; + } + } + + protected override void DoWaitForCompleteStop() + { + try + { + closeProcess(); + } + catch(Exception error) + { + log(MessageType.CatastrophicError, "DoWaitForCompleteStop()", "Svc stop leaked: " + error.ToMessageWithType(), error); + throw error; + } + } + + #endregion + + #region .pvt .impl + + private void processExited(object sender, EventArgs args) + { + if (Status!=ControlStatus.Active) return;//do not use Running here as it also checks for Starting + + try + { + closeProcess(); + } + catch(Exception error) + { + log(MessageType.CatastrophicError, "processExited()", "Process exited leaked: " + error.ToMessageWithType(), error); + throw error; + } + } + + private void startProcess() + { + var rel = Guid.NewGuid(); + var exe = System.IO.Path.Combine(ComponentDirector.RunPath, ExeFile); + var args = ExeArgs; + + m_Process = new Process(); + m_Process.StartInfo.FileName = exe; + m_Process.StartInfo.WorkingDirectory = ComponentDirector.RunPath; + m_Process.StartInfo.Arguments = args; + m_Process.StartInfo.UseShellExecute = false; + m_Process.StartInfo.CreateNoWindow = true; + m_Process.StartInfo.RedirectStandardInput = true; + m_Process.StartInfo.RedirectStandardOutput = true; + m_Process.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; + m_Process.EnableRaisingEvents = true;//this must be true to get events + m_Process.Exited += processExited; + + + log(MessageType.Info, "startProcess()", "Starting '{0}'/'{1}'".Args(exe, args), null, rel); + m_Process.Start(); + log(MessageType.Info, "startProcess()", "Process Started. Waiting for OK.", null, rel); + + var watch = Stopwatch.StartNew(); + while (!m_Process.HasExited && + (m_Process.StandardOutput==null || m_Process.StandardOutput.EndOfStream) && + watch.ElapsedMilliseconds < APP_PROCESS_LAUNCH_TIMEOUT_MS) + { + Thread.Sleep(500); + } + + if (m_Process.HasExited) + throw new AHGOVException(StringConsts.AHGOV_APP_PROCESS_CRASHED_AT_STARTUP_ERROR.Args(Name, exe, args)); + + if (m_Process.StandardOutput==null) + throw new AHGOVException(StringConsts.AHGOV_APP_PROCESS_STD_OUT_NULL_ERROR.Args(Name, exe, args)); + + if (!m_Process.StandardOutput.EndOfStream) + { + if (m_Process.StandardOutput.Read()=='O' && + m_Process.StandardOutput.Read()=='K' && + m_Process.StandardOutput.Read()=='.') + { + log(MessageType.Info, "startProcess()", "Started and returned OK. '{0}'/'{1}'".Args(exe, args), null, rel); + return;//success + } + } + + throw new AHGOVException(StringConsts.AHGOV_APP_PROCESS_NO_SUCCESS_AT_STARTUP_ERROR.Args(Name, exe, args)); + } + + private void closeProcess() + { + var rel = Guid.NewGuid(); + + if (m_Process!=null) + { + if (!m_Process.HasExited) + { + log(MessageType.Info, "closeProcess()", "Waiting for app process to exit...", null, rel); + // m_Process.StandardInput.WriteLine("");//Gracefully tells Application to exit + m_Process.WaitForExit(); + log(MessageType.Info, "closeProcess()", "App process exited", null, rel); + } + m_Process.Close(); + m_Process = null; + } + } + + + internal void log(MessageType type, string from, string text, Exception error = null, Guid? related = null) + { + ComponentDirector.log(type, "ManagedApp({0}).{1}".Args(Name, from), text, error, related); + } + #endregion + } +} diff --git a/src/Agni/AppModel/IAgniApplication.cs b/src/Agni/AppModel/IAgniApplication.cs new file mode 100644 index 0000000..15031ac --- /dev/null +++ b/src/Agni/AppModel/IAgniApplication.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Environment; +using NFX.ApplicationModel; +using NFX.DataAccess; + +using Agni.Identification; + +namespace Agni.AppModel +{ + + /// + /// Denotes system application/process types that this app container has, i.e.: HostGovernor, WebServer, etc. + /// + public enum SystemApplicationType + { + Unspecified = 0, + HostGovernor, + ZoneGovernor, + WebServer, + GDIDAuthority, + ServiceHost, + ProcessHost, + SecurityAuthority, + TestRig, + Tool + } + + + /// + /// Defines a contract for applications + /// + public interface IAgniApplication : IApplication + { + /// + /// Returns the name that uniquely identifies this application in the metabase. Every process/executable must provide its unique application name in metabase + /// + string MetabaseApplicationName { get; } + + /// + /// References system-related functionality + /// + IAgniSystem TheSystem { get; } + + /// + /// References application configuration root used to boot this application instance + /// + IConfigSectionNode BootConfigRoot { get; } + + /// + /// Denotes system application/process type that this app container has, i.e.: HostGovernor, WebServer, etc. + /// + SystemApplicationType SystemApplicationType { get; } + + /// + /// References distributed lock manager + /// + Locking.ILockManager LockManager { get; } + + /// + /// References distributed GDID provider + /// + IGDIDProvider GDIDProvider { get; } + + /// + /// References distributed process manager + /// + Workers.IProcessManager ProcessManager { get; } + + /// + /// References dynamic host manager + /// + Dynamic.IHostManager DynamicHostManager { get; } + } +} diff --git a/src/Agni/AppModel/MasterIntfs.cs b/src/Agni/AppModel/MasterIntfs.cs new file mode 100644 index 0000000..7267e36 --- /dev/null +++ b/src/Agni/AppModel/MasterIntfs.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Time; + +using Agni.Identification; + +namespace Agni.AppModel +{ + + /// + /// Defines rollef-up status per certain entity (i.e. Region, NOC, ZOne etc.) + /// + public interface IOperationalStatus + { + string Status { get; } + + int HostsTotal { get; } + int ProcessorCoresUserUsedPct { get; } + int ProcessorCoresSysUsedPct { get; } + int ProcessorCoresTotal { get; } + int MemoryGBUsed { get; } + int MemoryGBTotal { get; } + + long Errors { get;} + } + + /// + /// Provides access to dynamic Agni status/operations. + /// This Entity is similar to metabase, however unlike the metabse which is read-only configuration, this is a portal for working with Agni dynamic (changing) + /// information, such as: get statistics, broadcast messages etc. + /// + public interface IAgniSystem : IApplicationComponent, IDisposable + { + bool Available { get; } + + + /// + /// Returns the status of the system + /// + IOperationalStatus Status { get; } + + //todo In future add ability to get Zones,NOCS,Regions, their statitics etc... + } + + +} diff --git a/src/Agni/AppModel/NOPAgniSystem.cs b/src/Agni/AppModel/NOPAgniSystem.cs new file mode 100644 index 0000000..eae5946 --- /dev/null +++ b/src/Agni/AppModel/NOPAgniSystem.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.ApplicationModel; + +namespace Agni.AppModel +{ + public class NOPAgniSystem : ApplicationComponent, IAgniSystem + { + private static NOPAgniSystem s_Instance = new NOPAgniSystem(); + + private NOPAgniSystem() { } + + public static NOPAgniSystem Instance { get { return s_Instance; } } + + public bool Available { get { return false; } } + + public override string ComponentCommonName { get { return AgniSystemBase.AGNI_COMPONENT_NAME; } } + + public IOperationalStatus Status + { + get + { + throw new NotImplementedException(); + } + } + } +} \ No newline at end of file diff --git a/src/Agni/AppModel/Terminal/AppRemoteTerminal.cs b/src/Agni/AppModel/Terminal/AppRemoteTerminal.cs new file mode 100644 index 0000000..02cc39a --- /dev/null +++ b/src/Agni/AppModel/Terminal/AppRemoteTerminal.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Reflection; +using System.Runtime.Serialization; + +using NFX; +using NFX.Security; +using NFX.Environment; +using NFX.ApplicationModel; + +using Agni.Contracts; +using Agni.Security.Permissions.Admin; + + +namespace Agni.AppModel.Terminal +{ + + /// + /// Provides basic app-management capabilities + /// + [Serializable] + public class AppRemoteTerminal : ApplicationComponent, IRemoteTerminal, INamed, IDeserializationCallback, IConfigurable + { + public const string MARKUP_PRAGMA = ""; + + public const string CONFIG_APP_REMOTE_TERMINAL_SECTION = "remote-terminal"; + + public const string HELP_ARGS = "/?"; + + + private static object s_IDLock = new Object(); + private static int s_ID; + internal static Registry s_Registry = new Registry(); + + + /// + /// Makes an instance of remote terminal which is configured under app/remote-terminal section. + /// If section is not defined then makes AppRemoteTerminal instance + /// + public static AppRemoteTerminal MakeNewTerminal() + { + return FactoryUtils.MakeAndConfigure(App.ConfigRoot[CONFIG_APP_REMOTE_TERMINAL_SECTION], typeof(AppRemoteTerminal)); + } + + + public AppRemoteTerminal() : base() + { + lock (s_IDLock) + { + s_ID++; + m_ID = s_ID; + } + m_Name = new ELink((ulong)m_ID, null).Link; + m_Vars = new Vars(); + m_ScriptRunner = new ScriptRunner(); + } + + + + protected override void Destructor() + { + s_Registry.Unregister(this); + base.Destructor(); + } + + public void OnDeserialization(object sender) + { + lock (s_IDLock) + { + if (this.m_ID > s_ID) s_ID = m_ID; + } + s_Registry.Register(this); + } + + private int m_ID; + private string m_Name; + private string m_Who; + private DateTime m_WhenConnected; + private DateTime m_WhenInteracted; + private Vars m_Vars; + private ScriptRunner m_ScriptRunner; + + public override string ComponentCommonName { get { return "term-" + m_Name; } } + + /// + /// Returns unique terminal session ID + /// + public int ID { get { return m_ID; } } + + /// + /// Returns unique terminal session ID as an alpha link + /// + public string Name { get { return m_Name; } } + + /// + /// Returns description about who is connected + /// + public string Who { get { return m_Who; } } + + /// + /// Returns UTC timestamp of connection initiation + /// + public DateTime WhenConnected { get { return m_WhenConnected; } } + + + /// + /// Returns UTC timestamp of last interaction + /// + public DateTime WhenInteracted { get { return m_WhenInteracted; } } + + + /// + /// Provides cmdlets + /// + public virtual IEnumerable Cmdlets { get { return CmdletFinder.Common; } } + + /// + /// Provides variable resolver + /// + public virtual Vars Vars { get { return m_Vars; } } + + public void Configure(IConfigSectionNode fromNode) + { + m_ScriptRunner.Configure(fromNode[ScriptRunner.CONFIG_SCRIPT_RUNNER_SECTION]); + ConfigAttribute.Apply(this, fromNode); + DoConfigure(fromNode); + } + + public virtual RemoteTerminalInfo Connect(string who) + { + m_Who = who ?? SysConsts.UNKNOWN_ENTITY; + m_WhenConnected = App.TimeSource.UTCNow; + m_WhenInteracted = App.TimeSource.UTCNow; + + s_Registry.Register(this); + + return new RemoteTerminalInfo + { + TerminalName = Name, + WelcomeMsg = "Connected to '[{0}]{1}'@'{2}' on {3:G} {4:T} UTC. Session '{5}'".Args(AgniSystem.MetabaseApplicationName, + App.Name, + AgniSystem.HostName, + App.TimeSource.Now, + App.TimeSource.UTCNow, + Name), + Host = AgniSystem.HostName, + AppName = App.Name, + ServerLocalTime = App.TimeSource.Now, + ServerUTCTime = App.TimeSource.UTCNow + }; + } + + [AppRemoteTerminalPermission] + public virtual string Execute(string command) + { + m_WhenInteracted = App.TimeSource.UTCNow; + + if (command == null) return string.Empty; + + command = command.Trim(); + + if (command.IsNullOrWhiteSpace()) return string.Empty; + + if (!command.EndsWith("}") && command.Contains(HELP_ARGS)) + { + return getHelp(command.Replace(HELP_ARGS,"").Trim()); + } + + if (!command.EndsWith("}")) command += "{}"; + + var cmd = TerminalUtils.ParseCommand(command, m_Vars); + var result = new MemoryConfiguration(); + result.EnvironmentVarResolver = cmd.EnvironmentVarResolver; + m_ScriptRunner.Execute(cmd, result); + + return DoExecute(result.Root); + } + + public string Execute(IConfigSectionNode command) + { + if(command==null || !command.Exists) return string.Empty; + return DoExecute(command); + } + + public virtual string Disconnect() + { + return "Good bye!"; + } + + protected virtual void DoConfigure(IConfigSectionNode fromNode) {} + + protected virtual string DoExecute(IConfigSectionNode command) + { + var cname = command.Name.ToLowerInvariant(); + + var tp = Cmdlets.FirstOrDefault(cmd => cmd.Name.EqualsIgnoreCase(cname)); + + if (tp == null) + return StringConsts.RT_CMDLET_DONTKNOW_ERROR.Args(cname); + + //Check cmdlet security + NFX.Security.Permission.AuthorizeAndGuardAction(tp); + NFX.Security.Permission.AuthorizeAndGuardAction(tp.GetMethod(nameof(Execute))); + + var cmdlet = Activator.CreateInstance(tp, this, command) as Cmdlet; + if (cmdlet == null) + throw new AgniException(StringConsts.RT_CMDLET_ACTIVATION_ERROR.Args(cname)); + + using (cmdlet) + { + return cmdlet.Execute(); + } + } + + private string getHelp(string cmd) + { + if (cmd.IsNullOrEmpty()) return string.Empty; + + var target = Cmdlets.FirstOrDefault(c => c.Name.EqualsOrdIgnoreCase(cmd)); + if (target == null) return string.Empty; + + var result = new StringBuilder(1024); + result.AppendLine(AppRemoteTerminal.MARKUP_PRAGMA); + result.AppendLine(""); + + string help; + using (var inst = Activator.CreateInstance(target, this, null) as Cmdlet) + { + try + { + help = inst.GetHelp(); + } + catch (Exception error) + { + help = "Error getting help: " + error.ToMessageWithType(); + } + result.AppendLine(" - {1}".Args(cmd, help)); + } + result.AppendLine(""); + + return result.ToString(); + } + } +} diff --git a/src/Agni/AppModel/Terminal/Cmdlet.cs b/src/Agni/AppModel/Terminal/Cmdlet.cs new file mode 100644 index 0000000..5b6530b --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlet.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal +{ + /// + /// Provides generalizatin for commandlet - terminal command handler + /// + public abstract class Cmdlet : DisposableObject + { + protected Cmdlet(AppRemoteTerminal terminal, IConfigSectionNode args) + { + m_Terminal = terminal; + m_Args = args; + } + + protected AppRemoteTerminal m_Terminal; + protected IConfigSectionNode m_Args; + + public abstract string Execute(); + + public abstract string GetHelp(); + } +} diff --git a/src/Agni/AppModel/Terminal/CmdletFinder.cs b/src/Agni/AppModel/Terminal/CmdletFinder.cs new file mode 100644 index 0000000..dd1a1af --- /dev/null +++ b/src/Agni/AppModel/Terminal/CmdletFinder.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Reflection; + +using NFX; + +namespace Agni.AppModel.Terminal +{ + /// + /// Provides enumerations of well-known cmdlets and facilities to find process-specific cmdlets + /// + public static class CmdletFinder + { + /// + /// Provides common cmdlets supported by all servers + /// + public static IEnumerable Common + { + get + { + return FindByNamespace(typeof(AppRemoteTerminal), "Agni.AppModel.Terminal.Cmdlets"); + } + } + + /// + /// Provides cmdlets supported by host governor + /// + public static IEnumerable HGov + { + get + { + return FindByNamespace(typeof(AppRemoteTerminal), "Agni.AppModel.HostGovernor.Cmdlets"); + } + } + + /// + /// Provides cmdlets supported by zone governor + /// + public static IEnumerable ZGov + { + get + { + return FindByNamespace(typeof(AppRemoteTerminal), "Agni.AppModel.ZoneGovernor.Cmdlets"); + } + } + + + /// + /// Finds cmdlets in assembly that contains the specified type, optionally filtering the cmdlet types + /// + public static IEnumerable FindByNamespace(Type assemblyContainingType, string nsFilter = null) + { + return nsFilter.IsNullOrWhiteSpace() ? Find(assemblyContainingType) + : Find(assemblyContainingType, (t) => t.Namespace.EqualsSenseCase(nsFilter)); + } + + /// + /// Finds cmdlets in assembly that contains the specified type, optionally filtering the cmdlet types + /// + public static IEnumerable Find(Type assemblyContainingType, Func filter = null) + { + if (assemblyContainingType==null) assemblyContainingType = typeof(CmdletFinder); + var asm = Assembly.GetAssembly(assemblyContainingType); + var cmdlets = asm.GetTypes() + .Where(t => !t.IsAbstract && typeof(Cmdlet).IsAssignableFrom(t)); + + return filter!=null? cmdlets.Where( t => filter(t)) : cmdlets; + } + + } +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Appl.cs b/src/Agni/AppModel/Terminal/Cmdlets/Appl.cs new file mode 100644 index 0000000..fc135fb --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Appl.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; +using NFX.ApplicationModel; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + public class Appl : Cmdlet + { + private const string VAL = "|{0}\n" ; + + public const string CONFIG_STOP_NOW_HR_ATTR = "stop-now-hour"; + + + public Appl(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var app = AgniSystem.Application; + + var stopNowHour = m_Args.AttrByName(CONFIG_STOP_NOW_HR_ATTR).ValueAsInt(-10); + if (stopNowHour == app.LocalizedTime.Hour) + { + var text = StringConsts.APPL_CMD_STOPPING_INFO.Args(m_Terminal.Name, m_Terminal.WhenConnected, m_Terminal.Who); + App.Log.Write( new NFX.Log.Message + { + Type = NFX.Log.MessageType.Warning, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = "{0}.StopNow".Args(GetType().FullName), + Text = text + }); + App.Instance.Stop(); + return text;//noone may see this as app may terminate faster than response delivered + } + + + + + var sb = new StringBuilder(1024); + sb.AppendLine(AppRemoteTerminal.MARKUP_PRAGMA); + sb.AppendLine(""); + sb.AppendLine("Application Container"); + sb.AppendLine("----------------------------------------------------------------------------"); + sb.AppendFormat("Name "+VAL, app.Name ); + sb.AppendFormat("Host "+VAL, AgniSystem.HostName ); + sb.AppendFormat("Parent Zone Governor "+VAL, AgniSystem.ParentZoneGovernorPrimaryHostName ?? "[none, this host is top level]" ); + sb.AppendFormat("Role "+VAL, AgniSystem.HostMetabaseSection.RoleName ); + sb.AppendFormat("Role Apps "+VAL, AgniSystem.HostMetabaseSection.Role.AppNames.Aggregate("",(r,a)=>r+a+", ") ); + sb.AppendFormat("Metabase App "+VAL, app.MetabaseApplicationName ); + sb.AppendFormat("Instance ID "+VAL, app.InstanceID ); + sb.AppendFormat("Start Time "+VAL, app.StartTime ); + sb.AppendFormat("Running Time "+VAL, app.LocalizedTime - app.StartTime ); + sb.AppendFormat("Type "+VAL, app.GetType().FullName ); + sb.AppendFormat("Active "+VAL, app.Active); + sb.AppendFormat("Boot Conf Root "+VAL, app.BootConfigRoot ); + sb.AppendFormat("Conf Root "+VAL, app.ConfigRoot ); + sb.AppendFormat("Data Store "+VAL, app.DataStore.GetType().FullName ); + sb.AppendFormat("Glue "+VAL, app.Glue.GetType().FullName ); + sb.AppendFormat("Instrumentation. "+VAL, app.Instrumentation.GetType().FullName ); + sb.AppendFormat("Localized Time "+VAL, app.LocalizedTime ); + sb.AppendFormat("Time Location "+VAL, app.TimeLocation ); + sb.AppendFormat("Log "+VAL, app.Log.GetType().FullName ); + sb.AppendFormat("Object Store "+VAL, app.ObjectStore.GetType().FullName ); + sb.AppendFormat("Security Manager "+VAL, app.SecurityManager.GetType().FullName ); + sb.AppendFormat("Module Root "+VAL, app.ModuleRoot.GetType().FullName ); + sb.AppendFormat("TimeSource "+VAL, app.TimeSource.GetType().FullName ); + + var lwarning = app.Log.LastWarning; + sb.AppendFormat("Last Warning "+VAL, lwarning!=null ? "{0} {1} {2} {3}".Args(lwarning.TimeStamp, lwarning.Guid, lwarning.From, lwarning.Text) : string.Empty ); + + var lerror = app.Log.LastError; + sb.AppendFormat("Last Error "+VAL, lerror!=null ? "{0} {1} {2} {3}".Args(lerror.TimeStamp, lerror.Guid, lerror.From, lerror.Text) : string.Empty ); + + var lcatastrophe = app.Log.LastCatastrophe; + sb.AppendFormat("Last Catastrophe "+VAL, lcatastrophe!=null ? "{0} {1} {2} {3}".Args(lcatastrophe.TimeStamp, lcatastrophe.Guid, lcatastrophe.From, lcatastrophe.Text) : string.Empty ); + + + sb.AppendLine(); + sb.AppendLine("Root Components"); + sb.AppendLine("----------------------------------------------------------------------------"); + var all = ApplicationComponent.AllComponents; + foreach(var cmp in all.Where(cmp => !(cmp.ComponentDirector is IApplicationComponent))) + { + string name = null; + var named = cmp as INamed; + if (named!=null) name = named.Name; + + sb.AppendLine("SID: {0,-4:D4} {1} {2} {3} {4}".Args(cmp.ComponentSID, + fdt(cmp.ComponentStartTime), + cmp.GetType().FullName, + cmp.ComponentCommonName, name)); + + var children = all.Where(c => object.ReferenceEquals(c.ComponentDirector, cmp)); + foreach(var child in children) + { + string cname = null; + var cnamed = child as INamed; + if (cnamed!=null) cname = cnamed.Name; + sb.AppendLine(" -> {0,-6:D4} {1} {2} {3} {4}".Args(child.ComponentSID, + fdt(child.ComponentStartTime), + child.GetType().Name, + child.ComponentCommonName, + cname)); + } + + sb.AppendLine(); + } + + sb.AppendLine(" * Note: this command only outputs root components and 1 level of their immediate children"); + sb.AppendLine(""); + + return sb.ToString(); + } + + + public override string GetHelp() + { + return +@"Displays the status of the Application Container: + Pass stop-now-hour=app_local_hour to stop the app. + This is needed so that an inadvertent stopping of the app container + is precluded. The parameter must match the current local app time +"; + } + + internal static string fdt(DateTime date) + { + return "[{0:D2}/{1:D2} {2:D2}:{3:D2}:{4:D2}.{5:D3}]".Args(date.Month, + date.Day, + date.Hour, + date.Minute, + date.Second, + date.Millisecond); + } + + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/CMan.cs b/src/Agni/AppModel/Terminal/Cmdlets/CMan.cs new file mode 100644 index 0000000..d2539fc --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/CMan.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Glue; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + /// + /// Component Manager + /// + public class CMan : Cmdlet + { + + public const string CONFIG_SID_ATTR = "sid"; + public const string CONFIG_NAME_ATTR = "name"; + public const string CONFIG_PARAM_ATTR = "param"; + public const string CONFIG_VALUE_ATTR = "value"; + + + public static ApplicationComponent GetApplicationComponentBySIDorName(IConfigSectionNode args) + { + if (args==null || !args.Exists) return null; + + ApplicationComponent cmp = null; + var sid = args.AttrByName(CONFIG_SID_ATTR).ValueAsULong(); + var cname = args.AttrByName(CONFIG_NAME_ATTR).Value; + + if (sid>0) cmp = ApplicationComponent.GetAppComponentBySID(sid); + + if (cmp==null && cname.IsNotNullOrWhiteSpace()) cmp = ApplicationComponent.GetAppComponentByCommonName(cname); + return cmp; + } + + + public CMan(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var sid = m_Args.AttrByName(CONFIG_SID_ATTR).ValueAsULong(); + var cname = m_Args.AttrByName(CONFIG_NAME_ATTR).Value; + var param = m_Args.AttrByName(CONFIG_PARAM_ATTR).Value; + var value = m_Args.AttrByName(CONFIG_VALUE_ATTR).Value; + + var sb = new StringBuilder(1024); + sb.AppendLine(AppRemoteTerminal.MARKUP_PRAGMA); + sb.Append(""); + sb.AppendLine("Component Manager"); + sb.AppendLine("----------------------"); + + if (sid==0 && cname.IsNullOrWhiteSpace()) + listAll(sb); + else + { + ApplicationComponent cmp = null; + if (sid>0) cmp = ApplicationComponent.GetAppComponentBySID(sid); + if (cmp==null && cname.IsNotNullOrWhiteSpace()) cmp = ApplicationComponent.GetAppComponentByCommonName(cname); + if (cmp!=null) + details(sb, cmp, param, value); + else + sb.AppendFormat("Component with the supplied SID and CommonName was not found\n"); + } + + sb.AppendLine(""); + + return sb.ToString(); + } + + + + + + public override string GetHelp() + { + return +@"Prints component information and manages parameters. + Pass either sid or name for particular component. + Pass param and value to set the value in particular component. + Parameters: + sid = int - component instance SID + or + name=string - component common name + + param=string - name of parameter to set + value=object - new parameter value to set + +"; + } + + + private void listAll(StringBuilder sb) + { + var all = ApplicationComponent.AllComponents; + var root = all.Where(c => c.ComponentDirector==null); + sb.AppendLine("Root components w/o director:"); + sb.AppendLine(); + foreach(var cmp in root) + listOne(sb, all, cmp, 0); + + sb.AppendLine(); + + var other = all.Where(c => c.ComponentDirector!=null && !(c.ComponentDirector is ApplicationComponent)); + sb.AppendLine("Components with non-component director:"); + sb.AppendLine(); + foreach(var cmp in other) + listOne(sb, all, cmp, 0); + } + + private void listOne(StringBuilder sb, IEnumerable all, ApplicationComponent cmp, int level) + { + if (level>7) return;//cyclical ref + + var sp = level<=0?string.Empty : string.Empty.PadLeft(level*3); + var pfx0 = sp+"├▌"; + var pfx = sp+"├─"; + var pfxL = sp+"└─"; + + + sb.AppendLine(pfx0+"SID: {1:D4} {2} {3} {4} " + .Args( + level==0 ? "white" : level>1 ? "darkcyan" : "cyan", + cmp.ComponentSID, + Appl.fdt(cmp.ComponentStartTime), + cmp.ComponentCommonName, + (cmp is INamed ? ((INamed)cmp).Name : "" ))); + + if (cmp.ComponentDirector!=null && !(cmp.ComponentDirector is ApplicationComponent)) + sb.AppendLine(pfx+"Director: "+cmp.ComponentDirector.GetType().FullName); + + sb.AppendLine(pfxL+"{0} {1}".Args(cmp.GetType().FullName, System.IO.Path.GetFileName( cmp.GetType().Assembly.Location ))); + + + + var children = all.Where(c => object.ReferenceEquals(c.ComponentDirector, cmp)); + foreach(var child in children) + { + listOne(sb, all, child, level+1); + } + + if (level==0) sb.AppendLine(); + } + + + + private void details(StringBuilder sb, ApplicationComponent cmp, string param, string value) + { + if (param.IsNotNullOrWhiteSpace()) + { + sb.AppendLine("Trying to set parameter '{0}' to value '{1}'".Args(param, value ?? "")); + if (!NFX.ExternalParameterAttribute.SetParameter(cmp, param, value)) + { + sb.AppendLine("Parameter '{0}' set did NOT SUCCEED".Args(param)); + return; + } + sb.AppendLine("Parameter '{0}' set SUCCEEDED".Args(param)); + sb.AppendLine(); + } + + dumpDetails(sb, cmp, 0); + } + + private void dumpDetails(StringBuilder sb, ApplicationComponent cmp, int level) + { + if (level>7) return;//cyclical ref + + var pfx = level<=0?string.Empty : string.Empty.PadLeft(level)+"->"; + + sb.AppendLine(pfx+"SID: "+cmp.ComponentSID); + sb.AppendLine(pfx+"CommonName: "+cmp.ComponentCommonName); + sb.AppendLine(pfx+"Start Time (local): "+cmp.ComponentStartTime); + sb.AppendLine(pfx+"Type: "+cmp.GetType().FullName); + sb.AppendLine(pfx+"Assembly: "+cmp.GetType().Assembly.FullName); + sb.AppendLine(pfx+"Service: "+(cmp is NFX.ServiceModel.Service ? "Yes" : "No") ); + if (cmp is INamed) + sb.AppendLine(pfx+"Name: "+((INamed)cmp).Name); + + sb.AppendLine(pfx+"Interfaces: "+cmp.GetType() + .GetInterfaces() + .OrderBy(it=>it.FullName) + .Aggregate("",(r,i)=> + r+(typeof(IExternallyParameterized).IsAssignableFrom(i) ? + "{0}".Args(i.Name) : i.Name)+", ") ); + + sb.AppendLine(); + sb.AppendLine(); + sb.AppendLine(pfx+"Parameters: "); + sb.AppendLine(); + + var pars = NFX.ExternalParameterAttribute.GetParametersWithAttrs(cmp.GetType()); + foreach(var p in pars) + { + var nm = p.Item1; + object val; + if (!NFX.ExternalParameterAttribute.GetParameter(cmp, nm, out val)) val = "?"; + var tp = p.Item2; + var atr = p.Item3; + sb.AppendLine(pfx+"{0,-35}: {1,-10} ({2}) {3}".Args( + nm, + val==null ? "" : val, + tp.DisplayNameWithExpandedGenericArgs().Replace('<','(').Replace('>',')') + , atr.Groups==null?"*":atr.Groups.Aggregate("",(r,a)=>r+a+", "))); + } + + + sb.AppendLine(); + var dir = cmp.ComponentDirector; + sb.AppendLine(pfx+"Director: "+(dir==null? " -none- " : dir.GetType().FullName)); + if (dir is ApplicationComponent) + dumpDetails(sb, dir as ApplicationComponent, level+1); + } + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Conf.cs b/src/Agni/AppModel/Terminal/Cmdlets/Conf.cs new file mode 100644 index 0000000..d1bbcbb --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Conf.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public class Conf : Cmdlet + { + public Conf(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var conf = new LaconicConfiguration(); + conf.CreateFromNode( App.ConfigRoot ); + return conf.SaveToString(); + } + + public override string GetHelp() + { + return "Fetches current configuration tree"; + } + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Echo.cs b/src/Agni/AppModel/Terminal/Cmdlets/Echo.cs new file mode 100644 index 0000000..528af07 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Echo.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Glue; +using Agni.Workers; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + /// + /// Echo text + /// + public class Echo: Cmdlet + { + public Echo(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) { } + + public override string Execute() + { + return m_Args.ValueAsString(); + } + + public override string GetHelp() + { + return @"Echo text"; + } + } +} \ No newline at end of file diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Exec.cs b/src/Agni/AppModel/Terminal/Cmdlets/Exec.cs new file mode 100644 index 0000000..e67009c --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Exec.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Glue; +using Agni.Workers; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + /// + /// Execute commands + /// + public class Exec: Cmdlet + { + public const string CONFIG_TO_ATTR = "to"; + + public Exec(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) { } + + public override string Execute() + { + var to = m_Args.AttrByName(CONFIG_TO_ATTR).ValueAsString(); + var sb = new StringBuilder(); + foreach (var arg in m_Args.Children) + sb.Append(m_Terminal.Execute(arg)); + if (to.IsNullOrWhiteSpace()) + return sb.ToString(); + m_Terminal.Vars[to] = sb.ToString(); + return "OK"; + } + + public override string GetHelp() + { + return @"Execute commands"; + } + } +} \ No newline at end of file diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Gc.cs b/src/Agni/AppModel/Terminal/Cmdlets/Gc.cs new file mode 100644 index 0000000..e8f1146 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Gc.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; + +using NFX; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public class Gc : Cmdlet + { + public Gc(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var watch = Stopwatch.StartNew(); + var before = GC.GetTotalMemory(false); + System.GC.Collect(); + var after = GC.GetTotalMemory(false); + return "GC took {0} ms. and freed {1} bytes".Args(watch.ElapsedMilliseconds, before - after); + } + + public override string GetHelp() + { + return "Invokes garbage collector"; + } + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Glue.cs b/src/Agni/AppModel/Terminal/Cmdlets/Glue.cs new file mode 100644 index 0000000..7c563db --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Glue.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; +using NFX.Glue; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + public class Glue : Cmdlet + { + public const string CONFIG_BINDING_ATTR = "binding"; + private const string VAL = "|{0}\n" ; + + public Glue(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var bname = m_Args.AttrByName(CONFIG_BINDING_ATTR).Value; + + var glue = App.Glue; + + var attr = m_Args.AttrByName("client-log-level"); + if (attr.Exists) glue.ClientLogLevel = attr.ValueAsEnum(glue.ClientLogLevel); + + attr = m_Args.AttrByName("server-log-level"); + if (attr.Exists) glue.ServerLogLevel = attr.ValueAsEnum(glue.ServerLogLevel); + + attr = m_Args.AttrByName("server-instance-lock-timeout-ms"); + if (attr.Exists) glue.ServerInstanceLockTimeoutMs = attr.ValueAsInt(glue.ServerInstanceLockTimeoutMs); + + attr = m_Args.AttrByName("default-timeout-ms"); + if (attr.Exists) glue.DefaultTimeoutMs = attr.ValueAsInt(glue.DefaultTimeoutMs); + + attr = m_Args.AttrByName("default-dispatch-timeout-ms"); + if (attr.Exists) glue.DefaultDispatchTimeoutMs = attr.ValueAsInt(glue.DefaultDispatchTimeoutMs); + + + + + + + var sb = new StringBuilder(1024); + sb.AppendLine(AppRemoteTerminal.MARKUP_PRAGMA); + sb.AppendLine(""); + sb.AppendLine("Glue Status"); + sb.AppendLine("----------------------------------------------------------------------------"); + sb.AppendFormat("Glue "+VAL, glue.GetType().FullName ); + sb.AppendFormat("Bindings "+VAL, glue.Bindings.Count ); + sb.AppendFormat("Client Log Level "+VAL, glue.ClientLogLevel ); + sb.AppendFormat("Server Log Level "+VAL, glue.ServerLogLevel ); + sb.AppendFormat("Client Msg Inspectors "+VAL, glue.ClientMsgInspectors.Count ); + sb.AppendFormat("Server Msg Inspectors "+VAL, glue.ServerMsgInspectors.Count ); + sb.AppendFormat("Dflt.Timeout ms. "+VAL, glue.DefaultTimeoutMs ); + sb.AppendFormat("Dflt.Dispatch Timeout ms. "+VAL, glue.DefaultDispatchTimeoutMs ); + sb.AppendFormat("Localized Time "+VAL, glue.LocalizedTime ); + sb.AppendFormat("Time Location "+VAL, glue.TimeLocation ); + sb.AppendFormat("Providers "+VAL, glue.Providers.Count ); + sb.AppendFormat("Servers "+VAL, glue.Servers.Count ); + sb.AppendFormat("Srv Inst Lock Timeout ms. "+VAL, glue.ServerInstanceLockTimeoutMs ); + + + sb.AppendLine(); + if (bname.IsNullOrWhiteSpace()) + { + sb.AppendLine("Bindings"); + sb.AppendLine("Name Type CI SI CTr STr CDump SDump"); + sb.AppendLine("----------------------------------------------------------------------------"); + foreach(var binding in App.Glue.Bindings) + sb.AppendFormat("{0,-10} {1,-25} {2,1} {3,1} {4,4} {5,4} {6,8} {7,8} \n", + binding.Name, + binding.GetType().Name, + binding.InstrumentClientTransportStat ? ON : OFF, + binding.InstrumentServerTransportStat ? ON : OFF, + binding.ClientTransports.Count(), + binding.ServerTransports.Count(), + binding.ClientDump, + binding.ServerDump + ); + } + else + { + var binding = glue.Bindings[bname]; + if (binding==null) + sb.AppendFormat("Binding {0} not present in glue stack\n", bname); + else + { + sb.AppendLine("Binding Status"); + sb.AppendLine("----------------------------------------------------------------------------"); + sb.AppendFormat("Name "+VAL, binding.Name ); + sb.AppendFormat("Type "+VAL, binding.GetType().FullName ); + sb.AppendFormat("Provider "+VAL, binding.Provider==null?"":binding.Provider.GetType().FullName ); + sb.AppendFormat("Client Dump "+VAL, binding.ClientDump ); + sb.AppendFormat("Server Dump "+VAL, binding.ServerDump ); + sb.AppendFormat("Cl Tr Cnt Wait Threshold "+VAL, binding.ClientTransportCountWaitThreshold); + sb.AppendFormat("Cl Tr Ex Acq Timeout ms. "+VAL, binding.ClientTransportExistingAcquisitionTimeoutMs); + sb.AppendFormat("Cl Tr Idle Timeout ms. "+VAL, binding.ClientTransportIdleTimeoutMs); + sb.AppendFormat("Cl Tr Max Count "+VAL, binding.ClientTransportMaxCount); + sb.AppendFormat("Cl Tr Max Ex Acq Tmout ms."+VAL, binding.ClientTransportMaxExistingAcquisitionTimeoutMs); + sb.AppendFormat("Client Transport Count "+VAL, binding.ClientTransports.Count()); + sb.AppendFormat("Server Transport Count "+VAL, binding.ServerTransports.Count()); + sb.AppendFormat("Server Tr Idle Timeout ms."+VAL, binding.ServerTransportIdleTimeoutMs); + sb.AppendFormat("Localized Time "+VAL, binding.LocalizedTime ); + sb.AppendFormat("Time Location "+VAL, binding.TimeLocation ); + + sb.AppendLine(); + sb.AppendLine("Client Transports"); + sb.AppendLine("Node IdlSec RBy SBy Err RMsg SMsg"); + sb.AppendLine("----------------------------------------------------------------------------"); + foreach(var tran in binding.ClientTransports) + sb.AppendFormat("{0,-30} {1,6} {2,8} {3,8} {4,5} {5,5} {6,5}\n", + tran.Node, + tran.IdleAgeMs / 1000, + tran.StatBytesReceived, + tran.StatBytesSent, + tran.StatErrors, + tran.StatMsgReceived, + tran.StatMsgSent + ); + sb.AppendLine(); + sb.AppendLine("Server Transports"); + sb.AppendLine("Node IdlSec RBy SBy Err RMsg SMsg"); + sb.AppendLine("----------------------------------------------------------------------------"); + foreach(var tran in binding.ServerTransports) + sb.AppendFormat("{0,-30} {1,6} {2,8} {3,8} {4,5} {5,5} {6,5}\n", + tran.Node, + tran.IdleAgeMs / 1000, + tran.StatBytesReceived, + tran.StatBytesSent, + tran.StatErrors, + tran.StatMsgReceived, + tran.StatMsgSent + ); + + + } + } + + sb.AppendLine(); + sb.AppendLine("NOTE: For management use CMAN instead as it allows to set more parameters"); + sb.AppendLine(""); + + return sb.ToString(); + } + + + private const string ON = "X"; + private const string OFF = "-"; + + public override string GetHelp() + { + return +@"Dumps Glue status: + Pass binding=string to query the particular instance + Parameters: + binding=string - name of the binding of interest + client-log-level=MessageType - client side log level + server-log-level=MessageType - server side log level + server-instance-lock-timeout-ms=int - acquisition interval for + non-thread-safe servers + default-timeout-ms=int - default call timeout + default-dispatch-timeout-ms=int - default call dispatch timeout + +"; + } + + + private IEnumerable bindings(string bname) + { + return App.Glue.Bindings.Where(b=>bname.IsNullOrWhiteSpace() || b.Name.EqualsIgnoreCase(bname)); + } + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Help.cs b/src/Agni/AppModel/Terminal/Cmdlets/Help.cs new file mode 100644 index 0000000..f310592 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Help.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Reflection; + +using NFX; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public class Help : Cmdlet + { + public const string CONFIG_CMD_ATTR = "cmd"; + + public Help(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var cmdlets = m_Terminal.Cmdlets; + + var cmdName = m_Args.AttrByName(CONFIG_CMD_ATTR).Value; + + if (cmdName.IsNotNullOrWhiteSpace()) + cmdlets = cmdlets.Where( cmd => cmd.Name.EqualsIgnoreCase(cmdName) ); + + var result = new StringBuilder(1024); + result.AppendLine(AppRemoteTerminal.MARKUP_PRAGMA); + result.AppendLine(""); + + result.AppendLine(" Remote Terminal Help "); + result.AppendLine(); + result.AppendLine(" Commands get sent to server after ';' is typed at the end of line."); + result.AppendLine(" Use ' /?' for help."); + result.AppendLine(" Use 'exit;' to disconnect."); + result.AppendLine(" List of server cmdlets: "); + result.AppendLine(); + + foreach(var cmdlet in cmdlets.OrderBy(c=>c.Name)) + { + var name = cmdlet.Name.ToLower(); + var help = SysConsts.UNKNOWN_ENTITY; + + using(var inst = Activator.CreateInstance(cmdlet, m_Terminal, m_Args) as Cmdlet) + { + try + { + help = inst.GetHelp(); + } + catch(Exception error) + { + help = "Error getting help: " + error.ToMessageWithType(); + } + } + + result.AppendLine( " - {1}".Args( name, help) ); + } + + result.AppendLine(""); + return result.ToString(); + } + + public override string GetHelp() + { + return "Provides help on commandlets; help{cmd=name}"; + } + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Instr.cs b/src/Agni/AppModel/Terminal/Cmdlets/Instr.cs new file mode 100644 index 0000000..0bc154f --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Instr.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; +using NFX.Glue; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public enum InstrType { View, Glue, ClientGlue, ServerGlue, Metabase, Mb = Metabase } + + public class Instr : Cmdlet + { + + public const string CONFIG_TYPE_ATTR = "type"; + public const string CONFIG_ON_ATTR = "on"; + public const string CONFIG_BINDING_ATTR = "binding"; + + public Instr(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var type = m_Args.AttrByName(CONFIG_TYPE_ATTR).ValueAsEnum(InstrType.View); + var on = m_Args.AttrByName(CONFIG_ON_ATTR).ValueAsBool(); + var bname = m_Args.AttrByName(CONFIG_BINDING_ATTR).Value; + switch(type) + { + case InstrType.ClientGlue: + { + foreach(var binding in bindings(bname)) + binding.InstrumentClientTransportStat = on; + + break; + } + + case InstrType.ServerGlue: + { + foreach(var binding in bindings(bname)) + binding.InstrumentServerTransportStat = on; + + break; + } + + case InstrType.Glue: + { + foreach(var binding in bindings(bname)) + { + binding.InstrumentClientTransportStat = on; + binding.InstrumentServerTransportStat = on; + } + break; + } + + case InstrType.Metabase: + { + if (AgniSystem.IsMetabase) + { + var mb = AgniSystem.Metabase; + mb.InstrumentationEnabled = on; + } + break; + } + + default: + { + break;//dont change anything + } + } + + var sb = new StringBuilder(1024); + sb.AppendLine(AppRemoteTerminal.MARKUP_PRAGMA); + sb.AppendLine(""); + sb.AppendLine("Instrumentation Status"); + sb.AppendLine("----------------------"); + sb.AppendFormat("Enabled: {0} Overflown: {1} RecordCount: {2} MaxRecordCount: {3}\n", + App.Instrumentation.Enabled, + App.Instrumentation.Overflown, + App.Instrumentation.RecordCount, + App.Instrumentation.MaxRecordCount); + sb.AppendFormat("Interval: {0}ms. OS Interval: {1}ms.\n", + App.Instrumentation.ProcessingIntervalMS, + App.Instrumentation.OSInstrumentationIntervalMS); + sb.AppendLine(); + sb.AppendLine("Glue Bindings Instrumentation"); + sb.AppendLine("-----------------------------"); + foreach(var binding in App.Glue.Bindings) + sb.AppendFormat("{0,-10} {1,-25} Client: {2,6} Server: {3,6}\n", + binding.Name, + binding.GetType().Name, + binding.InstrumentClientTransportStat ? ON : OFF, + binding.InstrumentServerTransportStat ? ON : OFF); + sb.AppendLine(); + var mon = AgniSystem.IsMetabase && AgniSystem.Metabase.InstrumentationEnabled; + + sb.AppendLine(); + sb.AppendLine("Metabase Instrumentation"); + sb.AppendLine("-----------------------------"); + sb.AppendFormat("Enabled: {0}\n", mon ? ON : OFF); + + sb.AppendLine(); + sb.AppendLine("NOTE: For management use CMAN instead as it allows to set more parameters"); + + sb.AppendLine(""); + + return sb.ToString(); + } + + + private const string ON = "X"; + private const string OFF = "-"; + + public override string GetHelp() + { + return +@"Enables/disables instrumentation: + Pass type={View|Glue|ClientGlue|ServerGlue| + Metabase} to specify what to control. + Parameters: + on=bool - turn on/off + binding=string - optional name of binding to apply changes to +"; + } + + + private IEnumerable bindings(string bname) + { + return App.Glue.Bindings.Where(b=>bname.IsNullOrWhiteSpace() || b.Name.EqualsIgnoreCase(bname)); + } + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Log.cs b/src/Agni/AppModel/Terminal/Cmdlets/Log.cs new file mode 100644 index 0000000..c49e89f --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Log.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Log; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + + + public class Log : Cmdlet + { + public const string CONFIG_TYPE_ATTR = "type"; + public const string CONFIG_ON_ATTR = "on"; + public const string CONFIG_BINDING_ATTR = "binding"; + + public Log(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var type = m_Args.AttrByName(CONFIG_TYPE_ATTR).ValueAsEnum(InstrType.View); + var on = m_Args.AttrByName(CONFIG_ON_ATTR).ValueAsBool(); + var bname = m_Args.AttrByName(CONFIG_BINDING_ATTR).Value; + + var msg = new Message + { + From = m_Args.AttrByName("from").Value, + Source = m_Args.AttrByName("source").ValueAsInt(0), + Type = m_Args.AttrByName("type").ValueAsEnum(MessageType.Info), + Topic = m_Args.AttrByName("topic").ValueAsString("App Terminal"), + Text = m_Args.AttrByName("text").ValueAsString("-none-"), + Parameters = m_Args.AttrByName("parameters").Value + }; + App.Log.Write( msg ); + + return msg.ToString(); + } + + public override string GetHelp() + { + return +@"Writes to application log + Parameters: + from = string - name of code/component + source=int - int source + type=MessageType - standard log MessageType enum + topic=string - what msg relates to + text=string - msg text + parameters=string - msg parameters + +"; + } + + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Mbc.cs b/src/Agni/AppModel/Terminal/Cmdlets/Mbc.cs new file mode 100644 index 0000000..07f9ed4 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Mbc.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; + +using NFX; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public class Mbc : Cmdlet + { + public Mbc(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + if (!AgniSystem.IsMetabase) + return "Metabase is not allocated"; + + var result = new StringBuilder(); + AgniSystem.Metabase.DumpCacheStatus(result); + + return result.ToString(); + } + + public override string GetHelp() + { + return "Dumps status of metabase cache"; + } + } +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/NetSvc.cs b/src/Agni/AppModel/Terminal/Cmdlets/NetSvc.cs new file mode 100644 index 0000000..8cfe6ad --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/NetSvc.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using System.Threading.Tasks; + +using NFX; +using NFX.Environment; + + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + + public class NetSvc : Cmdlet + { + public const string CONFIG_HOST_ATTR = "host"; + public const string CONFIG_NET_ATTR = "net"; + public const string CONFIG_SVC_ATTR = "svc"; + public const string CONFIG_BINDING_ATTR = "binding"; + public const string CONFIG_FROM_ATTR = "from"; + + public NetSvc(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var watch = Stopwatch.StartNew(); + + var host = m_Args.AttrByName(CONFIG_HOST_ATTR).ValueAsString(); + var net = m_Args.AttrByName(CONFIG_NET_ATTR).ValueAsString(); + var svc = m_Args.AttrByName(CONFIG_SVC_ATTR).ValueAsString(); + var binding = m_Args.AttrByName(CONFIG_BINDING_ATTR).ValueAsString(null); + var from = m_Args.AttrByName(CONFIG_FROM_ATTR).ValueAsString(null); + + string node; + try + { + node = AgniSystem.Metabase.ResolveNetworkServiceToConnectString(host, net, svc, binding, from); + } + catch(Exception error) + { + return "ERROR: "+error.ToMessageWithType(); + } + + return +@"Resolved +Host: {0} +Net: {1} +Service: {2} +Binding: {3} +From: {4} + +into Glue node + {5} +Elapsed: {6} ms.".Args(host, net, svc, binding, from, node, watch.ElapsedMilliseconds ); + } + + public override string GetHelp() + { + return +@"Resolves network service call into Glue node. + Parameters: + host=path - metabase path of target host + net=name - network name + svc=name - service name + binding=name - optional, Glue binding + from=path - optional, metabase path to call-issuing host +"; + } + + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Perf.cs b/src/Agni/AppModel/Terminal/Cmdlets/Perf.cs new file mode 100644 index 0000000..a61d693 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Perf.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; + +using NFX; +using NFX.OS; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public class Perf : Cmdlet + { + public const string CONFIG_DURATION_ATTR = "duration"; + public const string CONFIG_SAMPLE_ATTR = "sample"; + + + public Perf(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var duration = m_Args.AttrByName(CONFIG_DURATION_ATTR).ValueAsInt(8000); + var sample = m_Args.AttrByName(CONFIG_SAMPLE_ATTR).ValueAsInt(500); + + if (duration<1000 || sample <100) return "Sampling rate must be > 100ms and duration >1000ms"; + + + var watch = Stopwatch.StartNew(); + + var lst = new List>(); + + while(watch.ElapsedMillisecondsduration=int_ms - specifies measurement interval length. + The value must be >1000ms + sample=int_ms - specifies measurement sampling rate. + The value must be > 100ms +"; + } + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Pilc.cs b/src/Agni/AppModel/Terminal/Cmdlets/Pilc.cs new file mode 100644 index 0000000..04ef11f --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Pilc.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using System.Threading.Tasks; + +using NFX; +using NFX.Environment; +using NFX.ApplicationModel; +using NFX.ApplicationModel.Pile; + + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + /// + /// Pile Cache + /// + public class Pilc : Cmdlet + { + public const string CONFIG_TABLE_ATTR = "table"; + public const string CONFIG_PURGE_ATTR = "purge"; + + public Pilc(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var cache = CMan.GetApplicationComponentBySIDorName(m_Args) as ICache; + + if (cache==null) + return "The specified component is not of ICache type"; + + var tName = m_Args.AttrByName(CONFIG_TABLE_ATTR).ValueAsString(); + var purge = m_Args.AttrByName(CONFIG_PURGE_ATTR).ValueAsBool(); + + var sb = new StringBuilder(1024); + sb.AppendLine(AppRemoteTerminal.MARKUP_PRAGMA); + sb.Append(""); + sb.AppendLine("Pile Cache"); + sb.AppendLine("----------------------"); + + if (purge) + sb.AppendLine("Purging:"); + + foreach(var tbl in cache.Tables.Where(t => tName==null || t.Name.EqualsOrdIgnoreCase(tName))) + { + sb.AppendFormatLine("{0,-48}| {1,8:n0}({2,8:n0})| {3,4:n0}%", tbl.Name, tbl.Count, tbl.Capacity, tbl.LoadFactor*100d); + if (purge) tbl.Purge(); + } + + sb.AppendLine(""); + return sb.ToString(); + } + + public override string GetHelp() + { + return +@"Pile Cache Manager + Parameters: + sid=id - PileCache component SID + or + name=name - PileCache component common name + + + table=name - optional specific table + purge=bool - if true, purges the table/s +"; + } + + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Proc.cs b/src/Agni/AppModel/Terminal/Cmdlets/Proc.cs new file mode 100644 index 0000000..ea262a8 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Proc.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Glue; +using Agni.Workers; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + /// + /// Process Manager + /// + public class Proc : Cmdlet + { + public const string CONFIG_ALLOC_SECTION = "alloc"; + public const string CONFIG_SPAWN_SECTION = "spawn"; + public const string CONFIG_DISPATCH_SECTION = "dispatch"; + public const string CONFIG_ENQUEUE_SECTION = "enqueue"; + public const string CONFIG_LIST_SECTION = "list"; + + public const string CONFIG_PROC_SECTION = "proc"; + public const string CONFIG_SIGNAL_SECTION = "signal"; + public const string CONFIG_TODO_SECTION = "todo"; + + public const string CONFIG_PID_ATTR = "pid"; + public const string CONFIG_ZONE_ATTR = "zone"; + public const string CONFIG_MUTEX_ATTR = "mutex"; + public const string CONFIG_HOSTSET_ATTR = "hostset"; + public const string CONFIG_SVC_ATTR = "svc"; + + public Proc(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) { } + + public override string Execute() + { + var sb = new StringBuilder(); + + var any = false; + foreach (var arg in m_Args.Children) + { + sb.AppendLine(execute(arg)); + any = true; + } + if (!any) sb.AppendLine(list(null)); + return sb.ToString(0, sb.Length - Environment.NewLine.Length); + } + + private string execute(IConfigSectionNode arg) + { + if (arg.IsSameName(CONFIG_ALLOC_SECTION)) return alloc(arg); + if (arg.IsSameName(CONFIG_SPAWN_SECTION)) return spawn(arg); + if (arg.IsSameName(CONFIG_DISPATCH_SECTION)) return dispatch(arg); + if (arg.IsSameName(CONFIG_ENQUEUE_SECTION)) return enqueue(arg); + if (arg.IsSameName(CONFIG_LIST_SECTION)) return list(arg); + return string.Empty; + } + + private string alloc(IConfigSectionNode arg) + { + var zone = arg.AttrByName(CONFIG_ZONE_ATTR).ValueAsString(); + var mutex = arg.AttrByName(CONFIG_MUTEX_ATTR).ValueAsString(); + + var pid = mutex.IsNullOrWhiteSpace() + ? AgniSystem.ProcessManager.Allocate(zone) + : AgniSystem.ProcessManager.AllocateMutex(zone, mutex); + + return pid.ToString(); + } + + private string spawn(IConfigSectionNode arg) + { + var pid = PID.Parse(arg.AttrByName(CONFIG_PID_ATTR).ValueAsString()); + var proc = arg[CONFIG_PROC_SECTION]; + AgniSystem.ProcessManager.Spawn(pid, proc); + return "OK"; + } + + private string dispatch(IConfigSectionNode arg) + { + var pid = PID.Parse(arg.AttrByName(CONFIG_PID_ATTR).ValueAsString()); + var signal = arg[CONFIG_SIGNAL_SECTION]; + var result = AgniSystem.ProcessManager.Dispatch(pid, signal); + return result.ToString(); + } + + private string enqueue(IConfigSectionNode arg) + { + var hostSet= arg.AttrByName(CONFIG_HOSTSET_ATTR).ValueAsString(); + var svc = arg.AttrByName(CONFIG_SVC_ATTR).ValueAsString(); + + var todo = arg[CONFIG_TODO_SECTION]; + AgniSystem.ProcessManager.Enqueue(hostSet, svc, todo); + return "OK"; + } + + private string list(IConfigSectionNode arg) + { + var sb = new StringBuilder(); + var zone = arg.AttrByName(CONFIG_ZONE_ATTR).ValueAsString(); + foreach(var process in AgniSystem.ProcessManager.List(zone, arg)) + sb.AppendFormatLine("{0}", process.PID); + return sb.ToString(); + } + + public override string GetHelp() + { + return @"Process Manager"; + } + } +} \ No newline at end of file diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Time.cs b/src/Agni/AppModel/Terminal/Cmdlets/Time.cs new file mode 100644 index 0000000..9964214 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Time.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using System.Reflection; + +using NFX; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public class Time : Cmdlet + { + public Time(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var result = new StringBuilder(0xff); + result.AppendLine("System Time:"); + result.AppendLine(" Machine Now: " + DateTime.Now); + result.AppendLine(" Machine UTCNow: " + DateTime.UtcNow); + result.AppendLine(" App Localized: " + App.LocalizedTime); + result.AppendLine(" App Location: " + App.TimeLocation); + result.AppendLine(" App Time Source: " + App.TimeSource.GetType().FullName); + result.AppendLine(" App Time Source Now: " + App.TimeSource.Now); + result.AppendLine(" App Time Source UTCNow: " + App.TimeSource.UTCNow); + result.AppendLine(" App Time Location: " + App.TimeSource.TimeLocation); + result.AppendLine(" App Start Time: " + App.StartTime ); + result.AppendLine(" App Running Time: " + (App.LocalizedTime - App.StartTime).ToString() ); + + return result.ToString(); + } + + public override string GetHelp() + { + return "Returns app container time"; + } + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Toil.cs b/src/Agni/AppModel/Terminal/Cmdlets/Toil.cs new file mode 100644 index 0000000..7d7b4ec --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Toil.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using System.Threading.Tasks; + +using NFX; +using NFX.Environment; + + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public enum ToilType { Cpu, Ram, RamCpu } + + public class Toil : Cmdlet + { + public const string CONFIG_TYPE_ATTR = "type"; + public const string CONFIG_TASKS_ATTR = "tasks"; + public const string CONFIG_DURATION_ATTR = "duration"; + public const string CONFIG_RAM_ATTR = "ram"; + + public Toil(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var watch = Stopwatch.StartNew(); + var type = m_Args.AttrByName(CONFIG_TYPE_ATTR).ValueAsEnum(ToilType.Cpu); + switch(type) + { + case ToilType.Ram: + { + doWork(false, true); + break; + } + + case ToilType.RamCpu: + { + doWork(true, true); + break; + } + + default: + { + doWork(true, false); + break; + } + } + + return "Elapsed: {0} ms.".Args( watch.ElapsedMilliseconds ); + } + + public override string GetHelp() + { + return +@"Loads machine with heavy work that simulates real conditions. + Pass type={Cpu|Ram|RamCpu} to specify the type of work. + Parameters: + tasks=int - specifies how many parallel tasks to run + duration=int_ms - for how long to toil the system + ram=int_mbytes - how many mbytes to allocate +"; + } + + + private void doWork(bool doCPU, bool doRAM) + { + var tasks = m_Args.AttrByName(CONFIG_TASKS_ATTR).ValueAsInt(4); + var duration = m_Args.AttrByName(CONFIG_DURATION_ATTR).ValueAsInt(3000); + var ram = m_Args.AttrByName(CONFIG_RAM_ATTR).ValueAsInt(64); + + if (tasks<1 || duration <1 || ram <1) return; + + Task[] tarr = new Task[tasks]; + + var watch = Stopwatch.StartNew(); + for(int i=0; i + { + while(watch.ElapsedMilliseconds < duration) + { + if (doRAM) + { + var list = new List(); + for(int count =0; count < ram; count++) + list.Add( new byte[1024*1024]); + } + if (!doCPU) System.Threading.Thread.Sleep(250); + } + } + ); + Task.WaitAll(tarr); + } + + + + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Ver.cs b/src/Agni/AppModel/Terminal/Cmdlets/Ver.cs new file mode 100644 index 0000000..0ec82ff --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Ver.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using System.Reflection; + +using NFX; +using NFX.Environment; + + +namespace Agni.AppModel.Terminal.Cmdlets +{ + + public class Ver : Cmdlet + { + public Ver(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) + { + + } + + public override string Execute() + { + var result = new StringBuilder(0xff); + result.AppendLine("Server Version/Build information:"); + result.AppendLine(" App: " + App.Name); + result.AppendLine(" NFX: " + BuildInformation.ForFramework); + result.AppendLine(" Agni: " + new BuildInformation( typeof(Agni.AgniSystem).Assembly )); + result.AppendLine(" Host: " + new BuildInformation( Assembly.GetEntryAssembly() )); + + return result.ToString(); + } + + public override string GetHelp() + { + return "Returns version/build information"; + } + } + +} diff --git a/src/Agni/AppModel/Terminal/Cmdlets/Who.cs b/src/Agni/AppModel/Terminal/Cmdlets/Who.cs new file mode 100644 index 0000000..97fc3e0 --- /dev/null +++ b/src/Agni/AppModel/Terminal/Cmdlets/Who.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Environment; + +namespace Agni.AppModel.Terminal.Cmdlets +{ + public class Who : Cmdlet + { + public Who(AppRemoteTerminal terminal, IConfigSectionNode args) : base(terminal, args) { } + + public override string Execute() + { + var result = new StringBuilder(0xff); + foreach (var t in AppRemoteTerminal.s_Registry.Values) + result.AppendFormat("{0}-{1}-{2}-{3}\n", t.Name, t.Who, t.WhenConnected, t.WhenInteracted); + return result.ToString(); + } + + public override string GetHelp() { return "Displays remote terminal sessions"; } + } +} diff --git a/src/Agni/AppModel/ZoneGovernor/Exceptions.cs b/src/Agni/AppModel/ZoneGovernor/Exceptions.cs new file mode 100644 index 0000000..917bb3e --- /dev/null +++ b/src/Agni/AppModel/ZoneGovernor/Exceptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +namespace Agni.AppModel.ZoneGovernor +{ + /// + /// Thrown to indicate AZGOV related problems + /// + [Serializable] + public class AZGOVException : AgniException + { + public AZGOVException() : base() { } + public AZGOVException(string message) : base(message) { } + public AZGOVException(string message, Exception inner) : base(message, inner) { } + protected AZGOVException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/AppModel/ZoneGovernor/ZoneGovernorServer.cs b/src/Agni/AppModel/ZoneGovernor/ZoneGovernorServer.cs new file mode 100644 index 0000000..e921193 --- /dev/null +++ b/src/Agni/AppModel/ZoneGovernor/ZoneGovernorServer.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +namespace Agni.AppModel.ZoneGovernor +{ + /// + /// Implements contracts trampoline that uses a singleton instance of ZoneGovernorService + /// + public sealed class ZoneGovernorServer : Agni.Contracts.IZoneTelemetryReceiver, + Agni.Contracts.IZoneLogReceiver, + Agni.Contracts.IZoneHostRegistry, + Agni.Contracts.IZoneHostReplicator, + Agni.Contracts.ILocker + { + public int SendTelemetry(string host, NFX.Instrumentation.Datum[] data) + { + return ZoneGovernorService.Instance.SendTelemetry(host, data); + } + + public int SendLog(string host, string appName, NFX.Log.Message[] data) + { + return ZoneGovernorService.Instance.SendLog(host, appName, data); + } + + public Contracts.HostInfo GetSubordinateHost(string hostName) + { + return ZoneGovernorService.Instance.GetSubordinateHost(hostName); + } + + public IEnumerable GetSubordinateHosts(string hostNameSearchPattern) + { + return ZoneGovernorService.Instance.GetSubordinateHosts(hostNameSearchPattern); + } + + public void RegisterSubordinateHost(Contracts.HostInfo host, Contracts.DynamicHostID? hid) + { + ZoneGovernorService.Instance.RegisterSubordinateHost(host, hid); + } + + public Contracts.DynamicHostID Spawn(string hostPath, string id = null) + { + return ZoneGovernorService.Instance.Spawn(hostPath, id); + } + + public void PostDynamicHostInfo(Contracts.DynamicHostID hid, DateTime stamp, string owner, int votes) + { + ZoneGovernorService.Instance.PostDynamicHostInfo(hid, stamp, owner, votes); + } + + public void PostHostInfo(Contracts.HostInfo host, Contracts.DynamicHostID? hid) + { + ZoneGovernorService.Instance.PostHostInfo(host, hid); + } + + public Contracts.DynamicHostInfo GetDynamicHostInfo(Contracts.DynamicHostID hid) + { + return ZoneGovernorService.Instance.GetDynamicHostInfo(hid); + } + + public Locking.LockTransactionResult ExecuteLockTransaction(Locking.Server.LockSessionData session, Locking.LockTransaction transaction) + { + return ZoneGovernorService.Instance.Locker.ExecuteLockTransaction(session, transaction); + } + + public bool EndLockSession(Locking.LockSessionID sessionID) + { + return ZoneGovernorService.Instance.Locker.EndLockSession(sessionID); + } + } +} diff --git a/src/Agni/AppModel/ZoneGovernor/ZoneGovernorService.cs b/src/Agni/AppModel/ZoneGovernor/ZoneGovernorService.cs new file mode 100644 index 0000000..fabba3a --- /dev/null +++ b/src/Agni/AppModel/ZoneGovernor/ZoneGovernorService.cs @@ -0,0 +1,606 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.IO; +using System.Threading; + +using NFX; +using NFX.Log; +using NFX.Environment; +using NFX.Collections; +using NFX.ApplicationModel; +using NFX.ServiceModel; +using NFX.Instrumentation; + +using Agni.Metabase; + +namespace Agni.AppModel.ZoneGovernor +{ + + /// + /// Provides Zone Governor Services - this is a singleton class + /// + public sealed class ZoneGovernorService : Service + { + #region CONSTS + public const string THREAD_NAME = "ZoneGovernorService"; + public const int THREAD_GRANULARITY_MS = 3120; + + public const string CONFIG_ZONE_GOVERNOR_SECTION = "zone-governor"; + + public const string CONFIG_SUB_INSTRUMENTATION_SECTION = "sub-instrumentation"; + public const string CONFIG_SUB_LOG_SECTION = "sub-log"; + + public const string CONFIG_REDUCE_DETAIL_SECTION = "reduce-detail"; + public const string CONFIG_TYPE_SECTION = "type"; + public const string CONFIG_LEVEL_ATTR = "level"; + + public const int SUB_HOST_MAX_AGE_SEC = 7/*min*/ * 60; + + #endregion + + #region Static + private static object s_InstanceLock = new object(); + private static volatile ZoneGovernorService s_Instance; + + + /// + /// Returns true to indicate that this process has zone governor instance + /// + public static bool IsZoneGovernor + { + get { return s_Instance!=null; } + } + + /// + /// Returns singleton instance or throws if service has not been allocated yet + /// + public static ZoneGovernorService Instance + { + get + { + var instance = s_Instance; + if (instance==null) + throw new AZGOVException(StringConsts.AZGOV_INSTANCE_NOT_ALLOCATED_ERROR); + + return instance; + } + } + #endregion + + #region .ctor/.dctor + /// + /// Creates a singleton instance or throws if instance is already created + /// + public ZoneGovernorService() : base(null) + { + if (!AgniSystem.IsMetabase) + throw new AZGOVException(StringConsts.METABASE_NOT_AVAILABLE_ERROR.Args(GetType().FullName+".ctor()")); + + lock(s_InstanceLock) + { + if (s_Instance!=null) + throw new AZGOVException(StringConsts.AZGOV_INSTANCE_ALREADY_ALLOCATED_ERROR); + + m_SubInstr = new InstrumentationService(this); + m_SubInstrReductionLevels = new Dictionary(); + m_SubInstrCallers = new ConcurrentDictionary(); + + m_SubLog = new LogService(this); + + m_SubHosts = new Registry(); + m_DynamicHostSlots = new Registry(); + + m_Locker = new Locking.Server.LockServerService( this ); + + s_Instance = this; + } + } + + protected override void Destructor() + { + lock(s_InstanceLock) + { + base.Destructor(); + s_Instance = null; + + m_Locker.Dispose(); + + var sis = m_SubInstr; + if (sis!=null) + { + m_SubInstr = null; + sis.Dispose(); + } + + var slg = m_SubLog; + if (slg != null) + { + m_SubLog = null; + slg.Dispose(); + } + } + } + + #endregion + + #region Fields + + private InstrumentationService m_SubInstr; + private Dictionary m_SubInstrReductionLevels; + private ConcurrentDictionary m_SubInstrCallers; + private int m_SubInstrCallerCount; + + private LogService m_SubLog; + + private Registry m_SubHosts; + private Registry m_DynamicHostSlots; + + private double m_CPULoadFactor = 1d; + + private Locking.Server.LockServerService m_Locker; + + private Thread m_Thread; + private AutoResetEvent m_WaitEvent; + + #endregion + + #region Properties + + public override string ComponentCommonName { get { return "zgov"; }} + + /// + /// A form of throttling. + /// Returns the coefficient 0.0 .. 1.0 that stipulates how much more work/load the node is ready to take. + /// The higher the CPU usage, the lower this number gets (closer to 0.0) + /// + public double CPULoadFactor { get { return m_CPULoadFactor;}} + + /// + /// Returns the number of active subordinate telemetry uploaders - + /// nodes that upload telemetry to this Zone Governor + /// + public int SubordinateInstrumentationCallerCount { get{ return m_SubInstrCallerCount; }} + + + /// + /// Returns instrumentation as reported by subordinate telemetry callers + /// + public IInstrumentation SubordinateInstrumentation { get { return m_SubInstr; } } + + /// + /// Returns locking server that this zgov hosts + /// + public Contracts.ILocker Locker { get { return m_Locker; } } + #endregion + + #region Public + /// + /// Called by subordinate nodes to report telemetry + /// + public int SendTelemetry(string host, NFX.Instrumentation.Datum[] data) + { + if (!Running || data==null) return 0; + var sis = m_SubInstr; + if (sis==null) return 0; + + for(var i=0; i + /// Called by subordinate nodes to report log + /// + public int SendLog(string host, string appName, NFX.Log.Message[] data) + { + if (!Running || data == null) return 0; + var slg = m_SubLog; + if (slg == null) return 0; + + foreach (var msg in data) + { + if (appName.IsNotNullOrWhiteSpace()) + msg.From = appName + "::" + msg.From; + slg.Write(msg); + } + + return (int)(Log.AgniZoneDestination.MAX_BUF_SIZE * m_CPULoadFactor); + } + + /// + /// Returns log messages from cyclical subordinate instrumentation buffer. + /// Please note that in order to use this property subordinate instrumentation service + /// must have its InstrumentationEnabled property set to true + /// + public IEnumerable GetSubordinateInstrumentationLogBuffer(bool asc) + { + return m_SubLog.GetInstrumentationBuffer(asc); + } + + /// + /// Registers /updates existing subordinate host information. This method implements IZoneHostRegistry contract + /// + public void RegisterSubordinateHost(Contracts.HostInfo host, Contracts.DynamicHostID? hid) + { + if (!Running || m_SubHosts==null || host==null) return; + + registerSubordinateHost(host, hid); + + var zHosts = AgniSystem.HostMetabaseSection.ParentZone.ZoneGovernorHosts.Where(hh => !AgniSystem.HostName.IsSameRegionPath(hh.RegionPath)); + foreach (var z in zHosts) + using (var cl = Contracts.ServiceClientHub.New(z)) + cl.Async_PostHostInfo(host, hid); + } + + /// + /// Returns registred subordinate hosts, optionally taking search pattern. This method implements IZoneHostRegistry contract. + /// Match pattern can contain up to one * wildcard and multiple ? wildcards + /// + public IEnumerable GetSubordinateHosts(string hostNameSearchPattern) + { + if (!Running || m_SubHosts==null) return Enumerable.Empty(); + + var matches = hostNameSearchPattern.IsNullOrWhiteSpace() + ? m_SubHosts + : m_SubHosts.Where(h => NFX.Parsing.Utils.MatchPattern(h.Name, hostNameSearchPattern, senseCase: false)); + + return matches.ToArray(); + } + + /// + /// Returns information for specified subordinate host or null + /// + public Contracts.HostInfo GetSubordinateHost(string hostName) + { + if (!Running || m_SubHosts==null || hostName.IsNullOrWhiteSpace()) return null; + + //safeguard trim around dynamic name + var i = hostName.IndexOf(Metabank.HOST_DYNAMIC_SUFFIX_SEPARATOR); + if (i>0 && i !AgniSystem.HostName.IsSameRegionPath(hh.RegionPath)); + foreach (var h in hosts) + using (var cl = Contracts.ServiceClientHub.New(h)) + cl.Async_PostDynamicHostInfo(hid, dhi.Stamp, dhi.Owner, dhi.Votes); + } + return hid; + } + + public void PostDynamicHostInfo(Contracts.DynamicHostID hid, DateTime stamp, string owner, int votes) + { + if (!Running || m_DynamicHostSlots == null) return; + var zone = AgniSystem.HostMetabaseSection.ParentZone; + + // TODO: Check zone + var dhi = m_DynamicHostSlots[hid.ID]; + var post = false; + if (dhi == null) + { + dhi = new Contracts.DynamicHostInfo(hid.ID); + post = true; + m_DynamicHostSlots.Register(dhi); + } + + if (dhi.Stamp > stamp || post) + { + dhi.Stamp = stamp; + dhi.Owner = owner; + dhi.Votes = votes; + post = true; + } + + if (dhi.Stamp == stamp) + dhi.Votes += 1; + + if (post) + { + var hosts = zone.ZoneGovernorHosts.Where(hh => !AgniSystem.HostName.IsSameRegionPath(hh.RegionPath)); + foreach (var h in hosts) + using (var cl = Contracts.ServiceClientHub.New(h)) + cl.Async_PostDynamicHostInfo(hid, dhi.Stamp, dhi.Owner, dhi.Votes); + } + } + + public void PostHostInfo(Contracts.HostInfo host, Contracts.DynamicHostID? hid) + { + if (!Running || m_SubHosts==null || host==null) return; + + registerSubordinateHost(host, hid); + } + + public Contracts.DynamicHostInfo GetDynamicHostInfo(Contracts.DynamicHostID hid) + { + if (!Running || m_DynamicHostSlots == null) return null; + + var zone = AgniSystem.HostMetabaseSection.ParentZone; + + // TODO: Check zone + var dhi = m_DynamicHostSlots[hid.ID]; + if (dhi == null) + { + throw new NotImplementedException(); + } + + return dhi; + } + #endregion + + #region Protected + + protected override void DoConfigure(IConfigSectionNode node) + { + string ip = "--"; + try + { + if (node == null) + node = App.ConfigRoot[CONFIG_ZONE_GOVERNOR_SECTION]; + + base.DoConfigure(node); + + ip = "A"; + + var siNode = node[CONFIG_SUB_INSTRUMENTATION_SECTION]; + m_SubInstr.Configure(siNode); + + ip = "B"; + + m_SubInstrReductionLevels.Clear(); + foreach (var tn in siNode[CONFIG_REDUCE_DETAIL_SECTION].Children.Where(cn => cn.IsSameName(CONFIG_TYPE_SECTION))) + { + var tname = tn.AttrByName(Configuration.CONFIG_NAME_ATTR).Value; + if (tname.IsNullOrWhiteSpace()) continue; + m_SubInstrReductionLevels[tname] = tn.AttrByName(CONFIG_LEVEL_ATTR).ValueAsInt(); + } + + ip = "C"; + + var slNode = node[CONFIG_SUB_LOG_SECTION]; + m_SubLog.Configure(slNode); + + ip = "D"; + + m_Locker.Configure(node[Locking.Server.LockServerService.CONFIG_LOCK_SERVER_SECTION]); + + ip = "E"; + + log(MessageType.Info, ".DoConfigure()", "Configured OK. ip=" + ip); + } + catch (Exception error) + { + var msg = "Error after '{0}' during ZoneGovernorService configuration: {1}".Args(ip, error.ToMessageWithType()); + log(MessageType.CatastrophicError, ".DoConfigure()", msg, error); + throw new AZGOVException(msg, error); + } + } + + protected override void DoStart() + { + try + { + m_CPULoadFactor = 1d; + m_SubInstrCallerCount = 0; + m_SubInstr.Start(); + + m_SubLog.Start(); + + m_Locker.Start(); + + m_WaitEvent = new AutoResetEvent(false); + + m_Thread = new Thread(threadSpin); + m_Thread.Name = THREAD_NAME; + m_Thread.Start(); + } + catch + { + if (m_Locker.Running) try { m_Locker.WaitForCompleteStop(); } catch { } + if (m_SubLog.Running) try { m_SubLog.WaitForCompleteStop(); } catch { } + if (m_SubInstr.Running) try { m_SubInstr.WaitForCompleteStop(); } catch { } + AbortStart(); + throw; + } + } + + protected override void DoSignalStop() + { + m_SubInstr.SignalStop(); + m_SubLog.SignalStop(); + m_Locker.SignalStop(); + } + + protected override void DoWaitForCompleteStop() + { + m_WaitEvent.Set(); + + m_Thread.Join(); + m_Thread = null; + + m_WaitEvent.Close(); + m_WaitEvent = null; + + m_SubInstr.WaitForCompleteStop(); + m_SubInstrCallers.Clear(); + m_SubInstrCallerCount = 0; + + m_SubLog.WaitForCompleteStop(); + + m_Locker.WaitForCompleteStop(); + + base.DoWaitForCompleteStop(); + } + + #endregion + + + #region .pvt .impl + + private void threadSpin() + { + const string FROM = "threadSpin()"; + + while (Running) + { + try + { + var now = App.TimeSource.UTCNow; + + updateCPULoadFactor(); + updateTelemetryCallers(now); + purgeOldSubHosts(now); + + m_WaitEvent.WaitOne(THREAD_GRANULARITY_MS); + } + catch(Exception error) + { //TODO restart loop?????? + log(MessageType.CatastrophicError, FROM, error.ToMessageWithType(), error); + } + } + + } + + private int m_CPU_1; + private int m_CPU_2; + private int m_CPU_3; + + private void updateCPULoadFactor() + { + var cpu = (NFX.OS.Computer.CurrentProcessorUsagePct + m_CPU_1 + m_CPU_2 + m_CPU_3) / 4; + m_CPU_3 = m_CPU_2; + m_CPU_2 = m_CPU_1; + m_CPU_1 = cpu; + + if (cpu<20) + m_CPULoadFactor = 1.0d; + else if (cpu<80) + m_CPULoadFactor = 1.0d - ((double)cpu / 100.0d); + else if (cpu<90) + m_CPULoadFactor = 0.075; + else + m_CPULoadFactor = 0.0d; + } + + private void purgeOldSubHosts(DateTime utcNow) + { + + var hosts = m_SubHosts.Where(h => (utcNow - h.UTCTimeStamp).TotalSeconds > SUB_HOST_MAX_AGE_SEC ).ToArray(); + foreach(var host in hosts) + m_SubHosts.Unregister( host );//expired + } + + private void updateTelemetryCallers(DateTime utcNow) + { + const int MAX_ACTIVE_AGE_MSEC = 40000; + + var delete = new List(); + + foreach(var kvp in m_SubInstrCallers) + if ((utcNow - kvp.Value).TotalMilliseconds > MAX_ACTIVE_AGE_MSEC) delete.Add(kvp.Key); + + DateTime dummy; + foreach(var key in delete) + m_SubInstrCallers.TryRemove(key, out dummy); + + m_SubInstrCallerCount = m_SubInstrCallers.Count(); + } + + private void registerSubordinateHost(Contracts.HostInfo host, Contracts.DynamicHostID? hid) + { + try + { + var shost = AgniSystem.Metabase.CatalogReg.NavigateHost(host.Name); + if (!shost.HasDirectOrIndirectParentZoneGovernor(AgniSystem.HostMetabaseSection, iAmZoneGovernor: false, transcendNOC: false)) + throw new AZGOVException(StringConsts.AZGOV_REGISTER_SUBORDINATE_HOST_PARENT_ERROR.Args(AgniSystem.HostName, host.Name)); + } + catch (Exception error) + { + throw new AZGOVException(StringConsts.AZGOV_REGISTER_SUBORDINATE_HOST_ERROR.Args(host.Name, error.ToMessageWithType()), error); + } + + m_SubHosts.RegisterOrReplace(host); + if (hid.HasValue && m_DynamicHostSlots != null) + { + var slot = m_DynamicHostSlots[hid.Value.ID]; + if (slot != null) slot.Host = host.Name; + } + + if (host.RandomSample.HasValue) + ExternalRandomGenerator.Instance.FeedExternalEntropySample(host.RandomSample.Value); + } + + internal void log(MessageType type, string from, string text, Exception error = null, Guid? related = null) + { + var msg = new NFX.Log.Message + { + Type = type, + Topic = SysConsts.LOG_TOPIC_ZONE_MANAGEMENT, + From = "{0}.{1}".Args(GetType().FullName, from), + Text = text, + Exception = error + }; + + if (related.HasValue) msg.RelatedTo = related.Value; + + App.Log.Write( msg ); + } + + #endregion + + + } +} diff --git a/src/Agni/Clients/Exceptions.cs b/src/Agni/Clients/Exceptions.cs new file mode 100644 index 0000000..b1670e8 --- /dev/null +++ b/src/Agni/Clients/Exceptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +namespace Agni.Clients +{ + /// + /// Thrown to indicate problems that arise while using clients (IAgniServiceClient implementors) + /// + [Serializable] + public class AgniClientException : AgniException + { + public AgniClientException() : base() { } + public AgniClientException(string message) : base(message) { } + public AgniClientException(string message, Exception inner) : base(message, inner) { } + protected AgniClientException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Clients/GDIDAuthority.cs b/src/Agni/Clients/GDIDAuthority.cs new file mode 100644 index 0000000..17c88a4 --- /dev/null +++ b/src/Agni/Clients/GDIDAuthority.cs @@ -0,0 +1,98 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 2/18/2015 8:21:59 PM at SEXTOD by Anton +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IGDIDAuthorityClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IGDIDAuthority server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class GDIDAuthority : ClientEndPoint, @Agni.@Contracts.@IGDIDAuthorityClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_AllocateBlock_0; + + //static .ctor + static GDIDAuthority() + { + var t = typeof(@Agni.@Contracts.@IGDIDAuthority); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_AllocateBlock_0 = new MethodSpec(t.GetMethod("AllocateBlock", new Type[]{ typeof(@System.@String), typeof(@System.@String), typeof(@System.@Int32), typeof(@System.@Nullable<@System.@UInt64>) })); + } + #endregion + + #region .ctor + public GDIDAuthority(string node, Binding binding = null) : base(node, binding) { ctor(); } + public GDIDAuthority(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public GDIDAuthority(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public GDIDAuthority(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IGDIDAuthority); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IGDIDAuthority.AllocateBlock'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@GDIDBlock' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@GDIDBlock @AllocateBlock(@System.@String @scopeName, @System.@String @sequenceName, @System.@Int32 @blockSize, @System.@Nullable<@System.@UInt64> @vicinity) + { + var call = Async_AllocateBlock(@scopeName, @sequenceName, @blockSize, @vicinity); + return call.GetValue<@Agni.@Contracts.@GDIDBlock>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IGDIDAuthority.AllocateBlock'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_AllocateBlock(@System.@String @scopeName, @System.@String @sequenceName, @System.@Int32 @blockSize, @System.@Nullable<@System.@UInt64> @vicinity) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_AllocateBlock_0, false, RemoteInstance, new object[]{@scopeName, @sequenceName, @blockSize, @vicinity}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/GDIDPersistenceRemoteLocation.cs b/src/Agni/Clients/GDIDPersistenceRemoteLocation.cs new file mode 100644 index 0000000..05fa8f3 --- /dev/null +++ b/src/Agni/Clients/GDIDPersistenceRemoteLocation.cs @@ -0,0 +1,127 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 5/19/2017 8:47:54 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IGDIDPersistenceRemoteLocationClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IGDIDPersistenceRemoteLocation server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class GDIDPersistenceRemoteLocation : ClientEndPoint, @Agni.@Contracts.@IGDIDPersistenceRemoteLocationClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_Read_0; + private static MethodSpec @s_ms_Write_1; + + //static .ctor + static GDIDPersistenceRemoteLocation() + { + var t = typeof(@Agni.@Contracts.@IGDIDPersistenceRemoteLocation); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_Read_0 = new MethodSpec(t.GetMethod("Read", new Type[]{ typeof(@System.@Byte), typeof(@System.@String), typeof(@System.@String) })); + @s_ms_Write_1 = new MethodSpec(t.GetMethod("Write", new Type[]{ typeof(@System.@String), typeof(@System.@String), typeof(@NFX.@DataAccess.@Distributed.@GDID) })); + } + #endregion + + #region .ctor + public GDIDPersistenceRemoteLocation(string node, Binding binding = null) : base(node, binding) { ctor(); } + public GDIDPersistenceRemoteLocation(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public GDIDPersistenceRemoteLocation(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public GDIDPersistenceRemoteLocation(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IGDIDPersistenceRemoteLocation); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IGDIDPersistenceRemoteLocation.Read'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Nullable<@NFX.@DataAccess.@Distributed.@GDID>' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Nullable<@NFX.@DataAccess.@Distributed.@GDID> @Read(@System.@Byte @authority, @System.@String @sequenceName, @System.@String @scopeName) + { + var call = Async_Read(@authority, @sequenceName, @scopeName); + return call.GetValue<@System.@Nullable<@NFX.@DataAccess.@Distributed.@GDID>>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IGDIDPersistenceRemoteLocation.Read'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Read(@System.@Byte @authority, @System.@String @sequenceName, @System.@String @scopeName) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Read_0, false, RemoteInstance, new object[]{@authority, @sequenceName, @scopeName}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IGDIDPersistenceRemoteLocation.Write'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public void @Write(@System.@String @sequenceName, @System.@String @scopeName, @NFX.@DataAccess.@Distributed.@GDID @value) + { + var call = Async_Write(@sequenceName, @scopeName, @value); + call.CheckVoidValue(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IGDIDPersistenceRemoteLocation.Write'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Write(@System.@String @sequenceName, @System.@String @scopeName, @NFX.@DataAccess.@Distributed.@GDID @value) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Write_1, false, RemoteInstance, new object[]{@sequenceName, @scopeName, @value}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/HostGovernor.cs b/src/Agni/Clients/HostGovernor.cs new file mode 100644 index 0000000..e4c8abe --- /dev/null +++ b/src/Agni/Clients/HostGovernor.cs @@ -0,0 +1,98 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 4/4/2017 8:51:29 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IHostGovernorClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IHostGovernor server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class HostGovernor : ClientEndPoint, @Agni.@Contracts.@IHostGovernorClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_GetHostInfo_0; + + //static .ctor + static HostGovernor() + { + var t = typeof(@Agni.@Contracts.@IHostGovernor); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_GetHostInfo_0 = new MethodSpec(t.GetMethod("GetHostInfo", new Type[]{ })); + } + #endregion + + #region .ctor + public HostGovernor(string node, Binding binding = null) : base(node, binding) { ctor(); } + public HostGovernor(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public HostGovernor(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public HostGovernor(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IHostGovernor); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IHostGovernor.GetHostInfo'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@HostInfo' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@HostInfo @GetHostInfo() + { + var call = Async_GetHostInfo(); + return call.GetValue<@Agni.@Contracts.@HostInfo>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IHostGovernor.GetHostInfo'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetHostInfo() + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetHostInfo_0, false, RemoteInstance, new object[]{}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/Locker.cs b/src/Agni/Clients/Locker.cs new file mode 100644 index 0000000..2fe667f --- /dev/null +++ b/src/Agni/Clients/Locker.cs @@ -0,0 +1,127 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 8/14/2016 20:08:43 at MIGHTY by opana +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@ILockerClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.ILocker server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class Locker : ClientEndPoint, @Agni.@Contracts.@ILockerClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_ExecuteLockTransaction_0; + private static MethodSpec @s_ms_EndLockSession_1; + + //static .ctor + static Locker() + { + var t = typeof(@Agni.@Contracts.@ILocker); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_ExecuteLockTransaction_0 = new MethodSpec(t.GetMethod("ExecuteLockTransaction", new Type[]{ typeof(@Agni.@Locking.@Server.@LockSessionData), typeof(@Agni.@Locking.@LockTransaction) })); + @s_ms_EndLockSession_1 = new MethodSpec(t.GetMethod("EndLockSession", new Type[]{ typeof(@Agni.@Locking.@LockSessionID) })); + } + #endregion + + #region .ctor + public Locker(string node, Binding binding = null) : base(node, binding) { ctor(); } + public Locker(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public Locker(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public Locker(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@ILocker); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.ILocker.ExecuteLockTransaction'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Locking.@LockTransactionResult' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Locking.@LockTransactionResult @ExecuteLockTransaction(@Agni.@Locking.@Server.@LockSessionData @session, @Agni.@Locking.@LockTransaction @transaction) + { + var call = Async_ExecuteLockTransaction(@session, @transaction); + return call.GetValue<@Agni.@Locking.@LockTransactionResult>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ILocker.ExecuteLockTransaction'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_ExecuteLockTransaction(@Agni.@Locking.@Server.@LockSessionData @session, @Agni.@Locking.@LockTransaction @transaction) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_ExecuteLockTransaction_0, false, RemoteInstance, new object[]{@session, @transaction}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.ILocker.EndLockSession'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Boolean' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Boolean @EndLockSession(@Agni.@Locking.@LockSessionID @sessionID) + { + var call = Async_EndLockSession(@sessionID); + return call.GetValue<@System.@Boolean>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ILocker.EndLockSession'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_EndLockSession(@Agni.@Locking.@LockSessionID @sessionID) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_EndLockSession_1, false, RemoteInstance, new object[]{@sessionID}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/LogReceiver.cs b/src/Agni/Clients/LogReceiver.cs new file mode 100644 index 0000000..6062f4a --- /dev/null +++ b/src/Agni/Clients/LogReceiver.cs @@ -0,0 +1,159 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 6/1/2017 8:48:15 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@ILogReceiverClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.ILogReceiver server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class LogReceiver : ClientEndPoint, @Agni.@Contracts.@ILogReceiverClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_SendLog_0; + private static MethodSpec @s_ms_GetByID_1; + private static MethodSpec @s_ms_List_2; + + //static .ctor + static LogReceiver() + { + var t = typeof(@Agni.@Contracts.@ILogReceiver); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_SendLog_0 = new MethodSpec(t.GetMethod("SendLog", new Type[]{ typeof(@NFX.@Log.@Message) })); + @s_ms_GetByID_1 = new MethodSpec(t.GetMethod("GetByID", new Type[]{ typeof(@System.@Guid), typeof(@System.@String) })); + @s_ms_List_2 = new MethodSpec(t.GetMethod("List", new Type[]{ typeof(@System.@String), typeof(@System.@DateTime), typeof(@System.@DateTime), typeof(@System.@Nullable<@NFX.@Log.@MessageType>), typeof(@System.@String), typeof(@System.@String), typeof(@System.@String), typeof(@System.@Nullable<@System.@Guid>), typeof(@System.@Int32) })); + } + #endregion + + #region .ctor + public LogReceiver(string node, Binding binding = null) : base(node, binding) { ctor(); } + public LogReceiver(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public LogReceiver(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public LogReceiver(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@ILogReceiver); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.ILogReceiver.SendLog'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// + public void @SendLog(@NFX.@Log.@Message @data) + { + var call = Async_SendLog(@data); + if (call.CallStatus != CallStatus.Dispatched) + throw new ClientCallException(call.CallStatus, "Call failed: 'LogReceiver.SendLog'"); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ILogReceiver.SendLog'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg. + /// + public CallSlot Async_SendLog(@NFX.@Log.@Message @data) + { + var request = new @Agni.@Contracts.@RequestMsg_ILogReceiver_SendLog(s_ts_CONTRACT, @s_ms_SendLog_0, true, RemoteInstance) + { + MethodArg_0_data = @data, + }; + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.ILogReceiver.GetByID'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@NFX.@Log.@Message' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @NFX.@Log.@Message @GetByID(@System.@Guid @id, @System.@String @channel) + { + var call = Async_GetByID(@id, @channel); + return call.GetValue<@NFX.@Log.@Message>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ILogReceiver.GetByID'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetByID(@System.@Guid @id, @System.@String @channel) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetByID_1, false, RemoteInstance, new object[]{@id, @channel}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.ILogReceiver.List'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Collections.@Generic.@IEnumerable<@NFX.@Log.@Message>' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Collections.@Generic.@IEnumerable<@NFX.@Log.@Message> @List(@System.@String @archiveDimensionsFilter, @System.@DateTime @startDate, @System.@DateTime @endDate, @System.@Nullable<@NFX.@Log.@MessageType> @type, @System.@String @host, @System.@String @channel, @System.@String @topic, @System.@Nullable<@System.@Guid> @relatedTo, @System.@Int32 @skipCount) + { + var call = Async_List(@archiveDimensionsFilter, @startDate, @endDate, @type, @host, @channel, @topic, @relatedTo, @skipCount); + return call.GetValue<@System.@Collections.@Generic.@IEnumerable<@NFX.@Log.@Message>>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ILogReceiver.List'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_List(@System.@String @archiveDimensionsFilter, @System.@DateTime @startDate, @System.@DateTime @endDate, @System.@Nullable<@NFX.@Log.@MessageType> @type, @System.@String @host, @System.@String @channel, @System.@String @topic, @System.@Nullable<@System.@Guid> @relatedTo, @System.@Int32 @skipCount) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_List_2, false, RemoteInstance, new object[]{@archiveDimensionsFilter, @startDate, @endDate, @type, @host, @channel, @topic, @relatedTo, @skipCount}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/Pinger.cs b/src/Agni/Clients/Pinger.cs new file mode 100644 index 0000000..4c2762e --- /dev/null +++ b/src/Agni/Clients/Pinger.cs @@ -0,0 +1,98 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 4/4/2017 8:51:29 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IPingerClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IPinger server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class Pinger : ClientEndPoint, @Agni.@Contracts.@IPingerClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_Ping_0; + + //static .ctor + static Pinger() + { + var t = typeof(@Agni.@Contracts.@IPinger); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_Ping_0 = new MethodSpec(t.GetMethod("Ping", new Type[]{ })); + } + #endregion + + #region .ctor + public Pinger(string node, Binding binding = null) : base(node, binding) { ctor(); } + public Pinger(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public Pinger(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public Pinger(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IPinger); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IPinger.Ping'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public void @Ping() + { + var call = Async_Ping(); + call.CheckVoidValue(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IPinger.Ping'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Ping() + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Ping_0, false, RemoteInstance, new object[]{}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/ProcessController.cs b/src/Agni/Clients/ProcessController.cs new file mode 100644 index 0000000..38cf0e2 --- /dev/null +++ b/src/Agni/Clients/ProcessController.cs @@ -0,0 +1,214 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 5/19/2017 8:47:54 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IProcessControllerClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IProcessController server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class ProcessController : ClientEndPoint, @Agni.@Contracts.@IProcessControllerClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_Spawn_0; + private static MethodSpec @s_ms_Dispatch_1; + private static MethodSpec @s_ms_Get_2; + private static MethodSpec @s_ms_GetDescriptor_3; + private static MethodSpec @s_ms_List_4; + + //static .ctor + static ProcessController() + { + var t = typeof(@Agni.@Contracts.@IProcessController); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_Spawn_0 = new MethodSpec(t.GetMethod("Spawn", new Type[]{ typeof(@Agni.@Workers.@ProcessFrame) })); + @s_ms_Dispatch_1 = new MethodSpec(t.GetMethod("Dispatch", new Type[]{ typeof(@Agni.@Workers.@SignalFrame) })); + @s_ms_Get_2 = new MethodSpec(t.GetMethod("Get", new Type[]{ typeof(@Agni.@Workers.@PID) })); + @s_ms_GetDescriptor_3 = new MethodSpec(t.GetMethod("GetDescriptor", new Type[]{ typeof(@Agni.@Workers.@PID) })); + @s_ms_List_4 = new MethodSpec(t.GetMethod("List", new Type[]{ typeof(@System.@Int32) })); + } + #endregion + + #region .ctor + public ProcessController(string node, Binding binding = null) : base(node, binding) { ctor(); } + public ProcessController(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public ProcessController(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public ProcessController(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IProcessController); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IProcessController.Spawn'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public void @Spawn(@Agni.@Workers.@ProcessFrame @frame) + { + var call = Async_Spawn(@frame); + call.CheckVoidValue(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IProcessController.Spawn'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Spawn(@Agni.@Workers.@ProcessFrame @frame) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Spawn_0, false, RemoteInstance, new object[]{@frame}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IProcessController.Dispatch'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Workers.@SignalFrame' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Workers.@SignalFrame @Dispatch(@Agni.@Workers.@SignalFrame @signal) + { + var call = Async_Dispatch(@signal); + return call.GetValue<@Agni.@Workers.@SignalFrame>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IProcessController.Dispatch'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Dispatch(@Agni.@Workers.@SignalFrame @signal) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Dispatch_1, false, RemoteInstance, new object[]{@signal}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IProcessController.Get'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Workers.@ProcessFrame' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Workers.@ProcessFrame @Get(@Agni.@Workers.@PID @pid) + { + var call = Async_Get(@pid); + return call.GetValue<@Agni.@Workers.@ProcessFrame>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IProcessController.Get'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Get(@Agni.@Workers.@PID @pid) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Get_2, false, RemoteInstance, new object[]{@pid}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IProcessController.GetDescriptor'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Workers.@ProcessDescriptor' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Workers.@ProcessDescriptor @GetDescriptor(@Agni.@Workers.@PID @pid) + { + var call = Async_GetDescriptor(@pid); + return call.GetValue<@Agni.@Workers.@ProcessDescriptor>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IProcessController.GetDescriptor'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetDescriptor(@Agni.@Workers.@PID @pid) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetDescriptor_3, false, RemoteInstance, new object[]{@pid}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IProcessController.List'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Collections.@Generic.@IEnumerable<@Agni.@Workers.@ProcessDescriptor>' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Collections.@Generic.@IEnumerable<@Agni.@Workers.@ProcessDescriptor> @List(@System.@Int32 @processorID) + { + var call = Async_List(@processorID); + return call.GetValue<@System.@Collections.@Generic.@IEnumerable<@Agni.@Workers.@ProcessDescriptor>>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IProcessController.List'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_List(@System.@Int32 @processorID) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_List_4, false, RemoteInstance, new object[]{@processorID}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/RemoteTerminal.cs b/src/Agni/Clients/RemoteTerminal.cs new file mode 100644 index 0000000..258abbb --- /dev/null +++ b/src/Agni/Clients/RemoteTerminal.cs @@ -0,0 +1,156 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 2/18/2015 8:21:59 PM at SEXTOD by Anton +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IRemoteTerminalClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IRemoteTerminal server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class RemoteTerminal : ClientEndPoint, @Agni.@Contracts.@IRemoteTerminalClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_Connect_0; + private static MethodSpec @s_ms_Execute_1; + private static MethodSpec @s_ms_Disconnect_2; + + //static .ctor + static RemoteTerminal() + { + var t = typeof(@Agni.@Contracts.@IRemoteTerminal); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_Connect_0 = new MethodSpec(t.GetMethod("Connect", new Type[]{ typeof(@System.@String) })); + @s_ms_Execute_1 = new MethodSpec(t.GetMethod("Execute", new Type[]{ typeof(@System.@String) })); + @s_ms_Disconnect_2 = new MethodSpec(t.GetMethod("Disconnect", new Type[]{ })); + } + #endregion + + #region .ctor + public RemoteTerminal(string node, Binding binding = null) : base(node, binding) { ctor(); } + public RemoteTerminal(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public RemoteTerminal(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public RemoteTerminal(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IRemoteTerminal); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IRemoteTerminal.Connect'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@RemoteTerminalInfo' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@RemoteTerminalInfo @Connect(@System.@String @who) + { + var call = Async_Connect(@who); + return call.GetValue<@Agni.@Contracts.@RemoteTerminalInfo>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IRemoteTerminal.Connect'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Connect(@System.@String @who) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Connect_0, false, RemoteInstance, new object[]{@who}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IRemoteTerminal.Execute'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@String' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@String @Execute(@System.@String @command) + { + var call = Async_Execute(@command); + return call.GetValue<@System.@String>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IRemoteTerminal.Execute'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Execute(@System.@String @command) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Execute_1, false, RemoteInstance, new object[]{@command}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IRemoteTerminal.Disconnect'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@String' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@String @Disconnect() + { + var call = Async_Disconnect(); + return call.GetValue<@System.@String>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IRemoteTerminal.Disconnect'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Disconnect() + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Disconnect_2, false, RemoteInstance, new object[]{}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/TelemetryReceiver.cs b/src/Agni/Clients/TelemetryReceiver.cs new file mode 100644 index 0000000..9c7b707 --- /dev/null +++ b/src/Agni/Clients/TelemetryReceiver.cs @@ -0,0 +1,101 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 6/28/2017 12:01:46 AM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@ITelemetryReceiverClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.ITelemetryReceiver server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class TelemetryReceiver : ClientEndPoint, @Agni.@Contracts.@ITelemetryReceiverClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_SendDatums_0; + + //static .ctor + static TelemetryReceiver() + { + var t = typeof(@Agni.@Contracts.@ITelemetryReceiver); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_SendDatums_0 = new MethodSpec(t.GetMethod("SendDatums", new Type[]{ typeof(@NFX.@Instrumentation.@Datum[]) })); + } + #endregion + + #region .ctor + public TelemetryReceiver(string node, Binding binding = null) : base(node, binding) { ctor(); } + public TelemetryReceiver(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public TelemetryReceiver(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public TelemetryReceiver(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@ITelemetryReceiver); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.ITelemetryReceiver.SendDatums'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// + public void @SendDatums(@NFX.@Instrumentation.@Datum[] @data) + { + var call = Async_SendDatums(@data); + if (call.CallStatus != CallStatus.Dispatched) + throw new ClientCallException(call.CallStatus, "Call failed: 'TelemetryReceiver.SendDatums'"); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ITelemetryReceiver.SendDatums'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg. + /// + public CallSlot Async_SendDatums(@NFX.@Instrumentation.@Datum[] @data) + { + var request = new @Agni.@Contracts.@RequestMsg_ITelemetryReceiver_SendDatums(s_ts_CONTRACT, @s_ms_SendDatums_0, true, RemoteInstance) + { + MethodArg_0_data = @data, + }; + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/Tester.cs b/src/Agni/Clients/Tester.cs new file mode 100644 index 0000000..539176c --- /dev/null +++ b/src/Agni/Clients/Tester.cs @@ -0,0 +1,98 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 5/26/2015 7:14:44 PM at SEXTOD by Anton +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@ITesterClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.ITester server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class Tester : ClientEndPoint, @Agni.@Contracts.@ITesterClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_TestEcho_0; + + //static .ctor + static Tester() + { + var t = typeof(@Agni.@Contracts.@ITester); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_TestEcho_0 = new MethodSpec(t.GetMethod("TestEcho", new Type[]{ typeof(@System.@Object) })); + } + #endregion + + #region .ctor + public Tester(string node, Binding binding = null) : base(node, binding) { ctor(); } + public Tester(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public Tester(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public Tester(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@ITester); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.ITester.TestEcho'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Object' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Object @TestEcho(@System.@Object @data) + { + var call = Async_TestEcho(@data); + return call.GetValue<@System.@Object>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ITester.TestEcho'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_TestEcho(@System.@Object @data) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_TestEcho_0, false, RemoteInstance, new object[]{@data}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/TodoQueue.cs b/src/Agni/Clients/TodoQueue.cs new file mode 100644 index 0000000..b976f06 --- /dev/null +++ b/src/Agni/Clients/TodoQueue.cs @@ -0,0 +1,98 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 1/25/2017 11:43:46 PM at SEXTOD by Anton +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@ITodoQueueClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.ITodoQueue server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class TodoQueue : ClientEndPoint, @Agni.@Contracts.@ITodoQueueClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_Enqueue_0; + + //static .ctor + static TodoQueue() + { + var t = typeof(@Agni.@Contracts.@ITodoQueue); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_Enqueue_0 = new MethodSpec(t.GetMethod("Enqueue", new Type[]{ typeof(@Agni.@Workers.@TodoFrame[]) })); + } + #endregion + + #region .ctor + public TodoQueue(string node, Binding binding = null) : base(node, binding) { ctor(); } + public TodoQueue(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public TodoQueue(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public TodoQueue(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@ITodoQueue); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.ITodoQueue.Enqueue'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Int32' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Int32 @Enqueue(@Agni.@Workers.@TodoFrame[] @todos) + { + var call = Async_Enqueue(@todos); + return call.GetValue<@System.@Int32>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.ITodoQueue.Enqueue'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Enqueue(@Agni.@Workers.@TodoFrame[] @todos) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Enqueue_0, false, RemoteInstance, new object[]{@todos}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/Tools/AgniGluecCompiler.cs b/src/Agni/Clients/Tools/AgniGluecCompiler.cs new file mode 100644 index 0000000..be1f51c --- /dev/null +++ b/src/Agni/Clients/Tools/AgniGluecCompiler.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Reflection; + +using NFX; +using NFX.Environment; +using NFX.Glue; +using NFX.Glue.Tools; + +/* + Usage pattern: + gluec Agni.dll -c "Agni.Clients.Tools.AgniGluecCompiler, Agni" -o ns-suffix="" cl-suffix="" ns-root="Agni.Clients" +*/ + + +namespace Agni.Clients.Tools +{ + public sealed class AgniGluecCompiler : CSharpGluecCompiler + { + public AgniGluecCompiler(Assembly asm) : base(asm) {} + + + protected override string FileHeader() + { + return +@"//Generated by {0} +{1}".Args( GetType().FullName, base.FileHeader() ); + } + + protected override void BeforeClientClass(StringBuilder sb, Type tc, string cname, string iname) + { + sb.AppendLine("// This implementation needs {0}, so".Args(iname)); + sb.AppendLine("// it can be used with ServiceClientHub class"); + sb.AppendLine(); + } + + protected override string GetClientInterfaceName(Type tc) + { + return TypeToStr(tc) + "Client"; + } + + } +} diff --git a/src/Agni/Clients/WebMessageSystem.cs b/src/Agni/Clients/WebMessageSystem.cs new file mode 100644 index 0000000..72ac6de --- /dev/null +++ b/src/Agni/Clients/WebMessageSystem.cs @@ -0,0 +1,343 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 7/22/2017 1:10:48 PM at SEXTOD by Anton +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IWebMessageSystemClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IWebMessageSystem server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class WebMessageSystem : ClientEndPoint, @Agni.@Contracts.@IWebMessageSystemClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_SendMessage_0; + private static MethodSpec @s_ms_GetMailboxInfo_1; + private static MethodSpec @s_ms_UpdateMailboxMessageStatus_2; + private static MethodSpec @s_ms_UpdateMailboxMessagesStatus_3; + private static MethodSpec @s_ms_UpdateMailboxMessagePublication_4; + private static MethodSpec @s_ms_GetMailboxMessageHeaders_5; + private static MethodSpec @s_ms_GetMailboxMessageCount_6; + private static MethodSpec @s_ms_FetchMailboxMessage_7; + private static MethodSpec @s_ms_FetchMailboxMessageAttachment_8; + + //static .ctor + static WebMessageSystem() + { + var t = typeof(@Agni.@Contracts.@IWebMessageSystem); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_SendMessage_0 = new MethodSpec(t.GetMethod("SendMessage", new Type[]{ typeof(@Agni.@WebMessaging.@AgniWebMessage) })); + @s_ms_GetMailboxInfo_1 = new MethodSpec(t.GetMethod("GetMailboxInfo", new Type[]{ typeof(@Agni.@WebMessaging.@MailboxID) })); + @s_ms_UpdateMailboxMessageStatus_2 = new MethodSpec(t.GetMethod("UpdateMailboxMessageStatus", new Type[]{ typeof(@Agni.@WebMessaging.@MailboxMsgID), typeof(@Agni.@WebMessaging.@MsgStatus), typeof(@System.@String), typeof(@System.@String) })); + @s_ms_UpdateMailboxMessagesStatus_3 = new MethodSpec(t.GetMethod("UpdateMailboxMessagesStatus", new Type[]{ typeof(@System.@Collections.@Generic.@IEnumerable<@Agni.@WebMessaging.@MailboxMsgID>), typeof(@Agni.@WebMessaging.@MsgStatus), typeof(@System.@String), typeof(@System.@String) })); + @s_ms_UpdateMailboxMessagePublication_4 = new MethodSpec(t.GetMethod("UpdateMailboxMessagePublication", new Type[]{ typeof(@Agni.@WebMessaging.@MailboxMsgID), typeof(@Agni.@WebMessaging.@MsgPubStatus), typeof(@System.@String), typeof(@System.@String) })); + @s_ms_GetMailboxMessageHeaders_5 = new MethodSpec(t.GetMethod("GetMailboxMessageHeaders", new Type[]{ typeof(@Agni.@WebMessaging.@MailboxID), typeof(@System.@String) })); + @s_ms_GetMailboxMessageCount_6 = new MethodSpec(t.GetMethod("GetMailboxMessageCount", new Type[]{ typeof(@Agni.@WebMessaging.@MailboxID), typeof(@System.@String) })); + @s_ms_FetchMailboxMessage_7 = new MethodSpec(t.GetMethod("FetchMailboxMessage", new Type[]{ typeof(@Agni.@WebMessaging.@MailboxMsgID) })); + @s_ms_FetchMailboxMessageAttachment_8 = new MethodSpec(t.GetMethod("FetchMailboxMessageAttachment", new Type[]{ typeof(@Agni.@WebMessaging.@MailboxMsgID), typeof(@System.@Int32) })); + } + #endregion + + #region .ctor + public WebMessageSystem(string node, Binding binding = null) : base(node, binding) { ctor(); } + public WebMessageSystem(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public WebMessageSystem(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public WebMessageSystem(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IWebMessageSystem); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.SendMessage'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@MsgSendInfo[]' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@MsgSendInfo[] @SendMessage(@Agni.@WebMessaging.@AgniWebMessage @msg) + { + var call = Async_SendMessage(@msg); + return call.GetValue<@Agni.@Contracts.@MsgSendInfo[]>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.SendMessage'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_SendMessage(@Agni.@WebMessaging.@AgniWebMessage @msg) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_SendMessage_0, false, RemoteInstance, new object[]{@msg}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.GetMailboxInfo'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@WebMessaging.@MailboxInfo' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@WebMessaging.@MailboxInfo @GetMailboxInfo(@Agni.@WebMessaging.@MailboxID @xid) + { + var call = Async_GetMailboxInfo(@xid); + return call.GetValue<@Agni.@WebMessaging.@MailboxInfo>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.GetMailboxInfo'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetMailboxInfo(@Agni.@WebMessaging.@MailboxID @xid) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetMailboxInfo_1, false, RemoteInstance, new object[]{@xid}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.UpdateMailboxMessageStatus'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// + public void @UpdateMailboxMessageStatus(@Agni.@WebMessaging.@MailboxMsgID @mid, @Agni.@WebMessaging.@MsgStatus @status, @System.@String @folders, @System.@String @adornments) + { + var call = Async_UpdateMailboxMessageStatus(@mid, @status, @folders, @adornments); + if (call.CallStatus != CallStatus.Dispatched) + throw new ClientCallException(call.CallStatus, "Call failed: 'WebMessageSystem.UpdateMailboxMessageStatus'"); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.UpdateMailboxMessageStatus'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg. + /// + public CallSlot Async_UpdateMailboxMessageStatus(@Agni.@WebMessaging.@MailboxMsgID @mid, @Agni.@WebMessaging.@MsgStatus @status, @System.@String @folders, @System.@String @adornments) + { + var request = new @Agni.@Contracts.@RequestMsg_IWebMessageSystem_UpdateMailboxMessageStatus(s_ts_CONTRACT, @s_ms_UpdateMailboxMessageStatus_2, true, RemoteInstance) + { + MethodArg_0_mid = @mid, + MethodArg_1_status = @status, + MethodArg_2_folders = @folders, + MethodArg_3_adornments = @adornments, + }; + return DispatchCall(request); + } + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.UpdateMailboxMessagesStatus'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// + public void @UpdateMailboxMessagesStatus(@System.@Collections.@Generic.@IEnumerable<@Agni.@WebMessaging.@MailboxMsgID> @mids, @Agni.@WebMessaging.@MsgStatus @status, @System.@String @folders, @System.@String @adornments) + { + var call = Async_UpdateMailboxMessagesStatus(@mids, @status, @folders, @adornments); + if (call.CallStatus != CallStatus.Dispatched) + throw new ClientCallException(call.CallStatus, "Call failed: 'WebMessageSystem.UpdateMailboxMessagesStatus'"); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.UpdateMailboxMessagesStatus'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg. + /// + public CallSlot Async_UpdateMailboxMessagesStatus(@System.@Collections.@Generic.@IEnumerable<@Agni.@WebMessaging.@MailboxMsgID> @mids, @Agni.@WebMessaging.@MsgStatus @status, @System.@String @folders, @System.@String @adornments) + { + var request = new @Agni.@Contracts.@RequestMsg_IWebMessageSystem_UpdateMailboxMessagesStatus(s_ts_CONTRACT, @s_ms_UpdateMailboxMessagesStatus_3, true, RemoteInstance) + { + MethodArg_0_mids = @mids, + MethodArg_1_status = @status, + MethodArg_2_folders = @folders, + MethodArg_3_adornments = @adornments, + }; + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.UpdateMailboxMessagePublication'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// + public void @UpdateMailboxMessagePublication(@Agni.@WebMessaging.@MailboxMsgID @mid, @Agni.@WebMessaging.@MsgPubStatus @status, @System.@String @oper, @System.@String @description) + { + var call = Async_UpdateMailboxMessagePublication(@mid, @status, @oper, @description); + if (call.CallStatus != CallStatus.Dispatched) + throw new ClientCallException(call.CallStatus, "Call failed: 'WebMessageSystem.UpdateMailboxMessagePublication'"); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.UpdateMailboxMessagePublication'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg. + /// + public CallSlot Async_UpdateMailboxMessagePublication(@Agni.@WebMessaging.@MailboxMsgID @mid, @Agni.@WebMessaging.@MsgPubStatus @status, @System.@String @oper, @System.@String @description) + { + var request = new @Agni.@Contracts.@RequestMsg_IWebMessageSystem_UpdateMailboxMessagePublication(s_ts_CONTRACT, @s_ms_UpdateMailboxMessagePublication_4, true, RemoteInstance) + { + MethodArg_0_mid = @mid, + MethodArg_1_status = @status, + MethodArg_2_oper = @oper, + MethodArg_3_description = @description, + }; + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.GetMailboxMessageHeaders'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@MessageHeaders' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@MessageHeaders @GetMailboxMessageHeaders(@Agni.@WebMessaging.@MailboxID @xid, @System.@String @query) + { + var call = Async_GetMailboxMessageHeaders(@xid, @query); + return call.GetValue<@Agni.@Contracts.@MessageHeaders>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.GetMailboxMessageHeaders'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetMailboxMessageHeaders(@Agni.@WebMessaging.@MailboxID @xid, @System.@String @query) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetMailboxMessageHeaders_5, false, RemoteInstance, new object[]{@xid, @query}); + return DispatchCall(request); + } + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.GetMailboxMessageCount'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@MessageHeaders' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public int @GetMailboxMessageCount(@Agni.@WebMessaging.@MailboxID @xid, @System.@String @query) + { + var call = Async_GetMailboxMessageCount(@xid, @query); + return call.GetValue(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.GetMailboxMessageCount'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetMailboxMessageCount(@Agni.@WebMessaging.@MailboxID @xid, @System.@String @query) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetMailboxMessageCount_6, false, RemoteInstance, new object[]{@xid, @query}); + return DispatchCall(request); + } + + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.FetchMailboxMessage'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@WebMessaging.@AgniWebMessage' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@WebMessaging.@AgniWebMessage @FetchMailboxMessage(@Agni.@WebMessaging.@MailboxMsgID @mid) + { + var call = Async_FetchMailboxMessage(@mid); + return call.GetValue<@Agni.@WebMessaging.@AgniWebMessage>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.FetchMailboxMessage'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_FetchMailboxMessage(@Agni.@WebMessaging.@MailboxMsgID @mid) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, s_ms_FetchMailboxMessage_7, false, RemoteInstance, new object[]{@mid}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IWebMessageSystem.FetchMailboxMessageAttachment'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@NFX.@Web.@Messaging.@Message.@Attachment' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @NFX.@Web.@Messaging.@Message.@Attachment @FetchMailboxMessageAttachment(@Agni.@WebMessaging.@MailboxMsgID @mid, @System.@Int32 @attachmentIndex) + { + var call = Async_FetchMailboxMessageAttachment(@mid, @attachmentIndex); + return call.GetValue<@NFX.@Web.@Messaging.@Message.@Attachment>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IWebMessageSystem.FetchMailboxMessageAttachment'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_FetchMailboxMessageAttachment(@Agni.@WebMessaging.@MailboxMsgID @mid, @System.@Int32 @attachmentIndex) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, s_ms_FetchMailboxMessageAttachment_8, false, RemoteInstance, new object[]{@mid, @attachmentIndex}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/ZoneHostRegistry.cs b/src/Agni/Clients/ZoneHostRegistry.cs new file mode 100644 index 0000000..4ddc539 --- /dev/null +++ b/src/Agni/Clients/ZoneHostRegistry.cs @@ -0,0 +1,185 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 4/5/2017 7:30:44 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IZoneHostRegistryClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IZoneHostRegistry server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class ZoneHostRegistry : ClientEndPoint, @Agni.@Contracts.@IZoneHostRegistryClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_GetSubordinateHosts_0; + private static MethodSpec @s_ms_GetSubordinateHost_1; + private static MethodSpec @s_ms_RegisterSubordinateHost_2; + private static MethodSpec @s_ms_Spawn_3; + + //static .ctor + static ZoneHostRegistry() + { + var t = typeof(@Agni.@Contracts.@IZoneHostRegistry); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_GetSubordinateHosts_0 = new MethodSpec(t.GetMethod("GetSubordinateHosts", new Type[]{ typeof(@System.@String) })); + @s_ms_GetSubordinateHost_1 = new MethodSpec(t.GetMethod("GetSubordinateHost", new Type[]{ typeof(@System.@String) })); + @s_ms_RegisterSubordinateHost_2 = new MethodSpec(t.GetMethod("RegisterSubordinateHost", new Type[]{ typeof(@Agni.@Contracts.@HostInfo), typeof(@System.@Nullable<@Agni.@Contracts.@DynamicHostID>) })); + @s_ms_Spawn_3 = new MethodSpec(t.GetMethod("Spawn", new Type[]{ typeof(@System.@String), typeof(@System.@String) })); + } + #endregion + + #region .ctor + public ZoneHostRegistry(string node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneHostRegistry(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneHostRegistry(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public ZoneHostRegistry(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IZoneHostRegistry); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneHostRegistry.GetSubordinateHosts'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Collections.@Generic.@IEnumerable<@Agni.@Contracts.@HostInfo>' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Collections.@Generic.@IEnumerable<@Agni.@Contracts.@HostInfo> @GetSubordinateHosts(@System.@String @hostNameSearchPattern) + { + var call = Async_GetSubordinateHosts(@hostNameSearchPattern); + return call.GetValue<@System.@Collections.@Generic.@IEnumerable<@Agni.@Contracts.@HostInfo>>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneHostRegistry.GetSubordinateHosts'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetSubordinateHosts(@System.@String @hostNameSearchPattern) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetSubordinateHosts_0, false, RemoteInstance, new object[]{@hostNameSearchPattern}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneHostRegistry.GetSubordinateHost'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@HostInfo' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@HostInfo @GetSubordinateHost(@System.@String @hostName) + { + var call = Async_GetSubordinateHost(@hostName); + return call.GetValue<@Agni.@Contracts.@HostInfo>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneHostRegistry.GetSubordinateHost'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetSubordinateHost(@System.@String @hostName) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetSubordinateHost_1, false, RemoteInstance, new object[]{@hostName}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneHostRegistry.RegisterSubordinateHost'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public void @RegisterSubordinateHost(@Agni.@Contracts.@HostInfo @host, @System.@Nullable<@Agni.@Contracts.@DynamicHostID> @hid) + { + var call = Async_RegisterSubordinateHost(@host, @hid); + call.CheckVoidValue(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneHostRegistry.RegisterSubordinateHost'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_RegisterSubordinateHost(@Agni.@Contracts.@HostInfo @host, @System.@Nullable<@Agni.@Contracts.@DynamicHostID> @hid) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_RegisterSubordinateHost_2, false, RemoteInstance, new object[]{@host, @hid}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneHostRegistry.Spawn'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@DynamicHostID' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@DynamicHostID @Spawn(@System.@String @hostPath, @System.@String @id) + { + var call = Async_Spawn(@hostPath, @id); + return call.GetValue<@Agni.@Contracts.@DynamicHostID>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneHostRegistry.Spawn'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_Spawn(@System.@String @hostPath, @System.@String @id) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_Spawn_3, false, RemoteInstance, new object[]{@hostPath, @id}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/ZoneHostReplicator.cs b/src/Agni/Clients/ZoneHostReplicator.cs new file mode 100644 index 0000000..5375544 --- /dev/null +++ b/src/Agni/Clients/ZoneHostReplicator.cs @@ -0,0 +1,162 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 4/5/2017 7:30:44 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IZoneHostReplicatorClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IZoneHostReplicator server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class ZoneHostReplicator : ClientEndPoint, @Agni.@Contracts.@IZoneHostReplicatorClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_PostDynamicHostInfo_0; + private static MethodSpec @s_ms_GetDynamicHostInfo_1; + private static MethodSpec @s_ms_PostHostInfo_2; + + //static .ctor + static ZoneHostReplicator() + { + var t = typeof(@Agni.@Contracts.@IZoneHostReplicator); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_PostDynamicHostInfo_0 = new MethodSpec(t.GetMethod("PostDynamicHostInfo", new Type[]{ typeof(@Agni.@Contracts.@DynamicHostID), typeof(@System.@DateTime), typeof(@System.@String), typeof(@System.@Int32) })); + @s_ms_GetDynamicHostInfo_1 = new MethodSpec(t.GetMethod("GetDynamicHostInfo", new Type[]{ typeof(@Agni.@Contracts.@DynamicHostID) })); + @s_ms_PostHostInfo_2 = new MethodSpec(t.GetMethod("PostHostInfo", new Type[]{ typeof(@Agni.@Contracts.@HostInfo), typeof(@System.@Nullable<@Agni.@Contracts.@DynamicHostID>) })); + } + #endregion + + #region .ctor + public ZoneHostReplicator(string node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneHostReplicator(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneHostReplicator(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public ZoneHostReplicator(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IZoneHostReplicator); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneHostReplicator.PostDynamicHostInfo'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// + public void @PostDynamicHostInfo(@Agni.@Contracts.@DynamicHostID @id, @System.@DateTime @stamp, @System.@String @owner, @System.@Int32 @votes) + { + var call = Async_PostDynamicHostInfo(@id, @stamp, @owner, @votes); + if (call.CallStatus != CallStatus.Dispatched) + throw new ClientCallException(call.CallStatus, "Call failed: 'ZoneHostReplicator.PostDynamicHostInfo'"); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneHostReplicator.PostDynamicHostInfo'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg. + /// + public CallSlot Async_PostDynamicHostInfo(@Agni.@Contracts.@DynamicHostID @id, @System.@DateTime @stamp, @System.@String @owner, @System.@Int32 @votes) + { + var request = new @Agni.@Contracts.@RequestMsg_IZoneHostReplicator_PostDynamicHostInfo(s_ts_CONTRACT, @s_ms_PostDynamicHostInfo_0, true, RemoteInstance) + { + MethodArg_0_id = @id, + MethodArg_1_stamp = @stamp, + MethodArg_2_owner = @owner, + MethodArg_3_votes = @votes, + }; + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneHostReplicator.GetDynamicHostInfo'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@Agni.@Contracts.@DynamicHostInfo' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @Agni.@Contracts.@DynamicHostInfo @GetDynamicHostInfo(@Agni.@Contracts.@DynamicHostID @hid) + { + var call = Async_GetDynamicHostInfo(@hid); + return call.GetValue<@Agni.@Contracts.@DynamicHostInfo>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneHostReplicator.GetDynamicHostInfo'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_GetDynamicHostInfo(@Agni.@Contracts.@DynamicHostID @hid) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_GetDynamicHostInfo_1, false, RemoteInstance, new object[]{@hid}); + return DispatchCall(request); + } + + + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneHostReplicator.PostHostInfo'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// + public void @PostHostInfo(@Agni.@Contracts.@HostInfo @host, @System.@Nullable<@Agni.@Contracts.@DynamicHostID> @hid) + { + var call = Async_PostHostInfo(@host, @hid); + if (call.CallStatus != CallStatus.Dispatched) + throw new ClientCallException(call.CallStatus, "Call failed: 'ZoneHostReplicator.PostHostInfo'"); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneHostReplicator.PostHostInfo'. + /// This is a one-way call per contract specification, meaning - the server sends no acknowledgement of this call receipt and + /// there is no result that server could return back to the caller. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg. + /// + public CallSlot Async_PostHostInfo(@Agni.@Contracts.@HostInfo @host, @System.@Nullable<@Agni.@Contracts.@DynamicHostID> @hid) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_PostHostInfo_2, true, RemoteInstance, new object[]{@host, @hid}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/ZoneLogReceiver.cs b/src/Agni/Clients/ZoneLogReceiver.cs new file mode 100644 index 0000000..23e6e32 --- /dev/null +++ b/src/Agni/Clients/ZoneLogReceiver.cs @@ -0,0 +1,98 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 5/19/2017 8:47:54 PM at CNONIM-PC by cnonim +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IZoneLogReceiverClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IZoneLogReceiver server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class ZoneLogReceiver : ClientEndPoint, @Agni.@Contracts.@IZoneLogReceiverClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_SendLog_0; + + //static .ctor + static ZoneLogReceiver() + { + var t = typeof(@Agni.@Contracts.@IZoneLogReceiver); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_SendLog_0 = new MethodSpec(t.GetMethod("SendLog", new Type[]{ typeof(@System.@String), typeof(@System.@String), typeof(@NFX.@Log.@Message[]) })); + } + #endregion + + #region .ctor + public ZoneLogReceiver(string node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneLogReceiver(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneLogReceiver(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public ZoneLogReceiver(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IZoneLogReceiver); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneLogReceiver.SendLog'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Int32' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Int32 @SendLog(@System.@String @host, @System.@String @appName, @NFX.@Log.@Message[] @data) + { + var call = Async_SendLog(@host, @appName, @data); + return call.GetValue<@System.@Int32>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneLogReceiver.SendLog'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_SendLog(@System.@String @host, @System.@String @appName, @NFX.@Log.@Message[] @data) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_SendLog_0, false, RemoteInstance, new object[]{@host, @appName, @data}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Clients/ZoneTelemetryReceiver.cs b/src/Agni/Clients/ZoneTelemetryReceiver.cs new file mode 100644 index 0000000..ff0c53f --- /dev/null +++ b/src/Agni/Clients/ZoneTelemetryReceiver.cs @@ -0,0 +1,98 @@ +//Generated by Agni.Clients.Tools.AgniGluecCompiler + +/* Auto generated by Glue Client Compiler tool (gluec) +on 2/18/2015 8:21:59 PM at SEXTOD by Anton +Do not modify this file by hand if you plan to regenerate this file again by the tool as manual changes will be lost +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +using NFX.Glue; +using NFX.Glue.Protocol; + + +namespace Agni.Clients +{ +// This implementation needs @Agni.@Contracts.@IZoneTelemetryReceiverClient, so +// it can be used with ServiceClientHub class + + /// + /// Client for glued contract Agni.Contracts.IZoneTelemetryReceiver server. + /// Each contract method has synchronous and asynchronous versions, the later denoted by 'Async_' prefix. + /// May inject client-level inspectors here like so: + /// client.MsgInspectors.Register( new YOUR_CLIENT_INSPECTOR_TYPE()); + /// + public class ZoneTelemetryReceiver : ClientEndPoint, @Agni.@Contracts.@IZoneTelemetryReceiverClient + { + + #region Static Members + + private static TypeSpec s_ts_CONTRACT; + private static MethodSpec @s_ms_SendTelemetry_0; + + //static .ctor + static ZoneTelemetryReceiver() + { + var t = typeof(@Agni.@Contracts.@IZoneTelemetryReceiver); + s_ts_CONTRACT = new TypeSpec(t); + @s_ms_SendTelemetry_0 = new MethodSpec(t.GetMethod("SendTelemetry", new Type[]{ typeof(@System.@String), typeof(@NFX.@Instrumentation.@Datum[]) })); + } + #endregion + + #region .ctor + public ZoneTelemetryReceiver(string node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneTelemetryReceiver(Node node, Binding binding = null) : base(node, binding) { ctor(); } + public ZoneTelemetryReceiver(IGlue glue, string node, Binding binding = null) : base(glue, node, binding) { ctor(); } + public ZoneTelemetryReceiver(IGlue glue, Node node, Binding binding = null) : base(glue, node, binding) { ctor(); } + + //common instance .ctor body + private void ctor() + { + + } + + #endregion + + public override Type Contract + { + get { return typeof(@Agni.@Contracts.@IZoneTelemetryReceiver); } + } + + + + #region Contract Methods + + /// + /// Synchronous invoker for 'Agni.Contracts.IZoneTelemetryReceiver.SendTelemetry'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning '@System.@Int32' or WrappedExceptionData instance. + /// ClientCallException is thrown if the call could not be placed in the outgoing queue. + /// RemoteException is thrown if the server generated exception during method execution. + /// + public @System.@Int32 @SendTelemetry(@System.@String @host, @NFX.@Instrumentation.@Datum[] @data) + { + var call = Async_SendTelemetry(@host, @data); + return call.GetValue<@System.@Int32>(); + } + + /// + /// Asynchronous invoker for 'Agni.Contracts.IZoneTelemetryReceiver.SendTelemetry'. + /// This is a two-way call per contract specification, meaning - the server sends the result back either + /// returning no exception or WrappedExceptionData instance. + /// CallSlot is returned that can be queried for CallStatus, ResponseMsg and result. + /// + public CallSlot Async_SendTelemetry(@System.@String @host, @NFX.@Instrumentation.@Datum[] @data) + { + var request = new RequestAnyMsg(s_ts_CONTRACT, @s_ms_SendTelemetry_0, false, RemoteInstance, new object[]{@host, @data}); + return DispatchCall(request); + } + + + #endregion + + }//class +}//namespace diff --git a/src/Agni/Contracts/BaseIntfs.cs b/src/Agni/Contracts/BaseIntfs.cs new file mode 100644 index 0000000..dab0310 --- /dev/null +++ b/src/Agni/Contracts/BaseIntfs.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Agni.Contracts +{ + + /// + /// Marker interface that denotes contracts that represent services + /// + public interface IAgniService { } + + /// + /// Marker interface for service clients that get managed by ServiceClient class + /// + public interface IAgniServiceClient : IAgniService, IDisposable + { + /// + /// Specifies timeout for the whole service call + /// + int TimeoutMs{ get; set;} + + /// + /// Indicates that this client instance should take a hold of the underlying data transmission mechanism (such as Glue transport) + /// and keep it reserved for subsequent calls. This may reduce latency for cases when many calls need to be executed in order. + /// The transport is reserved until this property is either reset to false or client instance is disposed. + /// For service clients which are based on Glue refer to NFX.Glue.ClientEndPoint.ReserveTransport + /// + bool ReserveTransport{ get; set;} + } + +} diff --git a/src/Agni/Contracts/IGDIDAuthority.cs b/src/Agni/Contracts/IGDIDAuthority.cs new file mode 100644 index 0000000..27248ce --- /dev/null +++ b/src/Agni/Contracts/IGDIDAuthority.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Glue; +using NFX.DataAccess.Distributed; +using Agni.Security.Permissions.Admin; + + +namespace Agni.Contracts +{ + /// + /// Represents a Global Distributed ID Authority contract + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IGDIDAuthority : IAgniService + { + GDIDBlock AllocateBlock(string scopeName, string sequenceName, int blockSize, ulong? vicinity = GDID.COUNTER_MAX); + } + + /// + /// Contract for client for IGDIDAuthority service + /// + public interface IGDIDAuthorityClient : IAgniServiceClient, IGDIDAuthority + { + + } + + + /// + /// Provides Global Distributed ID block allocated by authority + /// + [Serializable] + public sealed class GDIDBlock + { + public string ScopeName { get; internal set;} + public string SequenceName { get; internal set;} + public int Authority { get; internal set;} + public string AuthorityHost{ get; internal set;} + public uint Era { get; internal set;} + public ulong StartCounterInclusive { get; internal set;} + public int BlockSize { get; internal set;} + public DateTime ServerUTCTime { get; internal set;} + + [NonSerialized] + internal int __Remaining;//how much left per block + } + + + /// + /// Represents a backup location where GDID Authority persists its data + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IGDIDPersistenceRemoteLocation : IAgniService + { + GDID? Read(byte authority, string sequenceName, string scopeName); + void Write(string sequenceName, string scopeName, GDID value); + } + + /// + /// Contract for client for IGDIDPersistenceRemoteLocation service + /// + public interface IGDIDPersistenceRemoteLocationClient : IAgniServiceClient, IGDIDPersistenceRemoteLocation + { + + } +} diff --git a/src/Agni/Contracts/IHostGovernor.cs b/src/Agni/Contracts/IHostGovernor.cs new file mode 100644 index 0000000..29bd914 --- /dev/null +++ b/src/Agni/Contracts/IHostGovernor.cs @@ -0,0 +1,26 @@ +using System; + +using NFX; +using NFX.Glue; + +namespace Agni.Contracts +{ + /// + /// Returns information about the host + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IHostGovernor : IAgniService + { + HostInfo GetHostInfo(); + } + + + /// + /// Contract for client of IPinger svc + /// + public interface IHostGovernorClient : IAgniServiceClient, IHostGovernor + { + CallSlot Async_GetHostInfo(); + } +} \ No newline at end of file diff --git a/src/Agni/Contracts/ILocker.cs b/src/Agni/Contracts/ILocker.cs new file mode 100644 index 0000000..9f1d949 --- /dev/null +++ b/src/Agni/Contracts/ILocker.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Glue; +using NFX.Log; + +using Agni.Locking; +using Agni.Locking.Server; + +namespace Agni.Contracts +{ + + /// + /// Implemented by ZoneGovernors, provide distributed lock manager services + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface ILocker : IAgniService + { + LockTransactionResult ExecuteLockTransaction(LockSessionData session, LockTransaction transaction); + bool EndLockSession(LockSessionID sessionID); + } + + + /// + /// Contract for client of ILocker svc + /// + public interface ILockerClient : IAgniServiceClient, ILocker + { + CallSlot Async_ExecuteLockTransaction(LockSessionData session, LockTransaction transaction); + CallSlot Async_EndLockSession(LockSessionID sessionID); + } + + +} \ No newline at end of file diff --git a/src/Agni/Contracts/ILogReceiver.cs b/src/Agni/Contracts/ILogReceiver.cs new file mode 100644 index 0000000..05a6d16 --- /dev/null +++ b/src/Agni/Contracts/ILogReceiver.cs @@ -0,0 +1,52 @@ +using System; +using System.Reflection; +using System.Collections.Generic; + +using NFX.Glue; +using NFX.Glue.Protocol; +using NFX.Log; + +namespace Agni.Contracts +{ + + /// + /// Implemented by ILogReceiver, receive log data. + /// This contract is singleton for efficiency + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface ILogReceiver : IAgniService + { + [OneWay] + [ArgsMarshalling(typeof(RequestMsg_ILogReceiver_SendLog))] + void SendLog(Message data); + + Message GetByID(Guid id, string channel = null); + + IEnumerable List(string archiveDimensionsFilter, DateTime startDate, DateTime endDate, MessageType? type = null, + string host = null, string channel = null, string topic = null, + Guid? relatedTo = null, + int skipCount = 0); + } + + /// + /// Contract for client of ILogReceiver svc + /// + public interface ILogReceiverClient : IAgniServiceClient, ILogReceiver + { + CallSlot Async_SendLog(Message data); + CallSlot Async_GetByID(Guid id, string channel = null); + CallSlot Async_List(string archiveDimensionsFilter, DateTime startDate, DateTime endDate, MessageType? type = null, + string host = null, string channel = null, string topic = null, + Guid? relatedTo = null, + int skipCount = 0); + } + + public sealed class RequestMsg_ILogReceiver_SendLog : RequestMsg + { + public RequestMsg_ILogReceiver_SendLog(MethodInfo method, Guid? instance) : base(method, instance) { } + public RequestMsg_ILogReceiver_SendLog(TypeSpec contract, MethodSpec method, bool oneWay, Guid? instance) : base(contract, method, oneWay, instance) { } + + public Message MethodArg_0_data; + } +} \ No newline at end of file diff --git a/src/Agni/Contracts/IPinger.cs b/src/Agni/Contracts/IPinger.cs new file mode 100644 index 0000000..541acc7 --- /dev/null +++ b/src/Agni/Contracts/IPinger.cs @@ -0,0 +1,26 @@ +using System; + +using NFX; +using NFX.Glue; + +namespace Agni.Contracts +{ + /// + /// Used to see if the host responds at all + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IPinger : IAgniService + { + void Ping(); + } + + + /// + /// Contract for client of IPinger svc + /// + public interface IPingerClient : IAgniServiceClient, IPinger + { + CallSlot Async_Ping(); + } +} \ No newline at end of file diff --git a/src/Agni/Contracts/IProcessController.cs b/src/Agni/Contracts/IProcessController.cs new file mode 100644 index 0000000..e11629d --- /dev/null +++ b/src/Agni/Contracts/IProcessController.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; + +using NFX.Glue; + +using Agni.Workers; + +namespace Agni.Contracts +{ + + /// + /// Controls the spawning and execution of processes. + /// Dispatches signals + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IProcessController : IAgniService + { + void Spawn(ProcessFrame frame); + + SignalFrame Dispatch(SignalFrame signal); + + ProcessFrame Get(PID pid); + + ProcessDescriptor GetDescriptor(PID pid); + + IEnumerable List(int processorID); + } + + + /// + /// Contract for client of ITodoQueue svc + /// + public interface IProcessControllerClient : IAgniServiceClient, IProcessController + { + CallSlot Async_Spawn(ProcessFrame frame); + CallSlot Async_Dispatch(SignalFrame signal); + + CallSlot Async_Get(PID pid); + CallSlot Async_GetDescriptor(PID pid); + CallSlot Async_List(int processorID); + } +} \ No newline at end of file diff --git a/src/Agni/Contracts/IRemoteTerminal.cs b/src/Agni/Contracts/IRemoteTerminal.cs new file mode 100644 index 0000000..0d70556 --- /dev/null +++ b/src/Agni/Contracts/IRemoteTerminal.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Glue; +using NFX.Serialization.JSON; +using Agni.Security.Permissions.Admin; + + +namespace Agni.Contracts +{ + /// + /// Represents a contract for working with remote entities using terminal/command approach + /// + [Glued] + [AuthenticationSupport] + [RemoteTerminalOperatorPermission] + [LifeCycle(ServerInstanceMode.Stateful, SysConsts.REMOTE_TERMINAL_TIMEOUT_MS)] + public interface IRemoteTerminal : IAgniService + { + [Constructor] + RemoteTerminalInfo Connect(string who); + + string Execute(string command); + + [Destructor] + string Disconnect(); + } + + /// + /// Contract for client of IRemoteTerminal svc + /// + public interface IRemoteTerminalClient : IAgniServiceClient, IRemoteTerminal { } + + + /// + /// Provides info about remote terminal to connecting clients + /// + [Serializable] + public sealed class RemoteTerminalInfo + { + public RemoteTerminalInfo(){} + public RemoteTerminalInfo(JSONDataMap map) + { + TerminalName = map["TerminalName"].AsString(); + WelcomeMsg = map["WelcomeMsg"].AsString(); + Host = map["Host"].AsString(); + AppName = map["AppName"].AsString(); + ServerLocalTime = map["ServerLocalTime"].AsDateTime(); + ServerUTCTime = map["ServerUTCTime"].AsDateTime(); + } + + public string TerminalName { get; internal set;} + public string WelcomeMsg { get; internal set;} + public string Host { get; internal set;} + public string AppName { get; internal set;} + public DateTime ServerLocalTime { get; internal set;} + public DateTime ServerUTCTime { get; internal set;} + } +} diff --git a/src/Agni/Contracts/ITelemetryReceiver.cs b/src/Agni/Contracts/ITelemetryReceiver.cs new file mode 100644 index 0000000..0c35dd4 --- /dev/null +++ b/src/Agni/Contracts/ITelemetryReceiver.cs @@ -0,0 +1,39 @@ +using System; +using System.Reflection; +using System.Collections.Generic; + +using NFX.Glue; +using NFX.Glue.Protocol; +using NFX.Instrumentation; + +namespace Agni.Contracts +{ + /// + /// Implemented by ITelemetryReceiver, receive datum data. + /// This contract is singleton for efficiency + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface ITelemetryReceiver : IAgniService + { + [OneWay] + [ArgsMarshalling(typeof(RequestMsg_ITelemetryReceiver_SendDatums))] + void SendDatums(params Datum[] data); + } + + /// + /// Contract for client of ITelemetryReceiver svc + /// + public interface ITelemetryReceiverClient : IAgniServiceClient, ITelemetryReceiver + { + CallSlot Async_SendDatums(params Datum[] data); + } + + public sealed class RequestMsg_ITelemetryReceiver_SendDatums : RequestMsg + { + public RequestMsg_ITelemetryReceiver_SendDatums(MethodInfo method, Guid? instance) : base(method, instance) { } + public RequestMsg_ITelemetryReceiver_SendDatums(TypeSpec contract, MethodSpec method, bool oneWay, Guid? instance) : base(contract, method, oneWay, instance) { } + + public Datum[] MethodArg_0_data; + } +} \ No newline at end of file diff --git a/src/Agni/Contracts/ITester.cs b/src/Agni/Contracts/ITester.cs new file mode 100644 index 0000000..2f68558 --- /dev/null +++ b/src/Agni/Contracts/ITester.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Glue; +using NFX.Log; + +using Agni.Locking; +using Agni.Locking.Server; + +namespace Agni.Contracts +{ + + /// + /// Used for various testing + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface ITester : IAgniService + { + object TestEcho(object data); + } + + + /// + /// Contract for client of ITester svc + /// + public interface ITesterClient : IAgniServiceClient, ITester + { + CallSlot Async_TestEcho(object data); + } + + +} \ No newline at end of file diff --git a/src/Agni/Contracts/ITodoQueue.cs b/src/Agni/Contracts/ITodoQueue.cs new file mode 100644 index 0000000..3eddb0b --- /dev/null +++ b/src/Agni/Contracts/ITodoQueue.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Glue; +using NFX.Log; + +using Agni.Workers; + +namespace Agni.Contracts +{ + + /// + /// Sends todos to the queue on the remote host + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface ITodoQueue : IAgniService + { + /// + /// Enqueues todos in the order. Returns the maximum size of the next call to Enqueue() + /// which reflects the ability of the remote queue to enqueue more work. + /// This may be used for dynamic flow control. + /// Calling this method with empty or null array just returns the status + /// + int Enqueue(TodoFrame[] todos); + } + + + /// + /// Contract for client of ITodoQueue svc + /// + public interface ITodoQueueClient : IAgniServiceClient, ITodoQueue + { + CallSlot Async_Enqueue(TodoFrame[] todos); + } +} \ No newline at end of file diff --git a/src/Agni/Contracts/IWebMessageSystem.cs b/src/Agni/Contracts/IWebMessageSystem.cs new file mode 100644 index 0000000..f617f74 --- /dev/null +++ b/src/Agni/Contracts/IWebMessageSystem.cs @@ -0,0 +1,151 @@ +using System; +using System.Reflection; +using System.Collections.Generic; + +using NFX.Glue; +using NFX.Glue.Protocol; +using NFX.DataAccess.Distributed; + +using Agni.WebMessaging; +using NFX.Web.Messaging; + +namespace Agni.Contracts +{ + /// + /// Represents a distributed messaging system akin to GMail/Live etc. + /// Consumers should use AgniWebMessageSink via MessageService instance to route messages + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IWebMessageSystem : IAgniService + { + /// + /// Sends message into the system routing it to appropriate mailbox/es. + /// The function does not guarantee to return all of the mailboxes (as there may be too many of them). + /// The return is used to quickly pick who the message was routed to. + /// Null is returned if the message could not be routed to any recipient + /// + MsgSendInfo[] SendMessage(AgniWebMessage msg); + + /// + /// Returns information about the specified mailbox or null if not found + /// + MailboxInfo GetMailboxInfo(MailboxID xid); + + /// + /// Updates the particular mailbox message status + /// + [OneWay] + [ArgsMarshalling(typeof(RequestMsg_IWebMessageSystem_UpdateMailboxMessageStatus))] + void UpdateMailboxMessageStatus(MailboxMsgID mid, MsgStatus status, string folders, string adornments); + + /// + /// Updates mailbox messages status + /// + [OneWay] + [ArgsMarshalling(typeof(RequestMsg_IWebMessageSystem_UpdateMailboxMessagesStatus))] + void UpdateMailboxMessagesStatus(IEnumerable mids, MsgStatus status, string folders, string adornments); + + /// + /// Updates the particular mailbox message publication status along with operator and description + /// + [OneWay] + [ArgsMarshalling(typeof(RequestMsg_IWebMessageSystem_UpdateMailboxMessagePublication))] + void UpdateMailboxMessagePublication(MailboxMsgID mid, MsgPubStatus status, string oper, string description); + + /// + /// Gets the mailbox message headers without body or attachments. + /// The query format is implementation-specific, by default the system should fetch up to 32 latest unread messages + /// + MessageHeaders GetMailboxMessageHeaders(MailboxID xid, string query); + + /// + /// Gets the mailbox messages count + /// + int GetMailboxMessageCount(MailboxID xid, string query); + + /// + /// Fetches mailbox message by id or null if not found + /// + AgniWebMessage FetchMailboxMessage(MailboxMsgID mid); + + /// + /// Fetches an attachment for the specified message by id and attachment index or null if not found + /// + AgniWebMessage.Attachment FetchMailboxMessageAttachment(MailboxMsgID mid, int attachmentIndex); + } + + /// + /// Contract for client of IWebMessageStore svc + /// + public interface IWebMessageSystemClient : IAgniServiceClient, IWebMessageSystem + { + CallSlot Async_SendMessage(AgniWebMessage data); + CallSlot Async_GetMailboxMessageHeaders(MailboxID xid, string query); + CallSlot Async_FetchMailboxMessage(MailboxMsgID mid); + CallSlot Async_FetchMailboxMessageAttachment(MailboxMsgID mid, int attachmentIndex); + } + + /// + /// Returns messages without body/attachments aka "headers" with additional metadata + /// + public class MessageHeaders + { + public MessageHeaders(MailboxID xid, AgniWebMessage[] headers) { Mailbox = xid; Headers = headers;} + + public readonly MailboxID Mailbox; + public readonly AgniWebMessage[] Headers; + + //future: additional inromation returned for paging etc.. + } + + /// + /// Result of the message write operation + /// + public struct MsgSendInfo + { + public MsgSendInfo(MsgChannelWriteResult writeResult, MailboxMsgID? delivered, int addresseeIdx) + { + WriteResult = writeResult; + Delivered = delivered; + AddresseeIdx =addresseeIdx; + } + + public readonly MsgChannelWriteResult WriteResult; + public readonly MailboxMsgID? Delivered; + public readonly int AddresseeIdx; + } + + public sealed class RequestMsg_IWebMessageSystem_UpdateMailboxMessageStatus : RequestMsg + { + public RequestMsg_IWebMessageSystem_UpdateMailboxMessageStatus(MethodInfo method, Guid? instance) : base(method, instance) { } + public RequestMsg_IWebMessageSystem_UpdateMailboxMessageStatus(TypeSpec contract, MethodSpec method, bool oneWay, Guid? instance) : base(contract, method, oneWay, instance) { } + + public MailboxMsgID MethodArg_0_mid; + public MsgStatus MethodArg_1_status; + public string MethodArg_2_folders; + public string MethodArg_3_adornments; + } + + public sealed class RequestMsg_IWebMessageSystem_UpdateMailboxMessagesStatus : RequestMsg + { + public RequestMsg_IWebMessageSystem_UpdateMailboxMessagesStatus(MethodInfo method, Guid? instance) : base(method, instance) { } + public RequestMsg_IWebMessageSystem_UpdateMailboxMessagesStatus(TypeSpec contract, MethodSpec method, bool oneWay, Guid? instance) : base(contract, method, oneWay, instance) { } + + public IEnumerable MethodArg_0_mids; + public MsgStatus MethodArg_1_status; + public string MethodArg_2_folders; + public string MethodArg_3_adornments; + } + + public sealed class RequestMsg_IWebMessageSystem_UpdateMailboxMessagePublication : RequestMsg + { + public RequestMsg_IWebMessageSystem_UpdateMailboxMessagePublication(MethodInfo method, Guid? instance) : base(method, instance) { } + public RequestMsg_IWebMessageSystem_UpdateMailboxMessagePublication(TypeSpec contract, MethodSpec method, bool oneWay, Guid? instance) : base(contract, method, oneWay, instance) { } + + public MailboxMsgID MethodArg_0_mid; + public MsgPubStatus MethodArg_1_status; + public string MethodArg_2_oper; + public string MethodArg_3_description; + } +} \ No newline at end of file diff --git a/src/Agni/Contracts/IZoneHostRegistry.cs b/src/Agni/Contracts/IZoneHostRegistry.cs new file mode 100644 index 0000000..5769aa1 --- /dev/null +++ b/src/Agni/Contracts/IZoneHostRegistry.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Glue; +using NFX.Environment; +using NFX.Instrumentation; +using NFX.Log; +using NFX.OS; + +using Agni.Metabase; + +namespace Agni.Contracts +{ + /// + /// Implemented by ZoneGovernors, receives host status/network identification data from subordinate nodes (another zone governors or other hosts). + /// This contract is singleton for efficiency + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IZoneHostRegistry : IAgniService + { + + /// + /// Returns information for specified subordinate host/s name/s depending on hostName query parameter. + /// Match pattern can contain up to one * wildcard and multiple ? wildcards + /// + IEnumerable GetSubordinateHosts(string hostNameSearchPattern); + + /// + /// Returns information for specified subordinate host or null + /// + HostInfo GetSubordinateHost(string hostName); + + /// + /// Sends host registration/status update information from subordinate hosts + /// + void RegisterSubordinateHost(HostInfo host, DynamicHostID? hid = null); + + DynamicHostID Spawn(string hostPath, string id = null); + } + + /// + /// Client contract for IZoneHostRegistry svc + /// + public interface IZoneHostRegistryClient : IAgniServiceClient, IZoneHostRegistry + { + CallSlot Async_Spawn(string hostPath, string id = null); + } + + /// + /// Provides information about zone gov subordinate host + /// + [Serializable] + public class HostInfo : INamed + { + private static int s_MyEntropyMs = 3 + (DateTime.UtcNow.Millisecond % 98); + + private HostInfo() { } + + public static HostInfo ForThisHost() + { + var cpu = Computer.CurrentProcessorUsagePct; + var ram = Computer.GetMemoryStatus(); + var result = new HostInfo + { + m_Name = AgniSystem.HostName,//the HostName does not have spaces in dynamic host name + m_UTCTimeStamp = App.TimeSource.UTCNow, + + m_LastWarning = App.Log.LastWarning.ThisOrNewSafeWrappedException(), + m_LastError = App.Log.LastError.ThisOrNewSafeWrappedException(), + m_LastCatastrophe = App.Log.LastCatastrophe.ThisOrNewSafeWrappedException(), + + m_CurrentCPULoad = cpu, + m_CurrentRAMStatus = ram, + m_NetInfo = HostNetInfo.ForThisHost(), + + m_SystemBuildInfo = BuildInformation.ForFramework, + m_AgniBuildInfo = AgniSystem.CoreBuildInfo + }; + + + if (result.UTCTimeStamp.Millisecond % s_MyEntropyMs == 0) + { + int entropy = + unchecked(641 * cpu * ((int)ram.AvailablePhysicalBytes % 65171)); + //we purposely want to have "low" entropy so higher bytes are USUALLY 0 + + NFX.ExternalRandomGenerator.Instance.FeedExternalEntropySample(entropy); + } + + if ( + (cpu > 25 && cpu < 50 && (cpu & 0x1) == 0) || (cpu > 0 && cpu % 7 == 0) + ) + result.m_RandomSample = NFX.ExternalRandomGenerator.Instance.NextRandomInteger; + + return result; + } + + private string m_Name; + private DateTime m_UTCTimeStamp; + + private int m_CurrentCPULoad; + private MemoryStatus m_CurrentRAMStatus; + + private Message m_LastWarning; + private Message m_LastError; + private Message m_LastCatastrophe; + + private HostNetInfo m_NetInfo; + + private int? m_RandomSample; + private BuildInformation m_SystemBuildInfo; + private BuildInformation m_AgniBuildInfo; + + + /// + /// Host name (path) INCLUDING DYNAMIC HOST NAME suffix + /// + public string Name { get { return m_Name; } } + + /// + /// UTC time stamp of the object status + /// + public DateTime UTCTimeStamp { get { return m_UTCTimeStamp; } } + + /// + /// Returns last log warning message or null + /// + public Message LastWarning { get { return m_LastWarning; } } + + /// + /// Returns last log error message or null + /// + public Message LastError { get { return m_LastError; } } + + /// + /// Returns last log catastrophe message or null + /// + public Message LastCatastrophe { get { return m_LastCatastrophe; } } + + + /// + /// Current CPU load + /// + public int CurrentCPULoad { get { return m_CurrentCPULoad; } } + + /// + /// Current memory status + /// + public MemoryStatus CurrentRAMStatus { get { return m_CurrentRAMStatus; } } + + + /// + /// Returns network information for host + /// + public HostNetInfo NetInfo { get { return m_NetInfo; } } + + + /// + /// Sometimes (randomly) captures random integer from ExternalRandomGenerator on the machine, or null + /// + public int? RandomSample { get { return m_RandomSample; } } + + /// + /// Returns build info for system core (NFX or AFX...) + /// + public BuildInformation SystemBuildInfo { get { return m_SystemBuildInfo; } } + + /// + /// Returns build info for Agni core + /// + public BuildInformation AgniBuildInfo { get { return m_AgniBuildInfo; } } + } + + [Serializable] + public struct DynamicHostID + { + public DynamicHostID(string id, string zone) + { + Zone = zone; + ID = id; + } + + public readonly string Zone; + public readonly string ID; + + public override string ToString() { return "HID[{0}:{1}]".Args(ID, Zone); } + public override int GetHashCode() { return ID.GetHashCode(); } + } + + [Serializable] + public sealed class DynamicHostInfo : INamed + { + public DynamicHostInfo(string id) + { + m_ID = id; + } + + private string m_ID; + + public string ID { get { return m_ID; } } + public DateTime Stamp { get; internal set; } + public string Owner { get; internal set; } + public string Host { get; internal set; } + internal int Votes { get; set; } + + string INamed.Name { get { return m_ID; } } + } +} diff --git a/src/Agni/Contracts/IZoneHostReplicator.cs b/src/Agni/Contracts/IZoneHostReplicator.cs new file mode 100644 index 0000000..bbdba5b --- /dev/null +++ b/src/Agni/Contracts/IZoneHostReplicator.cs @@ -0,0 +1,38 @@ +using System; +using System.Reflection; + +using NFX; +using NFX.Glue; +using NFX.Glue.Protocol; + +namespace Agni.Contracts +{ + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IZoneHostReplicator : IAgniService + { + [OneWay] + [ArgsMarshalling(typeof(RequestMsg_IZoneHostReplicator_PostDynamicHostInfo))] + void PostDynamicHostInfo(DynamicHostID id, DateTime stamp, string owner, int votes); + DynamicHostInfo GetDynamicHostInfo(DynamicHostID hid); + [OneWay] + void PostHostInfo(HostInfo host, DynamicHostID? hid); + } + + public interface IZoneHostReplicatorClient : IAgniServiceClient, IZoneHostReplicator + { + CallSlot Async_PostDynamicHostInfo(DynamicHostID hid, DateTime stamp, string owner, int votes); + CallSlot Async_PostHostInfo(HostInfo host, DynamicHostID? hid); + } + + public sealed class RequestMsg_IZoneHostReplicator_PostDynamicHostInfo : RequestMsg + { + public RequestMsg_IZoneHostReplicator_PostDynamicHostInfo(MethodInfo method, Guid? instance) : base(method, instance) { } + public RequestMsg_IZoneHostReplicator_PostDynamicHostInfo(TypeSpec contract, MethodSpec method, bool oneWay, Guid? instance) : base(contract, method, oneWay, instance) { } + + public DynamicHostID MethodArg_0_id; + public DateTime MethodArg_1_stamp; + public string MethodArg_2_owner; + public int MethodArg_3_votes; + } +} diff --git a/src/Agni/Contracts/IZoneLogReceiver.cs b/src/Agni/Contracts/IZoneLogReceiver.cs new file mode 100644 index 0000000..d8005d4 --- /dev/null +++ b/src/Agni/Contracts/IZoneLogReceiver.cs @@ -0,0 +1,41 @@ + + + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Glue; +using NFX.Log; + + +namespace Agni.Contracts +{ + + /// + /// Implemented by ZoneGovernors, receive log data from subordinate nodes (another zone governors or other hosts). + /// This contract is singleton for efficiency + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IZoneLogReceiver : IAgniService + { + /// + /// Sends log batch from a named subordinate host. + /// Returns the receiver condition - a number of expected Message instances in the next call. + /// Keep in mind that a large number may be returned and Glue buffer limit may not be sufficient for large send, so + /// impose a limit on the caller side (i.e. 200 max message instances per call) + /// The busier the receiver gets, the lower is the number. This is a form of throttling/flow control + /// + int SendLog(string host, string appName, Message[] data); + } + + + /// + /// Contract for client of IZoneLogReceiver svc + /// + public interface IZoneLogReceiverClient : IAgniServiceClient, IZoneLogReceiver { } + + +} diff --git a/src/Agni/Contracts/IZoneTelemetryReceiver.cs b/src/Agni/Contracts/IZoneTelemetryReceiver.cs new file mode 100644 index 0000000..31c4b9a --- /dev/null +++ b/src/Agni/Contracts/IZoneTelemetryReceiver.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Glue; +using NFX.Instrumentation; + + +namespace Agni.Contracts +{ + + /// + /// Implemented by ZoneGovernors, receive telemetry data from subordinate nodes (another zone governors or other hosts). + /// This contract is singleton for efficiency + /// + [Glued] + [LifeCycle(ServerInstanceMode.Singleton)] + public interface IZoneTelemetryReceiver : IAgniService + { + /// + /// Sends telemetry batch from named subordinate host. + /// Returns the receiver condition - a number of expected Datum instances in the next call. + /// Keep in mind that a large number may be returned and Glue buffer limit may not be sufficient for large send, so + /// impose a limit on the caller side (i.e. 200 max datum instances per call) + /// The busier the receiver gets, the lower is the number. This is a form of throttling/flow control + /// + int SendTelemetry(string host, Datum[] data); + } + + + /// + /// Contract for client of IZoneTelemetryReceiver svc + /// + public interface IZoneTelemetryReceiverClient : IAgniServiceClient, IZoneTelemetryReceiver { } + + +} diff --git a/src/Agni/Contracts/ServiceClientHub.cs b/src/Agni/Contracts/ServiceClientHub.cs new file mode 100644 index 0000000..1ca1536 --- /dev/null +++ b/src/Agni/Contracts/ServiceClientHub.cs @@ -0,0 +1,735 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Glue; +using NFX.Glue.Protocol; +using NFX.Environment; +using NFX.ApplicationModel; +using NFX.Serialization.BSON; + +namespace Agni.Contracts +{ + /// + /// Represents a centralized manager and client factory for obtaining IAgniServiceClient-implementing instances + /// that are most likely Glue-based but do not have to be. + /// Custom solutions may derive from this class and override specifics (i.e. inject some policies in all service call clients) + /// + public class ServiceClientHub : ApplicationComponent + { + #region CONSTS + public const string CONFIG_SERVICE_CLIENT_HUB_SECTION = "service-client-hub"; + public const string CONFIG_MAP_SECTION = "map"; + + public const string CONFIG_LOCAL_SECTION = "local"; + public const string CONFIG_GLOBAL_SECTION = "global"; + public const string CONFIG_OPTIONS_SECTION = "options"; + + public const string CONFIG_MAP_CLIENT_CONTRACT_ATTR = "client-contract"; + public const string CONFIG_MAP_IMPLEMENTOR_ATTR = "implementor"; + #endregion + + #region Inner Types + /// + /// Provides mapping information for service contract + /// + [Serializable] + public sealed class ContractMapping : INamed + { + public ContractMapping(IConfigSectionNode config) + { + try + { + var cname = config.AttrByName(CONFIG_MAP_CLIENT_CONTRACT_ATTR).Value; + if (cname.IsNullOrWhiteSpace()) throw new Clients.AgniClientException(CONFIG_MAP_CLIENT_CONTRACT_ATTR + " is unspecified"); + m_Contract = Type.GetType(cname, true); + + m_Local = new Data(config, config[CONFIG_LOCAL_SECTION], false); + m_Global = new Data(config, config[CONFIG_GLOBAL_SECTION], true); + } + catch (Exception error) + { + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_MAPPING_CTOR_ERROR.Args( + config.ToLaconicString(NFX.CodeAnalysis.Laconfig.LaconfigWritingOptions.Compact), + error.ToMessageWithType()), error); + } + } + + + private Type m_Contract; + //------------------------------ + private Data m_Local; + private Data m_Global; + + + public sealed class Data + { + + internal Data(IConfigSectionNode baseConfig, IConfigSectionNode config, bool global) + { + var cname = config.AttrByName(CONFIG_MAP_IMPLEMENTOR_ATTR).ValueAsString(baseConfig.AttrByName(CONFIG_MAP_IMPLEMENTOR_ATTR).Value); + if (cname.IsNullOrWhiteSpace()) throw new Clients.AgniClientException(CONFIG_MAP_IMPLEMENTOR_ATTR + " is unspecified"); + m_Implementor = Type.GetType(cname, true); + + if (!m_Implementor.GetInterfaces().Any(i => i == typeof(IAgniServiceClient))) + throw new Clients.AgniClientException("Implementor {0} is not IAgniServiceClient".Args(m_Implementor.FullName)); + + ConfigAttribute.Apply(this, baseConfig); + ConfigAttribute.Apply(this, config); + + //service may be null + + if (m_Net.IsNullOrWhiteSpace()) + m_Net = global ? SysConsts.NETWORK_INTERNOC : SysConsts.NETWORK_NOCGOV; + + if (m_Binding.IsNullOrWhiteSpace()) + m_Binding = SysConsts.DEFAULT_BINDING; + + if (m_Options == null) + m_Options = "options{ }".AsLaconicConfig(); + } + + private Type m_Implementor; + [Config] private string m_Service = string.Empty; + [Config] private string m_Net; + [Config] private string m_Binding; + [Config] private int m_CallTimeoutMs = 0; + [Config] private bool m_ReserveTransport = false; + [Config(CONFIG_OPTIONS_SECTION)] private IConfigSectionNode m_Options; + + public Type Implementor { get { return m_Implementor; } } + public string Service { get { return m_Service; } } + public string Net { get { return m_Net; } } + public string Binding { get { return m_Binding; } } + public int CallTimeoutMs { get { return m_CallTimeoutMs; } } + public bool ReserveTransport { get { return m_ReserveTransport; } } + public IConfigSectionNode Options { get { return m_Options; } } + } + + public string Name { get { return m_Contract.AssemblyQualifiedName; } } + public Type Contract { get { return m_Contract; } } + public Data Local { get { return m_Local; } } + public Data Global { get { return m_Global; } } + + public override string ToString() + { + return "Mapping[{0} -> Local: {1}; Global: {2}]".Args(m_Contract.FullName, Local.Implementor.FullName, Global.Implementor.FullName); + } + + public override bool Equals(object obj) + { + var cm = obj as ContractMapping; + if (cm == null) return false; + return this.m_Contract == cm.m_Contract; + } + + public override int GetHashCode() + { + return m_Contract.GetHashCode(); + } + } + #endregion + + #region .static/.ctor + private static object s_Lock = new object(); + private static volatile ServiceClientHub s_Instance; + + /// + /// Returns the singleton instance of the ServiceClient or its derivative as configured + /// + public static ServiceClientHub Instance + { + get + { + var instance = s_Instance; + if (instance != null) return instance; + + lock (s_Lock) + { + if (s_Instance == null) + s_Instance = makeInstance(); + } + + return s_Instance; + } + } + + private static ServiceClientHub makeInstance() + { + string tpn = typeof(ServiceClientHub).FullName; + try + { + var mbNode = AgniSystem.Metabase.ServiceClientHubConfNode as ConfigSectionNode; + var appNode = App.ConfigRoot[SysConsts.APPLICATION_CONFIG_ROOT_SECTION][CONFIG_SERVICE_CLIENT_HUB_SECTION] as ConfigSectionNode; + + var effectiveConf = new MemoryConfiguration(); + effectiveConf.CreateFromMerge(mbNode, appNode); + var effective = effectiveConf.Root; + + tpn = effective.AttrByName(FactoryUtils.CONFIG_TYPE_ATTR).ValueAsString(typeof(ServiceClientHub).FullName); + + return FactoryUtils.Make(effective, typeof(ServiceClientHub), new object[] { effective }); + } + catch (Exception error) + { + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_HUB_SINGLETON_CTOR_ERROR.Args(tpn, error.ToMessageWithType()), error); + } + } + + + /// + /// Makes an appropriate implementor for requested service client contract type. + /// Pass svcName parameter in cases when the requested contract may get implemented by more than one network service. + /// The call is thread-safe. The caller should Dispose() the returned instance after it has been used + /// + public static TServiceClient New(string toHost, string fromHost = null, string svcName = null) where TServiceClient : class, IAgniServiceClient + { + return Instance.GetNew(toHost, fromHost, svcName); + } + + /// + /// Makes an appropriate implementor for requested service client contract type. + /// Pass svcName parameter in cases when the requested contract may get implemented by more than one network service. + /// The call is thread-safe. The caller should Dispose() the returned instance after it has been used + /// + public static TServiceClient New(Metabase.Metabank.SectionHost toHost, Metabase.Metabank.SectionHost fromHost = null, string svcName = null) where TServiceClient : class, IAgniServiceClient + { + return Instance.GetNew(toHost.RegionPath, fromHost != null ? fromHost.RegionPath : null, svcName); + } + + /// + /// Tries to resolve contract type to implementor and tests network service resolvability. Throws in case of error + /// + public static void TestSetupOf(string toHost, string fromHost = null, string svcName = null) where TServiceClient : class, IAgniServiceClient + { + Instance.RunTestSetupOf(toHost, fromHost, svcName); + } + + + /// + /// Performs a call by making the appropriate client and retries if the ClientCallException arises, + /// otherwise fails if no backup hosts are left + /// + /// The body of the client call, must not be null + /// The array of hosts to retry against, left to right + /// Optional functor that returns true for call exceptions that should abort the retry process + /// Optional + /// Optional + public static TResult CallWithRetry(Func callBody, + IEnumerable hosts, + Func abortFilter = null, + string fromHost = null, + string svcName = null + ) where TServiceClient : class, IAgniServiceClient + { + if (callBody == null) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + typeof(ServiceClientHub).Name + ".CallWithRetry<{0}>(callBody==null)".Args(typeof(TServiceClient).Name)); + + if (hosts == null || !hosts.Any()) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + typeof(ServiceClientHub).Name + ".CallWithRetry<{0}>(hosts==null|[0])".Args(typeof(TServiceClient).Name)); + + foreach (var host in hosts) + { + using (var client = New(host, fromHost, svcName)) + try + { + return callBody(client); + } + catch (Exception error) + { + App.Log.Write( + new NFX.Log.Message + { + Type = NFX.Log.MessageType.Error, + Topic = SysConsts.LOG_TOPIC_SVC, + From = "{0}.CallWithRetry()".Args(Instance.GetType().Name), + Source = 248, + Text = StringConsts.AGNI_SVC_CLIENT_HUB_RETRY_CALL_HOST_ERROR.Args(typeof(TServiceClient).FullName, host), + Exception = error + } + ); + + Instrumentation.ServiceClientHubRetriableCallError.Happened(typeof(TServiceClient), host); + + var shouldAbort = !(error is ClientCallException); + if (abortFilter != null) shouldAbort = abortFilter(client, error); + if (shouldAbort) throw; + //otherwise eat the exception and retry + } + } + + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_HUB_CALL_RETRY_FAILED_ERROR.Args(typeof(TServiceClient).Name, hosts.Count())); + } + + + /// + /// Performs a call by making the appropriate client and retries if the ClientCallException arises, + /// otherwise fails if no backup hosts are left + /// + /// The body of the client call, must not be null + /// The array of hosts to retry against, left to right + /// Optional functor that returns true for call exceptions that should abort the retry process + /// Optional + /// Optional + public static void CallWithRetry(Action callBody, + IEnumerable hosts, + Func abortFilter = null, + string fromHost = null, + string svcName = null) where TServiceClient : class, IAgniServiceClient + { + if (callBody == null) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + typeof(ServiceClientHub).Name + ".CallWithRetry<{0}>(callBody==null)".Args(typeof(TServiceClient).Name)); + + if (hosts == null || !hosts.Any()) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + typeof(ServiceClientHub).Name + ".CallWithRetry<{0}>(hosts==null|[0])".Args(typeof(TServiceClient).Name)); + + foreach (var host in hosts) + { + using (var client = New(host, fromHost, svcName)) + try + { + callBody(client); + return; + } + catch (Exception error) + { + App.Log.Write( + new NFX.Log.Message + { + Type = NFX.Log.MessageType.Error, + Topic = SysConsts.LOG_TOPIC_SVC, + From = "{0}.CallWithRetry()".Args(Instance.GetType().Name), + Source = 248, + Text = StringConsts.AGNI_SVC_CLIENT_HUB_RETRY_CALL_HOST_ERROR.Args(typeof(TServiceClient).FullName, host), + Exception = error + } + ); + + Instrumentation.ServiceClientHubRetriableCallError.Happened(typeof(TServiceClient), host); + + var shouldAbort = !(error is ClientCallException); + if (abortFilter != null) shouldAbort = abortFilter(client, error); + if (shouldAbort) throw; + //otherwise eat the exception and retry + } + } + + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_HUB_CALL_RETRY_FAILED_ERROR.Args(typeof(TServiceClient).Name, hosts.Count())); + } + + + /// + /// Performs a call by making the appropriate client and retries if the ClientCallException arises, + /// otherwise fails if no backup hosts are left + /// + /// The body of the client call, must not be null + /// The array of hosts to retry against, left to right + /// Optional functor that returns true for call exceptions that should abort the retry process + /// Optional + /// Optional + public static Task CallWithRetryAsync(Func> callBody, + IEnumerable hosts, + Func abortFilter = null, + string fromHost = null, + string svcName = null + ) where TServiceClient : class, IAgniServiceClient + { + if (callBody == null) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + typeof(ServiceClientHub).Name + ".CallWithRetry<{0}>(callBody==null)".Args(typeof(TServiceClient).Name)); + + if (hosts == null || !hosts.Any()) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + typeof(ServiceClientHub).Name + ".CallWithRetry<{0}>(hosts==null|[0])".Args(typeof(TServiceClient).Name)); + + var hostsCopy = hosts.ToArray(); + var hostsEnum = ((IEnumerable)hostsCopy).GetEnumerator(); + + var tcs = new TaskCompletionSource(); + + callWithRetryAsync(tcs, callBody, hostsEnum, hosts.Count(), abortFilter, fromHost, svcName); + + return tcs.Task; + } + + private static void callWithRetryAsync( + TaskCompletionSource tcs, + Func> callBody, + IEnumerator hosts, + int hostsCount, + Func abortFilter = null, + string fromHost = null, + string svcName = null + ) where TServiceClient : class, IAgniServiceClient + { + Task t = null; + TServiceClient client = null; + string host = null; + + while (hosts.MoveNext()) + { + try + { + host = hosts.Current; + client = New(host, fromHost, svcName); + t = callBody(client); + break; + } + catch (Exception ex) + { + App.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Error, + Topic = SysConsts.LOG_TOPIC_SVC, + From = "{0}.CallWithRetryAsync()".Args(Instance.GetType().Name), + Source = 304, + Text = StringConsts.AGNI_SVC_CLIENT_HUB_RETRY_CALL_HOST_ERROR.Args(typeof(TServiceClient).FullName, host), + Exception = ex + }); + + Instrumentation.ServiceClientHubRetriableCallError.Happened(typeof(TServiceClient), host); + + var shouldAbort = !(ex is ClientCallException); + if (abortFilter != null) shouldAbort = abortFilter(client, ex); + if (shouldAbort) + { + tcs.TrySetException(ex); + return; + } + } + finally + { + if (client != null && t == null) + DisposableObject.DisposeAndNull(ref client); + } + } + + if (t != null) + { + t.ContinueWith(antecedent => + { + try + { + var err = antecedent.Exception; + if (err == null) + { + tcs.SetResult(antecedent.Result); + } + else + { + var innerException = err.GetBaseException(); + + App.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Error, + Topic = SysConsts.LOG_TOPIC_SVC, + From = "{0}.CallWithRetryAsync()".Args(Instance.GetType().Name), + Source = 304, + Text = StringConsts.AGNI_SVC_CLIENT_HUB_RETRY_CALL_HOST_ERROR.Args(typeof(TServiceClient).FullName, host), + Exception = innerException + }); + + Instrumentation.ServiceClientHubRetriableCallError.Happened(typeof(TServiceClient), host); + + var shouldAbort = !(innerException is ClientCallException); + if (abortFilter != null) shouldAbort = abortFilter(client, innerException); + if (shouldAbort) + { + tcs.TrySetException(innerException); + return; + } + else + { + callWithRetryAsync(tcs, callBody, hosts, hostsCount, abortFilter, fromHost, svcName); + } + } + } + finally + { + DisposableObject.DisposeAndNull(ref client); + } + }, TaskContinuationOptions.ExecuteSynchronously); + + return; + } + + tcs.TrySetException(new Clients.AgniClientException( + StringConsts.AGNI_SVC_CLIENT_HUB_CALL_RETRY_FAILED_ERROR.Args(typeof(TServiceClient).Name, hostsCount))); + } + + + /// + /// Performs a call by making the appropriate client and retries if the ClientCallException arises, + /// otherwise fails if no backup hosts are left + /// + /// The body of the client call, must not be null + /// The array of hosts to retry against, left to right + /// Optional functor that returns true for call exceptions that should abort the retry process + /// Optional + /// Optional + public static Task CallWithRetryAsync(Func callBody, + IEnumerable hosts, + Func abortFilter = null, + string fromHost = null, + string svcName = null + ) where TServiceClient : class, IAgniServiceClient + { + return CallWithRetryAsync(cl => + callBody(cl) + .ContinueWith(antecedent => + { + var err = antecedent.Exception; + if (err != null) throw err; + + return true; + } + ), + hosts, abortFilter, fromHost, svcName); + } + + /// + /// Override to configure custom memebers. + /// The default implementation populates the CacheMap registry + /// + protected ServiceClientHub(IConfigSectionNode config) + { + ConfigAttribute.Apply(this, config); + foreach (var nmapping in config.Children.Where(c => c.IsSameName(CONFIG_MAP_SECTION))) + { + var mapping = new ContractMapping(nmapping); + m_CachedMap.Register(mapping); + } + } + #endregion + + #region Fields + private Registry m_CachedMap = new Registry(); + #endregion + + + #region Public + /// + /// Returns the registry of cached ContractMapping instances + /// + public IRegistry CachedMap { get { return m_CachedMap; } } + + + /// + /// Instance version of static ByContract(). See ByContract{TServiceClient}() + /// + public TServiceClient GetNew(string toHost, string fromHost = null, string svcName = null) where TServiceClient : class, IAgniServiceClient + { + Type tcontract = typeof(TServiceClient); + + if (toHost.IsNullOrWhiteSpace()) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".GetByContract<{0}>(host==null|empty)".Args(tcontract.Name)); + + if (fromHost.IsNullOrWhiteSpace()) fromHost = AgniSystem.HostName; + + if (fromHost.IsNullOrWhiteSpace()) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".GetByContract<{0}>(fromHost==null|empty & AgnySystem is not avail)".Args(tcontract.Name)); + + + ContractMapping mapping = MapContractToImplementation(tcontract); + + bool isGlobal; + Node node = ResolveNetworkService(mapping, toHost, fromHost, svcName, out isGlobal); + + var result = MakeClientInstance(mapping, isGlobal, node); + + SetupClientInstance(mapping, isGlobal, result, toHost, fromHost); + + return result; + } + + + /// + /// Tries to resolve contract type to implementor and tests network service resolvability. Throws in case of error + /// + public void RunTestSetupOf(string toHost, string fromHost = null, string svcName = null) where TServiceClient : class, IAgniServiceClient + { + Type tcontract = typeof(TServiceClient); + + if (toHost.IsNullOrWhiteSpace()) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".TestByContract<{0}>(host==null|empty)".Args(tcontract.Name)); + + if (fromHost.IsNullOrWhiteSpace()) fromHost = AgniSystem.HostName; + + if (fromHost.IsNullOrWhiteSpace()) + throw new Clients.AgniClientException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".TestByContract<{0}>(fromHost==null|empty & AgniSystem is not avail)".Args(tcontract.Name)); + + + ContractMapping mapping = MapContractToImplementation(tcontract); + + bool isGlobal; + Node node = ResolveNetworkService(mapping, toHost, fromHost, svcName, out isGlobal); + } + #endregion + + #region Protected + protected ContractMapping MapContractToImplementation(Type tContract) + { + try + { + var mapped = DoMapContractToImplementation(tContract); + if (mapped == null) throw new Clients.AgniClientException("Map['{0}'] -> ".Args(tContract.FullName)); + return mapped; + } + catch (Exception error) + { + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_HUB_MAPPING_ERROR.Args(tContract.FullName, error.ToMessageWithType()), error); + } + } + + protected Node ResolveNetworkService(ContractMapping mapping, string toHost, string fromHost, string svcName, out bool isGlobal) + { + try { return DoResolveNetworkService(mapping, toHost, fromHost, svcName, out isGlobal); } + catch (Exception error) + { + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_HUB_NET_RESOLVE_ERROR.Args(mapping, error.ToMessageWithType()), error); + } + } + + protected TServiceClient MakeClientInstance(ContractMapping mapping, bool isGlobal, Node node) where TServiceClient : IAgniServiceClient + { + try { return DoMakeClientInstance(mapping, isGlobal, node); } + catch (Exception error) + { + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_HUB_MAKE_INSTANCE_ERROR.Args(mapping, error.ToMessageWithType()), error); + } + } + + protected void SetupClientInstance(ContractMapping mapping, bool isGlobal, IAgniServiceClient instance, string toHost, string fromHost) + { + try { DoSetupClientInstance(mapping, isGlobal, instance, toHost, fromHost); } + catch (Exception error) + { + throw new Clients.AgniClientException(StringConsts.AGNI_SVC_CLIENT_HUB_SETUP_INSTANCE_ERROR.Args(mapping, error.ToMessageWithType()), error); + } + } + + /// + /// Override to map tContract into ContractMapping object. The default implementation uses cached registry of mappings + /// + protected virtual ContractMapping DoMapContractToImplementation(Type tContract) + { + return m_CachedMap[tContract.AssemblyQualifiedName]; + } + + /// + /// Override to resolve ContractMapping into physical connection parameters. + /// The default implementation uses metabase's ResolveNetworkService + /// + protected virtual Node DoResolveNetworkService(ContractMapping mapping, string toHost, string fromHost, string svcName, out bool isGlobal) + { + isGlobal = !AgniSystem.Metabase.CatalogReg.ArePathsInSameNOC(fromHost, toHost); + + var mappingData = isGlobal ? mapping.Global : mapping.Local; + + if (svcName.IsNullOrWhiteSpace()) svcName = mappingData.Service; + + return AgniSystem.Metabase.ResolveNetworkService(toHost, mappingData.Net, svcName, mappingData.Binding, fromHost); + } + + protected static readonly Type[] CTOR_SIG_GLUE_ENDPOINT = new Type[] { typeof(NFX.Glue.Node), typeof(NFX.Glue.Binding) }; + + /// + /// Override to make an instance of client per ContractMapping. + /// The default implementation uses activator + /// + protected virtual TServiceClient DoMakeClientInstance(ContractMapping mapping, bool isGlobal, Node node) where TServiceClient : IAgniServiceClient + { + var mappingData = isGlobal ? mapping.Global : mapping.Local; + + TServiceClient result; + + try + { + if (mappingData.Implementor.GetConstructor(CTOR_SIG_GLUE_ENDPOINT) != null) + result = (TServiceClient)Activator.CreateInstance(mappingData.Implementor, node, null); //Glue endpoint signature: node, binding + else + result = (TServiceClient)Activator.CreateInstance(mappingData.Implementor, node);//otherwise must have NODE as a sole parameter + } + catch (Exception error) + { + if (error is System.Reflection.TargetInvocationException) throw error.InnerException; + throw new Clients.AgniClientException("Implementor '{0}' must have .ctor(Glue.Node) or .ctor(GlueNode,Glue.Binding). Error: {1}".Args(mappingData.Implementor.FullName, error.ToMessageWithType())); + } + + return result; + } + + /// + /// Override to setup the instance of client after it has been allocated. + /// The default implementation injects headers, timeoutes, and inspectors for Glue.ClientEndPoints + /// + protected virtual void DoSetupClientInstance(ContractMapping mapping, bool isGlobal, IAgniServiceClient instance, string toHost, string fromHost) + { + var mappingData = isGlobal ? mapping.Global : mapping.Local; + + var tms = mappingData.CallTimeoutMs; + if (tms > 0) instance.TimeoutMs = tms; + instance.ReserveTransport = mappingData.ReserveTransport; + + + var gep = instance as ClientEndPoint; + if (gep != null) + { + var icfg = mappingData.Options; + if (icfg != null) + { + MsgInspectorConfigurator.ConfigureClientInspectors(gep.MsgInspectors, icfg); + HeaderConfigurator.ConfigureHeaders(gep.Headers, icfg); + } + } + } + #endregion + } + + + namespace Instrumentation + { + + [Serializable] + public abstract class ServiceClientHubEvent : NFX.Instrumentation.Event, NFX.Instrumentation.INetInstrument + { + protected ServiceClientHubEvent(string src) : base(src) { } + } + + [Serializable] + public abstract class ServiceClientHubErrorEvent : ServiceClientHubEvent, NFX.Instrumentation.IErrorInstrument + { + protected ServiceClientHubErrorEvent(string src) : base(src) { } + } + + + //todo Need to add the counter for successful calls as well, however be carefull, + //as to many details may create much instrumentation data (dont include contract+toHost)? + //or have level of detalization setting + + + [Serializable] + [BSONSerializable("D2BDA600-7B13-4701-BC8F-C9E72A26CED8")] + public class ServiceClientHubRetriableCallError : ServiceClientHubErrorEvent + { + protected ServiceClientHubRetriableCallError(string src) : base(src) { } + + public static void Happened(Type tContract, string toName) + { + var inst = ExecutionContext.Application.Instrumentation; + if (inst.Enabled) + inst.Record(new ServiceClientHubRetriableCallError("{0}::{1}".Args(tContract.FullName, toName))); + } + + public override string Description { get { return "Service client hub retriable call failed"; } } + + + protected override NFX.Instrumentation.Datum MakeAggregateInstance() + { + return new ServiceClientHubRetriableCallError(this.Source); + } + } + + } +} diff --git a/src/Agni/Coordination/Exceptions.cs b/src/Agni/Coordination/Exceptions.cs new file mode 100644 index 0000000..3870f45 --- /dev/null +++ b/src/Agni/Coordination/Exceptions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.Coordination +{ + /// + /// Thrown to indicate coordination-related problems + /// + [Serializable] + public class CoordinationException : AgniException + { + public CoordinationException() : base() { } + public CoordinationException(string message) : base(message) { } + public CoordinationException(string message, Exception inner) : base(message, inner) { } + protected CoordinationException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Coordination/HostSet.Builder.cs b/src/Agni/Coordination/HostSet.Builder.cs new file mode 100644 index 0000000..4037aa5 --- /dev/null +++ b/src/Agni/Coordination/HostSet.Builder.cs @@ -0,0 +1,145 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NFX; +using NFX.Time; +using NFX.Environment; + +using Agni.Metabase; +using Agni.Contracts; +using System.Collections; +using NFX.Log; + +namespace Agni.Coordination +{ + public partial class HostSet + { + /// + /// Provides the default HostSet.Builder that finds and makes instance of the matching HostSet + /// + public class Builder + { + + private static object s_InstanceLock = new object(); + private static volatile Builder s_Instance; + + /// + /// Returns a singleton builder instance as configured in the root metabase config, + /// hence the instance is injectable + /// + public static Builder Instance + { + get + { + if(s_Instance!=null) return s_Instance; + lock(s_InstanceLock) + { + if(s_Instance!=null) return s_Instance; + var node = AgniSystem.Metabase.RootConfig[Metabank.CONFIG_HOST_SET_BUILDER_SECTION]; + s_Instance = FactoryUtils.Make(node, typeof(Builder), new [] {node}); + } + + return s_Instance; + } + } + + protected Builder(IConfigSectionNode config) { } + + + /// + /// Tries to find a named host set starting at the requested cluster level. + /// Throws if not found. + /// + public THostSet FindAndBuild(string setName, string clusterPath, bool searchParent = true, bool transcendNoc = false) + where THostSet : HostSet + { + var result = TryFindAndBuild(setName, clusterPath, searchParent, transcendNoc); + + if (result==null) + throw new CoordinationException(StringConsts.HOST_SET_BUILDER_CONFIG_FIND_ERROR + .Args(GetType().Name, setName, clusterPath, searchParent, transcendNoc)); + return result; + } + + /// + /// Tries to find a named host set starting at the requested cluster level. + /// Returns null if not found. + /// + public THostSet TryFindAndBuild(string setName, string clusterPath, bool searchParent = true, bool transcendNoc = false) + where THostSet : HostSet + { + if (setName.IsNullOrWhiteSpace()) + throw new CoordinationException(StringConsts.ARGUMENT_ERROR+GetType().Name+".ctor(setName==null|empty)"); + + if (clusterPath.IsNullOrWhiteSpace()) + throw new CoordinationException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".ctor(clusterPath==null|empty)"); + + var result = DoTryFindAndBuild(setName, clusterPath, searchParent, transcendNoc); + return result; + } + + protected virtual THostSet DoTryFindAndBuild(string setName, string clusterPath, bool searchParent, bool transcendNoc) + where THostSet : HostSet + { + THostSet result = null; + + string actualPath; + var cnode = DoTryFindConfig(setName, clusterPath, searchParent, transcendNoc, out actualPath); + + if (cnode!=null) + result = FactoryUtils.Make(cnode, typeof(THostSet), new object[]{ setName, clusterPath, actualPath, cnode }); + + return result; + } + + protected virtual IConfigSectionNode DoTryFindConfig(string setName, string clusterPath, bool searchParent, bool transcendNoc, out string actualPath) + { + actualPath = null; + + Metabank.SectionRegionBase level = AgniSystem.Metabase.CatalogReg[clusterPath]; + if (!(level is Metabank.SectionZone || level is Metabank.SectionNOC)) return null; + IConfigSectionNode result = null; + + while(level!=null) + { + result = level.LevelConfig + .Children + .Where( c => c.IsSameName(Metabank.CONFIG_HOST_SET_SECTION) && c.IsSameNameAttr(setName) ) + .FirstOrDefault(); + + if (result!=null) break; + if (!searchParent) break; + + var zone = level as Metabank.SectionZone; + if (zone != null) + { + var zparent = zone.ParentZone; + if (zparent != null) + { + level = zparent; + continue; + } + + level = zone.NOC; + continue; + } + + if (!transcendNoc) break; + + var noc = level as Metabank.SectionNOC; + if (noc != null) + level = noc.ParentNOCZone; + } + + if (result!=null) + actualPath = level.RegionPath; + + return result; + } + } + } +} diff --git a/src/Agni/Coordination/HostSet.cs b/src/Agni/Coordination/HostSet.cs new file mode 100644 index 0000000..29565e3 --- /dev/null +++ b/src/Agni/Coordination/HostSet.cs @@ -0,0 +1,302 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NFX; +using NFX.Time; +using NFX.Environment; + +using Agni.Metabase; +using Agni.Contracts; +using System.Collections; +using NFX.Log; + +namespace Agni.Coordination +{ + /// + /// Represents a set of hosts that perform some common work. + /// Pramarily used for sharding work among hosts in the set. + /// The data is fed from the Metabase, and supports static and dynamic sets. + /// Static sets have a metabase-fixed number of hosts, whereas dynamic sets + /// may include dynamic hosts (as allocated by IaaS provider). + /// The dynamic sets are not supported until IaaS providers are implemented + /// + public partial class HostSet : DisposableObject, INamed + { + #region CONSTS + public const MessageType DEFAULT_LOG_LEVEL = MessageType.Warning; + + public const string CONFIG_HEARTBEAT_INTERVAL_SEC = "heartbeat-interval-sec"; + public const int DEFAULT_HEARTBEAT_INTERVAL_SEC = 3 * 60; + public const int MIN_HEARTBEAT_INTERVAL_SEC = 30; + #endregion + + #region Inner + public sealed class Host : INamed, IOrdered + { + public Host(Metabank.SectionHost host, int o) { m_Section = host; m_Order = o; } + private Metabank.SectionHost m_Section; + private int m_Order; + + public string Name { get { return m_Section.RegionPath; } } + public Metabank.SectionHost Section { get { return m_Section; } } + public int Order { get { return m_Order; } } + public DateTime? LastDownTime { get; internal set; } + } + + public struct HostPair : IEnumerable + { + public HostPair(Metabank.SectionHost p, Metabank.SectionHost s) { Primary = p; Secondary = s; } + public readonly Metabank.SectionHost Primary; + public readonly Metabank.SectionHost Secondary; + + public bool Assigned { get { return Primary != null; } } + + public IEnumerator GetEnumerator() + { + if (Primary != null) yield return Primary; + if (Secondary != null) yield return Secondary; + } + + IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } + } + #endregion + + #region Static + /// + /// Shortcut to HostSet.Builder.Instance.FindAndBuild().... + /// Tries to find a named host set starting at the requested cluster level. + /// Throws if not found. + /// + public static THostSet FindAndBuild(string setName, string clusterPath, bool searchParent = true, bool transcendNoc = false) + where THostSet : HostSet + { + return Builder.Instance.FindAndBuild(setName, clusterPath, searchParent, transcendNoc); + } + + /// + /// Shortcut to HostSet.Builder.Instance.TryFindAndBuild().... + /// Tries to find a named host set starting at the requested cluster level. + /// Returns null if not found. + /// + public static THostSet TryFindAndBuild(string setName, string clusterPath, bool searchParent = true, bool transcendNoc = false) + where THostSet : HostSet + { + return Builder.Instance.TryFindAndBuild(setName, clusterPath, searchParent, transcendNoc); + } + #endregion + + #region .ctor + protected HostSet(string setName, string reqPath, string path, IConfigSectionNode config) + { + m_Name = setName; + m_RequestedPath = path; + m_Path = path; + m_Dynamic = false; + + var mb = AgniSystem.Metabase; + + foreach (var hnode in config.Children.Where(c => c.IsSameName(Metabank.CONFIG_HOST_SET_HOST_SECTION))) + { + var n = hnode.AttrByName(Configuration.CONFIG_NAME_ATTR).Value; + var o = hnode.AttrByName(Configuration.CONFIG_ORDER_ATTR).ValueAsInt(); + if (n.IsNullOrWhiteSpace()) continue; + + var hp = Metabank.RegCatalog.JoinPathSegments(path, n); + var hsect = mb.CatalogReg.NavigateHost(hp); + + if (hsect.Dynamic) + m_Dynamic = true; + + m_DeclaredHosts.Register(new Host(hsect, o)); + } + + BuildHostList(); + + var heartbeatSec = config.AttrByName(CONFIG_HEARTBEAT_INTERVAL_SEC).ValueAsInt(DEFAULT_HEARTBEAT_INTERVAL_SEC); + if (heartbeatSec < 0) heartbeatSec = 0; + + if (heartbeatSec > 0 && heartbeatSec < MIN_HEARTBEAT_INTERVAL_SEC) heartbeatSec = MIN_HEARTBEAT_INTERVAL_SEC; + + if (heartbeatSec > 0) + { + m_HeartbeatScan = new Event(App.EventTimer, + body: e => DoHeartbeat(), + interval: TimeSpan.FromSeconds(heartbeatSec + ExternalRandomGenerator.Instance.NextScaledRandomInteger(-5, 5)), + bodyAsyncModel: EventBodyAsyncModel.AsyncTask, + enabled: false) + { + StartDate = App.TimeSource.UTCNow.AddSeconds(10), + TimeLocation = TimeLocation.UTC + }; + m_HeartbeatScan.Enabled = true; + } + + ConfigAttribute.Apply(this, config); + } + + protected override void Destructor() + { + DisposeAndNull(ref m_HeartbeatScan); + base.Destructor(); + } + #endregion + + #region Fields + private string m_Name; + private string m_RequestedPath; + private string m_Path; + + private bool m_Dynamic; + private Event m_HeartbeatScan; + private OrderedRegistry m_DeclaredHosts = new OrderedRegistry(); + private Host[] m_Hosts; + #endregion + + #region Properties + [Config(Default = DEFAULT_LOG_LEVEL)] + public MessageType LogLevel { get; set; } + + /// + /// Returns HostSet Name + /// + public string Name { get { return m_Name; } } + + /// + /// Returns the region path that was requested + /// + public string RequestedPath { get { return m_RequestedPath; } } + + /// + /// Returns the actual resolved region path at which the set operates + /// + public string Path { get { return m_Path; } } + + /// + /// True to indicate that the number of hosts in the set is flexible + /// + public bool Dynamic { get { return m_Dynamic; } } + + /// + /// The hosts that are declared in set + /// + public IOrderedRegistry DeclaredHosts { get { return m_DeclaredHosts; } } + #endregion + + #region Public + /// + /// Assigns a worker from the set for the supplied sharding key. + /// If key is null then a random member is assigned. + /// Returns null if there is no host available for assignment + /// + public virtual HostPair AssignHost(object shardingKey) + { + var hosts = m_Hosts;//thread-safe copy, as during excecution another may swap + + if (hosts == null || hosts.Length == 0) return new HostPair(); + + if (shardingKey == null) shardingKey = ExternalRandomGenerator.Instance.NextRandomInteger; + + var idx = (uint)MDB.ShardingUtils.ObjectToShardingID(shardingKey) % hosts.Length; + + var idx1 = -1L; + for (var c = 0; c < hosts.Length; c++) + { + var current = idx; + idx = (current + 1) % hosts.Length; + var host = hosts[current]; + if (!host.LastDownTime.HasValue) + { + idx1 = current; + break; + } + } + + var idx2 = -1L; + for (var c = 0; c < hosts.Length; c++) + { + var current = idx; + idx = (current + 1) % hosts.Length; + var host = hosts[current]; + if (!host.LastDownTime.HasValue && current != idx1) + { + idx2 = current; + break; + } + } + + return new HostPair(idx1 >= 0 ? hosts[idx1].Section : null, idx2 >= 0 ? hosts[idx2].Section : null); + } + + /// + /// Refreshes the list of hosts in the set + /// + public void Refresh() + { + BuildHostList(); + Task.Factory.StartNew(DoHeartbeat); + } + #endregion + + #region Protected + /// + /// Override to determine the lists of hosts in the set. + /// In case of dynamic hosts this method may be called many times + /// + protected virtual void BuildHostList() + { + m_Hosts = m_DeclaredHosts.OrderedValues.ToArray(); + } + + protected virtual void DoHeartbeat() + { + var hosts = m_Hosts; + + foreach (var host in hosts) + { + try + { + using (var pinger = ServiceClientHub.New(host.Section)) + pinger.Ping(); + + host.LastDownTime = null; + } + catch (Exception error) + { + host.LastDownTime = App.TimeSource.UTCNow; + //todo instrument + Log(MessageType.Error, "heartbeat()", "Sending heartbeat to '{0}' failed: {1}".Args(host.Section.RegionPath, error.ToMessageWithType()), error); + } + } + } + + protected Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + var logMessage = new Message + { + Topic = SysConsts.LOG_TOPIC_HOST_SET, + Text = message ?? string.Empty, + Type = type, + From = "{0}.{1}".Args(this.GetType().Name, from), + Exception = error, + Parameters = parameters + }; + if (relatedMessageID.HasValue) logMessage.RelatedTo = relatedMessageID.Value; + + App.Log.Write(logMessage); + + return logMessage.Guid; + } + #endregion + } +} diff --git a/src/Agni/Coordination/MDBShardWorkSet.cs b/src/Agni/Coordination/MDBShardWorkSet.cs new file mode 100644 index 0000000..4c6c147 --- /dev/null +++ b/src/Agni/Coordination/MDBShardWorkSet.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using Agni.MDB; + +namespace Agni.Coordination +{ + /// + /// Coordinates work on multiple shards in the MDB area + /// + public class MDBShardWorkSet : WorkSet + { + public MDBShardWorkSet(MDBArea area, string name = null) + : this(null, area, name) + { } + + public MDBShardWorkSet(string path, MDBArea area, string name = null) : base(path, "{0}.{1}".Args(area.NonNull(text: "area==null)").Name, name)) + { + Area = area; + Touch(); + } + + public readonly MDBArea Area; + + protected override void AssignWorkSegment() + { + m_TotalWorkCount = Area.AllShards.Count(); + m_MyWorkCount = TaskUtils.AssignWorkSegment(m_TotalWorkCount, WorkerCount, MyIndex, out m_MyFirstWorkIndex); + } + + protected override IEnumerator GetSegmentEnumerator() + { + if (m_MyFirstWorkIndex<0 || m_MyWorkCount<=0) + return Enumerable.Empty().GetEnumerator(); + + return Area.AllShards + .Skip(m_MyFirstWorkIndex) + .Take(m_MyWorkCount) + .GetEnumerator(); + } + } + +} diff --git a/src/Agni/Coordination/WorkSet.cs b/src/Agni/Coordination/WorkSet.cs new file mode 100644 index 0000000..c6b7c95 --- /dev/null +++ b/src/Agni/Coordination/WorkSet.cs @@ -0,0 +1,307 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.ApplicationModel; + +using Agni.Locking; +using SrvVar=Agni.Locking.Server.Variable; + +namespace Agni.Coordination +{ + + /// + /// Facilitates distributed processing coordination of the set of work which consists of TItems, + /// by using lock manager to exchange the status with other workers working on the same set. + /// Sets are identified by Name within the cluster Path. + /// This class is abstract and must be inherited-from to specify the actual work partitioning - + /// as required by the particular task (i.e. Process all users that...) by overriding AssignWorkSegment(). + /// The system retains the lock session for the duration of this instance existence, please dispose + /// both the instance and the enumerable returned by GetEnumerator(). + /// This class IS NOT thread safe. + /// + public abstract class WorkSet : ApplicationComponent, INamed, IEnumerable + { + public const string WORKSET_NS = "~WORKSET~"; + + protected WorkSet(string path, string name) + { + if (name.IsNullOrWhiteSpace()) + throw new CoordinationException(StringConsts.ARGUMENT_ERROR+"WorkSet.ctor(name=null|empty)"); + + if (path.IsNullOrWhiteSpace()) + path = AgniSystem.HostMetabaseSection.ParentZone.RegionPath; + + m_Path = path; + m_Name = name; + } + + protected override void Destructor() + { + DisposeAndNull(ref m_Session); + base.Destructor(); + } + + + + private string m_Path; + private string m_Name; + private LockSession m_Session; + + + + private string m_Round; + + private int m_WorkerCount; + private int m_MyIndex; + + protected int m_TotalWorkCount; + protected int m_MyFirstWorkIndex = -1; + protected int m_MyWorkCount; + + /// + /// The cluster path/level where the workset is going to execute. + /// The workset.Name is unique within this path + /// + public string Path { get { return m_Path;}} + + /// + /// Returns the workset name which uniquely identifies the set at the cluster level + /// + public string Name { get { return m_Name;}} + + + /// + /// Override to specify the average anticipated time of one item processing + /// + public virtual int OneItemProcessingTimeSec { get { return 5;} } + + + /// + /// Specifies the duration of the work round - this is when a worker gets assigned a position in the set. + /// The periodic repositioning is needed to ensure the 100% eventual coverage of the whole set as some workers may fail. + /// It also controls how often the whole work set is Assigned into portions between workers + /// + public virtual int RoundDurationSec { get { return 15 * 60; }} + + + /// + /// Returns the total number of workers in the set including this one + /// + public int WorkerCount{ get{ return m_WorkerCount; }} + + /// + /// Returns the index of this worker in the set + /// + public int MyIndex{ get{ return m_MyIndex; }} + + /// + /// Returns the total number of work units int the set + /// + public int TotalWorkCount { get { return m_TotalWorkCount; }} + + + /// + /// Returns the index of the first work item that this worker is assigned to + /// + public int MyFirstWorkIndex { get { return m_MyFirstWorkIndex; }} + + /// + /// Returns the number of work items to be processed by this worker starting at MyFirstWorkIndex + /// + public int MyWorkCount { get { return m_MyWorkCount; }} + + + + + + IEnumerator IEnumerable.GetEnumerator() { return this.GetEnumerator(); } + /// + /// Enumerates the work items which are assigned just to this worker. + /// The WorkSet class tries to coordinate the work of multiple workers so that they + /// do not intersect + /// + public IEnumerator GetEnumerator() + { + refresh(); + + var hasWork = m_MyFirstWorkIndex >= 0 && m_MyWorkCount > 0; + + var source = hasWork ? GetSegmentEnumerator() : Enumerable.Empty().GetEnumerator(); + return new enumerator(this, source); + } + + + /// + /// Call this method when the instance is not enumerated for a long time. + /// Enumeration refreshes the worker registration automatically. + /// + public void Touch() + { + try + { + refresh(); + } + catch (Exception error) + { + throw new CoordinationException(error.ToMessageWithType(), error); + } + } + + /// + /// Override to estimate the whole work by setting TotalWorkUnits, + /// and depending on TotalWorkUnits, set ThisFirstUnitIndex, and ThisLastUnitIndex + /// + protected abstract void AssignWorkSegment(); + + /// + /// Override to provide source enumeration of work items which depends on ThisFirstUnitIndex, ThisLastUnitIndex and TotalWorkUnits + /// + protected abstract IEnumerator GetSegmentEnumerator(); + + + private class enumerator : DisposableObject, IEnumerator + { + public enumerator(WorkSet wset, IEnumerator source) + { + m_WorkSet = wset; + m_Source = source; + } + + protected override void Destructor() + { + m_Source.Dispose(); + } + + private WorkSet m_WorkSet; + private IEnumerator m_Source; + private TItem m_Current; + + object IEnumerator.Current{ get{ return this.Current;} } + + public TItem Current{ get{ return m_Current;} } + + public bool MoveNext() + { + m_WorkSet.refresh(); + + if (m_Source.MoveNext()) + { + m_Current = m_Source.Current; + return true; + } + + return false; + } + + public void Reset() + { + throw new NotSupportedException("WorkSet.enumerator.Reset()"); + } + } + + private DateTime m_LastWorkAssign; + private DateTime m_LastRegistration; + + private void refresh() + { + var now = App.TimeSource.UTCNow; + + + var sinceReg = now - m_LastRegistration; + var sinceAssign = now - m_LastWorkAssign; + var needReg = sinceReg.TotalSeconds > 10 + ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 20); + var needAssign = sinceAssign.TotalSeconds > RoundDurationSec; + + if (needAssign) + m_Round = NFX.ExternalRandomGenerator.Instance.NextRandomWebSafeString(); + + //in case of lock server failure, we need to update who we are, where, and what we do + if (needReg || needAssign) + { + m_LastRegistration = now; + registerThisWorker(); + } + else + ensureSession(); + + + if (needAssign) + { + m_LastWorkAssign = now; + AssignWorkSegment(); + } + } + + private DateTime m_LastSession; + + private void ensureSession() + { + const int SESSION_DURATION_ITEMS = 10; + + var locker = AgniSystem.LockManager; + + if (m_Session!=null) + { + var passed = App.TimeSource.UTCNow - m_LastRegistration; + if (passed.TotalSeconds < ((SESSION_DURATION_ITEMS * 0.25) * OneItemProcessingTimeSec)) return; + //just ping; + var result = locker.ExecuteLockTransaction(m_Session, LockTransaction.PingAnyReliability); + if (result.ErrorCause==LockErrorCause.Unspecified) + { + m_LastSession = App.TimeSource.UTCNow; + return; + } + DisposeAndNull(ref m_Session); + } + + var timeoutSec = SESSION_DURATION_ITEMS * OneItemProcessingTimeSec; + var descr = "'{0}'@'{1}'".Args(Name, Path); + m_Session = locker.MakeSession(Path, Name, descr, timeoutSec); + m_LastSession = App.TimeSource.UTCNow; + } + + private void registerThisWorker() + { + ensureSession(); + + var table = this.GetType().AssemblyQualifiedName; + + var value = "{0}::{1}".Args(m_Round, AgniSystem.HostName);//round must be the first - it ensures different sorting every time when round changes + var descr = "{0} {1}".Args(GetType().Name, value); + var script = new LockTransaction(descr, WORKSET_NS, 0, 0.0d, + LockOp.SelectVarValue("Workers", table, Name, ignoreThisSession: true, abortIfNotFound: false, selectMany: true), + LockOp.AnywayContinueAfter( LockOp.DeleteVar(table, Name), resetAbort: true ), + LockOp.Assert( LockOp.SetVar(table, Name, value, allowDuplicates: true) ) + ); + var result = AgniSystem.LockManager.ExecuteLockTransaction(m_Session, script); + if (result.ErrorCause!=LockErrorCause.Unspecified) + { + //todo che delat? + //Interumentatiopn + logging + //Log() + } + + var vars = result["Workers"] as SrvVar[]; + if (vars != null) + { + var values = new List(vars.Select(v => v.Value.AsString())); + values.Add(value);//add myself + values.Sort(StringComparer.InvariantCulture);//must have pre-determined order + + m_WorkerCount = values.Count; + m_MyIndex = values.IndexOf(value);//my index in the set + } + else + { + m_WorkerCount = 1; + m_MyIndex = 0; + } + } + } + +} diff --git a/src/Agni/Dynamic/Exceptions.cs b/src/Agni/Dynamic/Exceptions.cs new file mode 100644 index 0000000..63e8215 --- /dev/null +++ b/src/Agni/Dynamic/Exceptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +namespace Agni.Dynamic +{ + /// + /// Thrown to indicate workers related problems + /// + [Serializable] + public class DynamicException : AgniException + { + public DynamicException() : base() {} + public DynamicException(string message) : base(message) { } + public DynamicException(string message, Exception inner) : base(message, inner) { } + protected DynamicException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Dynamic/HostManager.cs b/src/Agni/Dynamic/HostManager.cs new file mode 100644 index 0000000..d8b52c1 --- /dev/null +++ b/src/Agni/Dynamic/HostManager.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.ServiceModel; +using NFX.Environment; + +using Agni.AppModel; +using Agni.Metabase; + +namespace Agni.Dynamic +{ + public class HostManager : ServiceWithInstrumentationBase, IHostManagerImplementation + { + #region CONSTS + private static readonly TimeSpan INSTRUMENTATION_INTERVAL = TimeSpan.FromMilliseconds(3700); + #endregion + + #region .ctor + public HostManager(IAgniApplication director) : base(director) { } + + protected override void Destructor() + { + DisposableObject.DisposeAndNull(ref m_InstrumentationEvent); + base.Destructor(); + } + #endregion + + #region Fields + private bool m_InstrumentationEnabled; + private NFX.Time.Event m_InstrumentationEvent; + + private NFX.Collections.NamedInterlocked m_Stats = new NFX.Collections.NamedInterlocked(); + #endregion + + #region Properties + /// + /// Implements IInstrumentable + /// + [Config(Default = false)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOCKING, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled + { + get { return m_InstrumentationEnabled; } + set + { + m_InstrumentationEnabled = value; + if (m_InstrumentationEvent == null) + { + if (!value) return; + m_Stats.Clear(); + m_InstrumentationEvent = new NFX.Time.Event(App.EventTimer, null, e => AcceptManagerVisit(this, e.LocalizedTime), INSTRUMENTATION_INTERVAL); + } + else + { + if (value) return; + DisposableObject.DisposeAndNull(ref m_InstrumentationEvent); + m_Stats.Clear(); + } + } + } + #endregion + + #region Public + public Contracts.DynamicHostID Spawn(Metabank.SectionHost host, string id) + { + if (!host.Dynamic) throw new DynamicException("TODO"); + + var hosts = host.ParentZone.ZoneGovernorHosts; + return Contracts.ServiceClientHub.CallWithRetry + ( + (controller) => controller.Spawn(host.RegionPath, id), + hosts.Select(h => h.RegionPath) + ); + } + + public string GetHostName(Contracts.DynamicHostID hid) + { + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(hid.Zone); + var hosts = zone.ZoneGovernorHosts; + return Contracts.ServiceClientHub.CallWithRetry + ( + (controller) => controller.GetDynamicHostInfo(hid), + hosts.Select(h => h.RegionPath) + ).Host; + } + #endregion + } +} diff --git a/src/Agni/Dynamic/Intfs.cs b/src/Agni/Dynamic/Intfs.cs new file mode 100644 index 0000000..f6962f6 --- /dev/null +++ b/src/Agni/Dynamic/Intfs.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Instrumentation; +using NFX.Log; + +using Agni.Metabase; + +namespace Agni.Dynamic +{ + public interface IHostManager + { + Contracts.DynamicHostID Spawn(Metabank.SectionHost host, string id); + string GetHostName(Contracts.DynamicHostID hid); + } + + public interface IHostManagerImplementation : IHostManager, IApplicationComponent, IDisposable, IConfigurable, IInstrumentable + { + } +} diff --git a/src/Agni/Glue/AppTermBinding.cs b/src/Agni/Glue/AppTermBinding.cs new file mode 100644 index 0000000..a4a56cb --- /dev/null +++ b/src/Agni/Glue/AppTermBinding.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using NFX.Glue; +using NFX.Glue.Native; +using NFX.Glue.Protocol; + +namespace Agni.Glue +{ + /// + /// Provides synchronous Glue communication channel over TCP with IRemoteTerminal server using + /// text protocol stable to runtime version changes. + /// This binding is not expected to process many messages per second as it is used for admin purposes only, + /// the protocol is json-based framed in Glue + /// + public sealed class AppTermBinding : SyncBinding + { + #region Static Members + + public const int FRAME_FORMAT_APPTERM = 0x41505054;//APPT + + public static readonly TypeSpec TYPE_CONTRACT; + public static readonly MethodSpec METHOD_CONNECT; + public static readonly MethodSpec METHOD_EXECUTE; + public static readonly MethodSpec METHOD_DISCONNECT; + + //static .ctor + static AppTermBinding() + { + var t = typeof(@Agni.@Contracts.@IRemoteTerminal); + TYPE_CONTRACT = new TypeSpec(t); + METHOD_CONNECT = new MethodSpec(t.GetMethod("Connect", new Type[]{ typeof(@System.@String) })); + METHOD_EXECUTE = new MethodSpec(t.GetMethod("Execute", new Type[]{ typeof(@System.@String) })); + METHOD_DISCONNECT = new MethodSpec(t.GetMethod("Disconnect", new Type[]{ })); + } + #endregion + + public AppTermBinding(string name) : base (name) + { + + } + + public AppTermBinding(IGlueImplementation glue, string name) : base(glue, name) + { + } + + public override int FrameFormat => FRAME_FORMAT_APPTERM; + + + protected override ClientTransport MakeNewClientTransport(ClientEndPoint client) + { + return new AppTermClientTransport(this, client.Node); + } + + protected override ServerTransport OpenServerEndpoint(ServerEndPoint epoint) + { + var cfg = ConfigNode.NavigateSection(CONFIG_SERVER_TRANSPORT_SECTION); + if (!cfg.Exists) cfg = ConfigNode; + + var ipep = SyncBinding.ToIPEndPoint(epoint.Node); + var transport = new AppTermServerTransport(this, epoint, ipep.Address, ipep.Port); + transport.Configure(cfg); + transport.Start(); + + return transport; + } + } +} diff --git a/src/Agni/Glue/AppTermClientTransport.cs b/src/Agni/Glue/AppTermClientTransport.cs new file mode 100644 index 0000000..f54c36e --- /dev/null +++ b/src/Agni/Glue/AppTermClientTransport.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using NFX; +using NFX.Glue; +using NFX.Serialization.JSON; +using NFX.Glue.Native; +using NFX.Glue.Protocol; + +namespace Agni.Glue +{ + /// + /// Implements client transport for application terminal contract. + /// READ THIS: this binding processes a few messages a second at best, no need to implement complex optimizations + /// like copy-free code etc. + /// + public sealed class AppTermClientTransport : SyncClientTransport + { + public AppTermClientTransport(SyncBinding binding, Node node) : base(binding, node) + { + } + + + + protected override void DoEncodeRequest(MemoryStream ms, RequestMsg msg) + { + if (msg.Contract!=typeof(Contracts.IRemoteTerminal)) + throw new ProtocolException(StringConsts.GLUE_BINDING_UNSUPPORTED_FUNCTION_ERROR.Args(nameof(AppTermBinding), + nameof(Contracts.IRemoteTerminal), + msg.Contract.FullName)); + + var request = msg as RequestAnyMsg; + if (request==null) + throw new ProtocolException(StringConsts.GLUE_BINDING_UNSUPPORTED_FUNCTION_ERROR.Args(nameof(AppTermBinding), + nameof(RequestAnyMsg), + msg.GetType().FullName)); + + + var data = new JSONDataMap(); + data["id"] = request.RequestID.ID; + data["instance"] = request.RemoteInstance?.ToString("D"); + data["method"] = request.MethodName; + data["one-way"] = false;//reserved for future use + + if (request.Arguments!=null && request.Arguments.Length>0) + data["command"] = request.Arguments[0]; + else + data["command"] = null; + + //Handle headers, this binding allows ONLY for AuthenticationHeader with supported tokens/credentials + if (request.HasHeaders) + { + if (request.Headers.Count>1) + throw new ProtocolException(StringConsts.GLUE_BINDING_UNSUPPORTED_FUNCTION_ERROR.Args(nameof(AppTermBinding), + "1 AuthenticationHeader", + request.Headers.Count)); + var ahdr = request.Headers[0] as AuthenticationHeader; + if (ahdr==null) + throw new ProtocolException(StringConsts.GLUE_BINDING_UNSUPPORTED_FUNCTION_ERROR.Args(nameof(AppTermBinding), + "1 AuthenticationHeader", + request.Headers[0].GetType().FullName)); + + if (ahdr.Token.Assigned) + data["auth-token"] = Security.AgniAuthenticationTokenSerializer.Serialize(ahdr.Token); + + if (ahdr.Credentials!=null) + { + var src = ahdr.Credentials as NFX.Security.IStringRepresentableCredentials; + if (src==null) + throw new ProtocolException(StringConsts.GLUE_BINDING_UNSUPPORTED_FUNCTION_ERROR.Args(nameof(AppTermBinding), + "IStringRepresentableCredentials", + ahdr.Credentials.GetType().FullName)); + data["auth-cred"] = src.RepresentAsString(); + } + } + + var json = data.ToJSON(JSONWritingOptions.Compact); + var utf8 = Encoding.UTF8.GetBytes(json); + ms.Write(utf8, 0, utf8.Length); + } + + protected override ResponseMsg DoDecodeResponse(WireFrame frame, MemoryStream ms) + { + var utf8 = ms.GetBuffer(); + var json = Encoding.UTF8.GetString(utf8, (int)ms.Position, (int)ms.Length - (int)ms.Position); + var data = json.JSONToDataObject() as JSONDataMap; + + if (data==null) + throw new ProtocolException(StringConsts.GLUE_BINDING_RESPONSE_ERROR.Args(nameof(AppTermBinding),"data==null")); + + + var reqID = new FID( data["request-id"].AsULong(handling: ConvertErrorHandling.Throw) ); + var instance = data["instance"].AsNullableGUID(handling: ConvertErrorHandling.Throw); + + object returnValue = data["return"]; + if (returnValue==null || returnValue is string) + { + //return as-is + } else if (returnValue is JSONDataMap map)//error or Remote Terminal + { + var errorContent = map["error-content"].AsString(); + if (errorContent!=null) + returnValue = WrappedExceptionData.FromBase64(errorContent); + else + returnValue = new Contracts.RemoteTerminalInfo(map); + + } else throw new ProtocolException(StringConsts.GLUE_BINDING_RESPONSE_ERROR.Args(nameof(AppTermBinding), "data.return is "+returnValue.GetType().FullName)); + + + var result = new ResponseMsg(reqID, instance, returnValue); + + return result; + } + } +} diff --git a/src/Agni/Glue/AppTermServerTransport.cs b/src/Agni/Glue/AppTermServerTransport.cs new file mode 100644 index 0000000..48a664f --- /dev/null +++ b/src/Agni/Glue/AppTermServerTransport.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Net; +using System.IO; +using System.Reflection; + +using NFX; +using NFX.Glue; +using NFX.Glue.Native; +using NFX.Glue.Protocol; +using NFX.Serialization; +using NFX.Serialization.JSON; + +namespace Agni.Glue +{ + /// + /// Implements server transport for application terminal contract. + /// READ THIS: this binding processes a few messages a second at best, no need to implement complex optimizations + /// like copy-free code etc. + /// + public sealed class AppTermServerTransport : SyncServerTransport + { + public AppTermServerTransport(SyncBinding binding, + ServerEndPoint serverEndpoint, + IPAddress localAddr, int port) : base(binding, serverEndpoint, localAddr, port) + { + + } + + + protected override RequestMsg DoDecodeRequest(WireFrame frame, MemoryStream ms, ISerializer serializer) + { + var utf8 = ms.GetBuffer(); + var json = Encoding.UTF8.GetString(utf8, (int)ms.Position, (int)ms.Length - (int)ms.Position); + var data = JSONReader.DeserializeDataObject(json) as JSONDataMap; + + if (data==null) + throw new ProtocolException(StringConsts.GLUE_BINDING_REQUEST_ERROR.Args(nameof(AppTermBinding),"data==null")); + + var reqID = new FID( data["request-id"].AsULong(handling: ConvertErrorHandling.Throw) ); //kuda ego vstavit? + var instance = data["instance"].AsNullableGUID(handling: ConvertErrorHandling.Throw); + var oneWay = data["one-way"].AsBool(); + var method = data["method"].AsString(); + + MethodSpec mspec; + if (method.EqualsOrdIgnoreCase(nameof(Contracts.IRemoteTerminal.Connect))) mspec = AppTermBinding.METHOD_CONNECT; + else if (method.EqualsOrdIgnoreCase(nameof(Contracts.IRemoteTerminal.Execute))) mspec = AppTermBinding.METHOD_EXECUTE; + else if (method.EqualsOrdIgnoreCase(nameof(Contracts.IRemoteTerminal.Disconnect))) mspec = AppTermBinding.METHOD_DISCONNECT; + else + throw new ProtocolException(StringConsts.GLUE_BINDING_REQUEST_ERROR.Args(nameof(AppTermBinding),"unknown method `{0}`".Args(method))); + + var args = data["command"]==null ? new object[0] : new object[]{ data["command"].AsString() }; + + var result = new RequestAnyMsg(AppTermBinding.TYPE_CONTRACT, mspec, oneWay, instance, args); + + var autht = data["auth-token"].AsString(); + if (autht!=null) + { + var hdr = new AuthenticationHeader(Security.AgniAuthenticationTokenSerializer.Deserialize(autht)); + result.Headers.Add(hdr); + } + var authc = data["auth-cred"].AsString(); + if (authc!=null) + { + var hdr = new AuthenticationHeader(NFX.Security.IDPasswordCredentials.FromBasicAuth(authc)); + result.Headers.Add(hdr); + } + + return result; + } + + protected override void DoEncodeResponse(MemoryStream ms, ResponseMsg msg, ISerializer serializer) + { + var data = new JSONDataMap(); + data["id"] = msg.RequestID.ID; + data["instance"] = msg.RemoteInstance?.ToString("D"); + + if (msg.ReturnValue is string rstr) + data["return"] = rstr; + else if (msg.ReturnValue is Contracts.RemoteTerminalInfo rtrm) + data["return"] = rtrm; + else if (msg.ReturnValue is WrappedExceptionData wed) + data["return"] = new JSONDataMap{ {"error-content", wed.ToBase64()}}; + else + throw new ProtocolException(StringConsts.GLUE_BINDING_RESPONSE_ERROR.Args(nameof(AppTermBinding),"unsupported ReturnValue `{0}`".Args(msg.ReturnValue))); + + var json = data.ToJSON(JSONWritingOptions.Compact); + var utf8 = Encoding.UTF8.GetBytes(json); + ms.Write(utf8, 0, utf8.Length); + } + } +} diff --git a/src/Agni/Hosts/agdida/GDIDAuthorityRemoteTerminal.cs b/src/Agni/Hosts/agdida/GDIDAuthorityRemoteTerminal.cs new file mode 100644 index 0000000..6643768 --- /dev/null +++ b/src/Agni/Hosts/agdida/GDIDAuthorityRemoteTerminal.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Agni.AppModel.Terminal; + +namespace Agni.Hosts.agdida +{ + /// + /// Implements GDID Authority remote terminal + /// + public class GDIDAuthorityRemoteTerminal : AppRemoteTerminal + { + public GDIDAuthorityRemoteTerminal() : base() { } + } +} diff --git a/src/Agni/Hosts/agdida/ProgramBody.cs b/src/Agni/Hosts/agdida/ProgramBody.cs new file mode 100644 index 0000000..de38b6e --- /dev/null +++ b/src/Agni/Hosts/agdida/ProgramBody.cs @@ -0,0 +1,97 @@ +using System; +using System.Threading; + +using NFX; +using NFX.IO; + +using Agni.AppModel; +using Agni.Identification; + +namespace Agni.Hosts.agdida +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + AgniSystem.MetabaseApplicationName = SysConsts.APP_NAME_GDIDA; + + run(args); + + Environment.ExitCode = 0; + } + catch (Exception error) + { + Console.WriteLine(error.ToString()); + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + const string FROM = "AGDIDA.Program"; + + using (var app = new AgniServiceApplication(SystemApplicationType.GDIDAuthority, args, null)) + { + try + { + using (var authority = new GDIDAuthorityService()) + { + authority.Configure(null); + authority.Start(); + try + { + // WARNING: Do not modify what this program reads/writes from/to standard IO streams because + // AHGOV uses those particular string messages for its protocol + Console.WriteLine("OK."); //<-- AHGOV protocol, AHGOV waits for this token to assess startup situation + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + Console.WriteLine("Waiting for line to terminate..."); + + var abortableConsole = new TerminalUtils.AbortableLineReader(); + try + { + while (app.Active) + { + if (abortableConsole.Line != null) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Info, + Topic = SysConsts.LOG_TOPIC_ID_GEN, + From = FROM, + Text = "Main loop received CR|LF. Exiting..." + }); + break; //<-- AHGOV protocol, AHGOV sends a when it is time to shut down + } + Thread.Sleep(1000); + } + } + finally + { + abortableConsole.Abort(); + } + } + finally + { + authority.WaitForCompleteStop(); + } + } + } + catch (Exception error) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.CatastrophicError, + Topic = SysConsts.LOG_TOPIC_ID_GEN, + From = FROM, + Text = "Exception leaked in run(): " + error.ToMessageWithType(), + Exception = error + }); + + throw error; + } + }//using app + } + } +} diff --git a/src/Agni/Hosts/agdida/Welcome.txt b/src/Agni/Hosts/agdida/Welcome.txt new file mode 100644 index 0000000..deb47df --- /dev/null +++ b/src/Agni/Hosts/agdida/Welcome.txt @@ -0,0 +1,6 @@ + +Agni GDID Authority +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 19 2018 + + \ No newline at end of file diff --git a/src/Agni/Hosts/ahgov/HGovRemoteTerminal.cs b/src/Agni/Hosts/ahgov/HGovRemoteTerminal.cs new file mode 100644 index 0000000..24713ff --- /dev/null +++ b/src/Agni/Hosts/ahgov/HGovRemoteTerminal.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Agni.AppModel.Terminal; + +namespace Agni.Hosts.ahgov +{ + /// + /// Implements Host Governor remote terminal + /// + public class HGovRemoteTerminal : AppRemoteTerminal + { + public HGovRemoteTerminal() : base() + { + + } + + public override IEnumerable Cmdlets + { + get + { + var local = CmdletFinder.FindByNamespace(typeof(HGovRemoteTerminal), "Agni.Hosts.ahgov.Cmdlets"); + return base.Cmdlets.Concat(CmdletFinder.HGov).Concat(local); + } + } + } +} diff --git a/src/Agni/Hosts/ahgov/ProgramBody.cs b/src/Agni/Hosts/ahgov/ProgramBody.cs new file mode 100644 index 0000000..03e6c7d --- /dev/null +++ b/src/Agni/Hosts/ahgov/ProgramBody.cs @@ -0,0 +1,100 @@ +using System; +using System.Threading; + +using NFX; +using NFX.IO; + +using Agni.AppModel; +using Agni.AppModel.HostGovernor; + +namespace Agni.Hosts.ahgov +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + AgniSystem.MetabaseApplicationName = SysConsts.APP_NAME_HGOV; + + run(args); + + Environment.ExitCode = 0; + } + catch (Exception error) + { + Console.WriteLine(error.ToString()); + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + const string FROM = "AHGOV.Program"; + + using (var app = new AgniServiceApplication(SystemApplicationType.HostGovernor, args, null)) + { + try + { + var fromARD = app.CommandArgs[SysConsts.ARD_PARENT_CMD_PARAM].Exists; + var updateProblem = app.CommandArgs[SysConsts.ARD_UPDATE_PROBLEM_CMD_PARAM].Exists; + using (var governor = new HostGovernorService(fromARD, updateProblem)) + { + governor.Configure(null); + governor.Start(); + try + { + // WARNING: Do not modify what this program reads/writes from/to standard IO streams because + // ARD uses those particular string messages for its protocol + Console.WriteLine("OK."); //<-- ARD protocol, ARD waits for this token to assess startup situation + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + Console.WriteLine("Waiting for line to terminate..."); + + + var abortableConsole = new TerminalUtils.AbortableLineReader(); + try + { + while (app.Active && !governor.NeedsProcessRestart) + { + if (abortableConsole.Line != null) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Info, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = FROM, + Text = "Main loop received CR|LF. Exiting..." + }); + break; //<-- ARD protocol, ARD sends a when it is time to shut down + } + Thread.Sleep(1000); + } + } + finally + { + abortableConsole.Abort(); + } + } + finally + { + governor.WaitForCompleteStop(); + } + }//using governor + } + catch (Exception error) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.CatastrophicError, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = FROM, + Text = "Exception leaked in run(): " + error.ToMessageWithType(), + Exception = error + }); + + throw error; + } + }//using APP + } + } +} diff --git a/src/Agni/Hosts/ahgov/Welcome.txt b/src/Agni/Hosts/ahgov/Welcome.txt new file mode 100644 index 0000000..49978fd --- /dev/null +++ b/src/Agni/Hosts/ahgov/Welcome.txt @@ -0,0 +1,6 @@ + +Agni Host Governor +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/Hosts/aph/ProcessHostRemoteTerminal.cs b/src/Agni/Hosts/aph/ProcessHostRemoteTerminal.cs new file mode 100644 index 0000000..ace7a5f --- /dev/null +++ b/src/Agni/Hosts/aph/ProcessHostRemoteTerminal.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Agni.AppModel.Terminal; + +namespace Agni.Hosts.aph +{ + /// + /// Implements Process Host remote terminal + /// + public class ProcessHostRemoteTerminal : AppRemoteTerminal + { + public ProcessHostRemoteTerminal() : base() { } + } +} diff --git a/src/Agni/Hosts/aph/ProgramBody.cs b/src/Agni/Hosts/aph/ProgramBody.cs new file mode 100644 index 0000000..d0cb90a --- /dev/null +++ b/src/Agni/Hosts/aph/ProgramBody.cs @@ -0,0 +1,111 @@ +using System; +using System.Threading; + +using NFX; +using NFX.IO; + +using Agni.AppModel; +using Agni.Workers.Server; + +namespace Agni.Hosts.aph +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + //Specify the name in boot conf under: + //agni + //{ + // metabase + // { + // app-name="XXXXXXXX" + // } + //} + + //OR + //Inject from command line: aph -agni app-name= + + + //DO NOT USE static process assignment + //AgniSystem.MetabaseApplicationName = Agni.SysConsts.APP_NAME_APH; + + run(args); + + Environment.ExitCode = 0; + } + catch (Exception error) + { + Console.WriteLine(error.ToString()); + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + const string FROM = "APH.Program"; + + using (var app = new AgniServiceApplication(SystemApplicationType.ProcessHost, args, null)) + { + try + { + using (var procHost = new ProcessControllerService(null)) + { + procHost.Configure(null); + procHost.Start(); + try + { + // WARNING: Do not modify what this program reads/writes from/to standard IO streams because + // AHGOV uses those particular string messages for its protocol + Console.WriteLine("OK."); //<-- AHGOV protocol, AHGOV waits for this token to assess startup situation + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + Console.WriteLine("Waiting for line to terminate..."); + + var abortableConsole = new TerminalUtils.AbortableLineReader(); + try + { + while (app.Active) + { + if (abortableConsole.Line != null) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Info, + Topic = Agni.SysConsts.LOG_TOPIC_WWW, + From = FROM, + Text = "Main loop received CR|LF. Exiting..." + }); + break; //<-- AHGOV protocol, AHGOV sends a when it is time to shut down + } + Thread.Sleep(1000); + } + } + finally + { + abortableConsole.Abort(); + } + } + finally + { + procHost.WaitForCompleteStop(); + } + } + } + catch (Exception error) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.CatastrophicError, + Topic = Agni.SysConsts.LOG_TOPIC_SVC, + From = FROM, + Text = "Exception leaked in run(): " + error.ToMessageWithType(), + Exception = error + }); + + throw error; + } + }//using app + } + } +} diff --git a/src/Agni/Hosts/aph/Welcome.txt b/src/Agni/Hosts/aph/Welcome.txt new file mode 100644 index 0000000..90a6e8b --- /dev/null +++ b/src/Agni/Hosts/aph/Welcome.txt @@ -0,0 +1,6 @@ + +Agni Process Host +Copyright (c) 2017-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/Hosts/ash/ProgramBody.cs b/src/Agni/Hosts/ash/ProgramBody.cs new file mode 100644 index 0000000..3b3c321 --- /dev/null +++ b/src/Agni/Hosts/ash/ProgramBody.cs @@ -0,0 +1,111 @@ +using System; +using System.Threading; + +using NFX; +using NFX.IO; +using NFX.ServiceModel; + +using Agni.AppModel; + +namespace Agni.Hosts.ash +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + //Specify the name in boot conf under: + //agni + //{ + // metabase + // { + // app-name="XXXXXXXX" + // } + //} + + //OR + //Inject from command line: ash -agni app-name= + + + //DO NOT USE static process assignment + //AgniSystem.MetabaseApplicationName = Agni.SysConsts.APP_NAME_ASH; + + run(args); + + Environment.ExitCode = 0; + } + catch (Exception error) + { + Console.WriteLine(error.ToString()); + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + const string FROM = "ASH.Program"; + + using (var app = new AgniServiceApplication(SystemApplicationType.ServiceHost, args, null)) + { + try + { + using (var svcHost = new CompositeServiceHost(null)) + { + svcHost.Configure(null); + svcHost.Start(); + try + { + // WARNING: Do not modify what this program reads/writes from/to standard IO streams because + // AHGOV uses those particular string messages for its protocol + Console.WriteLine("OK."); //<-- AHGOV protocol, AHGOV waits for this token to assess startup situation + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + Console.WriteLine("Waiting for line to terminate..."); + + var abortableConsole = new TerminalUtils.AbortableLineReader(); + try + { + while (app.Active) + { + if (abortableConsole.Line != null) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Info, + Topic = Agni.SysConsts.LOG_TOPIC_WWW, + From = FROM, + Text = "Main loop received CR|LF. Exiting..." + }); + break; //<-- AHGOV protocol, AHGOV sends a when it is time to shut down + } + Thread.Sleep(1000); + } + } + finally + { + abortableConsole.Abort(); + } + } + finally + { + svcHost.WaitForCompleteStop(); + } + } + } + catch (Exception error) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.CatastrophicError, + Topic = Agni.SysConsts.LOG_TOPIC_SVC, + From = FROM, + Text = "Exception leaked in run(): " + error.ToMessageWithType(), + Exception = error + }); + + throw error; + } + }//using app + } + } +} diff --git a/src/Agni/Hosts/ash/ServiceHostRemoteTerminal.cs b/src/Agni/Hosts/ash/ServiceHostRemoteTerminal.cs new file mode 100644 index 0000000..d7fdd92 --- /dev/null +++ b/src/Agni/Hosts/ash/ServiceHostRemoteTerminal.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Agni.AppModel.Terminal; + +namespace Agni.Hosts.ash +{ + /// + /// Implements Service Host remote terminal + /// + public class ServiceHostRemoteTerminal : AppRemoteTerminal + { + public ServiceHostRemoteTerminal() : base() { } + + } +} diff --git a/src/Agni/Hosts/ash/Welcome.txt b/src/Agni/Hosts/ash/Welcome.txt new file mode 100644 index 0000000..877bbd4 --- /dev/null +++ b/src/Agni/Hosts/ash/Welcome.txt @@ -0,0 +1,6 @@ + +Agni Service Host +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/Hosts/aws/ProgramBody.cs b/src/Agni/Hosts/aws/ProgramBody.cs new file mode 100644 index 0000000..944a2ca --- /dev/null +++ b/src/Agni/Hosts/aws/ProgramBody.cs @@ -0,0 +1,112 @@ +using System; +using System.Threading; + +using NFX; +using NFX.IO; +using NFX.Wave; + +using Agni; +using Agni.AppModel; + +namespace Agni.Hosts.aws +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + //Specify the name in boot conf under: + //agni + //{ + // metabase + // { + // app-name="XXXXXXXX" + // } + //} + + //OR + //Inject from command line: aws -agni app-name= + + + //DO NOT USE static process assignment + //AgniSystem.MetabaseApplicationName = Agni.SysConsts.APP_NAME_AGDIDA; + + run(args); + + Environment.ExitCode = 0; + } + catch (Exception error) + { + Console.WriteLine(error.ToString()); + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + const string FROM = "AWS.Program"; + + using (var app = new AgniServiceApplication(SystemApplicationType.WebServer, args, null)) + { + try + { + using (var wwwServer = new WaveServer(app.MetabaseApplicationName)) + { + wwwServer.Configure(null); + wwwServer.Start(); + try + { + // WARNING: Do not modify what this program reads/writes from/to standard IO streams because + // AHGOV uses those particular string messages for its protocol + Console.WriteLine("OK."); //<-- AHGOV protocol, AHGOV waits for this token to assess startup situation + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + Console.WriteLine("Waiting for line to terminate..."); + + var abortableConsole = new TerminalUtils.AbortableLineReader(); + try + { + while (app.Active) + { + if (abortableConsole.Line != null) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Info, + Topic = Agni.SysConsts.LOG_TOPIC_WWW, + From = FROM, + Text = "Main loop received CR|LF. Exiting..." + }); + break; //<-- AHGOV protocol, AHGOV sends a when it is time to shut down + } + Thread.Sleep(1000); + } + } + finally + { + abortableConsole.Abort(); + } + } + finally + { + wwwServer.WaitForCompleteStop(); + } + } + } + catch (Exception error) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.CatastrophicError, + Topic = Agni.SysConsts.LOG_TOPIC_WWW, + From = FROM, + Text = "Exception leaked in run(): " + error.ToMessageWithType(), + Exception = error + }); + + throw error; + } + }//using app + } + } +} diff --git a/src/Agni/Hosts/aws/WebServerRemoteTerminal.cs b/src/Agni/Hosts/aws/WebServerRemoteTerminal.cs new file mode 100644 index 0000000..1992cdf --- /dev/null +++ b/src/Agni/Hosts/aws/WebServerRemoteTerminal.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Agni.AppModel.Terminal; + +namespace Agni.Hosts.aws +{ + /// + /// Implements Web Server remote terminal + /// + public class WebServerRemoteTerminal : AppRemoteTerminal + { + public WebServerRemoteTerminal() : base() { } + + } +} diff --git a/src/Agni/Hosts/aws/Welcome.txt b/src/Agni/Hosts/aws/Welcome.txt new file mode 100644 index 0000000..866439c --- /dev/null +++ b/src/Agni/Hosts/aws/Welcome.txt @@ -0,0 +1,6 @@ + +Agni Web Server +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/Hosts/azgov/ProgramBody.cs b/src/Agni/Hosts/azgov/ProgramBody.cs new file mode 100644 index 0000000..d677c13 --- /dev/null +++ b/src/Agni/Hosts/azgov/ProgramBody.cs @@ -0,0 +1,98 @@ +using System; +using System.Threading; + +using NFX; +using NFX.IO; + +using Agni.AppModel; +using Agni.AppModel.ZoneGovernor; + +namespace Agni.Hosts.azgov +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + Agni.AgniSystem.MetabaseApplicationName = Agni.SysConsts.APP_NAME_ZGOV; + + run(args); + + Environment.ExitCode = 0; + } + catch (Exception error) + { + Console.WriteLine(error.ToString()); + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + const string FROM = "AZGOV.Program"; + + using (var app = new AgniServiceApplication(SystemApplicationType.ZoneGovernor, args, null)) + { + try + { + using (var governor = new ZoneGovernorService()) + { + governor.Configure(null); + governor.Start(); + try + { + // WARNING: Do not modify what this program reads/writes from/to standard IO streams because + // AHGOV uses those particular string messages for its protocol + Console.WriteLine("OK."); //<-- AHGOV protocol, AHGOV waits for this token to assess startup situation + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + Console.WriteLine("Waiting for line to terminate..."); + + + var abortableConsole = new TerminalUtils.AbortableLineReader(); + try + { + while (app.Active) + { + if (abortableConsole.Line != null) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.Info, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = FROM, + Text = "Main loop received CR|LF. Exiting..." + }); + break; //<-- AHGOV protocol, AHGOV sends a when it is time to shut down + } + Thread.Sleep(1000); + } + } + finally + { + abortableConsole.Abort(); + } + } + finally + { + governor.WaitForCompleteStop(); + } + }//using governor + } + catch (Exception error) + { + app.Log.Write(new NFX.Log.Message + { + Type = NFX.Log.MessageType.CatastrophicError, + Topic = SysConsts.LOG_TOPIC_ZONE_MANAGEMENT, + From = FROM, + Text = "Exception leaked in run(): " + error.ToMessageWithType(), + Exception = error + }); + + throw error; + } + }//using APP + } + } +} diff --git a/src/Agni/Hosts/azgov/Welcome.txt b/src/Agni/Hosts/azgov/Welcome.txt new file mode 100644 index 0000000..f9fabdf --- /dev/null +++ b/src/Agni/Hosts/azgov/Welcome.txt @@ -0,0 +1,6 @@ + +Agni Zone Governor +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/Hosts/azgov/ZGovRemoteTerminal.cs b/src/Agni/Hosts/azgov/ZGovRemoteTerminal.cs new file mode 100644 index 0000000..006c4d7 --- /dev/null +++ b/src/Agni/Hosts/azgov/ZGovRemoteTerminal.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Agni.AppModel.Terminal; + +namespace Agni.Hosts.azgov +{ + /// + /// Implements Zone Governor remote terminal + /// + public class ZGovRemoteTerminal : AppRemoteTerminal + { + public ZGovRemoteTerminal() : base() { } + + public override IEnumerable Cmdlets + { + get + { + var local = CmdletFinder.FindByNamespace(typeof(ZGovRemoteTerminal), "Agni.Hosts.azgov.Cmdlets"); + return base.Cmdlets.Concat(CmdletFinder.ZGov).Concat(local); + } + } + } +} \ No newline at end of file diff --git a/src/Agni/Identification/Exceptions.cs b/src/Agni/Identification/Exceptions.cs new file mode 100644 index 0000000..fc313aa --- /dev/null +++ b/src/Agni/Identification/Exceptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +namespace Agni.Identification +{ + /// + /// Thrown to indicate GDID generation related problems + /// + [Serializable] + public class GDIDException : AgniException + { + public GDIDException() : base() { } + public GDIDException(string message) : base(message) { } + public GDIDException(string message, Exception inner) : base(message, inner) { } + protected GDIDException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Identification/GDIDAuthorityService.cs b/src/Agni/Identification/GDIDAuthorityService.cs new file mode 100644 index 0000000..140b528 --- /dev/null +++ b/src/Agni/Identification/GDIDAuthorityService.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Threading; + +using NFX; +using NFX.Log; +using NFX.Collections; +using NFX.ServiceModel; +using NFX.Environment; +using NFX.DataAccess.Distributed; +using NFX.ApplicationModel; + +using Agni.Contracts; + +namespace Agni.Identification +{ + /// + /// Generates Global Distributed IDs - singleton, only one instance of this service may be allocated per process + /// + public sealed class GDIDAuthorityService : GDIDAuthorityServiceBase, IGDIDAuthority + { + #region Inner Classes + private class scope : INamed + { + public string Name { get; set; } + public Registry Sequences = new Registry(); + } + + internal class sequence : INamed + { + public string Name { get ; set; } + public uint Era; + public ulong Value; + public bool New; + } + #endregion + + + #region Static + private static object s_InstanceLock = new object(); + private static volatile GDIDAuthorityService s_Instance; + + /// + /// Returns singleton instance or throws if service has not been allocated yet + /// + public static GDIDAuthorityService Instance + { + get + { + var instance = s_Instance; + if (instance==null) + throw new GDIDException(StringConsts.GDIDAUTH_INSTANCE_NOT_ALLOCATED_ERROR); + + return instance; + } + } + + #endregion + + #region .ctor/.dctor + /// + /// Creates a singleton instance or throws if instance is already created + /// + public GDIDAuthorityService() : base() + { + lock(s_InstanceLock) + { + if (s_Instance!=null) + throw new GDIDException(StringConsts.GDIDAUTH_INSTANCE_ALREADY_ALLOCATED_ERROR); + + s_Instance = this; + } + } + + protected override void Destructor() + { + lock(s_InstanceLock) + { + base.Destructor(); + s_Instance = null; + } + } + #endregion + + #region Fields + private byte[] m_AuthorityIDs; + private Registry m_Scopes = new Registry(); + #endregion + + #region Properties + + public override string ComponentCommonName { get { return "gdida"; }} + + [Config("$"+CONFIG_AUTHORITY_IDS_ATTR)] + public byte[] AuthorityIDs + { + get{ return m_AuthorityIDs; } + set + { + CheckServiceInactive(); + + if (value==null) + { + m_AuthorityIDs = null; + return; + } + + foreach(var id in value) + if (id<0 || id>GDID.AUTHORITY_MAX) + throw new GDIDException(StringConsts.GDIDAUTH_IDS_INVALID_AUTHORITY_VALUE_ERROR.Args(id)); + + m_AuthorityIDs = value; + Log(MessageType.Warning, "AuthorityIDs.set()", StringConsts.GDIDAUTH_AUTHORITY_ASSIGNMENT_WARNING.Args(value.ToDumpString(DumpFormat.Hex))); + } + } + + #endregion + + #region Public + /// + /// Performs block allocation + /// + public GDIDBlock AllocateBlock(string scopeName, string sequenceName, int blockSize, ulong? vicinity) + { + if (!Running) + throw new GDIDException(StringConsts.GDIDAUTH_INSTANCE_NOT_RUNNING_ERROR); + + CheckNameValidity(scopeName); + CheckNameValidity(sequenceName); + + if (blockSize<=0) + throw new GDIDException(StringConsts.ARGUMENT_ERROR+"AllocateBlock(blockSize<=0)"); + + scopeName = scopeName.ToUpperInvariant();//different cases for readability + sequenceName = sequenceName.ToLowerInvariant(); + + if (blockSize>MAX_BLOCK_SIZE) blockSize = MAX_BLOCK_SIZE; + + //get a subsequent authority index + var now = DateTime.Now; + var idx = (((now.DayOfYear * 24) + now.Hour) & CoreConsts.ABS_HASH_MASK) % this.m_AuthorityIDs.Length; + byte authority = this.m_AuthorityIDs[idx]; + + return allocate(authority, scopeName, sequenceName, blockSize, vicinity); + } + #endregion + + #region Protected + protected override void DoStart() + { + if (m_AuthorityIDs==null || m_AuthorityIDs.Length<1) + throw new GDIDException(StringConsts.GDIDAUTH_IDS_INVALID_AUTHORITY_VALUE_ERROR.Args("")); + base.DoStart(); + } + #endregion + + #region .pvt + + private GDIDBlock allocate(byte authority, string scopeName, string sequenceName, int blockSize, ulong? vicinity) + { + var scopeKey = "{0}://{1}".Args(AuthorityPathSeg(authority), scopeName); + + var scope = m_Scopes.GetOrRegister(scopeKey,(key) => new scope{Name = key}, scopeKey); + + var sequence = scope.Sequences.GetOrRegister(sequenceName,(_) => new sequence{Name = sequenceName, New=true}, 0);//with NEW=TRUE + + var result = new GDIDBlock() + { + ScopeName = scopeName, + SequenceName = sequenceName, + Authority = authority, + AuthorityHost = AgniSystem.HostName, + BlockSize = blockSize, + ServerUTCTime = App.TimeSource.UTCNow + }; + + lock(scope) + { + //0. If just allocated then need to read from disk + if (sequence.New) + { + sequence.New = false; + var id = ReadFromLocations(authority, scopeName, sequenceName); + sequence.Era = id.Era; + sequence.Value = id.Value; + } + + //1. make a local copy of vars, that may mutate but dont get committed until written to disk + var era = sequence.Era; + var value = sequence.Value; + + //1.1 make sure that GDID.Zero is never returned + if (authority==0 && era==0 && value==0) value = 1;//Don't start value from Zero in ERA=0 and Autority=0 + + if (value >= GDID.COUNTER_MAX - (ulong)(blockSize + 1))//its time to update ERA (+1 for safeguard/edge case) + { + if (era==uint.MaxValue-4)//ALERT, with some room + { + Log(MessageType.CriticalAlert, "allocate()", StringConsts.GDIDAUTH_ERA_EXHAUSTED_ALERT.Args(scopeName, sequenceName)); + } + if (era==uint.MaxValue) //hard stop + { + var txt = StringConsts.GDIDAUTH_ERA_EXHAUSTED_ERROR.Args(scopeName, sequenceName); + Log(MessageType.CatastrophicError, "allocate()", txt); + throw new GDIDException( txt ); + } + + era++; + value = 0; + Instrumentation.AuthEraPromotedEvent.Happened( scopeName, sequenceName ); + Log(MessageType.Warning, "allocate()", StringConsts.GDIDAUTH_ERA_PROMOTED_WARNING.Args(scopeName, sequenceName, era)); + } + + result.Era = era; + result.StartCounterInclusive = value; + + value = value + (ulong)blockSize; + + //2. Try to write to disk, if it fails we could not allocate anything and will bail out with exception + WriteToLocations(authority, scopeName, sequenceName, new _id(era, value)); + + //3. only after write to disk succeeds do we commit the changes back into the sequence instance + sequence.Era = era; + sequence.Value = value; + } + + return result; + } + #endregion + + } + + +} diff --git a/src/Agni/Identification/GDIDAuthorityServiceBase.Locations.cs b/src/Agni/Identification/GDIDAuthorityServiceBase.Locations.cs new file mode 100644 index 0000000..cbd9334 --- /dev/null +++ b/src/Agni/Identification/GDIDAuthorityServiceBase.Locations.cs @@ -0,0 +1,159 @@ +using System; +using System.Text; +using System.IO; + +using NFX; +using NFX.Environment; + + + +namespace Agni.Identification{ public partial class GDIDAuthorityServiceBase { + + + public abstract class PersistenceLocation : INamed, IOrdered + { + protected PersistenceLocation(IConfigSectionNode node) + { + if (node==null) + throw new GDIDException(StringConsts.ARGUMENT_ERROR + "PersistenceLocation(node=null)"); + + Name = node.AttrByName(Configuration.CONFIG_NAME_ATTR).Value; + Order = node.AttrByName(Configuration.CONFIG_ORDER_ATTR).ValueAsInt(); + if (Name.IsNullOrWhiteSpace()) + throw new GDIDException(StringConsts.ARGUMENT_ERROR + "PersistenceLocation(name=null|empty)"); + } + + public string Name { get; private set; } + public int Order { get; private set; } + + public abstract string Validate(); + + public abstract void Write(byte authority, string scopeName, string sequenceName, _id data); + public abstract _id? Read(byte authority, string scopeName, string sequenceName); + + public override string ToString() + { + return "{0}('{1}',#{2})".Args(GetType().Name, Name, Order); + } + } + + + + public sealed class DiskPersistenceLocation : PersistenceLocation + { + public const string CONFIG_PATH_ATTR = "path"; + + public DiskPersistenceLocation(IConfigSectionNode node) : base(node) + { + DiskPath = node.AttrByName(CONFIG_PATH_ATTR).Value; + if (DiskPath.IsNullOrWhiteSpace()) + throw new GDIDException(StringConsts.ARGUMENT_ERROR + "DiskPersistenceLocation(path=null|empty)"); + } + + public string DiskPath { get; private set; } + + public override string ToString() + { + return "{0} -> '{1}'".Args(base.ToString(), DiskPath); + } + + public override string Validate() + { + if (DiskPath.IsNullOrWhiteSpace()) + return "Path is null"; + + if (DiskPath.Length > MAX_DISK_PATH_LENGTH) + return "`{0}` is too long".Args(DiskPath.TakeFirstChars(30)); + + if (!Directory.Exists(DiskPath)) + return "Path `{0}` does not exist".Args(DiskPath); + + return null; + } + + public override void Write(byte authority, string scopeName, string sequenceName, _id data) + { + var fname = getFileName(DiskPath, authority, scopeName, sequenceName); + + if (fname.Length>MAX_PATH_LENGTH) + throw new GDIDException(StringConsts.GDIDAUTH_DISK_PATH_TOO_LONG_ERROR.Args(fname)); + + var authDir = Path.Combine(DiskPath, AuthorityPathSeg(authority)); + if (!Directory.Exists(authDir)) + Directory.CreateDirectory(authDir); + + var scopeDir = Path.Combine(authDir, scopeName); + if (!Directory.Exists(scopeDir)) + Directory.CreateDirectory(scopeDir); + + using(var fs = new FileStream(fname, FileMode.Create, FileAccess.Write, FileShare.None, 0xff, FileOptions.WriteThrough)) + { + var buf = Encoding.ASCII.GetBytes( data.ToString() ); + fs.Write(buf, 0, buf.Length); + fs.Flush(true); + } + } + + public override _id? Read(byte authority, string scopeName, string sequenceName) + { + var fname = getFileName(DiskPath, authority, scopeName, sequenceName); + + if (fname.Length>MAX_PATH_LENGTH) return null; + if (!File.Exists(fname)) return null; + + + using(var fs = new FileStream(fname, FileMode.Open, FileAccess.Read, FileShare.None)) + using(var reader = new StreamReader(fs, Encoding.ASCII)) + { + var sid = reader.ReadToEnd(); + _id id = new _id(sid); //this throws + return id; + } + } + + private static string getFileName(string dpath, byte authority, string scope, string seq) + { + return Path.Combine(dpath, AuthorityPathSeg(authority), scope, seq); + } + + }//disk + + + + public sealed class RemotePersistenceLocation : PersistenceLocation + { + public const string CONFIG_HOST_ATTR = "host"; + + public RemotePersistenceLocation(IConfigSectionNode node) : base(node) + { + Host = node.AttrByName(CONFIG_HOST_ATTR).Value; + if (Host.IsNullOrWhiteSpace()) + throw new GDIDException(StringConsts.ARGUMENT_ERROR + "RemotePersistenceLocation(host=null|empty)"); + } + + public string Host { get; private set; } + + public override string ToString() + { + return "{0} -> '{1}'".Args(base.ToString(), Host); + } + + public override string Validate() + { + throw new NotImplementedException(); + } + + public override _id? Read(byte authority, string scopeName, string sequenceName) + { + throw new NotImplementedException(); + } + + public override void Write(byte authority, string scopeName, string sequenceName, _id data) + { + throw new NotImplementedException(); + } + } //Remote + + + +}} diff --git a/src/Agni/Identification/GDIDAuthorityServiceBase.cs b/src/Agni/Identification/GDIDAuthorityServiceBase.cs new file mode 100644 index 0000000..f96238e --- /dev/null +++ b/src/Agni/Identification/GDIDAuthorityServiceBase.cs @@ -0,0 +1,288 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Threading; + +using NFX; +using NFX.Log; +using NFX.Collections; +using NFX.ServiceModel; +using NFX.Environment; +using NFX.DataAccess.Distributed; +using NFX.ApplicationModel; + +using Agni.Contracts; + +namespace Agni.Identification +{ + /// + /// Base for GDIDAuthority and GDIDPersistenceLocation services + /// + public abstract partial class GDIDAuthorityServiceBase : Service + { + #region CONSTS + public const int MAX_PATH_LENGTH = 240;//dictated by Windows + public const int MAX_NAME_LENGTH = 80;// ns[80] + seq[80] = 160 240 - 160 = 80 for disk path prefix + public const int MAX_DISK_PATH_LENGTH = MAX_PATH_LENGTH - ( 1 + MAX_NAME_LENGTH + MAX_NAME_LENGTH + 3);// 76 = 240 - (aid[1] + ns[80] + seq[80] + separators[3]) + + public const int MAX_BLOCK_SIZE = 1024;//this number effects performance greatly, however increasing it is dangerous as it may lead to + // too many wasted IDs if requesting process requests too many than does not use them up + + public const string CONFIG_GDID_AUTHORITY_SECTION = "gdid-authority"; + + public const string CONFIG_AUTHORITY_IDS_ATTR = "authority-ids"; + + public const string CONFIG_PERSISTENCE_SECTION = "persistence"; + public const string CONFIG_LOCATION_SECTION = "location"; + #endregion + + #region Inner Classes + public struct _id + { + private const string SEPARATOR = "::"; + + public _id(uint era, ulong value) { Era = era; Value = value;} + public _id(string content) + { + try + { + if (content.IsNullOrWhiteSpace()) + throw new GDIDException(""); + + var i = content.IndexOf(SEPARATOR); + if (i<=0) throw new GDIDException("no "+SEPARATOR); + + var sera = content.Substring(0, i); + var sid = content.Substring(i+SEPARATOR.Length); + + Era = uint.Parse(sera); + Value = ulong.Parse(sid); + } + catch(Exception error) + { + throw new GDIDException(StringConsts.GDIDAUTH_ID_DATA_PARSING_ERROR.Args(content, error.ToMessageWithType()), error); + } + } + + public readonly uint Era; + public readonly ulong Value; + + public override string ToString() + { + return Era.ToString()+SEPARATOR+Value.ToString(); + } + + public static bool operator >(_id left, _id right) + { + return left.Era > right.Era || (left.Era == right.Era && left.Value > right.Value); + } + + public static bool operator <(_id left, _id right) + { + return left.Era < right.Era || (left.Era == right.Era && left.Value < right.Value); + } + } + #endregion + + + #region Static + + /// + /// Checks the name for validity. Throws if name does not contain valid chars or over the max length + /// + public static void CheckNameValidity(string name) + { + if (name.IsNullOrWhiteSpace() || + name.Length>MAX_NAME_LENGTH) + throw new GDIDException(StringConsts.GDIDAUTH_NAME_INVALID_LEN_ERROR.Args(name, MAX_NAME_LENGTH)); + + for(var i=0; i='0' && c<='9') continue; + if (c>='A' && c<='Z') continue; + if (c>='a' && c<='z') continue; + if (i!=0 && (c=='_'||c=='.'||c=='-') && i!=name.Length-1) continue; + throw new GDIDException(StringConsts.GDIDAUTH_NAME_INVALID_CHARS_ERROR.Args(name)); + } + } + + public static string AuthorityPathSeg(byte authority) + { + return ((char)(authority<=9 ? '0'+authority : 'a'+authority-10)).ToString(); + } + + #endregion + + #region .ctor/.dctor + protected GDIDAuthorityServiceBase() : base(null) + { + } + #endregion + + #region Fields + + private OrderedRegistry m_Locations = new OrderedRegistry(); + + #endregion + + #region Properties + + /// + /// Specifies where and how the service persists data on disk + /// + public IRegistry PersistenceLocations { get {return m_Locations;} } + + #endregion + + + #region Protected + + protected override void DoConfigure(IConfigSectionNode node) + { + if (node==null) + node = App.ConfigRoot[CONFIG_GDID_AUTHORITY_SECTION]; + + base.DoConfigure(node); + ConfigAttribute.Apply( this, node ); + + foreach(var lnode in node[CONFIG_PERSISTENCE_SECTION].Children.Where(n=>n.IsSameName(CONFIG_LOCATION_SECTION))) + { + var location = FactoryUtils.Make(lnode, typeof(DiskPersistenceLocation), new object[] {lnode}); + m_Locations.Register(location); + } + } + + protected override void DoStart() + { + base.DoStart(); + + string error = null; + if (m_Locations.Count==0) error = ""; + else + foreach(var location in m_Locations) + { + var ve = location.Validate(); + if (ve.IsNotNullOrWhiteSpace()) + error += "Location '{0}'. Config error: {1} \n".Args(location.Name, ve); + } + + if (error!=null) + throw new GDIDException(StringConsts.GDIDAUTH_LOCATIONS_CONFIG_ERROR + error); + } + + //blocking call and throws if can not write to any devices + protected void WriteToLocations(byte authority, string scope, string seq, _id id) + { + var guid = Guid.NewGuid(); + + StringBuilder errors = null; + var totalFailure = true; + + foreach(var location in m_Locations.OrderedValues) + try + { + location.Write(authority, scope, seq, id); + totalFailure = false; + } + catch(Exception error) + { + if (errors==null) errors = new StringBuilder(); + errors.AppendLine( "Path '{0}'. Exception: {1}".Args(location, error.ToMessageWithType()) ); + Log(MessageType.CriticalAlert, "WriteToLocations()", location.ToString(), error, guid); + Instrumentation.AuthLocationWriteFailureEvent.Happened(location.ToString());//Location-level + } + + if (totalFailure) + { + var txt = StringConsts.GDIDAUTH_LOCATION_PERSISTENCE_FAILURE_ERROR + ( errors!=null ? errors.ToString() : "no locations"); + + Log(MessageType.CatastrophicError, "WriteToLocations()", txt, null, guid); + + Instrumentation.AuthLocationWriteTotalFailureEvent.Happened();//TOTAL-LEVEL(for all locations) + + throw new GDIDException(txt); + } + } + + + //blocking call + protected _id ReadFromLocations(byte authority, string scope, string seq) + { + var guid = Guid.NewGuid(); + + StringBuilder errors = null; + _id? result = null; + + var onlyErrors = true; + var first = true; + + foreach(var location in m_Locations.OrderedValues) + try + { + var got = location.Read(authority, scope, seq); + onlyErrors = false; + if (!got.HasValue) continue; + + if (!result.HasValue || got.Value > result.Value) + { + result = got.Value;//take the maximum + if (!first) + { + if (errors==null) errors = new StringBuilder(); + var txt = "Location '{0}' had a later sequence value '{1}' than prior location".Args(location, got.Value); + errors.AppendLine(txt); + Log(MessageType.CriticalAlert, "ReadFromLocations()", txt, null, guid); + } + } + first = false; + } + catch(Exception error) + { + if (errors==null) errors = new StringBuilder(); + var txt = "Error at location '{0}': {1}".Args(location, error.ToMessageWithType()); + errors.AppendLine(txt); + Log(MessageType.CriticalAlert, "ReadFromLocations()", txt, null, guid); + Instrumentation.AuthLocationReadFailureEvent.Happened(location.ToString());//LOCATION-LEVEL + throw; + } + + + if (!result.HasValue && onlyErrors) + { + var txt = StringConsts.GDIDAUTH_LOCATIONS_READ_FAILURE_ERROR + ( errors!=null ? errors.ToString() : "no locations"); + + Log(MessageType.CatastrophicError, "ReadFromLocations()", txt, null, guid); + + Instrumentation.AuthLocationReadTotalFailureEvent.Happened();//TOTAL-LEVEL + + throw new GDIDException(txt); + } + + return result ?? new _id(0, 0); + } + + protected void Log(MessageType type, string from, string msg, Exception error = null, Guid? batch = null) + { + var lm = new NFX.Log.Message{ + Type = type, + Topic = SysConsts.LOG_TOPIC_ID_GEN, + From = "{0}.{1}".Args(GetType().Name, from), + Text = msg, + Exception = error + }; + + if (batch.HasValue) + lm.RelatedTo = batch.Value; + + App.Log.Write( lm ); + } + + #endregion + + } + + +} diff --git a/src/Agni/Identification/GDIDGenerator.cs b/src/Agni/Identification/GDIDGenerator.cs new file mode 100644 index 0000000..dec7b57 --- /dev/null +++ b/src/Agni/Identification/GDIDGenerator.cs @@ -0,0 +1,643 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NFX; +using NFX.Log; +using NFX.ServiceModel; +using NFX.DataAccess; +using NFX.DataAccess.Distributed; +using NFX.Environment; +using NFX.ApplicationModel; + +using Agni.Contracts; + +namespace Agni.Identification +{ + /// + /// Generates Global Distributed IDs (GDID). + /// This class is thread safe (for calling Generate) + /// + public sealed class GDIDGenerator : ApplicationComponent, IConfigurable, IGDIDProvider + { + #region CONSTS + public const string CONFIG_GDID_SECTION = "gdid"; + public const string CONFIG_AUTHORITY_SECTION = "authority"; + + #endregion + + #region Inner Classes + + internal class scope : INamed + { + public string Name { get; set; } + public Registry Sequences = new Registry(); + } + + internal class sequence : INamed + { + internal scope Scope {get; set; } + public string Name { get ; set; } + public GDIDBlock Block; + + //the following are used for LWM condition handling + public volatile GDIDBlock NextBlock; + public volatile Task FetchingNextBlock; + public DateTime LastAllocationUTC = DateTime.UtcNow.AddSeconds(-1000); + + public double avgSinceLastAlloc1; + public double avgSinceLastAlloc2; + public double avgSinceLastAlloc3; + } + + /// + /// Describes a status of the named sequence + /// + public class SequenceInfo : ISequenceInfo + { + internal SequenceInfo(sequence seq) + { + Name = seq.Name; + var block = seq.Block; + if (block!=null) + { + Authority = block.Authority; + AuthorityHost = block.AuthorityHost; + Era = block.Era; + Value = block.StartCounterInclusive + (ulong)(block.BlockSize - block.__Remaining); + UTCTime = block.ServerUTCTime; + BlockSize = block.BlockSize; + Remaining = block.__Remaining; + } + NextBlock = seq.NextBlock!=null; + } + + public readonly string Name; + public readonly int Authority; + public readonly string AuthorityHost; + public readonly uint Era; + public readonly ulong Value; + public readonly DateTime UTCTime; + public readonly int BlockSize; + public readonly int Remaining; + public readonly bool NextBlock; + + + uint ISequenceInfo.Era + { + get { return this.Era; } + } + + ulong ISequenceInfo.ApproximateCurrentValue + { + get { return Value; } + } + + DateTime ISequenceInfo.IssueUTCDate + { + get { return UTCTime; } + } + + string ISequenceInfo.IssuerName + { + get { return "[{0}] {1}".Args(Authority, AuthorityHost); } + } + + int ISequenceInfo.RemainingPreallocation + { + get { return Remaining; } + } + + int ISequenceInfo.TotalPreallocation + { + get { return BlockSize; } + } + + string INamed.Name + { + get { return Name; } + } + } + + + /// + /// Provides information about the host that runs GDID generation authority + /// + public sealed class AuthorityHost : INamed, IEquatable + { + /// + /// Name of the host and distance from the caller. If distance =0 then it will be calculated + /// + public AuthorityHost(string name, int distanceKm = 0) + { + m_Name = name.IsNullOrWhiteSpace() ? SysConsts.NULL : name; + if (AgniSystem.IsMetabase) + m_DistanceKm = distanceKm>0 ? distanceKm : (int)AgniSystem.Metabase.CatalogReg.GetDistanceBetweenPaths(AgniSystem.HostName, name); + else + m_DistanceKm = distanceKm; + + } + private string m_Name; + private int m_DistanceKm; + + + /// + /// Host name + /// + public string Name{ get {return m_Name;} } + + /// + /// Relative distance to the destination host from this machine + /// + public int DistanceKm { get {return m_DistanceKm;} } + + /// + /// Returns enumeration of Global Distributed ID generation Authorities. + /// The distance is computed, and can not be specified in config + /// + public static IEnumerable FromConfNode(IConfigSectionNode parentNode) + { + if (parentNode==null || !parentNode.Exists) + yield break; + + foreach(var anode in parentNode.Children.Where(n=>n.IsSameName(CONFIG_AUTHORITY_SECTION))) + { + var name = anode.AttrByName(Metabase.Metabank.CONFIG_NAME_ATTR).Value; + + if (name.IsNullOrWhiteSpace()) + name = anode.AttrByName(Metabase.Metabank.CONFIG_NETWORK_ROUTING_HOST_ATTR).Value; + + if (name.IsNullOrWhiteSpace()) + continue; + + yield return new Identification.GDIDGenerator.AuthorityHost(name); + } + } + + + public bool Equals(AuthorityHost other) + { + if (other==null) return false; + + return Name.IsSameRegionPath(other.Name); + } + + public override bool Equals(object obj) + { + return this.Equals(obj as AuthorityHost); + } + + public override int GetHashCode() + { + return Name.GetRegionPathHashCode(); + } + + public override string ToString() + { + return Name; + } + } + + + #endregion + + #region CONSTS + + /// + /// Specifies the level of "free" ID range in block below which reservation of the next block gets triggered + /// + public const double BLOCK_LOW_WATER_MARK = 0.25d; + + + #endregion + + + + #region .ctor + + public GDIDGenerator() : this(null, null, null, null) {} + public GDIDGenerator(string name, object director) : this(name, director, null, null) {} + public GDIDGenerator(string name, object director, string scopePrefix, string sequencePrefix) : base(director) + { + if (name.IsNullOrWhiteSpace()) + name = Guid.NewGuid().ToString(); + + m_Name = name; + + m_ScopePrefix = scopePrefix; + if (m_ScopePrefix!=null) m_ScopePrefix = m_ScopePrefix.Trim(); + + m_SequencePrefix = sequencePrefix; + if (m_SequencePrefix!=null) m_SequencePrefix = m_SequencePrefix.Trim(); + } + + + #endregion + + #region Fields + + private string m_Name; + private Registry m_Scopes = new Registry(); + private Registry m_AuthorityHosts = new Registry(); + + private bool m_BlockWasAllocated; + private string m_TestingAuthorityNode; + + private string m_ScopePrefix; + private string m_SequencePrefix; + + #endregion + + #region Properties + + + public override string ComponentCommonName { get { return "gdidgen-"+Name; }} + + /// + /// Name of the instance + /// + public string Name + { + get { return m_Name; } + } + + + /// + /// Returns immutable scope prefix set in .ctor or null. This name is prepended to all requests + /// + public string ScopePrefix { get{ return m_ScopePrefix;} } + + /// + /// Returns immutable sequence prefix set in .ctor or null. This name is prepended to all requests + /// + public string SequencePrefix { get{ return m_SequencePrefix;} } + + + /// + /// Returns host names that host GDID generation authority service + /// + public Registry AuthorityHosts + { + get { return m_AuthorityHosts; } + } + + /// + /// Returns the list of all scope names in the instance + /// + public IEnumerable SequenceScopeNames + { + get { return m_Scopes.Select(s => s.Name);} + } + + + /// + /// Gets/sets Authority Glue Node for testing. It can only be set once in the testing app container init before the first call to + /// Generate is made. When this setting is set then any authority nodes which would have been normally used will be + /// completely bypassed during block allocation + /// + public string TestingAuthorityNode + { + get { return m_TestingAuthorityNode;} + set + { + if (m_BlockWasAllocated) + throw new GDIDException(StringConsts.GDIDGEN_SET_TESTING_ERROR); + m_TestingAuthorityNode = value; + } + } + + /// + /// Returns true after block was allocated at least once + /// + public bool BlockWasAllocated + { + get { return m_BlockWasAllocated;} + } + + #endregion + + #region Public + + public void Configure(IConfigSectionNode node) + { + if (node!=null) + ConfigAttribute.Apply(this, node); + } + + + /// + /// Returns sequnce information enumerable for all sequences in the named scope + /// + public IEnumerable GetSequenceInfos(string scopeName) + { + if (scopeName==null) + throw new GDIDException(StringConsts.ARGUMENT_ERROR+GetType().Name+".GetSequenceInfos(scopeName=null)"); + + if (m_ScopePrefix!=null) + scopeName = m_ScopePrefix + scopeName; + + var scope = m_Scopes[scopeName]; + if (scope==null) + return Enumerable.Empty(); + + return scope.Sequences.Select(s => new SequenceInfo(s)); + } + + IEnumerable IUniqueSequenceProvider.GetSequenceInfos(string scopeName) + { + return this.GetSequenceInfos(scopeName); + } + + ulong IUniqueSequenceProvider.GenerateOneSequenceID(string scopeName, string sequenceName, int blockSize, ulong? vicinity, bool noLWM) + { + return GenerateOneGDID(scopeName, sequenceName, blockSize, vicinity, noLWM).ID; + } + + ConsecutiveUniqueSequenceIDs IUniqueSequenceProvider.TryGenerateManyConsecutiveSequenceIDs(string scopeName, + string sequenceName, + int idCount, + ulong? vicinity, + bool noLWM) + { + var gdids = TryGenerateManyConsecutiveGDIDs(scopeName, sequenceName, idCount, vicinity, noLWM); + return new ConsecutiveUniqueSequenceIDs(gdids[0].ID, gdids.Length); + } + + + + /// + /// Generates a single Globally-Unique distributed ID (GDID) for the supplied sequence name. This method is thread-safe. + /// The algorithm also respects LWM value, once the number of free counters gets below LWM the asynchronous non-blocking acquisition from authority is triggered + /// + /// The name of scope where sequences are kept + /// The name of sequence within the scope for which ID to be obtained + /// If >0 requests to pre-allocate specified number of sequences, otherwise the generator will pre-allocate the most suitable/configured size + /// The location on ID counter scale, the authority may disregard this parameter + /// + /// When true, does not start async fetch of the next ID block while the current block reaches low-water-mark. + /// This may not be desired in some short-lived processes. + /// The provider may disregard this flag + /// + /// The GDID instance + public GDID GenerateOneGDID(string scopeName, string sequenceName, int blockSize=0, ulong? vicinity = GDID.COUNTER_MAX, bool noLWM = false) + { + if (scopeName==null || sequenceName==null) + throw new GDIDException(StringConsts.ARGUMENT_ERROR+GetType().Name+".GenerateOneGDID(scopeName|sequenceName=null)"); + + if (m_ScopePrefix!=null) scopeName = m_ScopePrefix + scopeName; + if (m_SequencePrefix!=null) sequenceName = m_SequencePrefix + sequenceName; + + scopeName = scopeName.Trim(); + sequenceName = sequenceName.Trim(); + + GDIDAuthorityService.CheckNameValidity(scopeName); + GDIDAuthorityService.CheckNameValidity(sequenceName); + + var scope = m_Scopes.GetOrRegister(scopeName, (snm) => new scope{Name = snm}, scopeName); + + var sequence = scope.Sequences.GetOrRegister(sequenceName, (_) => new sequence{Scope = scope, Name = sequenceName}, 0);//with block=NULL + + + lock(sequence) + { + var block = sequence.Block; + + if (block==null || block.__Remaining<=0)//need to get Next + { + block = sequence.NextBlock;//atomic + + if (block==null) + block = allocateBlock(sequence, blockSize, vicinity); + else + { + sequence.NextBlock = null; + sequence.FetchingNextBlock = null; + } + + sequence.Block = block; + block.__Remaining = block.BlockSize; + } + + + var counter = block.StartCounterInclusive + (ulong)(block.BlockSize - block.__Remaining); + block.__Remaining--; + + var result = new GDID(block.Era, block.Authority, counter); + + //check LWM + if (!noLWM && block.BlockSize>7 && sequence.FetchingNextBlock==null) + { + double curlvl = (double)block.__Remaining / (double)block.BlockSize; + if (curlvl <= BLOCK_LOW_WATER_MARK) //start fetching next block + { + sequence.FetchingNextBlock = Task.Factory.StartNew(()=> + { + try + { + var nextBlock = allocateBlock(sequence, blockSize, vicinity); + sequence.NextBlock = nextBlock;//atomic assignment + } + catch(Exception error) + { //todo Perf counter + log(MessageType.Error, GetType().Name+".Generate().Task{}", "Error getting NextBlock", error); + } + }); + } + } + + + return result; + }//lock + } + + + /// + /// Tries to generate the specified number of Globally-Unique distributed IDs (GDID) for the supplied sequence name. This method is thread-safe. + /// The method may generate less GUIDs than requested. All IDs come from the same authority + /// + /// The name of scope where sequences are kept + /// The name of sequence within the scope for which ID to be obtained + /// The dsired number of consecutive GDIDs + /// The location on ID counter scale, the authority may disregard this parameter + /// + /// When true, does not start async fetch of the next ID block while the current block reaches low-water-mark. + /// This may not be desired in some short-lived processes. + /// The provider may disregard this flag + /// + /// The GDID instance array which may be shorter than requested + public GDID[] TryGenerateManyConsecutiveGDIDs(string scopeName, string sequenceName, int gdidCount, ulong? vicinity = GDID.COUNTER_MAX, bool noLWM = false) + { + if (scopeName==null || sequenceName==null) + throw new GDIDException(StringConsts.ARGUMENT_ERROR+GetType().Name+".TryGenerateManyConsecutiveGDIDs(scopeName|sequenceName=null)"); + + if (gdidCount<=0) + throw new GDIDException(StringConsts.ARGUMENT_ERROR+GetType().Name+".TryGenerateManyConsecutiveGDIDs(gdidCount<=0)"); + + if (m_ScopePrefix!=null) scopeName = m_ScopePrefix + scopeName; + if (m_SequencePrefix!=null) sequenceName = m_SequencePrefix + sequenceName; + + scopeName = scopeName.Trim(); + sequenceName = sequenceName.Trim(); + + GDIDAuthorityService.CheckNameValidity(scopeName); + GDIDAuthorityService.CheckNameValidity(sequenceName); + + var scope = m_Scopes.GetOrRegister(scopeName, (snm) => new scope{Name = snm}, scopeName); + + var sequence = scope.Sequences.GetOrRegister(sequenceName, (_) => new sequence{Scope = scope, Name = sequenceName}, 0);//with block=NULL + + + lock(sequence) + { + var block = sequence.Block; + + if (block==null || block.__Remaining<=(gdidCount / 2))//get whole block + { + block = allocateBlock(sequence, gdidCount, vicinity); + block.__Remaining = block.BlockSize; + } + + var result = new GDID[ Math.Min(gdidCount, block.__Remaining) ]; + + for(var i=0; i Block size is: "+blockSize); + if (m_TestingAuthorityNode.IsNullOrWhiteSpace()) + return allocateBlockInSystem(seq.Scope.Name, seq.Name, blockSize, vicinity); + else + return allocateBlockInTesting(seq.Scope.Name, seq.Name, blockSize, vicinity); + } + + + private GDIDBlock allocateBlockInTesting(string scopeName, string sequenceName, int blockSize, ulong? vicinity) + { + Instrumentation.AllocBlockRequestedEvent.Happened(scopeName, sequenceName); + using(var cl = new Clients.GDIDAuthority(m_TestingAuthorityNode)) + { + try + { + var result = cl.AllocateBlock(scopeName, sequenceName, blockSize, vicinity); + Instrumentation.AllocBlockSuccessEvent.Happened(scopeName, sequenceName, m_TestingAuthorityNode); + return result; + } + catch + { + Instrumentation.AllocBlockFailureEvent.Happened(scopeName, sequenceName, blockSize, m_TestingAuthorityNode); + throw; + } + } + } + + private GDIDBlock allocateBlockInSystem(string scopeName, string sequenceName, int blockSize, ulong? vicinity) + { + Instrumentation.AllocBlockRequestedEvent.Happened(scopeName, sequenceName); + + GDIDBlock result = null; + var batch = Guid.NewGuid(); + var list = ""; + foreach(var node in this.AuthorityHosts.OrderBy(h => h.DistanceKm))//in the order of distances, closest first + { + list += (" Trying '{0}' at {1}km\n".Args(node.Name, node.DistanceKm)); + try + { + using(var cl = AgniSystem.IsMetabase + ? ServiceClientHub.New( node.Name ) + : new Clients.GDIDAuthority( node.Name ) + ) + result = cl.AllocateBlock(scopeName, sequenceName, blockSize, vicinity); + + Instrumentation.AllocBlockSuccessEvent.Happened(scopeName, sequenceName, node.Name); + } + catch(Exception error) + { + log(MessageType.Error, GetType().Name+".allocateBlock()", "Error invoking GDIDAuthority.AllocateBlock('{0}')".Args(node), error, batch); + + Instrumentation.AllocBlockFailureEvent.Happened(scopeName, sequenceName, blockSize, node.Name); + } + + if (result!=null) break; + } + + if (result==null) + { + if (list.IsNullOrWhiteSpace()) list = ""; + log(MessageType.Emergency, GetType().Name+".allocateBlock()", StringConsts.GDIDGEN_ALL_AUTHORITIES_FAILED_ERROR + list, batch: batch); + Instrumentation.AllocBlockRequestFailureEvent.Happened(scopeName, sequenceName); + throw new GDIDException(StringConsts.GDIDGEN_ALL_AUTHORITIES_FAILED_ERROR + list); + } + + return result; + } + + private void log(MessageType type, string from, string msg, Exception error = null, Guid? batch = null) + { + var lm = new NFX.Log.Message{ + Type = type, + Topic = SysConsts.LOG_TOPIC_ID_GEN, + From = from, + Text = msg, + Exception = error + }; + + if (batch.HasValue) + lm.RelatedTo = batch.Value; + + App.Log.Write( lm ); + } + + #endregion + + + + + + + + } + +} diff --git a/src/Agni/Identification/GDIDGlueServers.cs b/src/Agni/Identification/GDIDGlueServers.cs new file mode 100644 index 0000000..e90cead --- /dev/null +++ b/src/Agni/Identification/GDIDGlueServers.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Threading; + +using NFX; +using NFX.Log; +using NFX.Collections; +using NFX.ServiceModel; +using NFX.Environment; +using NFX.DataAccess.Distributed; +using NFX.ApplicationModel; + +using Agni.Contracts; + +namespace Agni.Identification +{ + /// + /// Implements GDIDAuthority contract trampoline that uses a singleton instance of GDIDAuthorityService to + /// allocate blocks + /// + public sealed class GDIDAuthority : IGDIDAuthority + { + /// + /// Implements IGDIDAuthority contract - allocates block + /// + public GDIDBlock AllocateBlock(string scopeName, string sequenceName, int blockSize, ulong? vicinity = GDID.COUNTER_MAX) + { + Instrumentation.AuthAllocBlockCalledEvent.Happened(scopeName, sequenceName); + return GDIDAuthorityService.Instance.AllocateBlock(scopeName, sequenceName, blockSize, vicinity); + } + } + + /// + /// Implements IGDIDPersistenceLocation contract trampoline that uses a singleton instance of GDIDPersistenceLocationService to + /// store gdids + /// + public sealed class GDIPersistenceRemoteLocation : IGDIDPersistenceRemoteLocation + { + public GDID? Read(byte authority, string sequenceName, string scopeName) + { + throw new NotImplementedException(); + } + + public void Write(string sequenceName, string scopeName, GDID value) + { + throw new NotImplementedException(); + } + } + + +} diff --git a/src/Agni/Identification/Instrumentation/Events.cs b/src/Agni/Identification/Instrumentation/Events.cs new file mode 100644 index 0000000..a4a7a6d --- /dev/null +++ b/src/Agni/Identification/Instrumentation/Events.cs @@ -0,0 +1,283 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Instrumentation; +using NFX.Serialization.BSON; + +namespace Agni.Identification.Instrumentation +{ + /// + /// Provides base for GDID events + /// + [Serializable] + public abstract class GDIDEvent : Event, IGDIDInstrument + { + protected GDIDEvent(string src) : base(src) { } + + public override string Description { get { return "Provides info about GDID events"; } } + + public override string ValueUnitName { get { return StringConsts.UNIT_NAME_TIME; } } + + } + + /// + /// Provides base for GDID events that happen in generator (client side) + /// + [Serializable] + public abstract class GDIDGeneratorEvent : GDIDEvent + { + protected GDIDGeneratorEvent(string src) : base(src) { } + } + + /// + /// Provides base for GDID events that happen in authority (server side) + /// + [Serializable] + public abstract class GDIDAuthorityEvent : GDIDEvent, INetInstrument + { + protected GDIDAuthorityEvent(string src) : base(src) { } + } + + /// + /// Generator requested new block allocation + /// + [Serializable] + [BSONSerializable("31A3C469-8A8B-43CE-9765-C538AC0DD574")] + public class AllocBlockRequestedEvent : GDIDGeneratorEvent, INetInstrument + { + protected AllocBlockRequestedEvent(string src) : base(src) { } + + public override string Description { get { return "Generator requested new block allocation"; } } + + public static void Happened(string scope, string seq) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AllocBlockRequestedEvent("{0}::{1}".Args(scope, seq))); + } + + protected override Datum MakeAggregateInstance() + { + return new AllocBlockRequestedEvent(this.Source); + } + } + + /// + /// Generator requested new block allocation completely failed + /// + [Serializable] + [BSONSerializable("1DA4AEF0-7D9A-4E12-856D-01E0BE61DA84")] + public class AllocBlockRequestFailureEvent : GDIDGeneratorEvent, INetInstrument, IErrorInstrument, ICatastropyInstrument + { + protected AllocBlockRequestFailureEvent(string src) : base(src) { } + + public override string Description { get { return "Generator requested new block allocation completely failed"; } } + + public static void Happened(string scope, string seq) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AllocBlockRequestFailureEvent("{0}::{1}".Args(scope, seq))); + } + + protected override Datum MakeAggregateInstance() + { + return new AllocBlockRequestFailureEvent(this.Source); + } + } + + /// + /// Generator successfully allocated new block + /// + [Serializable] + [BSONSerializable("9724C9E3-2CBF-45AD-861C-3FF818209232")] + public class AllocBlockSuccessEvent : GDIDGeneratorEvent, INetInstrument + { + protected AllocBlockSuccessEvent(string src) : base(src) { } + + public override string Description { get { return "Generator successfully allocated new block"; } } + + public static void Happened(string scope, string seq, string authority) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AllocBlockSuccessEvent("{0}::{1}::{2}".Args(scope, seq, authority))); + } + + protected override Datum MakeAggregateInstance() + { + return new AllocBlockSuccessEvent(this.Source); + } + } + + /// + /// Generator block allocation attempt failed + /// + [Serializable] + [BSONSerializable("A963B984-11EB-4333-A01F-F73EAF84D126")] + public class AllocBlockFailureEvent : GDIDGeneratorEvent, INetInstrument, IErrorInstrument + { + protected AllocBlockFailureEvent(string src) : base(src) { } + + public override string Description { get { return "Generator block allocation attempt failed"; } } + + public static void Happened(string scope, string seq, int block, string authority) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AllocBlockFailureEvent("{0}::{1}::{2}".Args(scope, seq, authority))); + } + + protected override Datum MakeAggregateInstance() + { + return new AllocBlockFailureEvent(this.Source); + } + } + + /// + /// Authority received block allocation call + /// + [Serializable] + [BSONSerializable("88D561E4-32C6-47A0-ACA1-E7E735D3D84E")] + public class AuthAllocBlockCalledEvent : GDIDAuthorityEvent, INetInstrument + { + protected AuthAllocBlockCalledEvent(string src) : base(src) { } + + public override string Description { get { return "Authority received block allocation call"; } } + + public static void Happened(string scope, string seq) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AuthAllocBlockCalledEvent("{0}.{1}".Args(scope, seq))); + } + + protected override Datum MakeAggregateInstance() + { + return new AuthAllocBlockCalledEvent(this.Source); + } + } + + /// + /// Authority location write failed + /// + [Serializable] + [BSONSerializable("6DB69416-5CFF-4378-9448-D887923540B5")] + public class AuthLocationWriteFailureEvent : GDIDAuthorityEvent, IErrorInstrument + { + protected AuthLocationWriteFailureEvent(string location) : base(location) { } + + public override string Description { get { return "Authority location write failed"; } } + + public static void Happened(string location) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AuthLocationWriteFailureEvent(location)); + } + + protected override Datum MakeAggregateInstance() + { + return new AuthLocationWriteFailureEvent(this.Source); + } + } + + /// + /// Authority sequence write to all locations failed + /// + [Serializable] + [BSONSerializable("BF0FE93A-5969-4A0C-9FEA-FDB8EBA86798")] + public class AuthLocationWriteTotalFailureEvent : GDIDAuthorityEvent, ICatastropyInstrument, IErrorInstrument + { + protected AuthLocationWriteTotalFailureEvent() : base(string.Empty) { } + + public override string Description { get { return "Authority sequence write to all locations failed"; } } + + public static void Happened() + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AuthLocationWriteTotalFailureEvent()); + } + + protected override Datum MakeAggregateInstance() + { + return new AuthLocationWriteTotalFailureEvent(); + } + } + + /// + /// Authority sequence read from location failed + /// + [Serializable] + [BSONSerializable("E29A3788-1372-47B7-B1EB-32740F93C9FF")] + public class AuthLocationReadFailureEvent : GDIDAuthorityEvent, IErrorInstrument + { + protected AuthLocationReadFailureEvent(string location) : base(location) { } + + public override string Description { get { return "Authority sequence read from location failed"; } } + + public static void Happened(string location) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AuthLocationReadFailureEvent(location)); + } + + protected override Datum MakeAggregateInstance() + { + return new AuthLocationReadFailureEvent(this.Source); + } + } + + /// + /// Authority sequence read failed for all locations + /// + [Serializable] + [BSONSerializable("A3CFF2DB-4D72-41EA-8F46-27B3BDE3B3B0")] + public class AuthLocationReadTotalFailureEvent : GDIDAuthorityEvent, ICatastropyInstrument, IErrorInstrument + { + protected AuthLocationReadTotalFailureEvent() : base(string.Empty) { } + + public override string Description { get { return "Authority sequence read failed for all locations"; } } + + public static void Happened() + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AuthLocationReadTotalFailureEvent()); + } + + protected override Datum MakeAggregateInstance() + { + return new AuthLocationReadTotalFailureEvent(); + } + } + + /// + /// Authority ERA promoted +1 + /// + [Serializable] + [BSONSerializable("065480FD-6198-45D2-AA15-F54EE71E3D16")] + public class AuthEraPromotedEvent : GDIDAuthorityEvent + { + protected AuthEraPromotedEvent(string src) : base(src) { } + + public override string Description { get { return "Authority ERA promoted +1"; } } + + public static void Happened(string scope, string seq) + { + var inst = App.Instrumentation; + if (inst.Enabled) + inst.Record(new AuthEraPromotedEvent("{0}:{1}".Args(scope, seq))); + } + + protected override Datum MakeAggregateInstance() + { + return new AuthEraPromotedEvent(this.Source); + } + } +} diff --git a/src/Agni/Instrumentation/AgniInstrumentationProvider.cs b/src/Agni/Instrumentation/AgniInstrumentationProvider.cs new file mode 100644 index 0000000..7b8353e --- /dev/null +++ b/src/Agni/Instrumentation/AgniInstrumentationProvider.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NFX; +using NFX.Log; +using NFX.Instrumentation; +using NFX.Environment; + +using Agni.Metabase; + +namespace Agni.Instrumentation +{ + /// + /// Reduces instrumentation data stream and uploads it to the higher-standing zone governor + /// + public class AgniInstrumentationProvider : InstrumentationProvider + { + #region CONSTS + public const string CONFIG_HOST_ATTR = "host"; + private const string LOG_TOPIC = "Log.AgniDestination"; + private const MessageType DEFAULT_LOG_LEVEL = MessageType.Warning; + #endregion + + #region .ctor + public AgniInstrumentationProvider() : base(null) {} + public AgniInstrumentationProvider(InstrumentationService director) : base(director) {} + #endregion + + #region Fields + private Metabank.SectionHost m_Host; + #endregion + + #region Properties + /// + /// Specifies the log level for operations performed by Pay System. + /// + [Config(Default = DEFAULT_LOG_LEVEL)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOG)] + public MessageType LogLevel { get; set; } + #endregion + + #region Protected + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + + m_Host = AgniSystem.Metabase.CatalogReg.NavigateHost(node.AttrByName(CONFIG_HOST_ATTR).Value); + } + + protected override object BeforeBatch() + { + return new List(); + } + + protected override void AfterBatch(object batchContext) + { + var datumList = batchContext as List; + if (datumList != null) + send(datumList.ToArray()); + + } + protected override void Write(Datum aggregatedDatum, object batchContext, object typeContext) + { + var datumList = batchContext as List; + if (datumList != null) + { + datumList.Add(aggregatedDatum); + if (datumList.Count>100) + { + send(datumList.ToArray()); + datumList.Clear(); + } + } + else + send(aggregatedDatum); + } + #endregion + + #region Private + private void send(params Datum[] data) + { + try + { + //TODO Cache the client instance, do not create client on every call + using (var client = Contracts.ServiceClientHub.New(m_Host)) + client.Async_SendDatums(data); + } + catch (Exception error) + { + throw new TelemetryArchiveException("{0}.Write".Args(GetType().Name), error); + } + } + + private Guid log( + MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + var logMessage = new Message + { + Type = type, + Topic = LOG_TOPIC, + From = "{0}.{1}".Args(GetType().FullName, from), + Text = message, + Exception = error, + Parameters = parameters + }; + + if (relatedMessageID.HasValue) logMessage.RelatedTo = relatedMessageID.Value; + + App.Log.Write(logMessage); + + return logMessage.Guid; + } + #endregion + } +} diff --git a/src/Agni/Instrumentation/AgniZoneInstrumentationProvider.cs b/src/Agni/Instrumentation/AgniZoneInstrumentationProvider.cs new file mode 100644 index 0000000..d24ac89 --- /dev/null +++ b/src/Agni/Instrumentation/AgniZoneInstrumentationProvider.cs @@ -0,0 +1,258 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NFX; +using NFX.Log; +using NFX.Instrumentation; + +namespace Agni.Instrumentation +{ + /// + /// Reduces instrumentation data stream and uploads it to the higher-standing zone governor + /// + public class AgniZoneInstrumentationProvider : InstrumentationProvider + { + #region CONSTS + public const string THREAD_NAME = "AgniZoneInstrumentationProvider"; + public const int THREAD_GRANULARITY_MS = 3750; + public const int GLUE_CALL_SLA_MS = 5137; + #endregion + + + private class Chunk : Queue + { + const int MAX_SIZE = 7; + + public void Push(Datum datum) + { + this.Enqueue(datum); + if (this.Count>MAX_SIZE) this.Dequeue(); + } + + public Datum Reaggregate() + { + if (this.Count==0) return null; + var first = this.Peek(); + var aggregated = first.Aggregate(this); + this.Clear(); + return aggregated; + } + } + + private class BySrc : Dictionary{} + private class ByType : Dictionary{} + + + #region .ctor + public AgniZoneInstrumentationProvider() : base(null) + { + } + + public AgniZoneInstrumentationProvider(InstrumentationService director) : base(director) + { + } + #endregion + + #region Fields + + private ByType m_ByType = new ByType(); + + private List m_Uploading; + private bool m_IAmRootHost; + + private Thread m_Thread; + private AutoResetEvent m_WaitEvent; + + private int m_ZGovCallTimeoutMs; + #endregion + + + #region Properties + + /// + /// Overrides default service timeout when set to value greater than 0 + /// + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_GLUE)] + public int ZGovCallTimeoutMs + { + get { return m_ZGovCallTimeoutMs;} + set { m_ZGovCallTimeoutMs = value >0 ? value : 0;} + } + + #endregion + + #region Protected + protected override void Write(Datum aggregatedDatum, object batchContext, object typeContext) + { + if (!Running) return; + if (aggregatedDatum==null) return; + if (m_IAmRootHost) return; + if (AgniSystem.IsMetabase) return; + + + var t = aggregatedDatum.GetType(); + + BySrc bySrc; + if (!m_ByType.TryGetValue(t, out bySrc)) + { + bySrc = new BySrc(); + m_ByType[t] = bySrc; + } + + Chunk chunk; + if (!bySrc.TryGetValue(aggregatedDatum.Source, out chunk)) + { + chunk = new Chunk(); + bySrc[aggregatedDatum.Source] = chunk; + } + + chunk.Push(aggregatedDatum); + if (m_Uploading!=null) return; + + + var toUpload = new List(); + foreach(var kvpT in m_ByType) + foreach(var kvpS in kvpT.Value) + { + var reaggr = kvpS.Value.Reaggregate(); + if (reaggr==null) continue; + toUpload.Add(reaggr); + } + + m_Uploading = toUpload; + } + + protected override void DoStart() + { + m_WaitEvent = new AutoResetEvent(false); + + m_Thread = new Thread(threadSpin); + m_Thread.Name = THREAD_NAME; + m_Thread.Start(); + + m_IAmRootHost = AgniSystem.ParentZoneGovernorPrimaryHostName==null; + } + + protected override void DoSignalStop() + { + } + + protected override void DoWaitForCompleteStop() + { + m_WaitEvent.Set(); + + m_Thread.Join(); + m_Thread = null; + + m_WaitEvent.Close(); + m_WaitEvent = null; + + m_ByType = new ByType(); + } + #endregion + + + #region .pvt + private void threadSpin() + { + const string FROM = "threadSpin()"; + const int MAX_FAILURES_PER_RANK = 3; + + int MAX_SEND_BATCH = 1024; + + while(Running) + { + if (m_Uploading==null) + { + m_WaitEvent.WaitOne(THREAD_GRANULARITY_MS); + continue; + } + + var sendNextTime = MAX_SEND_BATCH; + var zgHost = AgniSystem.ParentZoneGovernorPrimaryHostName; + var client = Contracts.ServiceClientHub.New( zgHost ); + if (m_ZGovCallTimeoutMs>0) client.TimeoutMs = m_ZGovCallTimeoutMs; + + try + { + var alreadySent = 0; + var failures = 0; + while(Running && alreadySentMAX_SEND_BATCH) sendNextTime = MAX_SEND_BATCH; + } + catch(Exception error) + { + //todo INSTRUMENT errors + + if (error is NFX.Glue.MessageSizeException) + { + if (MAX_SEND_BATCH>128) + { + MAX_SEND_BATCH = (int)(MAX_SEND_BATCH * 0.75d); + if (sendNextTime>MAX_SEND_BATCH) sendNextTime = MAX_SEND_BATCH; + continue; + } + } + + failures++; + if (failures>MAX_FAILURES_PER_RANK) + { + failures = 0; + var phost = AgniSystem.Metabase.CatalogReg.NavigateHost(zgHost).ParentZoneGovernorPrimaryHost(transcendNOC: true);// if not found here, go to the NOC higher than this one + if (phost==null) + {//we came to very root - data lost + log(MessageType.Error, FROM+".retryTop", StringConsts.INSTR_SEND_TELEMETRY_TOP_LOST_ERROR.Args(error.ToMessageWithType())); + break; + } + zgHost = phost.RegionPath; + client.Dispose(); + client = Contracts.ServiceClientHub.New( zgHost ); + client.TimeoutMs = GLUE_CALL_SLA_MS; + } + continue; + } + + alreadySent += toSend.Length; + } + + } + catch(Exception error) + { + log(MessageType.Error, FROM, error.ToMessageWithType(), error); + } + finally + { + m_Uploading = null; + client.Dispose(); + } + }//while Running + } + + internal void log(MessageType type, string from, string text, Exception error = null, Guid? related = null) + { + var msg = new NFX.Log.Message + { + Type = type, + Topic = SysConsts.LOG_TOPIC_INSTRUMENTATION, + From = "{0}.{1}".Args(GetType().FullName, from), + Text = text, + Exception = error + }; + + if (related.HasValue) msg.RelatedTo = related.Value; + + App.Log.Write( msg ); + } + #endregion + } +} diff --git a/src/Agni/Instrumentation/Exceptions.cs b/src/Agni/Instrumentation/Exceptions.cs new file mode 100644 index 0000000..1cdc705 --- /dev/null +++ b/src/Agni/Instrumentation/Exceptions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.Instrumentation +{ + /// + /// Thrown to indicate log archive related problems + /// + [Serializable] + public class TelemetryArchiveException : AgniException + { + public TelemetryArchiveException() : base() {} + public TelemetryArchiveException(string message) : base(message) {} + public TelemetryArchiveException(string message, Exception inner) : base(message, inner) { } + protected TelemetryArchiveException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Instrumentation/Server/TelemetryArchiveStore.cs b/src/Agni/Instrumentation/Server/TelemetryArchiveStore.cs new file mode 100644 index 0000000..fb137a2 --- /dev/null +++ b/src/Agni/Instrumentation/Server/TelemetryArchiveStore.cs @@ -0,0 +1,45 @@ +using System; + +using NFX; +using NFX.Log; +using NFX.ApplicationModel; +using NFX.Environment; +using System.Collections.Generic; +using NFX.Instrumentation; + +namespace Agni.Instrumentation.Server +{ + /// + /// Represents a base for entities that archive instrumentation data + /// + public abstract class TelemetryArchiveStore : ApplicationComponent + { + protected TelemetryArchiveStore(TelemetryReceiverService director, IConfigSectionNode node) : base(director) + { + ConfigAttribute.Apply(this, node); + } + + /// + /// References service that this store is under + /// + public TelemetryReceiverService ArchiveService { get { return (TelemetryReceiverService)ComponentDirector;} } + + public abstract object BeginTransaction(); + + public abstract void CommitTransaction(object transaction); + + public abstract void RollbackTransaction(object transaction); + + public abstract void Put(Datum[] data, object transaction); + + protected Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + return ArchiveService.Log(type, "{0}.{1}".Args(GetType().Name, from), message, error, relatedMessageID, parameters); + } + } +} diff --git a/src/Agni/Instrumentation/Server/TelemetryReceiverService.cs b/src/Agni/Instrumentation/Server/TelemetryReceiverService.cs new file mode 100644 index 0000000..a317ef9 --- /dev/null +++ b/src/Agni/Instrumentation/Server/TelemetryReceiverService.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; + +using NFX; +using NFX.Environment; +using NFX.Instrumentation; +using NFX.ServiceModel; +using NFX.Log; + +namespace Agni.Instrumentation.Server +{ + /// + /// Glue adapter for Contracts.ITelemetryReceiver + /// + public sealed class TelemetryReceiverServer : Contracts.ITelemetryReceiver + { + public void SendDatums(params Datum[] data) + { + TelemetryReceiverService.Instance.SendDatums(data); + } + } + + + /// + /// Provides server implementation of Contracts.ITelemetryReceiver + /// + public sealed class TelemetryReceiverService : ServiceWithInstrumentationBase, Contracts.ITelemetryReceiver + { + #region CONSTS + public const string CONFIG_ARCHIVE_STORE_SECTION = "archive-store"; + + public const MessageType DEFAULT_LOG_LEVEL = MessageType.Warning; + #endregion + + #region STATIC/.ctor + private static object s_Lock = new object(); + private static volatile TelemetryReceiverService s_Instance; + + internal static TelemetryReceiverService Instance + { + get + { + var instance = s_Instance; + if (instance == null) + throw new TelemetryArchiveException("{0} is not allocated".Args(typeof(TelemetryReceiverService).FullName)); + return instance; + } + } + + public TelemetryReceiverService() : this(null) { } + + public TelemetryReceiverService(object director) : base(director) + { + LogLevel = MessageType.Warning; + + lock (s_Lock) + { + if (s_Instance != null) + throw new TelemetryArchiveException("{0} is already allocated".Args(typeof(TelemetryReceiverService).FullName)); + + s_Instance = this; + } + } + + protected override void Destructor() + { + base.Destructor(); + DisposeAndNull(ref m_ArchiveStore); + s_Instance = null; + } + #endregion + + #region Fields + private TelemetryArchiveStore m_ArchiveStore; + #endregion + + #region Properties + [Config] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOG, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled { get; set; } + + [Config(Default = DEFAULT_LOG_LEVEL)] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_WORKER, CoreConsts.EXT_PARAM_GROUP_LOG)] + public MessageType LogLevel { get; set; } + #endregion + + #region Public + public void SendDatums(params Datum[] data) + { + var transaction = m_ArchiveStore.BeginTransaction(); + try + { + m_ArchiveStore.Put(data, transaction); + m_ArchiveStore.CommitTransaction(transaction); + } + catch (Exception error) + { + m_ArchiveStore.RollbackTransaction(transaction); + + Log(MessageType.CatastrophicError, "put('{0}')".Args(data.Length), error.ToMessageWithType(), error); + + throw new TelemetryArchiveException(StringConsts.TELEMETRY_ARCHIVE_PUT_TX_BODY_ERROR.Args(m_ArchiveStore.GetType().Name, error.ToMessageWithType()), error); + } + } + + public Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + var logMessage = new Message + { + Topic = SysConsts.LOG_TOPIC_WORKER, + Text = message ?? string.Empty, + Type = type, + From = "{0}.{1}".Args(this.GetType().Name, from), + Exception = error, + Parameters = parameters + }; + if (relatedMessageID.HasValue) logMessage.RelatedTo = relatedMessageID.Value; + + App.Log.Write(logMessage); + + return logMessage.Guid; + } + #endregion + + #region Protected + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + + DisposeAndNull(ref m_ArchiveStore); + + var storeNode = node[CONFIG_ARCHIVE_STORE_SECTION]; + if (storeNode.Exists) + m_ArchiveStore = FactoryUtils.Make(storeNode, args: new object[] { this, storeNode }); + } + + protected override void DoStart() + { + if (m_ArchiveStore == null) + throw new TelemetryArchiveException("{0} does not have archive store injected".Args(GetType().Name)); + + if (m_ArchiveStore is IService) ((IService)m_ArchiveStore).Start(); + base.DoStart(); + } + + protected override void DoSignalStop() + { + if (m_ArchiveStore is IService) ((IService)m_ArchiveStore).SignalStop(); + base.DoSignalStop(); + } + + protected override void DoWaitForCompleteStop() + { + if (m_ArchiveStore is IService) ((IService)m_ArchiveStore).WaitForCompleteStop(); + base.DoWaitForCompleteStop(); + } + #endregion + } +} diff --git a/src/Agni/KDB/Exceptions.cs b/src/Agni/KDB/Exceptions.cs new file mode 100644 index 0000000..e489775 --- /dev/null +++ b/src/Agni/KDB/Exceptions.cs @@ -0,0 +1,19 @@ +using System; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.KDB +{ + /// + /// Thrown to indicate KDB-related problems + /// + [Serializable] + public class KDBException : AgniException + { + public KDBException() : base() { } + public KDBException(string message) : base(message) { } + public KDBException(string message, Exception inner) : base(message, inner) { } + protected KDBException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/KDB/IKDBDataStore.cs b/src/Agni/KDB/IKDBDataStore.cs new file mode 100644 index 0000000..5f71342 --- /dev/null +++ b/src/Agni/KDB/IKDBDataStore.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.DataAccess; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; + + +namespace Agni.KDB +{ + + public struct KDBRecord where TResult : class + { + public static readonly KDBRecord Unassigned = new KDBRecord(); + + public KDBRecord(TResult value, int slidingExpirationDays, DateTime lastUseDate, DateTime? absoluteExpirationDateUTC) + { + Value = value; + SlidingExpirationDays = slidingExpirationDays; + LastUseDate = lastUseDate; + AbsoluteExpirationDateUTC = absoluteExpirationDateUTC; + } + + public readonly TResult Value; + public readonly int SlidingExpirationDays; + public readonly DateTime LastUseDate; + public readonly DateTime? AbsoluteExpirationDateUTC; + + public bool IsAssigned { get { return Value != null; } } + } + + /// + /// Stipulates a contract for KDBDataStore + /// + public interface IKDBDataStore : IDataStore + { + /// + /// Gets a row of data by key, or null if row with such key was not found or data is not Row + /// + /// Table. Required must be non-null valid identifier string less than 32 chars + /// Byte array key, must be non-null array with at least one element + Row Get(string table, byte[] key); + + /// + ///Gets a row of data projected in the specified typed model, or null if row with such key was not found or data is not Row + /// + /// Table. Required must be non-null valid identifier string less than 32 chars + /// Byte array key, must be non-null array with at least one element + /// If true then does not update the item's last use time + KDBRecord Get(string table, byte[] key, bool dontToch = false) where TRow : Row; + + + /// + ///Gets a raw byte[] of data or null if data does not exist or data is not raw byte[] but Row + /// + /// Table. Required must be non-null valid identifier string less than 32 chars + /// Byte array key, must be non-null array with at least one element + /// If true then does not update the item's last use time + KDBRecord GetRaw(string table, byte[] key, bool dontToch = false); + + + /// + /// Puts a row of data under the specified key + /// + /// Table. Required must be non-null valid identifier string less than 32 chars + /// Byte array key, must be non-null array with at least one element + /// Data object must be non-null + /// + /// When set, specifies the sliding expiration of the entry in days. + /// The system DOES NOT guarantee the instantaneous deletion of expired data + /// + /// + /// When set, specifies when garbage collector should auto-delete the value. + /// It does not guarantee that the value is deleted right at that date + /// + void Put(string table, byte[] key, Row value, int slidingExpirationDays = -1, DateTime? absoluteExpirationDateUtc = null); + + + /// + /// Puts a raw byte[] value under the specified key + /// + /// Table. Required must be non-null valid identifier string less than 32 chars + /// Byte array key, must be non-null array with at least one element + /// byte[] must be non-null + /// + /// When set, specifies the sliding expiration of the entry in days. + /// The system DOES NOT guarantee the instantaneous deletion of expired data + /// + /// + /// When set, specifies when garbage collector should auto-delete the value. + /// It does not guarantee that the value is deleted right at that date + /// + void PutRaw(string table, byte[] key, byte[] value, int slidingExpirationDays = -1, DateTime? absoluteExpirationDateUtc = null); + + /// + /// Deletes a row of data under the specified key returning true if deletion succeeded + /// + /// Table. Required must be non-null valid identifier string less than 32 chars + /// Byte array key, must be non-null array with at least one element + bool Delete(string table, byte[] key); + } + + public interface IKDBDataStoreImplementation : IKDBDataStore, IDataStoreImplementation + { + + } + +} diff --git a/src/Agni/KDB/Instrumentation/Gauges.cs b/src/Agni/KDB/Instrumentation/Gauges.cs new file mode 100644 index 0000000..7c4f90a --- /dev/null +++ b/src/Agni/KDB/Instrumentation/Gauges.cs @@ -0,0 +1,211 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Instrumentation; +using NFX.Serialization.BSON; + +namespace Agni.KDB.Instrumentation +{ + /// + /// Provides base for KDB long gauges + /// + [Serializable] + public abstract class KDBLongGauge : LongGauge, IDatabaseInstrument + { + protected KDBLongGauge(string src, long value) : base(src, value) { } + } + + /// + /// Provides Get hit count per table + /// + [Serializable] + [BSONSerializable("F60CD643-CC90-4590-A816-A382F06C41E9")] + public class GetHitCount : KDBLongGauge + { + public GetHitCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides Get hit count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new GetHitCount(this.Source, 0); } + } + + /// + /// Provides Get hit from fallback set count per table + /// + [Serializable] + [BSONSerializable("2B5EB443-412B-4E5B-94C3-04A5F96A8EC1")] + public class GetFallbackHitCount : KDBLongGauge + { + public GetFallbackHitCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides Get hit from fallback set count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new GetFallbackHitCount(this.Source, 0); } + } + + /// + /// Provides Get miss count per table + /// + [Serializable] + [BSONSerializable("97D5ACC4-CE8F-4ABD-851A-5546F101D982")] + public class GetMissCount : KDBLongGauge + { + public GetMissCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides Get miss count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new GetMissCount(this.Source, 0); } + } + + /// + /// Provides Get resulted in last use stamp update count per table + /// + [Serializable] + [BSONSerializable("B636682D-EAA8-4C3F-B886-14F6560AE467")] + public class GetTouchCount : KDBLongGauge + { + public GetTouchCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides Get resulted in last use stamp update count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new GetTouchCount(this.Source, 0); } + } + + /// + /// Provides put count per table + /// + [Serializable] + [BSONSerializable("0E47D988-22C6-438B-8451-6C23ED681D30")] + public class PutCount : KDBLongGauge + { + public PutCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides put count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new PutCount(this.Source, 0); } + } + + /// + /// Provides delete hit count per table + /// + [Serializable] + [BSONSerializable("7E4CE481-3AFF-44F2-9364-5AC59E5E0612")] + public class DeleteHitCount : KDBLongGauge + { + public DeleteHitCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides delete hit count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new DeleteHitCount(this.Source, 0); } + } + + /// + /// Provides delete fallback count per table + /// + [Serializable] + [BSONSerializable("DA220322-90A8-4BDA-89FF-991C5890286E")] + public class DeleteFallbackCount : KDBLongGauge + { + public DeleteFallbackCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides delete fallback count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new DeleteFallbackCount(this.Source, 0); } + } + + /// + /// Provides delete miss count per table + /// + [Serializable] + [BSONSerializable("EB587F38-1F5E-4A72-B7A7-2D38B4232F75")] + public class DeleteMissCount : KDBLongGauge + { + public DeleteMissCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides delete miss count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new DeleteMissCount(this.Source, 0); } + } + + /// + /// Provides error count per table + /// + [Serializable] + [BSONSerializable("B01A6C89-B9E4-47DE-ACC7-AC8BC4A6163D")] + public class ErrorCount : KDBLongGauge, IErrorInstrument + { + public ErrorCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides error count per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new ErrorCount(this.Source, 0); } + } + + /// + /// Provides number of records expired with expiration days per table + /// + [Serializable] + [BSONSerializable("DA2E5293-8ED0-49B8-B578-47F9B9D13A3A")] + public class SlidingExpirationCount : KDBLongGauge + { + public SlidingExpirationCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides number of records expired with expiration days per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_RECORD; } } + + protected override Datum MakeAggregateInstance() { return new SlidingExpirationCount(this.Source, 0); } + } + + /// + /// Provides number of records expired with absolute expiration per table + /// + [Serializable] + [BSONSerializable("CC6B3137-9783-40D3-B269-18CD15B8DA0A")] + public class AbsoluteExpirationCount : KDBLongGauge + { + public AbsoluteExpirationCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides number of records expired with absolute expiration per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_RECORD; } } + + protected override Datum MakeAggregateInstance() { return new AbsoluteExpirationCount(this.Source, 0); } + } + + /// + /// Provides number of records moved from fallback to current shard set per table + /// + [Serializable] + [BSONSerializable("E0FBB4E8-C74C-4185-A981-2FC3CA3E5607")] + public class MigrationCount : KDBLongGauge + { + public MigrationCount(string tbl, long value) : base(tbl, value) { } + + public override string Description { get { return "Provides number of records moved from fallback to current shard set per table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_RECORD; } } + + protected override Datum MakeAggregateInstance() { return new MigrationCount(this.Source, 0); } + } +} diff --git a/src/Agni/KDB/KDBConstraints.cs b/src/Agni/KDB/KDBConstraints.cs new file mode 100644 index 0000000..f996df6 --- /dev/null +++ b/src/Agni/KDB/KDBConstraints.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; + +namespace Agni.KDB +{ + public static class KDBConstraints + { + public const int MAX_TABLE_NAME_LEN = 32; + public const int MAX_KEY_LEN = 255; + + public static void CheckTableName(string table, string opName) + { + if (table.IsNullOrWhiteSpace()) throw new KDBException(StringConsts.KDB_TABLE_IS_NULL_OR_EMPTY_ERROR.Args(opName)); + + var len = table.Length; + if (len > MAX_TABLE_NAME_LEN) throw new KDBException(StringConsts.KDB_TABLE_MAX_LEN_ERROR.Args(opName, len, MAX_TABLE_NAME_LEN)); + for(var i = 0; i < len; i++) + { + var c = table[i]; + if ((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (i > 0 && c >= '0' && c <= '9') || + c == '_') continue; + + throw new KDBException(StringConsts.KDB_TABLE_CHARACTER_ERROR.Args(opName, table)); + } + } + + public static void CheckKey(byte[] key, string opName) + { + if (key == null || key.Length == 0) throw new KDBException(StringConsts.KDB_KEY_IS_NULL_OR_EMPTY_ERROR.Args(opName)); + + var len = key.Length; + if (len > MAX_KEY_LEN) throw new KDBException(StringConsts.KDB_KEY_MAX_LEN_ERROR.Args(opName, len, MAX_KEY_LEN)); + } + } +} diff --git a/src/Agni/Locking/Exceptions.cs b/src/Agni/Locking/Exceptions.cs new file mode 100644 index 0000000..067e211 --- /dev/null +++ b/src/Agni/Locking/Exceptions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.Locking +{ + /// + /// Thrown to indicate locking related problems + /// + [Serializable] + public class LockingException : AgniException + { + public LockingException() : base() { } + public LockingException(string message) : base(message) { } + public LockingException(string message, Exception inner) : base(message, inner) { } + protected LockingException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Locking/Instrumentation/BaseGauges.cs b/src/Agni/Locking/Instrumentation/BaseGauges.cs new file mode 100644 index 0000000..a36e396 --- /dev/null +++ b/src/Agni/Locking/Instrumentation/BaseGauges.cs @@ -0,0 +1,57 @@ +using System; + +using NFX.Instrumentation; + +namespace Agni.Locking.Instrumentation +{ + + [Serializable] + public abstract class LockingLongGauge : LongGauge, ILockingInstrument + { + protected LockingLongGauge(string src, long value) : base(src, value) {} + } + + [Serializable] + public abstract class LockingDoubleGauge : DoubleGauge, ILockingInstrument + { + protected LockingDoubleGauge(string src, double value) : base(src, value) {} + } + + + + + + + + + [Serializable] + public abstract class LockingServerGauge : LockingLongGauge + { + protected LockingServerGauge(string src, long value) : base(src, value) {} + } + + [Serializable] + public abstract class LockingClientGauge : LockingLongGauge + { + protected LockingClientGauge(string src, long value) : base(src, value) {} + } + + + + + + + [Serializable] + public abstract class LockingServerDoubleGauge : LockingDoubleGauge + { + protected LockingServerDoubleGauge(string src, double value) : base(src, value) {} + } + + + [Serializable] + public abstract class LockingClientDoubleGauge : LockingDoubleGauge + { + protected LockingClientDoubleGauge(string src, double value) : base(src, value) {} + } + +} diff --git a/src/Agni/Locking/Instrumentation/ClientGauges.cs b/src/Agni/Locking/Instrumentation/ClientGauges.cs new file mode 100644 index 0000000..304281e --- /dev/null +++ b/src/Agni/Locking/Instrumentation/ClientGauges.cs @@ -0,0 +1,45 @@ +using System; + +using NFX.Instrumentation; +using NFX.Serialization.BSON; + +namespace Agni.Locking.Instrumentation +{ + + + /// + /// Provides the count of LockSession instances in the LockManager + /// + [Serializable] + [BSONSerializable("C189E810-9971-4225-94C3-306FEFB40600")] + public sealed class LockSessions : LockingClientGauge + { + internal LockSessions(long value) : base(Datum.UNSPECIFIED_SOURCE, value) { } + + public override string Description { get { return "Provides the count of LockSession instances in the LockManager"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_SESSION; } } + + protected override Datum MakeAggregateInstance() { return new LockSessions(0); } + } + + + /// + /// Provides the number of times that locking transactions have been requested to be executed + /// + [Serializable] + [BSONSerializable("06A5C7D0-7A28-4E8C-B642-B089B33717C1")] + public sealed class LockTransactionRequests : LockingClientGauge + { + internal LockTransactionRequests(string src, long value) : base(src, value) { } + + public override string Description + { + get { return "Provides the number of times that locking transactions have been requested to be executed"; } + } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TRANSACTION; } } + + protected override Datum MakeAggregateInstance() { return new LockTransactionRequests(this.Source, 0); } + } +} diff --git a/src/Agni/Locking/Instrumentation/ServerGauges.cs b/src/Agni/Locking/Instrumentation/ServerGauges.cs new file mode 100644 index 0000000..7b6ad42 --- /dev/null +++ b/src/Agni/Locking/Instrumentation/ServerGauges.cs @@ -0,0 +1,151 @@ +using System; + +using NFX.Instrumentation; +using NFX.Serialization.BSON; + +namespace Agni.Locking.Instrumentation +{ + /// + /// Provides the count of processed server transactions + /// + [Serializable] + [BSONSerializable("615ECDF5-0593-4F56-AD6D-5C4164D68A1B")] + public sealed class ServerLockTransactions : LockingServerGauge + { + internal ServerLockTransactions(string src, long value) : base(src, value) { } + + public override string Description { get { return "Provides the count of processed server transactions "; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TRANSACTION; } } + + protected override Datum MakeAggregateInstance() { return new ServerLockTransactions(Source, 0); } + } + + /// + /// Provides the count of requests that end the session + /// + [Serializable] + [BSONSerializable("51A3BBC6-7B33-4F01-9FA8-AEE8CA6EB665")] + public sealed class ServerEndLockSessionCalls : LockingServerGauge + { + internal ServerEndLockSessionCalls(long value) : base(Datum.UNSPECIFIED_SOURCE, value) { } + + public override string Description { get { return "Provides the count of requests that end the session"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_CALL; } } + + protected override Datum MakeAggregateInstance() { return new ServerEndLockSessionCalls(0); } + } + + /// + /// Provides the current level of trust to locking server + /// + [Serializable] + [BSONSerializable("501689C3-B2AD-43C7-97D4-D6BDE7853EE4")] + public sealed class ServerTrustLevel : LockingServerDoubleGauge + { + internal ServerTrustLevel(double value) : base(Datum.UNSPECIFIED_SOURCE, value) { } + + public override string Description { get { return "Provides the current level of trust to locking server"; } } + + public override string ValueUnitName { get { return "trust"; } } + + protected override Datum MakeAggregateInstance() { return new ServerTrustLevel(0); } + } + + /// + /// Provides the current level of server calls which is considered a norm + /// + [Serializable] + [BSONSerializable("C93A4229-C261-466A-B7A2-D77845BA7AC3")] + public sealed class ServerCallsNorm : LockingServerDoubleGauge + { + internal ServerCallsNorm(double value) : base(Datum.UNSPECIFIED_SOURCE, value) { } + + public override string Description { get { return "Provides the current level of server calls which is considered a norm"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_CALL; } } + + protected override Datum MakeAggregateInstance() { return new ServerCallsNorm(0); } + } + + /// + /// Provides the number of records that have expired + /// + [Serializable] + [BSONSerializable("51C49154-C017-480D-9CA7-1E9E20771F23")] + public sealed class ServerExpiredRecords : LockingServerGauge + { + internal ServerExpiredRecords(long value) : base(Datum.UNSPECIFIED_SOURCE, value) { } + + public override string Description { get { return "Provides the number of records that have expired"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_RECORD; } } + + protected override Datum MakeAggregateInstance() { return new ServerExpiredRecords(0); } + } + + /// + /// Provides the number of sessions that have expired + /// + [Serializable] + [BSONSerializable("4D7295AC-FE3C-4382-B698-D4F00522FC52")] + public sealed class ServerExpiredSessions : LockingServerGauge + { + internal ServerExpiredSessions(long value) : base(Datum.UNSPECIFIED_SOURCE, value) { } + + public override string Description { get { return "Provides the number of sessions that have expired"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_SESSION; } } + + protected override Datum MakeAggregateInstance() { return new ServerExpiredSessions(0); } + } + + /// + /// Provides the number of empty tables removed from the server namespaces + /// + [Serializable] + [BSONSerializable("811ED7C5-E27F-474D-BC4D-7DE4A0303668")] + public sealed class ServerRemovedEmptyTables : LockingServerGauge + { + internal ServerRemovedEmptyTables(long value) : base(Datum.UNSPECIFIED_SOURCE, value) { } + + public override string Description { get { return " Provides the number of empty tables removed from the server namespaces"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TABLE; } } + + protected override Datum MakeAggregateInstance() { return new ServerRemovedEmptyTables(0); } + } + + /// + /// Provides the count of tables per namespace + /// + [Serializable] + [BSONSerializable("988FC14B-4631-472C-8252-E30AD78C8369")] + public sealed class ServerNamespaceTables : LockingServerGauge + { + internal ServerNamespaceTables(string nsname, long value) : base(nsname, value) { } + + public override string Description { get { return "Provides the count of tables per namespace"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TABLE; } } + + protected override Datum MakeAggregateInstance() { return new ServerNamespaceTables(this.Source, 0); } + } + + /// + /// Provides the count of committed records per namespace table + /// + [Serializable] + [BSONSerializable("2F85C218-D985-41F9-838F-FBED90BAC563")] + public sealed class ServerNamespaceTableRecordCount : LockingServerGauge + { + internal ServerNamespaceTableRecordCount(string nsTname, long value) : base(nsTname, value) { } + + public override string Description { get { return "Provides the count of committed records per namespace table"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_RECORD; } } + + protected override Datum MakeAggregateInstance() { return new ServerNamespaceTableRecordCount(this.Source, 0); } + } +} diff --git a/src/Agni/Locking/Intfs.cs b/src/Agni/Locking/Intfs.cs new file mode 100644 index 0000000..f22be97 --- /dev/null +++ b/src/Agni/Locking/Intfs.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Instrumentation; + + +namespace Agni.Locking +{ + /// + /// Defines operations of distributed lock manager + /// + public interface ILockManager + { + + /// + /// Creates a session of work with lock server at the specified path level and sharding id + /// + /// The level of the that has to be covered by the lock, the zone governors are looked up at that or higher level + /// The object used for work partitioning. Keep in mind that ALL logically-connected lock entities must use the same shardingID + /// Logical description of the session + /// The maximum session age in seconds, or null for default + /// New lock session registered with the manager. The session instance may be later looked-up by LockSessionID + LockSession MakeSession(string path, object shardingID, string description = null, int? maxAgeSec = null); + + /// + /// Executes a lock transaction in the secified session returning transaction result object even if lock could not be set. + /// The exception would indicate inability to deliver the transaction request to the server or other system problem + /// + LockTransactionResult ExecuteLockTransaction(LockSession session, LockTransaction transaction); + + /// + /// Executes a lock transaction in the secified session returning transaction result object even if lock could not be set. + /// The exception would indicate inability to deliver the transaction request to the server or other system problem + /// + Task ExecuteLockTransactionAsync(LockSession session, LockTransaction transaction); + + /// + /// Ends the remote lock session returning true if it was found remotely and destroyed + /// + bool EndLockSession(LockSession session); + + /// + /// Ends the remote lock session returning true if it was found remotely and destroyed + /// + Task EndLockSessionAsync(LockSession session); + + /// + /// Returns the session by id or null + /// + LockSession this[LockSessionID sid] { get; } + } + + /// + /// Denotes implementations of ILockManager + /// + public interface ILockManagerImplementation : ILockManager, IApplicationComponent, IDisposable, IConfigurable, IInstrumentable + { + + } + +} diff --git a/src/Agni/Locking/LocalTestingLockManager.cs b/src/Agni/Locking/LocalTestingLockManager.cs new file mode 100644 index 0000000..b81ef54 --- /dev/null +++ b/src/Agni/Locking/LocalTestingLockManager.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Environment; +using NFX.ServiceModel; +using NFX.Instrumentation; + +namespace Agni.Locking +{ + /// + /// Used for testing, facade for calling locking APIs from client code into server that is hosted in the same process + /// + public sealed class LocalTestingLockManager : LockManagerBase + { + public LocalTestingLockManager() : base() { } + + private Server.LockServerService m_Server; + + + protected override void DoStart() + { + m_Server = new Server.LockServerService(this); + m_Server.Start(); + } + + protected override void DoWaitForCompleteStop() + { + DisposableObject.DisposeAndNull(ref m_Server); + } + + protected override LockTransactionResult DoExecuteLockTransaction(LockSession session, LockTransaction transaction) + { + return m_Server.ExecuteLockTransaction(session.Data, transaction); + } + + protected override Task DoExecuteLockTransactionAsync(LockSession session, LockTransaction transaction) + { + return TaskUtils.AsCompletedTask( () => m_Server.ExecuteLockTransaction(session.Data, transaction) ); + } + + protected override bool DoEndLockSession(LockSession session) + { + return m_Server.EndLockSession(session.ID); + } + + protected override Task DoEndLockSessionAsync(LockSession session) + { + return TaskUtils.AsCompletedTask( () => m_Server.EndLockSession(session.ID ) ); + } + } + +} diff --git a/src/Agni/Locking/LockManager.cs b/src/Agni/Locking/LockManager.cs new file mode 100644 index 0000000..4b85294 --- /dev/null +++ b/src/Agni/Locking/LockManager.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Environment; +using NFX.ServiceModel; +using NFX.Instrumentation; + +namespace Agni.Locking +{ + /// + /// Facade for calling locking APIs from client code + /// + public sealed class LockManager : LockManagerBase + { + public LockManager() : base() { } + + + protected override LockTransactionResult DoExecuteLockTransaction(LockSession session, LockTransaction transaction) + { + return Contracts.ServiceClientHub.CallWithRetry + ( + (locker) => locker.ExecuteLockTransaction(session.Data, transaction), + session.ServerHosts + ); + } + + protected override Task DoExecuteLockTransactionAsync(LockSession session, LockTransaction transaction) + { + return Contracts.ServiceClientHub.CallWithRetryAsync + ( + (locker) => locker.Async_ExecuteLockTransaction(session.Data, transaction).AsTaskReturning(), + session.ServerHosts + ); + } + + protected override bool DoEndLockSession(LockSession session) + { + return Contracts.ServiceClientHub.CallWithRetry + ( + (locker) => locker.EndLockSession(session.Data.ID), + session.ServerHosts + ); + } + + protected override Task DoEndLockSessionAsync(LockSession session) + { + return Contracts.ServiceClientHub.CallWithRetryAsync + ( + (locker) => locker.Async_EndLockSession(session.Data.ID).AsTaskReturning(), + session.ServerHosts + ); + } + } + +} diff --git a/src/Agni/Locking/LockManagerBase.cs b/src/Agni/Locking/LockManagerBase.cs new file mode 100644 index 0000000..0cecdf5 --- /dev/null +++ b/src/Agni/Locking/LockManagerBase.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Environment; +using NFX.ServiceModel; +using NFX.Instrumentation; + +namespace Agni.Locking +{ + /// + /// Base for Facades used for calling locking APIs from client code + /// + public abstract class LockManagerBase : ServiceWithInstrumentationBase, ILockManagerImplementation + { + #region CONSTS + private static readonly TimeSpan INSTR_INTERVAL = TimeSpan.FromMilliseconds(3700); + + #endregion + + #region .ctor + + public LockManagerBase() : base() { } + + protected override void Destructor() + { + DisposableObject.DisposeAndNull(ref m_InstrumentationEvent); + base.Destructor(); + } + + #endregion + + + + #region Fields + + private ConcurrentDictionary m_Sessions = new ConcurrentDictionary(); + + + private bool m_InstrumentationEnabled; + private NFX.Time.Event m_InstrumentationEvent; + + private NFX.Collections.NamedInterlocked m_Stats = new NFX.Collections.NamedInterlocked(); + + #endregion + + #region Properties + + /// + /// Implements IInstrumentable + /// + [Config(Default=false)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOCKING, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled + { + get { return m_InstrumentationEnabled;} + set + { + m_InstrumentationEnabled = value; + if (m_InstrumentationEvent==null) + { + if (!value) return; + m_Stats.Clear(); + m_InstrumentationEvent = new NFX.Time.Event(App.EventTimer, null, e => AcceptManagerVisit(this, e.LocalizedTime), INSTR_INTERVAL); + } + else + { + if (value) return; + DisposableObject.DisposeAndNull(ref m_InstrumentationEvent); + m_Stats.Clear(); + } + } + } + + public LockSession this[LockSessionID sid] + { + get + { + LockSession session; + if (m_Sessions.TryGetValue(sid, out session)) return session; + return null; + } + } + + #endregion + + #region Pub + + public virtual LockSession MakeSession(string path, object shardingID, string description = null, int? maxAgeSec = null) + { + var session = new LockSession(this, path, shardingID, description, maxAgeSec); + m_Sessions[ session.ID ] = session; + + return session; + } + + public LockTransactionResult ExecuteLockTransaction(LockSession session, LockTransaction transaction) + { + if (!Running) return LockTransactionResult.CallFailed; + + if (session==null || transaction==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().Name+".ExecuteLockTransaction(session|tran==null)"); + + checkSessionExists(session); + + if (m_InstrumentationEnabled) + m_Stats.IncrementLong(session.Path); + + return DoExecuteLockTransaction(session, transaction); + } + + public Task ExecuteLockTransactionAsync(LockSession session, LockTransaction transaction) + { + if (!Running) return Task.FromResult(LockTransactionResult.CallFailed); + + if (session==null || transaction==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().Name+".ExecuteLockTransactionAsync(session|tran==null)"); + + checkSessionExists(session); + + if (m_InstrumentationEnabled) + m_Stats.IncrementLong(session.Path); + + return DoExecuteLockTransactionAsync(session, transaction); + } + + + public bool EndLockSession(LockSession session) + { + if (!Running) return false; + + if (session==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().Name+".EndLockSession(session==null)"); + + checkSessionExists(session); + + var ended = DoEndLockSession(session); + + LockSession d; + m_Sessions.TryRemove(session.ID, out d); + + return ended; + } + + public Task EndLockSessionAsync(LockSession session) + { + if (!Running) return Task.FromResult(false); + + if (session==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().Name+".EndLockSessionAsync(session==null)"); + + checkSessionExists(session); + + LockSession d; + m_Sessions.TryRemove(session.ID, out d); + + return DoEndLockSessionAsync(session); + } + + #endregion + + + #region Protected + + protected abstract LockTransactionResult DoExecuteLockTransaction(LockSession session, LockTransaction transaction); + protected abstract Task DoExecuteLockTransactionAsync(LockSession session, LockTransaction transaction); + protected abstract bool DoEndLockSession(LockSession session); + protected abstract Task DoEndLockSessionAsync(LockSession session); + + + protected override void DoStart() + { + base.DoStart(); + } + + protected override void DoWaitForCompleteStop() + { + base.DoWaitForCompleteStop(); + } + + protected override void DoAcceptManagerVisit(object manager, DateTime managerNow) + { + dumpStats(); + } + + #endregion + + + #region .pvt + + private void checkSessionExists(LockSession session) + { + var s = this[session.ID]; + if (s==null) throw new LockingException(StringConsts.LOCK_SESSION_NOT_ACTIVE_ERROR.Args(session.ID, session.Description)); + } + + private void dumpStats() + { + var instr = App.Instrumentation; + if (!instr.Enabled) return; + + + instr.Record( new Instrumentation.LockSessions( m_Sessions.Count ) ); + + foreach( var kvp in m_Stats.SnapshotAllLongs(0)) + instr.Record( new Instrumentation.LockTransactionRequests(kvp.Key, kvp.Value) ); + } + + #endregion + + + } + +} diff --git a/src/Agni/Locking/LockOps.Bases.cs b/src/Agni/Locking/LockOps.Bases.cs new file mode 100644 index 0000000..8d81b55 --- /dev/null +++ b/src/Agni/Locking/LockOps.Bases.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +using Agni.Locking.Server; + +namespace Agni.Locking +{ + /// + /// Represents operations of locking transaction + /// + public static partial class LockOp + { + [Serializable] + public abstract class Op + { + [NonSerialized] + protected string m_Path; + + /// + /// Invoked by the server to prepare this node before execution/before lock + /// + public virtual void Prepare(EvalContext ctx, string path) { m_Path = path + this.GetType().Name + "/"; } + } + + [Serializable] + public abstract class OperatorOp : Op + { + public abstract bool GetValue(EvalContext ctx); + } + + [Serializable] + public abstract class BinaryOperatorOp : OperatorOp + { + public BinaryOperatorOp(OperatorOp lop, OperatorOp rop) + { + if (lop==null||rop==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"BinOp(lop|rop=null)"); + + LeftOperand = lop; + RightOperand = rop; + } + public readonly OperatorOp LeftOperand; + public readonly OperatorOp RightOperand; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + LeftOperand.Prepare(ctx, m_Path); + RightOperand.Prepare(ctx, m_Path); + } + } + + [Serializable] + public abstract class UnaryOperatorOp : OperatorOp {} + + + [Serializable] + public abstract class StatementOp : Op + { + public abstract void Execute(EvalContext ctx); + } + + [Serializable] + public abstract class FlowOp : StatementOp + { + public FlowOp() { } + } + + /// + /// Represents a change to the state, returns true if change succeeds, false otherwise + /// + [Serializable] + public abstract class ChangeOp : UnaryOperatorOp + { + public ChangeOp(string table, string var) : base() + { + if (table.IsNullOrWhiteSpace() || var.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"Change(table|var==null|empty)"); + + Table = table; + Var = var; + } + + public readonly string Table; [NonSerialized]internal Server.Table _Table; + public readonly string Var; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + _Table = ctx.GetExistingOrMakeTableByName(Table); + } + } + + + [Serializable] + public abstract class SelectOp : StatementOp + { + public SelectOp(string intoName) + { + if (intoName.IsNullOrWhiteSpace() ) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"Select(intoName=null|empty)"); + + IntoName = intoName; + } + public readonly string IntoName; + } + + } + +} diff --git a/src/Agni/Locking/LockOps.Change.cs b/src/Agni/Locking/LockOps.Change.cs new file mode 100644 index 0000000..fa07b07 --- /dev/null +++ b/src/Agni/Locking/LockOps.Change.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +using Agni.Locking.Server; + +namespace Agni.Locking +{ + /// + /// Represents operations of locking transaction + /// + public static partial class LockOp + { + + + public static SetVarOp SetVar(string table, + string var, + object value, + string description = null, + DateTime? expirationTimeUTC = null, + bool allowDuplicates = false) + { + return new SetVarOp(table, var, value, description, expirationTimeUTC, allowDuplicates); + } + + /// + /// Sets the named variable in table, return true if var was set, false if conflict happened + /// + [Serializable] + public sealed class SetVarOp : ChangeOp + { + internal SetVarOp(string table, + string var, + object value, + string description, + DateTime? expirationTimeUTC, + bool allowDuplicates) : base(table, var) + { + Value = value; + Description = description; + ExpirationTimeUTC = expirationTimeUTC; + AllowDuplicates = allowDuplicates; + } + + public override bool GetValue(EvalContext ctx) + { + return _Table.SetVariable(ctx, Var, Value, Description, ExpirationTimeUTC, AllowDuplicates); + } + + public readonly object Value; + public readonly string Description; + public readonly DateTime? ExpirationTimeUTC; + public readonly bool AllowDuplicates; + + } + + + public static DeleteVarOp DeleteVar(string table, string var, object value = null) + { + return new DeleteVarOp(table, var, value); + } + /// + /// Deletes the named variable in table that this session has created, true if it was deleted, false if it did not exist + /// or was not created by this session + /// + [Serializable] + public sealed class DeleteVarOp : ChangeOp + { + internal DeleteVarOp(string table, string var, object value) : base(table, var) + { + Value = value; + } + + public readonly object Value; + + public override bool GetValue(EvalContext ctx) + { + return _Table.DeleteVariable(ctx, Var, Value); + } + } + + + + } + +} diff --git a/src/Agni/Locking/LockOps.Flow.cs b/src/Agni/Locking/LockOps.Flow.cs new file mode 100644 index 0000000..26cf9a4 --- /dev/null +++ b/src/Agni/Locking/LockOps.Flow.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +using Agni.Locking.Server; + +namespace Agni.Locking +{ + /// + /// Represents operations of locking transaction + /// + public static partial class LockOp + { + + public static AbortOp Abort() { return new AbortOp();} + /// + /// Unconditionally aborts the processing + /// + [Serializable] + public sealed class AbortOp : FlowOp + { + internal AbortOp() { } + + public override void Execute(EvalContext ctx) + { + ctx.Abort(this.m_Path); + } + } + + + public static AssertOp Assert(OperatorOp condition) { return new AssertOp(condition);} + /// + /// Asserts the condition and aborts if it is not true + /// + [Serializable] + public sealed class AssertOp : FlowOp + { + internal AssertOp(OperatorOp condition) + { + if (condition==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"Assert(condition=null)"); + + Condition = condition; + } + + public readonly OperatorOp Condition; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + Condition.Prepare(ctx, m_Path); + } + + public override void Execute(EvalContext ctx) + { + var val = Condition.GetValue(ctx); + if (ctx.Aborted) return; + if (!val) ctx.Abort( m_Path ); + } + } + + public static AnywayContinueAfterOp AnywayContinueAfter(ChangeOp operation, bool resetAbort = false) + { + return new AnywayContinueAfterOp(operation, resetAbort); + } + /// + /// Ignores the result of a change operation whether it returns true or false and continues + /// + [Serializable] + public sealed class AnywayContinueAfterOp : FlowOp + { + internal AnywayContinueAfterOp(ChangeOp operation, bool resetAbort) + { + if (operation==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"AnywayContinueAfter(operation=null)"); + Operation = operation; + } + + public readonly ChangeOp Operation; + public readonly bool ResetAbort; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + Operation.Prepare(ctx, m_Path); + } + + public override void Execute(EvalContext ctx) + { + Operation.GetValue(ctx); + //ignore the resulting abort + if (ResetAbort) ctx.ResetAbort(); + } + } + + + public static BlockOp Block(params StatementOp[] statements ) + { + return new BlockOp(statements); + } + /// + /// Block of statements + /// + [Serializable] + public sealed class BlockOp : FlowOp + { + internal BlockOp(StatementOp[] statements) + { + if (statements==null || statements.Length<1) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"BlockOp(statements=null|0)"); + Statements = statements; + } + + public readonly StatementOp[] Statements; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + foreach(var statement in Statements) statement.Prepare(ctx, m_Path); + } + + public override void Execute(EvalContext ctx) + { + foreach( var statement in Statements) + { + statement.Execute(ctx); + if (ctx.Aborted) return; + } + } + } + + + + public static IfOp If(OperatorOp condition, StatementOp then, StatementOp elze = null) + { + return new IfOp(condition, then, elze); + } + /// + /// Conditional statement + /// + [Serializable] + public sealed class IfOp : FlowOp + { + internal IfOp(OperatorOp condition, StatementOp then, StatementOp elze) + { + if (condition==null || then==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"IfOp(condition|then==null)"); + Condition = condition; + Then = then; + Else = elze; + } + + public readonly OperatorOp Condition; + public readonly StatementOp Then; + public readonly StatementOp Else; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + + Condition.Prepare(ctx, m_Path); + Then.Prepare(ctx, m_Path); + if (Else!=null) + Else.Prepare(ctx, m_Path); + } + + public override void Execute(EvalContext ctx) + { + var result = Condition.GetValue(ctx); + if (ctx.Aborted) return; + + if (result) + Then.Execute(ctx); + else + { + if (Else!=null) Else.Execute(ctx); + } + } + } + + + + + + + + } + +} diff --git a/src/Agni/Locking/LockOps.Logical.cs b/src/Agni/Locking/LockOps.Logical.cs new file mode 100644 index 0000000..232d515 --- /dev/null +++ b/src/Agni/Locking/LockOps.Logical.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +using Agni.Locking.Server; + +namespace Agni.Locking +{ + /// + /// Represents operations of locking transaction + /// + public static partial class LockOp + { + + + public static AndOp And(OperatorOp lop, OperatorOp rop) { return new AndOp(lop, rop);} + [Serializable] + public sealed class AndOp : BinaryOperatorOp + { + internal AndOp(OperatorOp lop, OperatorOp rop): base(lop, rop){} + + public override bool GetValue(EvalContext ctx) + { //common code not refactored to parent class for speed (less virt calls) + var lv = LeftOperand.GetValue(ctx); + if (ctx.Aborted) return false; + var rv = RightOperand.GetValue(ctx); + if (ctx.Aborted) return false; + return lv && rv; + } + } + + public static OrOp Or(OperatorOp lop, OperatorOp rop) { return new OrOp(lop, rop);} + [Serializable] + public sealed class OrOp : BinaryOperatorOp + { + internal OrOp(OperatorOp lop, OperatorOp rop) : base(lop, rop){} + + public override bool GetValue(EvalContext ctx) + { //common code not refactored to parent class for speed (less virt calls) + var lv = LeftOperand.GetValue(ctx); + if (ctx.Aborted) return false; + var rv = RightOperand.GetValue(ctx); + if (ctx.Aborted) return false; + return lv || rv; + } + } + + public static XorOp Xor(OperatorOp lop, OperatorOp rop) { return new XorOp(lop, rop);} + [Serializable] + public sealed class XorOp : BinaryOperatorOp + { + internal XorOp(OperatorOp lop, OperatorOp rop) : base(lop, rop){} + + public override bool GetValue(EvalContext ctx) + { //common code not refactored to parent class for speed (less virt calls) + var lv = LeftOperand.GetValue(ctx); + if (ctx.Aborted) return false; + var rv = RightOperand.GetValue(ctx); + if (ctx.Aborted) return false; + return lv ^ rv; + } + } + + public static NotOp Not(OperatorOp operand) { return new NotOp(operand);} + /// + /// Reverses the boolean value of the operand + /// + public sealed class NotOp : UnaryOperatorOp + { + internal NotOp(OperatorOp operand) + { + if (operand==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"Not(operand=null)"); + + Operand = operand; + } + public readonly OperatorOp Operand; + + public override bool GetValue(EvalContext ctx) + { + var v = Operand.GetValue(ctx); + if (ctx.Aborted) return false; + return !v; + } + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + Operand.Prepare(ctx, m_Path); + } + } + + + public static ExistsOp Exists(string table, string var, object value = null, bool ignoreThisSession=true) + { + return new ExistsOp(table, var, value, ignoreThisSession); + } + /// + /// Returns true if the named variable exists. + /// + [Serializable] + public sealed class ExistsOp : UnaryOperatorOp + { + internal ExistsOp(string table, string var, object value, bool ignoreThisSession) + { + if (table.IsNullOrWhiteSpace() || var.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"Exists(table|var=null|empty)"); + + Table = table; + Var = var; + IgnoreThisSession = ignoreThisSession; + } + public readonly string Table; [NonSerialized]internal Server.Table _Table; + public readonly string Var; + public readonly object Value; + public readonly bool IgnoreThisSession; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + _Table = ctx.GetExistingOrMakeTableByName(Table); + } + + public override bool GetValue(EvalContext ctx) + { + return _Table.Exists(ctx, Var, Value, IgnoreThisSession); + } + } + + public static TrueOp True{ get{ return new TrueOp();}} + /// + /// Dummy operator that always returns true + /// + [Serializable] + public sealed class TrueOp : UnaryOperatorOp + { + internal TrueOp() {} + public override bool GetValue(EvalContext ctx){ return true;} + } + + public static FalseOp False{ get{ return new FalseOp();}} + /// + /// Dummy operator that always returns false + /// + [Serializable] + public sealed class FalseOp : UnaryOperatorOp + { + internal FalseOp() {} + public override bool GetValue(EvalContext ctx){ return false;} + } + + + + } + +} diff --git a/src/Agni/Locking/LockOps.Select.cs b/src/Agni/Locking/LockOps.Select.cs new file mode 100644 index 0000000..8dc1c6d --- /dev/null +++ b/src/Agni/Locking/LockOps.Select.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +using Agni.Locking.Server; + +namespace Agni.Locking +{ + /// + /// Represents operations of locking transaction + /// + public static partial class LockOp + { + + public static SelectConstantValueOp SelectConstantValue(string intoName, object value) + { + return new SelectConstantValueOp(intoName, value); + } + /// + /// Returns the constant value as a named key which is returned back to the client + /// + [Serializable] + public sealed class SelectConstantValueOp : SelectOp + { + internal SelectConstantValueOp(string intoName, object value) : base(intoName) + { + Value = value; + } + public readonly object Value; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + } + + public override void Execute(EvalContext ctx) + { + ctx.AddData( IntoName, Value ); + } + } + + + + + public static SelectOperatorValueOp SelectOperatorValue(string intoName, OperatorOp operand) + { + return new SelectOperatorValueOp(intoName, operand); + } + /// + /// Returns the value of operator as a named key which is returned back to the client + /// + [Serializable] + public sealed class SelectOperatorValueOp : SelectOp + { + internal SelectOperatorValueOp(string intoName, OperatorOp operand) : base(intoName) + { + if (operand==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"SelectOperatorValue(operand=null)"); + + Operand = operand; + } + public readonly OperatorOp Operand; + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + Operand.Prepare(ctx, m_Path); + } + + public override void Execute(EvalContext ctx) + { + var val = Operand.GetValue(ctx); + if (ctx.Aborted) return; + ctx.AddData( IntoName, val ); + } + } + + + public static SelectVarValueOp SelectVarValue(string intoName, + string table, + string var, + bool ignoreThisSession=true, + bool abortIfNotFound=false, + bool selectMany=false) + { + return new SelectVarValueOp(intoName, table, var, ignoreThisSession, abortIfNotFound, selectMany); + } + + /// + /// Returns the value of a named variable either a a single or Variable[] object. + /// Optionally aborts if variable was not set + /// + [Serializable] + public sealed class SelectVarValueOp : SelectOp + { + internal SelectVarValueOp(string intoName, string table, string var, bool ignoreThisSession, bool abortIfNotFound, bool selectMany) : base(intoName) + { + if (table.IsNullOrWhiteSpace() || var.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"SelectVarValue(table|name=null|empty)"); + + Table = table; + Var = var; + IgnoreThisSession = ignoreThisSession; + AbortIfNotFound = abortIfNotFound; + SelectMany = selectMany; + } + public readonly string Table; [NonSerialized]internal Server.Table _Table; + public readonly string Var; + public readonly bool IgnoreThisSession; + public readonly bool AbortIfNotFound; + public readonly bool SelectMany; + + + public override void Prepare(EvalContext ctx, string path) + { + base.Prepare(ctx, path); + _Table = ctx.GetExistingOrMakeTableByName(Table); + } + + public override void Execute(EvalContext ctx) + { + var val = _Table.GetVariable(SelectMany, ctx, Var, IgnoreThisSession); + + if (AbortIfNotFound && val==null) + ctx.Abort( m_Path ); + else + ctx.AddData( IntoName, val ); + } + } + + + + + } + +} diff --git a/src/Agni/Locking/LockSession.cs b/src/Agni/Locking/LockSession.cs new file mode 100644 index 0000000..fb0daa7 --- /dev/null +++ b/src/Agni/Locking/LockSession.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +namespace Agni.Locking +{ + + /// + /// Identifies a locking session + /// + [Serializable] + public struct LockSessionID : IEquatable + { + internal LockSessionID(string host) + { + Host = host.IsNullOrWhiteSpace() ? AgniSystem.HostName : host; + ID = Guid.NewGuid(); + } + + public readonly string Host; + public readonly Guid ID; + + public bool Equals(LockSessionID other) + { + return this.ID.Equals(other.ID) && this.Host.EqualsOrdSenseCase(other.Host); + } + + public override int GetHashCode() + { + return ID.GetHashCode() ^ Host.GetHashCodeOrdSenseCase(); + } + + public override bool Equals(object obj) + { + if (obj is LockSessionID) return this.Equals((LockSessionID)obj); + return false; + } + + public override string ToString() + { + return ID.ToString() + "@" + Host;//faster than Args, perf is critical here as this is sused for registry lokup + } + } + + + /// + /// Represents a session on a remote lock server under which lock transactions gets executed. + /// Obtain an instance from local LockManager.MakeSession() + /// + public sealed class LockSession : DisposableObject, INamed, IEquatable + { + internal LockSession(ILockManagerImplementation manager, string path, object shardingID, string description = null, int? maxAgeSec = null) + { + if (path.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"LockSession.ctor(path==null|empty)"); + + if (shardingID==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"LockSession.ctor(shardingID==null)"); + + Data = new Server.LockSessionData(new LockSessionID( null ), description, maxAgeSec); + Manager = manager; + Path = path; + ShardingID = shardingID; + + try + { + var shardingHash = shardingID.GetHashCode(); + + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(Path); + var primaryZgovs = zone + .FindNearestParentZoneGovernors(iAmZoneGovernor: false, filter: (host) => !host.IsZGovLockFailover, transcendNOC: true) + .OrderBy( host => host.Name) + .ToArray(); + + if (primaryZgovs.Length<1) + throw new LockingException(StringConsts.LOCK_SESSION_PATH_LEVEL_NO_ZGOVS_ERROR.Args(Path)); + + var failoverZgovs = zone + .FindNearestParentZoneGovernors(iAmZoneGovernor: false, filter: (host) => host.IsZGovLockFailover, transcendNOC: true) + .OrderBy( host => host.Name) + .ToArray(); + + if (failoverZgovs.Length>0 && + (primaryZgovs.Length!=failoverZgovs.Length || + !primaryZgovs[0].ParentZone.IsLogicallyTheSame( failoverZgovs[0].ParentZone )) + ) + throw new LockingException(StringConsts.LOCK_SESSION_ZGOV_SETUP_ERROR.Args(primaryZgovs[0].ParentZone.RegionPath)); + + var idx = (shardingHash & CoreConsts.ABS_HASH_MASK) % primaryZgovs.Length; + ServerHostPrimary = primaryZgovs[idx].RegionPath; + if (failoverZgovs.Length>0) + ServerHostSecondary = failoverZgovs[idx].RegionPath; + + + } + catch(Exception error) + { + throw new LockingException(StringConsts.LOCK_SESSION_PATH_ERROR.Args(Path, error.ToMessageWithType() ) ,error); + } + } + + protected override void Destructor() + { + Manager.EndLockSession( this ); + } + + + internal readonly Server.LockSessionData Data; + + public string Name { get { return ID.ToString();}} + public LockSessionID ID { get { return Data.ID;}} + public string Description { get { return Data.Description;}} + public readonly string Path; + public readonly string ServerHostPrimary; + public readonly string ServerHostSecondary; + public readonly object ShardingID; + public int? MaxAgeSec { get { return Data.MaxAgeSec;}} + + /// + /// Returns lock server hosts for his session in the primary -> secondary(if any) sequence + /// + public IEnumerable ServerHosts + { + get + { + yield return ServerHostPrimary; + + if (ServerHostSecondary.IsNotNullOrWhiteSpace()) + yield return ServerHostSecondary; + } + } + + + /// + /// References manager that opened the session + /// + public readonly ILockManagerImplementation Manager; + + + public override int GetHashCode() + { + return ID.GetHashCode(); + } + + public override bool Equals(object obj) + { + var other = obj as LockSession; + return other!=null?this.Equals(other) : false; + } + + public bool Equals(LockSession other) + { + return other!=null && this.ID.Equals(other.ID); + } + + public override string ToString() + { + return "[{0}]{1}".Args(ID, Description); + } + } +} diff --git a/src/Agni/Locking/LockTransaction.cs b/src/Agni/Locking/LockTransaction.cs new file mode 100644 index 0000000..dd02266 --- /dev/null +++ b/src/Agni/Locking/LockTransaction.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +using Agni.Locking.Server; + +namespace Agni.Locking +{ + /// + /// Represents a transaction that gets atomically applied to the remote locking server + /// + [Serializable] + public sealed class LockTransaction : IEnumerable + { + /// + /// Represents transaction that pings any server regardless of reliability guarantees + /// + public static readonly LockTransaction PingAnyReliability = new LockTransaction(0, 0.0d); + + + /// + /// Creates a new transaction object for execution on a remote lock server + /// + public LockTransaction( + string description, + string namespaceName, + uint minRuntimeSec, + double minTrustLevel, + params LockOp.StatementOp[] statements + ) + { + if (description.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().FullName+".ctor(description=null|empty)"); + + if (namespaceName.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().FullName+".ctor(namespaceName=null|empty)"); + + if (minTrustLevel < 0d || minTrustLevel > 1.0d) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().FullName+".ctor(minTrustLevel must be 0d..1d)"); + + if (statements==null || statements.Length<1) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().FullName+".ctor(statements=null|empty)"); + + ID = Guid.NewGuid(); + Description = description; + Namespace = namespaceName; + MinimumRequiredRuntimeSec = minRuntimeSec; + MinimumRequiredTrustLevel = minTrustLevel; + Statements = statements; + } + + + /// + /// Creates PING transaction - a transaction that just touches the session and does nothing else + /// + public LockTransaction( + uint minRuntimeSec, + double minTrustLevel + ) + { + if (minTrustLevel < 0d || minTrustLevel > 1.0d) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().FullName+".ctor(minTrustLevel must be 0d..1d)"); + + ID = Guid.NewGuid(); + MinimumRequiredRuntimeSec = minRuntimeSec; + MinimumRequiredTrustLevel = minTrustLevel; + } + + /// + /// Unique ID assigned to this transaction at create time + /// + public readonly Guid ID; + + /// + /// Provides Textual description for transaction + /// + public readonly string Description; + + /// + /// Name of data/memory partition in lock server where all transactions get serialized, hence + /// where all changes are guaranteed to be atomic and coherent + /// + public readonly string Namespace; + + + /// + /// Specifies te minimum time that the server had to run for for transaction to succeed + /// + public readonly uint MinimumRequiredRuntimeSec; + + /// + /// Specifies 0..1 minimum trust that the server has to have for transaction to succeed. + /// If the server's level of trust is below this number, then transaction fails + /// + public readonly double MinimumRequiredTrustLevel; + + /// + /// The list of statemnts that get executed in order + /// + public readonly LockOp.StatementOp[] Statements; + + + + public IEnumerator GetEnumerator() + { + return ((IEnumerable)Statements).GetEnumerator(); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return Statements.GetEnumerator(); + } + + } + +} diff --git a/src/Agni/Locking/LockTransactionResult.cs b/src/Agni/Locking/LockTransactionResult.cs new file mode 100644 index 0000000..d6477f5 --- /dev/null +++ b/src/Agni/Locking/LockTransactionResult.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; + +namespace Agni.Locking +{ + /// + /// Denotes statuses of transaction execution + /// + public enum LockStatus + { + /// + /// The call could not be placed/did not return success + /// + CallFailed=0, + + /// + /// The requested transaction could not be executed as there are locking conflicts + /// + TransactionError, + + /// + /// The requested transaction was executed OK + /// + TransactionOK + } + + + /// + /// Provides transaction execution error causes + /// + public enum LockErrorCause + { + /// + /// Unknown/unspecified cause + /// + Unspecified = 0, + + /// + /// The part of transaction statement graph failed, see FailedStatement for details + /// + Statement, + + /// + /// The requirements for transaction execution are not met + /// + MinimumRequirements, + + /// + /// The requested transaction was executed OK + /// + SessionExpired + } + + + + /// + /// Represents a result of locking operation performed by ILockManager. + /// If transaction selects some results then this class enumerates all key value pairs that may contain duplicate names + /// + [Serializable] + public sealed class LockTransactionResult : IEnumerable> + { + + public static LockTransactionResult CallFailed + { + get { return new LockTransactionResult(Guid.Empty, null, LockStatus.CallFailed, LockErrorCause.Unspecified, null, 0, 0d, null); } + } + + + public LockTransactionResult(Guid tranID, string server, LockStatus status, LockErrorCause errorCause, string failedStatement, uint runtimeSec, double trustLevel, KeyValuePair[] data) + { + TransactionID = tranID; + ServerHost = server; + Status = status; + ErrorCause = errorCause; + FailedStatement = failedStatement; + ServerRuntimeSec = runtimeSec; + ServerTrustLevel = trustLevel; + m_Data = data; + } + + //the array is used instead of dictionary for serialization speed + private readonly KeyValuePair[] m_Data; + + + /// + /// Returns the original transaction ID + /// + public readonly Guid TransactionID; + + /// + /// Returns the host name of the host that serviced the call + /// + public readonly string ServerHost; + + /// + /// Returns the status of the call: whether call failed or transaction could not be executed + /// + public readonly LockStatus Status; + + /// + /// In case of error specifies the cause + /// + public readonly LockErrorCause ErrorCause; + + /// + /// Returns the description of failed statement or null if transactions succeeded + /// + public readonly string FailedStatement; + + /// + /// Returns for how many seconds the server has been running + /// + public readonly uint ServerRuntimeSec; + + /// + /// Returns the coefficient of trust 0..1(maximum) of the server state. + /// The value is computed based on the length of server uninterrupted runtime. + /// This value plays an important role in speculative locking when lock server crashes, the caller may examine + /// this returned value and based on it reject the fact that lock was taken, or the caller may specify + /// the LockTransaction.MinimumRequiredTrustLevel for the transaction to succeed + /// + public readonly double ServerTrustLevel; + + /// + /// Returns the count of returned variables + /// + public int Count {get{ return m_Data==null? 0 : m_Data.Length;}} + + /// + /// Returns the first variable value by case-sensitive name. The var names may not be unique, + /// as they are added during tran execution. Use instance of this class to enumerate all key value pairs + /// + public object this[string var] + { + get + { + if (m_Data==null) return null; + for (var i=0; i> GetEnumerator() + { + return m_Data==null ? Enumerable.Empty>().GetEnumerator() : ((IEnumerable>)m_Data).GetEnumerator(); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return m_Data==null ? Enumerable.Empty>().GetEnumerator() : m_Data.GetEnumerator(); + } + + public override string ToString() + { + return "{0}({1},'{2}','{3}')".Args(TransactionID, Status, ErrorCause, FailedStatement); + } + } + + + + +} diff --git a/src/Agni/Locking/NOPLockManager.cs b/src/Agni/Locking/NOPLockManager.cs new file mode 100644 index 0000000..cfac76f --- /dev/null +++ b/src/Agni/Locking/NOPLockManager.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + + +using NFX.ApplicationModel; + +namespace Agni.Locking +{ + public sealed class NOPLockManager : ApplicationComponent, ILockManagerImplementation + { + + private static NOPLockManager s_Instance = new NOPLockManager(); + + public static NOPLockManager Instance { get{ return s_Instance;}} + + + + public LockTransactionResult ExecuteLockTransaction(LockSession session, LockTransaction transaction) + { + return new LockTransactionResult(Guid.Empty, null, LockStatus.TransactionOK, LockErrorCause.Unspecified, null, 0, 0d, null); + } + + public Task ExecuteLockTransactionAsync(LockSession session, LockTransaction transaction) + { + return Task.FromResult( ExecuteLockTransaction(session, transaction) ); + } + + public bool EndLockSession(LockSession session) + { + return true; + } + + public Task EndLockSessionAsync(LockSession session) + { + return Task.FromResult( EndLockSession(session) ); + } + + public LockSession MakeSession(string path, object shardingID, string description = null, int? maxAgeSec = null) + { + return new LockSession(this, path, shardingID, description, maxAgeSec); + } + + + public LockSession this[LockSessionID sid] + { + get { return null; } + } + + + + public void Configure(NFX.Environment.IConfigSectionNode node) + { + + } + + public bool InstrumentationEnabled + { + get + { + return false; + } + set + { + + } + } + + public bool ExternalGetParameter(string name, out object value, params string[] groups) + { + value = null; + return false; + } + + public IEnumerable> ExternalParameters + { + get { return Enumerable.Empty>(); } + } + + public IEnumerable> ExternalParametersForGroups(params string[] groups) + { + return Enumerable.Empty>(); + } + + public bool ExternalSetParameter(string name, object value, params string[] groups) + { + return false; + } + + + } +} diff --git a/src/Agni/Locking/Server/EvalContext.cs b/src/Agni/Locking/Server/EvalContext.cs new file mode 100644 index 0000000..c2d5e54 --- /dev/null +++ b/src/Agni/Locking/Server/EvalContext.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Agni.Locking.Server +{ + /// + /// Holds context information during execution of LockTransaction graph + /// + public sealed class EvalContext + { + internal EvalContext(ServerLockSession session, Namespace ns, LockTransaction tran) + { + Session = session; + Namespace = ns; + Transaction = tran; + } + + private bool m_Aborted; + private string m_FailedStatement; + private List> m_Data; + + + /// + /// Lock Session that this request is under + /// + internal readonly ServerLockSession Session; + + /// + /// ID of a session that this request is under + /// + public LockSessionID SessionID { get{ return Session.ID;}} + + /// + /// Description of the session that this request is under + /// + public string SessionDescription { get { return Session.Data.Description; } } + + /// + /// Returns the transaction being evaluated + /// + public readonly LockTransaction Transaction; + + + internal readonly Namespace Namespace; + + + /// + /// Returns true when graph evaluation was aborted. + /// This flag is checked by all operations instead of throwing exception which is much slower + /// + public bool Aborted { get { return m_Aborted;} } + + /// + /// If aborted, returns the description of failed statement + /// + public string FailedStatement { get{ return m_FailedStatement;}} + + /// + /// Aborts the evaluation of lock tran graph. Conceptually similar to throw, however is 8-10x faster + /// + public void Abort(string failedStatement) + { + m_FailedStatement = failedStatement; + m_Aborted = true; + } + + /// + /// Resets abort condition + /// + public void ResetAbort() + { + m_FailedStatement = null; + m_Aborted = false; + } + + + public KeyValuePair[] Data {get{ return m_Data==null ? null : m_Data.ToArray();}} + + /// + /// Adds named key with value to result list + /// + public void AddData(string key, object value) + { + if (m_Data==null) m_Data = new List>(32); + m_Data.Add( new KeyValuePair(key, value) ); + } + + + internal Table GetExistingOrMakeTableByName(string name) + { + return Namespace.GetExistingOrMakeTableByName( name ); + } + + + } +} diff --git a/src/Agni/Locking/Server/LockServerService.cs b/src/Agni/Locking/Server/LockServerService.cs new file mode 100644 index 0000000..cbe7c85 --- /dev/null +++ b/src/Agni/Locking/Server/LockServerService.cs @@ -0,0 +1,513 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NFX; +using NFX.Log; +using NFX.Collections; +using NFX.Environment; +using NFX.ApplicationModel; +using NFX.ServiceModel; + +namespace Agni.Locking.Server +{ + /// + /// Implements lock server. Usually this service is activated by Zone gov process + /// + public sealed class LockServerService : ServiceWithInstrumentationBase, Contracts.ILocker + { + #region CONSTS + public const string THREAD_NAME = "LockServerService"; + + public const string CONFIG_LOCK_SERVER_SECTION = "lock-server"; + + public const int DEFAULT_SESSION_MAX_AGE_SEC = 60; + public const int MIN_SESSION_MAX_AGE_SEC = 1; + + #endregion + + #region .ctor + public LockServerService(object director) : base(director) + { + + } + #endregion + + #region Fields + private Thread m_Thread; + private AutoResetEvent m_WaitEvent; + + private DateTime m_StartTimeUTC; + private double m_TrustLevel; + + + private double m_ServerCallsNorm; + private int m_CurrentServerCalls; + + private int m_DefaultSessionMaxAgeSec = DEFAULT_SESSION_MAX_AGE_SEC; + + private Registry m_Sessions = new Registry(); + private Registry m_Namespaces = new Registry(); + + private bool m_InstrumentationEnabled; + private bool m_DetailedTableInstrumentation; + + private NamedInterlocked m_stat_ExecuteTranCalls = new NamedInterlocked(); + private int m_stat_EndLockSessionCalls; + private long m_stat_ExpiredRecords; + private int m_stat_ExpiredSessions; + private int m_stat_RemovedEmptyTables; + + #endregion + + #region Properties + + + + /// + /// Implements IInstrumentable + /// + [Config(Default=false)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOCKING, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled + { + get { return m_InstrumentationEnabled;} + set + { + if (m_InstrumentationEnabled!=value) + { + m_InstrumentationEnabled = value; + + m_stat_ExecuteTranCalls.Clear(); + m_stat_EndLockSessionCalls = 0; + m_stat_ExpiredRecords = 0; + m_stat_ExpiredSessions = 0; + m_stat_RemovedEmptyTables = 0; + } + } + } + + + /// + /// When true, instruments every table + /// + [Config(Default=false)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOCKING, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public bool DetailedTableInstrumentation + { + get { return m_DetailedTableInstrumentation;} + set { m_DetailedTableInstrumentation = value; } + } + + + /// + /// Returns when server started + /// + public DateTime StartTimeUTC{ get{ return m_StartTimeUTC;}} + + + /// + /// Returns the current norm for the number of calls - the trust level goes down when + /// server experiences a sharp call drop + /// + public int CurrentServerCallsNorm{ get{ return (int)m_ServerCallsNorm;}} + + /// + /// Returns the current trust level 0.0 .. 1.0 of this server + /// + public double CurrentTrustLevel{ get{ return m_TrustLevel; }} + + /// + /// Default session maximum age in seconds + /// + [Config(Default=DEFAULT_SESSION_MAX_AGE_SEC)] + public int DefaultSessionMaxAgeSec + { + get{ return m_DefaultSessionMaxAgeSec;} + set{ m_DefaultSessionMaxAgeSec = value < MIN_SESSION_MAX_AGE_SEC ? MIN_SESSION_MAX_AGE_SEC : value; } + } + + #endregion + + #region Public + + public LockTransactionResult ExecuteLockTransaction(LockSessionData session, LockTransaction transaction) + { + var result = executeLockTransaction(session, transaction); + + if (m_InstrumentationEnabled) + { + var key = result.Status==LockStatus.TransactionOK ? "OK" //for speed not to concat strings + : result.Status.ToString() + ':' + result.ErrorCause.ToString(); + m_stat_ExecuteTranCalls.IncrementLong(key); + } + + return result; + } + + private LockTransactionResult executeLockTransaction(LockSessionData session, LockTransaction transaction) + { + if (!Running) return LockTransactionResult.CallFailed; + + Interlocked.Increment(ref m_CurrentServerCalls); + + if (session==null || transaction==null) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().Name+".ExecuteLockTransaction(session|transaction==null)"); + + var isPing = transaction.Statements == null; + + if (!isPing && transaction.Namespace.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+GetType().Name+".ExecuteLockTransaction(transaction.Namespace==null|empty)"); + + var currentTrustLevel = CurrentTrustLevel; + var appRunTimeSec = (uint)(DateTime.UtcNow - m_StartTimeUTC).TotalSeconds; + + //insufficient runtime period length or trust level + if (transaction.MinimumRequiredRuntimeSec>appRunTimeSec || + transaction.MinimumRequiredTrustLevel>currentTrustLevel) + return new LockTransactionResult(transaction.ID, + AgniSystem.HostName, + LockStatus.TransactionError, + LockErrorCause.MinimumRequirements, + null, + appRunTimeSec, + currentTrustLevel, + null); + + + var sid = session.ID.ToString(); + var ss = m_Sessions.GetOrRegister(sid, (sd) => new ServerLockSession(sd), session); + lock(ss) + { + if (ss.Disposed) + return new LockTransactionResult(transaction.ID, + AgniSystem.HostName, + LockStatus.TransactionError, + LockErrorCause.SessionExpired, + null, + appRunTimeSec, + currentTrustLevel, + null); + + ss.m_LastInteractionUTC = App.TimeSource.UTCNow; + + if (isPing)//ping just touches session above + return new LockTransactionResult(transaction.ID, + AgniSystem.HostName, + LockStatus.TransactionOK, + LockErrorCause.Unspecified, + null, + appRunTimeSec, + currentTrustLevel, + null); + + var ns = m_Namespaces.GetOrRegister(transaction.Namespace, (_) => new Namespace(transaction.Namespace), null); + + var ectx = new EvalContext(ss, ns, transaction); + + prepareTransaction( ectx ); //prepare is not under the lock + + LockTransactionResult result; + + lock(ns) //execute is UNDER THE LOCK + result = executeTransaction( ectx, appRunTimeSec, currentTrustLevel ); + return result; + } + } + + public bool EndLockSession(LockSessionID sessionID) + { + if (!Running) return false; + + Interlocked.Increment(ref m_CurrentServerCalls); + + + var sid = sessionID.ToString(); + var ss = m_Sessions[sid]; + if (ss==null) return false; + + + if (m_InstrumentationEnabled) + Interlocked.Increment( ref m_stat_EndLockSessionCalls); + + + lock(ss) + { + //note: no need to update last interaction time + m_Sessions.Unregister(ss); + if (ss.Disposed) return false; + + ss.Dispose(); + return true; + } + } + + #endregion + + #region Protected + + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + } + + protected override void DoStart() + { + m_StartTimeUTC = DateTime.UtcNow; + m_CurrentServerCalls = 0; + m_ServerCallsNorm = 0.0d; + + m_WaitEvent = new AutoResetEvent(false); + + m_Thread = new Thread(threadSpin); + m_Thread.Name = THREAD_NAME; + m_Thread.Start(); + } + + protected override void DoWaitForCompleteStop() + { + m_TrustLevel = 0d; + + m_WaitEvent.Set(); + + m_Thread.Join(); + m_Thread = null; + + m_WaitEvent.Close(); + m_WaitEvent = null; + + m_Sessions.Clear(); + } + + #endregion + + #region .pvt + + private void prepareTransaction(EvalContext ctx) + { + var statements = ctx.Transaction.Statements; + for(var i=0; i< statements.Length; i++) + { + var statement = statements[i]; + statement.Prepare( ctx, i.ToString() + ":/"); + } + } + + //warning: this is called under the global lock on NS, do not block + private LockTransactionResult executeTransaction(EvalContext ctx, uint appRunTimeSec, double trustLevel) + { + var status = LockStatus.TransactionOK; + var errorCause = LockErrorCause.Unspecified; + + var statements = ctx.Transaction.Statements; + + for(var i=0; i m_ServerCallsNorm) + { + m_ServerCallsNorm = currentCalls;//norm is INSTANTLY adjusted to the number of calls made + } + else + { + m_ServerCallsNorm *= NORM_DECAY_FACTOR; + } + + if (m_ServerCallsNorm>1.0d) + { + var drop = m_ServerCallsNorm - currentCalls; + var ratio = drop / m_ServerCallsNorm; + + m_TrustLevel = 1.0d - ratio; + + if (m_TrustLevel>1.0d) m_TrustLevel = 1.0d; + } + else + { + m_ServerCallsNorm = 0.0d; + m_TrustLevel = 1.0d; + } + + if (m_TrustLevel < 0.9d) + NFX.ExternalRandomGenerator.Instance.FeedExternalEntropySample( (int)( m_TrustLevel * 1378459007 ) ); //0.0 .. 0.99 * some odd number 1b+ + } + + private void removeExpiredTableData(DateTime now) + { + long totalRemoved = 0; + foreach( var ns in m_Namespaces)//thread-safe snapshot + { + if (!Running) return; + + lock(ns) + { + foreach(var tbl in ns.Tables) + { + if (!Running) return; + + totalRemoved += tbl.RemoveExpired( now ); + + if (ns.RemoveTableIfEmpty( tbl ))//does not affect the foreach as it is snapshot based + if (m_InstrumentationEnabled) + Interlocked.Increment(ref m_stat_RemovedEmptyTables); + } + } + } + + + if (m_InstrumentationEnabled) + Interlocked.Add(ref m_stat_ExpiredRecords, totalRemoved); + } + + + private void removeExpiredSessions(DateTime now) + { + int totalTimedOut = 0; + foreach( var session in m_Sessions)//thread-safe snapshot + { + if (!Running) return; + + if (Monitor.TryEnter(session)) + try + { + var sessionMaxAge = session.Data.MaxAgeSec.HasValue ? session.Data.MaxAgeSec.Value : this.m_DefaultSessionMaxAgeSec; + if ((now - session.m_LastInteractionUTC).TotalSeconds > sessionMaxAge)//session times out + { + m_Sessions.Unregister( session ); + session.Dispose(); + totalTimedOut++; + } + } + finally + { + Monitor.Exit(session); + } + } + + if (m_InstrumentationEnabled) + Interlocked.Increment(ref m_stat_ExpiredSessions); + + } + + + private void dumpStats() + { + if (!App.Instrumentation.Enabled || !m_InstrumentationEnabled) return; + + foreach(var src in m_stat_ExecuteTranCalls.AllNames) + App.Instrumentation.Record( new Instrumentation.ServerLockTransactions(src, m_stat_ExecuteTranCalls.ExchangeLong(src, 0)) ); + + App.Instrumentation.Record( new Instrumentation.ServerEndLockSessionCalls( Interlocked.Exchange(ref m_stat_EndLockSessionCalls, 0)) ); + + App.Instrumentation.Record( new Instrumentation.ServerExpiredRecords( Interlocked.Exchange(ref m_stat_ExpiredRecords, 0)) ); + App.Instrumentation.Record( new Instrumentation.ServerExpiredSessions( Interlocked.Exchange(ref m_stat_ExpiredSessions, 0)) ); + App.Instrumentation.Record( new Instrumentation.ServerRemovedEmptyTables( Interlocked.Exchange(ref m_stat_RemovedEmptyTables, 0)) ); + + App.Instrumentation.Record( new Instrumentation.ServerTrustLevel( m_TrustLevel ) ); + App.Instrumentation.Record( new Instrumentation.ServerCallsNorm( m_ServerCallsNorm ) ); + + if (m_DetailedTableInstrumentation) + foreach(var ns in m_Namespaces) + { + App.Instrumentation.Record( new Instrumentation.ServerNamespaceTables(ns.Name, ns.Tables.Count ) ); + foreach(var tbl in ns.Tables) + App.Instrumentation.Record( new Instrumentation.ServerNamespaceTableRecordCount( ns.Name+"::"+tbl.Name, tbl.TotalRecordCount ) ); + } + } + + + internal void log(MessageType type, string from, string text, Exception error = null, Guid? related = null) + { + var msg = new NFX.Log.Message + { + Type = type, + Topic = SysConsts.LOG_TOPIC_LOCKING, + From = "{0}.{1}".Args(GetType().FullName, from), + Text = text, + Exception = error + }; + + if (related.HasValue) msg.RelatedTo = related.Value; + + App.Log.Write( msg ); + } + #endregion + + + } + +} diff --git a/src/Agni/Locking/Server/LockSessionData.cs b/src/Agni/Locking/Server/LockSessionData.cs new file mode 100644 index 0000000..76ccc17 --- /dev/null +++ b/src/Agni/Locking/Server/LockSessionData.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + + +using NFX; + +namespace Agni.Locking.Server +{ + /// + /// Contains data to establish a session on a remote lock server. + /// Clients should use LockSession instead, which is obtained from the local LockManager.MakeSession + /// + [Serializable] + public sealed class LockSessionData : INamed, IEquatable + { + internal LockSessionData(LockSessionID id, string description, int? maxAgeSec) + { + ID = id; + Description = description; + MaxAgeSec = maxAgeSec; + } + + public string Name { get { return ID.ToString();}} + public readonly LockSessionID ID; + public readonly string Description; + public readonly int? MaxAgeSec; + + public override int GetHashCode() + { + return ID.GetHashCode(); + } + + public override bool Equals(object obj) + { + var other = obj as LockSessionData; + return other!=null?this.Equals(other) : false; + } + + public bool Equals(LockSessionData other) + { + return other!=null && this.ID.Equals(other.ID); + } + + public override string ToString() + { + return "[{0}]{1}".Args(ID, Description); + } + } + + + + internal sealed class ServerLockSession : DisposableObject, INamed + { + internal ServerLockSession(LockSessionData data) + { + Data = data; + } + + protected override void Destructor() + { + foreach( var ns in m_MutatedNamespaces) + lock(ns) + { + foreach(var tbl in ns.Tables) + tbl.EndSession( this ); + } + } + + internal HashSet m_MutatedNamespaces = new HashSet(); + + + + public readonly LockSessionData Data; + + internal DateTime m_LastInteractionUTC; + + public string Name { get { return Data.Name; } } + + public LockSessionID ID { get{ return Data.ID;}} + + public override int GetHashCode() { return Data.GetHashCode(); } + + public override bool Equals(object obj) { return Data.Equals(((ServerLockSession)obj).Data); } + } + + + + +} diff --git a/src/Agni/Locking/Server/Namespace.cs b/src/Agni/Locking/Server/Namespace.cs new file mode 100644 index 0000000..1edee1f --- /dev/null +++ b/src/Agni/Locking/Server/Namespace.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; + +namespace Agni.Locking.Server +{ + /// + /// Represents a namespace that contains tables in locking server process + /// + internal sealed class Namespace : INamed + { + + internal Namespace(string name) + { + if (name.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"Namespace.ctor(name==null|empty)"); + + m_Name = name; + } + + private string m_Name; + private Registry m_Tables = new Registry
(); + + internal HashSet
m_MutatedTables = new HashSet
(); + + + public string Name {get{ return m_Name;}} + + public IRegistry
Tables{ get{ return m_Tables;} } + + public Table GetExistingOrMakeTableByName(string name) + { + return m_Tables.GetOrRegister(name, (_) => new Table(this, name), 1); + } + + public bool RemoveTableIfEmpty(Table tbl) + { + if (tbl.Count>0) return false; + return m_Tables.Unregister(tbl); + } + + public override int GetHashCode() + { + return m_Name.GetHashCodeOrdIgnoreCase(); + } + + public override bool Equals(object obj) + { + var other = obj as Namespace; + return other!=null && this.m_Name.EqualsOrdIgnoreCase(other.m_Name); + } + } + +} diff --git a/src/Agni/Locking/Server/Table.cs b/src/Agni/Locking/Server/Table.cs new file mode 100644 index 0000000..459bd35 --- /dev/null +++ b/src/Agni/Locking/Server/Table.cs @@ -0,0 +1,405 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; + +namespace Agni.Locking.Server +{ + /// + /// 2 VarLists in tentative and committed states + /// + internal sealed class DataSlot + { + + private bool m_Changing; + private VarList m_Data; + private VarList m_RollbackData; + + public VarList Data{ get{ return m_Data;}} + + public bool Changing { get{ return m_Changing;}} + + public void Change(VarList data) + { + if (!m_Changing) + { + m_RollbackData = m_Data; + m_Changing = true; + } + m_Data = data; + } + + /// + /// Returns how many records have added/removed since change started + /// + public int Commit() + { + if (!m_Changing) throw new LockingException(StringConsts.INTERNAL_IMPLEMENTATION_ERROR + "DataSlot.Commit(!changing)"); + + var nowCount = m_Data!=null? m_Data.Count : 0; + var wasCount = m_RollbackData!=null? m_RollbackData.Count : 0; + + m_RollbackData = null; + m_Changing = false; + + return nowCount - wasCount; + } + + public void Rollback() + { + if (!m_Changing) throw new LockingException(StringConsts.INTERNAL_IMPLEMENTATION_ERROR + "DataSlot.Rollback(!changing)"); + m_Data = m_RollbackData; + m_RollbackData = null; + m_Changing = false; + } + + } + + /// + /// Represents an IMMUTABLE variable kept on a lock server + /// + [Serializable] + public sealed class Variable + { + internal Variable( + LockSessionID sessionID, + Guid tranID, + DateTime setTimeUTC, + DateTime? expirationTimeUTC, + string description, + object value + ) + { + SessionID = sessionID; + TranID = tranID; + SetTimeUTC = setTimeUTC; + ExpirationTimeUTC = expirationTimeUTC; + Description = description; + Value = value; + } + + public readonly LockSessionID SessionID; + public readonly Guid TranID; + public readonly DateTime SetTimeUTC; + public readonly DateTime? ExpirationTimeUTC; + public readonly string Description; + public readonly object Value; + } + + + internal sealed class VarList : List + { + public VarList() : base() {} + public VarList(VarList existing) : base(existing){} + } + + + + + /// + /// Represents a table of locking server namespace. + /// This class is not thread-safe for tran execution as there is only 1 worker thread that executes the transaction graphs at any given time + /// + internal sealed class Table : INamed + { + + internal Table(Namespace ns, string name) + { + if (ns==null || name.IsNullOrWhiteSpace()) + throw new LockingException(StringConsts.ARGUMENT_ERROR+"Table.ctor(ns==null | name==null|empty)"); + + m_Namespace = ns; + m_Name = name; + } + + private Namespace m_Namespace; + private string m_Name; + internal Dictionary m_Data = new Dictionary(1024, StringComparer.Ordinal); + + private Dictionary> m_SessionOwnedData = new Dictionary>(); + + + private List m_PendingChanges = new List(); + + private int m_TotalRecordCount; + + public Namespace Namespace{ get{ return m_Namespace;}} + public string Name {get{ return m_Name;}} + + + /// + /// Returns the number of slots in the table + /// + public int Count { get{ return m_Data.Count; }} + + /// + /// Returns the total number of committed Variable records in all slots. + /// This property is THREAD safe (i.e. for instrumentation) + /// + public int TotalRecordCount{ get{ return m_TotalRecordCount;}} + + + internal void Commit(ServerLockSession session) + { + foreach(var ds in m_PendingChanges)//may have duplicates + if (ds.Changing) + { + var deltaRecords = ds.Commit(); + + m_TotalRecordCount += deltaRecords; + + HashSet set; + if (!m_SessionOwnedData.TryGetValue(session, out set)) + { + set = new HashSet(); + m_SessionOwnedData[session] = set; + } + set.Add( ds ); + + session.m_MutatedNamespaces.Add( m_Namespace ); + } + + m_PendingChanges.Clear(); + } + + internal void Rollback(ServerLockSession session) + { + foreach(var ds in m_PendingChanges)//may have duplicates + if (ds.Changing) ds.Rollback(); + m_PendingChanges.Clear(); + } + + public void EndSession(ServerLockSession session) + { + HashSet set; + if (!m_SessionOwnedData.TryGetValue(session, out set)) return; + foreach(var slot in set) + m_TotalRecordCount -= slot.Data.RemoveAll( v => v.SessionID.Equals(session.Data.ID) ); + + m_SessionOwnedData.Remove( session ); + } + + + /// + /// Returns true if named variable exists created by this or another session + /// + public bool Exists(EvalContext ctx, string name, object value, bool ignoreThisSession) + { + DataSlot slot; + if (!m_Data.TryGetValue(name, out slot)) return false; + + var list = slot.Data; + if (list==null) return false; + + if (!ignoreThisSession) + { + if (value==null) return list.Count>0; + return list.Any(e => e.Value!=null && value.Equals(e.Value)); + } + + for(var i=0; i + /// Returns named variable value or value array created by this or another session. + /// If not found then null is returned + /// + public object GetVariable(bool many, EvalContext ctx, string name, bool ignoreThisSession) + { + DataSlot slot; + if (!m_Data.TryGetValue(name, out slot)) return null; + + var list = slot.Data; + if (list==null) return null; + + if (many) + { + Variable[] result; + + if (!ignoreThisSession) + result = list.ToArray(); + else + result = list.Where(i => !i.SessionID.Equals(ctx.SessionID)).ToArray(); + + return result.Length>0 ? result : null; + } + //else Single + if (list.Count==0) return null; + if (!ignoreThisSession) return list[0]; + + return list.FirstOrDefault(i => !i.SessionID.Equals(ctx.SessionID)); + } + + + + /// + /// Sets named var + /// + public bool SetVariable(EvalContext ctx, string name, object value, string description, DateTime? expirationTimeUTC, bool allowDuplicates) + { + DataSlot slot; + VarList list; + if (!m_Data.TryGetValue(name, out slot)) + { + slot = new DataSlot(); + list = new VarList(); + + var var = new Variable(ctx.SessionID, + ctx.Transaction.ID, + App.TimeSource.UTCNow, + expirationTimeUTC, + getDescr(description, ctx), + value); + list.Add( var ); + slot.Change( list ); + + m_Data.Add(name, slot); + m_PendingChanges.Add( slot ); + m_Namespace.m_MutatedTables.Add( this ); + return true; + } + + list = slot.Data; + + if (list==null) + { + list = new VarList(); + + var var = new Variable(ctx.SessionID, + ctx.Transaction.ID, + App.TimeSource.UTCNow, + expirationTimeUTC, + getDescr(description, ctx), + value); + list.Add( var ); + slot.Change( list ); + + //20161102 + //m_Data.Add(name, slot); + m_PendingChanges.Add( slot ); + m_Namespace.m_MutatedTables.Add( this ); + return true; + } + + if (list.Count>0 && !allowDuplicates) return false; + + var newList = new VarList(list); + + var replaced = false; + for(var i=0; i + /// Deletes named var, true if found and removed + /// + public bool DeleteVariable(EvalContext ctx, string name, object value) + { + DataSlot slot; + if (!m_Data.TryGetValue(name, out slot)) + return false; + + var list = slot.Data; + if (list==null) return false; + + var anymatch = list.Any( v => v.SessionID.Equals(ctx.SessionID) && (value==null || value.Equals(v.Value))); + if (!anymatch) return false; + + VarList newList = new VarList(); + for(var i=0; i + /// Called on a server thread UNDER LOCK to purge outdated data + /// + public int RemoveExpired(DateTime now) + { + var removed = 0; + foreach(var kvp in m_Data) + { + var lst = kvp.Value.Data; + if (lst==null) continue; + for(var i=0; i + /// Sends log messages to log receiver + /// + public sealed class AgniDestination : Destination + { + #region CONSTS + public const string CONFIG_HOST_ATTR = "host"; + + private const MessageType DEFAULT_LOG_LEVEL = MessageType.Warning; + #endregion + + #region .ctor + public AgniDestination() : base() { } + public AgniDestination(string name) : base(name) { } + #endregion + + #region Fields + + private string m_HostName; + private ILogReceiverClient m_Client; + + #endregion + + #region Properties + + /// + /// Specifies the log level for logging operations of this destination + /// + [Config(Default = DEFAULT_LOG_LEVEL)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOG)] + public MessageType LogLevel { get; set; } + + + /// + /// Specifies the name of the host where the destination sends the data + /// + [Config] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOG)] + public string Host { get; set; } + + #endregion + + #region Protected + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + + //throws on bad host spec + AgniSystem.Metabase.CatalogReg.NavigateHost(Host); + } + + public override void Close() + { + base.Close(); + DisposeAndNull(ref m_Client); + } + + protected override void DoSend(Message entry) + { + var msg = entry.ThisOrNewSafeWrappedException(false); + + try + { + ensureClient(); + m_Client.Async_SendLog(msg); + } + catch (Exception error) + { + throw new LogArchiveException("{0}.DoSend: {1}".Args(GetType().Name, error.ToMessageWithType()), error); + } + } + #endregion + + #region Private + + private void ensureClient() + { + var hn = this.Host; + if (m_Client == null && !hn.EqualsOrdIgnoreCase(m_HostName)) + { + m_Client = ServiceClientHub.New(hn); + m_HostName = hn; + } + } + + #endregion + } +} diff --git a/src/Agni/Log/AgniZoneDestination.cs b/src/Agni/Log/AgniZoneDestination.cs new file mode 100644 index 0000000..d85a4f4 --- /dev/null +++ b/src/Agni/Log/AgniZoneDestination.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Log; +using NFX.Log.Destinations; + +namespace Agni.Log +{ + /// + /// Sends log messages to parent zone governor + /// + public sealed class AgniZoneDestination : Destination + { + public const int DEFAULT_BUF_SIZE = 15; + public const int MAX_BUF_SIZE = 0xff; + public const int FLUSH_INTERVAL_MS = 7341; + + + public AgniZoneDestination() : base() { } + public AgniZoneDestination(string name) : base(name) { } + + + private List m_Buf = new List(DEFAULT_BUF_SIZE); + private int m_BufSize = DEFAULT_BUF_SIZE; + private DateTime m_LastFlush; + private int m_ZGovCallTimeoutMs; + + + /// + /// Overrides default service timeout when set to value greater than 0 + /// + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_GLUE)] + public int ZGovCallTimeoutMs + { + get { return m_ZGovCallTimeoutMs;} + set { m_ZGovCallTimeoutMs = value >0 ? value : 0;} + } + + public override void Close() + { + flush(); + base.Close(); + } + + protected override void DoSend(Message entry) + { + var message = entry.Clone(); + if (message.Exception != null) + message.Exception = new WrappedException(new WrappedExceptionData(message.Exception, captureStack: false)); + m_Buf.Add(message); + if (m_Buf.Count > m_BufSize) flush(); + } + + protected override void DoPulse() + { + base.DoPulse(); + + var flushEvery = FLUSH_INTERVAL_MS + NFX.ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, 5000); + if ((Service.Now - m_LastFlush).TotalMilliseconds > flushEvery) + flush(); + } + + private void flush() + { + m_LastFlush = Service.Now; + + if (m_Buf.Count == 0) return; + if (!AgniSystem.IsMetabase) return; + + try + { + var myHost = AgniSystem.HostName; + var zgHost = AgniSystem.ParentZoneGovernorPrimaryHostName; + if (zgHost.IsNullOrWhiteSpace()) return; + + //TODO Cache the client instance, do not create client on every call + using (var client = Contracts.ServiceClientHub.New( zgHost )) + { + if (m_ZGovCallTimeoutMs>0) client.TimeoutMs = m_ZGovCallTimeoutMs; + var expect = client.SendLog(myHost, AgniSystem.MetabaseApplicationName, m_Buf.ToArray()); + if (expect > MAX_BUF_SIZE) expect = MAX_BUF_SIZE; + if (expect < 1) expect = 1; + m_BufSize = expect; + } + } + catch (Exception error) + { + log(MessageType.Error, ".flush()", StringConsts.LOG_SEND_TOP_LOST_ERROR.Args(error.ToMessageWithType())); + } + finally + { + m_Buf.Clear(); + } + } + + private void log(MessageType type, string from, string text, Exception error = null, Guid? related = null) + { + var msg = new NFX.Log.Message + { + Type = type, + Topic = SysConsts.LOG_TOPIC_INSTRUMENTATION, + From = "{0}.{1}".Args(GetType().FullName, from), + Text = text, + Exception = error + }; + + if (related.HasValue) msg.RelatedTo = related.Value; + + App.Log.Write(msg); + } + + } +} diff --git a/src/Agni/Log/Exceptions.cs b/src/Agni/Log/Exceptions.cs new file mode 100644 index 0000000..86034b3 --- /dev/null +++ b/src/Agni/Log/Exceptions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.Log +{ + /// + /// Thrown to indicate log archive related problems + /// + [Serializable] + public class LogArchiveException : AgniException + { + public LogArchiveException() : base() {} + public LogArchiveException(string message) : base(message) {} + public LogArchiveException(string message, Exception inner) : base(message, inner) { } + protected LogArchiveException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Log/Server/LogArchiveDimensionsMapper.cs b/src/Agni/Log/Server/LogArchiveDimensionsMapper.cs new file mode 100644 index 0000000..1395377 --- /dev/null +++ b/src/Agni/Log/Server/LogArchiveDimensionsMapper.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; + +using NFX.ApplicationModel; +using NFX.Environment; + +namespace Agni.Log.Server +{ + /// + /// Maps archive dimensions to/from model of the particular business system + /// + public class LogArchiveDimensionsMapper : ApplicationComponent + { + public LogArchiveDimensionsMapper(LogReceiverService director, IConfigSectionNode node) : base(director) + { + ConfigAttribute.Apply(this, node); + } + + public virtual Dictionary StoreMap(string archiveDimensions) { return null; } + public virtual Dictionary FilterMap(string archiveDimensions) { return null; } + } +} diff --git a/src/Agni/Log/Server/LogArchiveStore.cs b/src/Agni/Log/Server/LogArchiveStore.cs new file mode 100644 index 0000000..972e8a0 --- /dev/null +++ b/src/Agni/Log/Server/LogArchiveStore.cs @@ -0,0 +1,96 @@ +using System; + +using NFX; +using NFX.Log; +using NFX.ApplicationModel; +using NFX.Environment; +using System.Collections.Generic; + +namespace Agni.Log.Server +{ + /// + /// Represents a base for entities that archive log data + /// + public abstract class LogArchiveStore : ApplicationComponent + { + protected LogArchiveStore(LogReceiverService director, LogArchiveDimensionsMapper mapper, IConfigSectionNode node) : base(director) + { + ConfigAttribute.Apply(this, node); + m_Mapper = mapper; + } + + protected override void Destructor() + { + base.Destructor(); + DisposeAndNull(ref m_Mapper); + } + + private LogArchiveDimensionsMapper m_Mapper; + + /// + /// Maps archive dimensions to/from model of the particular business system + /// + public LogArchiveDimensionsMapper Mapper { get { return m_Mapper; } } + + /// + /// References service that this store is under + /// + public LogReceiverService ArchiveService { get { return (LogReceiverService)ComponentDirector;} } + + /// + /// Returns log message by ID + /// + public virtual Message GetByID(Guid id, string channel = null) + { + Message result; + if (!TryGetByID(id, out result, channel)) + throw new LogArchiveException(StringConsts.LOG_ARCHIVE_MESSAGE_NOT_FOUND_ERROR.Args(id)); + return result; + } + + /// + /// Starts transaction represented by return object + /// + public abstract object BeginTransaction(); + + /// + /// Commits the transaction started with BeginTransaction + /// + public abstract void CommitTransaction(object transaction); + + /// + /// Rolls back the transaction started with BeginTransaction + /// + public abstract void RollbackTransaction(object transaction); + + /// + /// Writes message to the store within transaction context + /// + public abstract void Put(Message message, object transaction); + + /// + /// Tries to fetch message by ID. Returns true if found + /// + public abstract bool TryGetByID(Guid id, out Message message, string channel = null); + + /// + /// Returns enumerable of messages according to dimension filter in laconic format + /// + public abstract IEnumerable List(string archiveDimensionsFilter, DateTime startDate, DateTime endDate, MessageType? type = null, + string host = null, string channel = null, string topic = null, + Guid? relatedTo = null, + int skipCount = 0); + + + + protected Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + return ArchiveService.Log(type, "{0}.{1}".Args(GetType().Name, from), message, error, relatedMessageID, parameters); + } + } +} diff --git a/src/Agni/Log/Server/LogReceiverService.cs b/src/Agni/Log/Server/LogReceiverService.cs new file mode 100644 index 0000000..804cc34 --- /dev/null +++ b/src/Agni/Log/Server/LogReceiverService.cs @@ -0,0 +1,199 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Environment; +using NFX.Log; +using NFX.ServiceModel; + +using Agni.MDB; + +namespace Agni.Log.Server +{ + /// + /// Glue adapter for Contracts.ILogReceiver + /// + public sealed class LogReceiverServer : Contracts.ILogReceiver + { + public void SendLog(Message data) + { + LogReceiverService.Instance.SendLog(data); + } + + public Message GetByID(Guid id, string channel = null) + { + return LogReceiverService.Instance.GetByID(id, channel); + } + + public IEnumerable List(string archiveDimensionsFilter, DateTime startDate, DateTime endDate, MessageType? type = null, + string host = null, string channel = null, string topic = null, + Guid? relatedTo = null, int skipCount = 0) + { + return LogReceiverService.Instance.List(archiveDimensionsFilter, startDate, endDate, type, host, channel, topic, relatedTo, skipCount); + } + } + + /// + /// Provides server implementation of Contracts.ILogReceiver + /// + public sealed class LogReceiverService : ServiceWithInstrumentationBase, Contracts.ILogReceiver + { + #region CONSTS + public const string CONFIG_ARCHIVE_MAPPER_SECTION = "archive-mapper"; + public const string CONFIG_ARCHIVE_STORE_SECTION = "archive-store"; + + public const MessageType DEFAULT_LOG_LEVEL = MessageType.Warning; + #endregion + + #region STATIC/.ctor + private static object s_Lock = new object(); + private static volatile LogReceiverService s_Instance; + + internal static LogReceiverService Instance + { + get + { + var instance = s_Instance; + if (instance == null) + throw new LogArchiveException("{0} is not allocated".Args(typeof(LogReceiverService).FullName)); + return instance; + } + } + + public LogReceiverService() : this(null) { } + + public LogReceiverService(object director) : base(director) + { + LogLevel = MessageType.Warning; + + lock (s_Lock) + { + if (s_Instance != null) + throw new LogArchiveException("{0} is already allocated".Args(typeof(LogReceiverService).FullName)); + + s_Instance = this; + } + } + + protected override void Destructor() + { + base.Destructor(); + DisposeAndNull(ref m_ArchiveStore); + s_Instance = null; + } + #endregion + + #region Fields + private LogArchiveStore m_ArchiveStore; + #endregion + + #region Properties + [Config] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOG, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled { get; set; } + + [Config(Default = DEFAULT_LOG_LEVEL)] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_WORKER, CoreConsts.EXT_PARAM_GROUP_LOG)] + public MessageType LogLevel { get; set; } + #endregion + + #region Public + public void SendLog(Message data) + { + var transaction = m_ArchiveStore.BeginTransaction(); + try + { + m_ArchiveStore.Put(data, transaction); + m_ArchiveStore.CommitTransaction(transaction); + } + catch (Exception error) + { + m_ArchiveStore.RollbackTransaction(transaction); + + Log(MessageType.CatastrophicError, "put('{0}', '{1}', '{2}')".Args(data.Host, data.From, data.Guid), error.ToMessageWithType(), error); + + throw new LogArchiveException(StringConsts.LOG_ARCHIVE_PUT_TX_BODY_ERROR.Args(m_ArchiveStore.GetType().Name, error.ToMessageWithType()), error); + } + } + + public Message GetByID(Guid id, string channel = null) + { + return m_ArchiveStore.GetByID(id, channel); + } + + public IEnumerable List(string archiveDimensionsFilter, DateTime startDate, DateTime endDate, MessageType? type = null, + string host = null, string channel = null, string topic = null, + Guid? relatedTo = null, int skipCount = 0) + { + return m_ArchiveStore.List(archiveDimensionsFilter, startDate, endDate, type, host, channel, topic, relatedTo, skipCount); + } + + public Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + var logMessage = new Message + { + Topic = SysConsts.LOG_TOPIC_WORKER, + Text = message ?? string.Empty, + Type = type, + From = "{0}.{1}".Args(this.GetType().Name, from), + Exception = error, + Parameters = parameters + }; + if (relatedMessageID.HasValue) logMessage.RelatedTo = relatedMessageID.Value; + + App.Log.Write(logMessage); + + return logMessage.Guid; + } + #endregion + + #region Protected + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + + LogArchiveDimensionsMapper mapper = null; + + DisposeAndNull(ref m_ArchiveStore); + + var mapperNode = node[CONFIG_ARCHIVE_MAPPER_SECTION]; + mapper = FactoryUtils.Make(mapperNode, defaultType: typeof(LogArchiveDimensionsMapper), args: new object[] { this, mapperNode }); + + var storeNode = node[CONFIG_ARCHIVE_STORE_SECTION]; + if (storeNode.Exists) + m_ArchiveStore = FactoryUtils.Make(storeNode, args: new object[] { this, mapper, storeNode }); + } + + protected override void DoStart() + { + if (m_ArchiveStore == null) + throw new LogArchiveException("{0} does not have archive store injected".Args(GetType().Name)); + + if (m_ArchiveStore is IService) ((IService)m_ArchiveStore).Start(); + base.DoStart(); + } + + protected override void DoSignalStop() + { + if (m_ArchiveStore is IService) ((IService)m_ArchiveStore).SignalStop(); + base.DoSignalStop(); + } + + protected override void DoWaitForCompleteStop() + { + if (m_ArchiveStore is IService) ((IService)m_ArchiveStore).WaitForCompleteStop(); + base.DoWaitForCompleteStop(); + } + #endregion + } +} diff --git a/src/Agni/MDB/CRUDOperations.cs b/src/Agni/MDB/CRUDOperations.cs new file mode 100644 index 0000000..6dd80e0 --- /dev/null +++ b/src/Agni/MDB/CRUDOperations.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Data; +using System.Threading.Tasks; + +using NFX; +using NFX.DataAccess; +using NFX.DataAccess.CRUD; + +namespace Agni.MDB +{ + /// + /// Provides facade for ICrudOperations and ICRUDTransactionOperations + /// executed against the particular shard returned by the MDB areas partition / routing + /// + public struct CRUDOperations : ICRUDOperations, ICRUDTransactionOperations + { + + internal CRUDOperations(MDBArea.Partition.Shard shard) + { + this.Shard = shard; + } + + /// + /// The shard that services this instance + /// + public readonly MDBArea.Partition.Shard Shard; + + + + public int Delete(Row row, IDataStoreKey key = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.Delete(row, key); + } + + public Task DeleteAsync(Row row, IDataStoreKey key = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.DeleteAsync(row, key); + } + + public int ExecuteWithoutFetch(params Query[] queries) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.ExecuteWithoutFetch(queries); + } + + public Task ExecuteWithoutFetchAsync(params Query[] queries) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.ExecuteWithoutFetchAsync(queries); + } + + public Schema GetSchema(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.GetSchema(query); + } + + public Task GetSchemaAsync(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.GetSchemaAsync(query); + } + + public int Insert(Row row, FieldFilterFunc filter = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.Insert(row, filter); + } + + public Task InsertAsync(Row row, FieldFilterFunc filter = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.InsertAsync(row, filter); + } + + public List Load(params Query[] queries) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.Load(queries); + } + + public Task> LoadAsync(params Query[] queries) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.LoadAsync(queries); + } + + public Row LoadOneRow(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.LoadOneRow(query); + } + + public Task LoadOneRowAsync(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.LoadOneRowAsync(query); + } + + public RowsetBase LoadOneRowset(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.LoadOneRowset(query); + } + + public Task LoadOneRowsetAsync(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.LoadOneRowsetAsync(query); + } + + public Cursor OpenCursor(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.OpenCursor(query); + } + + public Task OpenCursorAsync(Query query) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.OpenCursorAsync(query); + } + + public int Save(params RowsetBase[] rowsets) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.Save(rowsets); + } + + public Task SaveAsync(params RowsetBase[] rowsets) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.SaveAsync(rowsets); + } + + public bool SupportsTrueAsynchrony + { + get { return Shard.Area.PhysicalDataStore.SupportsTrueAsynchrony; } + } + + public int Update(Row row, IDataStoreKey key = null, FieldFilterFunc filter = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.Update(row, key, filter); + } + + public Task UpdateAsync(Row row, IDataStoreKey key = null, FieldFilterFunc filter = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.UpdateAsync(row, key, filter); + } + + public int Upsert(Row row, FieldFilterFunc filter = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.Upsert(row, filter); + } + + public Task UpsertAsync(Row row, FieldFilterFunc filter = null) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.UpsertAsync(row, filter); + } + + public CRUDTransaction BeginTransaction(IsolationLevel iso = IsolationLevel.ReadCommitted, TransactionDisposeBehavior behavior = TransactionDisposeBehavior.CommitOnDispose) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.BeginTransaction(iso, behavior); + } + + public Task BeginTransactionAsync(IsolationLevel iso = IsolationLevel.ReadCommitted, TransactionDisposeBehavior behavior = TransactionDisposeBehavior.CommitOnDispose) + { + using(new CRUDOperationCallContext{ConnectString = Shard.EffectiveConnectionString}) + return Shard.Area.PhysicalDataStore.BeginTransactionAsync(iso, behavior); + } + + public bool SupportsTransactions + { + get { return Shard.Area.PhysicalDataStore.SupportsTransactions; } + } + } +} diff --git a/src/Agni/MDB/ConnectStringBuilders.cs b/src/Agni/MDB/ConnectStringBuilders.cs new file mode 100644 index 0000000..a7f257f --- /dev/null +++ b/src/Agni/MDB/ConnectStringBuilders.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; + +namespace Agni.MDB +{ + public abstract class ConnectStringBuilderBase : IConfigStringBuilder + { + [Config] public string Host; + [Config] public string Network; + [Config] public string Service; + [Config] public string Binding; + + protected NFX.Glue.Node m_ResolvedNode; + protected string m_ResolvedService; + + public abstract string BuildString(); + + public void Configure(IConfigSectionNode node) + { + ConfigAttribute.Apply(this, node); + m_ResolvedNode = AgniSystem.Metabase.ResolveNetworkService(Host, Network, Service, Binding); + } + } + + public sealed class MongoDBConnectStringBuilder : ConnectStringBuilderBase + { + [Config] public string DB; + //todo specify more detailed parameters in future + + public override string BuildString() + { + return "mongo{{server='{0}:{1}' db='{2}'}}".Args(m_ResolvedNode.Host, m_ResolvedNode.Service, DB); + } + } + + public sealed class MySqlConnectStringBuilder : ConnectStringBuilderBase + { + [Config] public string DB; + [Config] public string UserID; + [Config] public string Password; + [Config] public string ConnectionLifeTimeSec; + //todo specify more detailed parameters in future + + public override string BuildString() + { + return "Server={0};Port={1};Database={2};Uid={3};Pwd={4};ConnectionLifeTime={5};".Args(m_ResolvedNode.Host, m_ResolvedNode.Service, DB, UserID, Password, ConnectionLifeTimeSec); + } + } +} diff --git a/src/Agni/MDB/Exceptions.cs b/src/Agni/MDB/Exceptions.cs new file mode 100644 index 0000000..c49090a --- /dev/null +++ b/src/Agni/MDB/Exceptions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.MDB +{ + /// + /// Thrown to indicate MDB-related problems + /// + [Serializable] + public class MDBException : AgniException + { + public MDBException() : base() { } + public MDBException(string message) : base(message) { } + public MDBException(string message, Exception inner) : base(message, inner) { } + protected MDBException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/MDB/IMDBDataStore.cs b/src/Agni/MDB/IMDBDataStore.cs new file mode 100644 index 0000000..e493339 --- /dev/null +++ b/src/Agni/MDB/IMDBDataStore.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.DataAccess; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; +using NFX.ApplicationModel.Pile; + + +namespace Agni.MDB +{ + /// + /// Stipulates a contract for MDBDataStore + /// + public interface IMDBDataStore : IDataStore + { + string SchemaName{ get;} + string BankName{ get;} + + IGDIDProvider GDIDGenerator{ get;} + + MDBCentralArea CentralArea{ get;} + + IRegistry Areas{ get;} + + /// + /// Pile big memory cache + /// + ICache Cache { get;} + + /// + /// Returns CRUDOperations facade connected to the appropriate database server within the named area + /// which services the shard computed from the briefcase GDID + /// + CRUDOperations PartitionedOperationsFor(string areaName, GDID idBriefcase); + + /// + /// Returns CRUDOperations facade connected to the appropriate shard within the central area as + /// determined by the the shardingID + /// + CRUDOperations CentralOperationsFor(object shardingID); + } +} diff --git a/src/Agni/MDB/LongDataOperation.cs b/src/Agni/MDB/LongDataOperation.cs new file mode 100644 index 0000000..8cdfd1c --- /dev/null +++ b/src/Agni/MDB/LongDataOperation.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Agni.MDB +{ + /// + /// Gets called by various routines that report long-running data operation progress and allow to cancel further processing. + /// + public delegate void LongDataOperationCallback(LongDataOperation operation); + + /// + /// Provides a context for long data operation, such as fetching data from many shards. + /// Callers may cancel operation and track overall progress via a LongDataOperationCallback. + /// This is conceptually similar to CancelationToken + /// + public sealed class LongDataOperation + { + public LongDataOperation() + { + m_StartTimeUTC = NFX.App.TimeSource.UTCNow; + } + + private DateTime m_StartTimeUTC; + private volatile int m_Total; + private int m_Current; + private volatile bool m_Canceled; + + private volatile bool m_HasResult; + private TResult m_Result; + + + /// + /// Allows to attach arbitrary object context + /// + public object Context; + + + /// + /// Returns the UTC timestamp of this operation + /// + public DateTime StartTimeUTC{ get{ return m_StartTimeUTC;} } + + + /// + /// The total amount of work + /// + public int Total{ get{return m_Total;}} + + /// + /// The current progress of work out of Total + /// + public int Current{ get{ return Thread.VolatileRead(ref m_Current);} } + + /// + /// True if operation was canceled via Cancel() + /// + public bool Canceled { get{ return m_Canceled;}} + + /// + /// True if operation was completed via a call to SetResult() + /// + public bool HasResult { get{ return m_HasResult;}} + + /// + /// The final result of operation which is available after completion or cancel + /// + public TResult Result{ get{ return m_Result;}} + + /// + /// Sets total amount of work + /// + public void SetTotal(int total){ m_Total = total;} + + /// + /// Sets current progress out of total + /// + public void SetCurrent(int current){ m_Current = current;} + + /// + /// Sets current progress by +1 via interlocked + /// + public int AdvanceCurrent() + { + return Interlocked.Increment(ref m_Current); + } + + /// + /// Call to cancel the further processing + /// + public void Cancel() { m_Canceled = true; } + + /// + /// Call to set result + /// + public void SetResult(TResult result) + { + m_HasResult = true; + m_Result = result; + Thread.MemoryBarrier(); + } + } +} diff --git a/src/Agni/MDB/MDBAppComponent.cs b/src/Agni/MDB/MDBAppComponent.cs new file mode 100644 index 0000000..5bac5d0 --- /dev/null +++ b/src/Agni/MDB/MDBAppComponent.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; + +namespace Agni.MDB +{ + /// + /// Base type for externally parametrized app components that are used throughout MDB implementation + /// + public abstract class MDBAppComponent : ApplicationComponent, IExternallyParameterized + { + protected MDBAppComponent(object director) : base(director) + { + + } + + /// + /// Returns named parameters that can be used to control this component + /// + public virtual IEnumerable> ExternalParameters{ get { return ExternalParameterAttribute.GetParameters(this); } } + + /// + /// Returns named parameters that can be used to control this component + /// + public virtual IEnumerable> ExternalParametersForGroups(params string[] groups) + { + return ExternalParameterAttribute.GetParameters(this, groups); + } + + /// + /// Gets external parameter value returning true if parameter was found + /// + public virtual bool ExternalGetParameter(string name, out object value, params string[] groups) + { + return ExternalParameterAttribute.GetParameter(this, name, out value, groups); + } + + /// + /// Sets external parameter value returning true if parameter was found and set + /// + public virtual bool ExternalSetParameter(string name, object value, params string[] groups) + { + return ExternalParameterAttribute.SetParameter(this, name, value, groups); + } + + } +} diff --git a/src/Agni/MDB/MDBArea.Partition.cs b/src/Agni/MDB/MDBArea.Partition.cs new file mode 100644 index 0000000..aa68be1 --- /dev/null +++ b/src/Agni/MDB/MDBArea.Partition.cs @@ -0,0 +1,269 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; + +namespace Agni.MDB +{ + + public abstract partial class MDBArea + { + /// + /// Represents partition within the area + /// + public sealed class Partition : MDBAppComponent, IComparable, IEquatable, IComparable + { + + /// + /// Denotes connection types Primary/Secondary + /// + public enum ShardBackendConnection{Primary=0, Secondary} + + /// + /// Represents a SHARD information for the DB particular host + /// + public sealed class Shard : MDBAppComponent, IComparable, IComparable, IEquatable + { + internal Shard(Partition part, IConfigSectionNode config) : base(part) + { + m_Partition = part; + m_Order = config.AttrByName(CONFIG_ORDER_ATTR).ValueAsInt(0); + + PrimaryHostConnectString = ConfigStringBuilder.Build(config, CONFIG_PRIMARY_CONNECT_STRING_ATTR); + SecondaryHostConnectString = ConfigStringBuilder.Build(config, CONFIG_SECONDARY_CONNECT_STRING_ATTR); + + if (PrimaryHostConnectString.IsNullOrWhiteSpace()) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_SHARD_CSTR_ERROR.Args(part.Area.Name, part.Order, CONFIG_PRIMARY_CONNECT_STRING_ATTR)); + + if (SecondaryHostConnectString.IsNullOrWhiteSpace()) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_SHARD_CSTR_ERROR.Args(part.Area.Name, part.Order, CONFIG_SECONDARY_CONNECT_STRING_ATTR)); + + } + + private Partition m_Partition; + private int m_Order; + + private ShardBackendConnection m_ConnectionType; + + + public MDBArea Area { get{ return m_Partition.Area;}} + public Partition Partition { get{ return m_Partition;}} + public int Order { get{ return m_Order;}} + + + public readonly string PrimaryHostConnectString; + public readonly string SecondaryHostConnectString; + + + /// + /// Returns Primary then secondary connect strings + /// + public IEnumerable ConnectStrings + { + get + { + yield return PrimaryHostConnectString; + yield return SecondaryHostConnectString; + } + } + + + /// + /// Returns either primary or secondary connect string + /// depending on connection type + /// + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_DATA)] + public ShardBackendConnection ConnectionType + { + get + { + return m_ConnectionType; + } + set + { + if (m_ConnectionType!=value) + { + m_ConnectionType = value; + //todo Instrument + } + } + } + + /// + /// Returns either primary or secondary connect string + /// depending on connection type + /// + public string EffectiveConnectionString + { + get + { + return m_ConnectionType==ShardBackendConnection.Primary ? + PrimaryHostConnectString : SecondaryHostConnectString; + } + } + + /// + /// Returns new wrapper for operations on this shard + /// + public CRUDOperations CRUDOperations + { + get{ return new CRUDOperations(this);} + } + + + public override bool Equals(object obj) + { + var other = obj as Shard; + return this.Equals(other); + } + + public bool Equals(Shard other) + { + if (other==null) return false; + return object.ReferenceEquals(this.m_Partition, other.m_Partition) && + this.m_Order == other.m_Order; + } + + public override int GetHashCode() + { + return m_Order.GetHashCode(); + } + + public override string ToString() + { + return "{0}->[{1}]".Args(m_Partition, m_Order); + } + + public int CompareTo(object obj) + { + var other = obj as Shard; + return this.CompareTo(other); + } + + public int CompareTo(Shard other) + { + if (other==null) return +1; + if (!object.ReferenceEquals(this.m_Partition, other.m_Partition)) return +1; + return this.m_Order.CompareTo(other.Order); + } + } + + + internal Partition(MDBArea area, IConfigSectionNode config) : base(area) + { + m_Area = area; + + if (area is MDBCentralArea) + { + m_StartGDID = new GDID(0, 0, 0); + } + else + { + var sgdid = config.AttrByName(CONFIG_START_GDID_ATTR).ValueAsString(); + + if (!GDID.TryParse(sgdid, out m_StartGDID)) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_PARTITION_GDID_ERROR.Args(area.Name, sgdid, "unparsable, expected 'era:0:counter'")); + + if (m_StartGDID.Authority!=0) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_PARTITION_GDID_ERROR.Args(area.Name, sgdid, "authority segment must be 0")); + } + //Shards + var shards = new List(); + foreach(var snode in config.Children.Where( cn => cn.IsSameName(CONFIG_SHARD_SECTION))) + { + var shard = new Shard(this, snode); + shards.Add( shard ); + } + + if (shards.Count==0) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_NO_PARTITION_SHARDS_ERROR.Args(area.Name, this.m_StartGDID)); + + if (shards.Count != shards.Distinct().Count()) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_DUPLICATE_PARTITION_SHARDS_ERROR.Args(area.Name, this.m_StartGDID)); + + shards.Sort(); + m_Shards = shards.ToArray(); + } + + private MDBArea m_Area; + private int m_Order; internal void __setOrder(int num) { m_Order = num;} + private GDID m_StartGDID; + private Shard[] m_Shards; + + + public MDBArea Area { get{ return m_Area; }} + public int Order { get{ return m_Order; }} + public GDID StartGDID { get{ return m_StartGDID;}} + public Shard[] Shards { get{ return m_Shards; }} + + + + /// + /// Returns CRUDOperations facade connected to the appropriate database server within this named area and partition + /// which services the shard computed from sharding id + /// + public CRUDOperations ShardedOperationsFor(object idSharding) + { + var shard = GetShardForID(idSharding); + return new CRUDOperations(shard); + } + + /// + /// Finds appropriate shard for ID. See MDB.ShardingUtils + /// + public Shard GetShardForID(object idSharding) + { + ulong subid = MDB.ShardingUtils.ObjectToShardingID(idSharding); + + return Shards[ subid % (ulong)Shards.Length ]; + } + + public override bool Equals(object obj) + { + var other = obj as Partition; + return this.Equals(other); + } + + public bool Equals(Partition other) + { + if (other==null) return false; + return + object.ReferenceEquals(this.m_Area, other.m_Area) && + GDIDRangeComparer.Instance.Compare(this.StartGDID, other.StartGDID)==0; + } + + public int CompareTo(object obj) + { + var other = obj as Partition; + return this.CompareTo(other); + } + + public int CompareTo(Partition other) + { + if (other==null) return +1; + if (!object.ReferenceEquals(this.m_Area, other.m_Area)) return +1; + return GDIDRangeComparer.Instance.Compare(this.StartGDID, other.StartGDID); + } + + public override int GetHashCode() + { + return this.m_StartGDID.GetHashCode(); + } + + public override string ToString() + { + return "Area('{0}').Partition('{1}' starting '{2}' shards {3})".Args(m_Area.Name, m_Order, m_StartGDID, m_Shards.Length); + } + + + + }//Partition + }//MDBArea + +} diff --git a/src/Agni/MDB/MDBArea.cs b/src/Agni/MDB/MDBArea.cs new file mode 100644 index 0000000..5a54bcc --- /dev/null +++ b/src/Agni/MDB/MDBArea.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; + +namespace Agni.MDB +{ + /// + /// Represents a general ancestor for CENTRAL or partitioned areas + /// + public abstract partial class MDBArea : MDBAppComponent, INamed + { + #region CONSTS + public const string CENTRAL_AREA_NAME = "CENTRAL"; + + public const string CONFIG_AREA_SECTION = "area"; + public const string CONFIG_PARTITION_SECTION = "partition"; + public const string CONFIG_SHARD_SECTION = "shard"; + public const string CONFIG_START_GDID_ATTR = "start-gdid"; + public const string CONFIG_ORDER_ATTR = "order"; + + public const string CONFIG_PRIMARY_CONNECT_STRING_ATTR = "primary-cs"; + public const string CONFIG_SECONDARY_CONNECT_STRING_ATTR = "secondary-cs"; + #endregion + + #region .ctor + protected MDBArea(MDBDataStore store, IConfigSectionNode node) : base(store) + { + if (store==null || node==null || !node.Exists) + throw new MDBException(StringConsts.ARGUMENT_ERROR+"MDBArea.ctor(store==null|node==null|!Exists)"); + + + ConfigAttribute.Apply(this, node); + + var dsnode = node[CommonApplicationLogic.CONFIG_DATA_STORE_SECTION]; + if (!dsnode.Exists) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_NO_DATASTORE_ERROR.Args(node.RootPath)); + + m_PhysicalDataStore = FactoryUtils.MakeAndConfigure(dsnode, args: new []{ this }); + } + + + protected override void Destructor() + { + DisposableObject.DisposeAndNull(ref m_PhysicalDataStore); + base.Destructor(); + } + #endregion + + #region Fields + private ICRUDDataStoreImplementation m_PhysicalDataStore; + #endregion + + #region Properties + + public MDBDataStore Store{ get { return (MDBDataStore)base.ComponentDirector;}} + public abstract string Name{ get;} + + /// + /// Physical data store that services the area + /// + public ICRUDDataStoreImplementation PhysicalDataStore + { + get { return m_PhysicalDataStore;} + } + + /// + /// Returns all ordered partitions of the area - one for central, or all actual partitions for partitioned area + /// + public abstract IEnumerable AllPartitions { get;} + + /// + /// Returns all ordered shards within ordered partitions + /// + public abstract IEnumerable AllShards { get;} + + #endregion + } + +} diff --git a/src/Agni/MDB/MDBCentralArea.cs b/src/Agni/MDB/MDBCentralArea.cs new file mode 100644 index 0000000..c7f4a9e --- /dev/null +++ b/src/Agni/MDB/MDBCentralArea.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; + +namespace Agni.MDB +{ + /// + /// Represents a single central area that has one central partition + /// + public sealed class MDBCentralArea : MDBArea + { + + internal MDBCentralArea(MDBDataStore store, IConfigSectionNode node) : base(store, node) + { + m_CentralPartition = new Partition(this, node); + } + + //Central has only one partition + private Partition m_CentralPartition; + + + public override string Name{ get{ return CENTRAL_AREA_NAME;}} + + /// + /// Returns a single partition of MDBCentralArea + /// + public Partition CentralPartition{ get{ return m_CentralPartition;}} + + public override IEnumerable AllPartitions { get { yield return m_CentralPartition;} } + + public override IEnumerable AllShards { get { return m_CentralPartition.Shards;}} + + + /// + /// Returns CRUDOperations facade connected to the appropriate database server within the CENTRAL area's partition + /// which services the shard computed from sharding id + /// + public CRUDOperations ShardedOperationsFor(object idSharding) + { + return m_CentralPartition.ShardedOperationsFor(idSharding); + } + + } + +} diff --git a/src/Agni/MDB/MDBDataStore.cs b/src/Agni/MDB/MDBDataStore.cs new file mode 100644 index 0000000..603c2df --- /dev/null +++ b/src/Agni/MDB/MDBDataStore.cs @@ -0,0 +1,284 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.ApplicationModel; +using NFX.ApplicationModel.Pile; +using NFX.Environment; +using NFX.DataAccess; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; +using NFX.ServiceModel; + +using Agni.Identification; + +namespace Agni.MDB +{ + /// + /// Represents a MDB data store - consisting of areas with partitions + /// + public class MDBDataStore : ServiceWithInstrumentationBase, IMDBDataStore, IDataStoreImplementation + { + + #region .ctor + public MDBDataStore() : this(null, null) { } + public MDBDataStore(string name, object director) : base(director) + { + Name = name.IsNullOrWhiteSpace() ? Guid.NewGuid().ToString() : name; + } + #endregion + + + #region Private Fields + + public string m_SchemaName; + public string m_BankName; + + private bool m_InstrumentationEnabled; + private StoreLogLevel m_LogLevel; + private string m_TargetName; + + + private GDIDGenerator m_GDIDGenerator; + + private MDBCentralArea m_CentralArea; + private Registry m_Areas = new Registry(); + + private LocalCache m_Cache; + + #endregion + + #region Properties + + [Config] + public string SchemaName + { + get{ return m_SchemaName;} + set + { + CheckServiceInactive(); + m_SchemaName = value; + } + } + + [Config] + public string BankName + { + get{ return m_BankName;} + set + { + CheckServiceInactive(); + m_BankName = value; + } + } + + + /// + /// Generates GDIDs + /// + public IGDIDProvider GDIDGenerator { get {return m_GDIDGenerator;} } + + + + [Config(Default=false)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_DATA, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled{ get{return m_InstrumentationEnabled;} set{m_InstrumentationEnabled = value;}} + + + [Config] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_DATA)] + public StoreLogLevel LogLevel + { + get { return m_LogLevel; } + set { m_LogLevel = value;} + } + + [Config] + public string TargetName + { + get { return m_TargetName; } + set + { + CheckServiceInactive(); + m_TargetName = value; + } + } + + /// + /// Returns the central area - the one that does not partition data + /// + public MDBCentralArea CentralArea { get{ return m_CentralArea;}} + + /// + /// Returns the areas (including the central area) + /// + public IRegistry Areas { get{ return m_Areas;}} + + + /// + /// Pile big memory cache. The store does not use this by itself, the implementors/derived stores should + /// use the cache to store business-specific data + /// + public ICache Cache + { + get{ return m_Cache;} + } + + #endregion + + #region Public + + public void TestConnection() + { + throw new NotImplementedException(); + } + + /// + /// Returns CRUDOperations facade connected to the appropriate database server within the named area + /// which services the shard computed from the briefcase GDID + /// + public CRUDOperations PartitionedOperationsFor(string areaName, GDID idBriefcase) + { + CheckServiceActive(); + var area = m_Areas[areaName] as MDBPartitionedArea; + if (area==null) + throw new MDBException(StringConsts.MDB_PARTITIONED_AREA_MISSING_ERROR.Args(areaName)); + + return area.PartitionedOperationsFor(idBriefcase); + } + + ///// + ///// Returns CRUDOperations facade connected to the appropriate database server within the named area + ///// which services the shard computed from the briefcase GDID + ///// + //public CRUDOperations PartitionedOperationsFor(string areaName, object shardingID, int volumeOffset = -1) + //{ + // CheckServiceActive(); + // var area = m_Areas[areaName] as MDBPartitionedArea; + // if (area == null) + // throw new MDBException(StringConsts.MDB_PARTITIONED_AREA_MISSING_ERROR.Args(areaName)); + + // #warning TODO + // var parts = area.AllPartitions; + // var count = parts.Count(); + // var volume = volumeOffset % count; + // if (volume < 0) + // volume = count - volume; + // return parts.Skip(volume).Single().ShardedOperationsFor(shardingID); + //} + + /// + /// Returns CRUDOperations facade connected to the appropriate shard within the central area as + /// determined by the the shardingID + /// + public CRUDOperations CentralOperationsFor(object shardingID) + { + CheckServiceActive(); + return m_CentralArea.ShardedOperationsFor(shardingID); + } + #endregion + + #region Protected + + public static string GetGDIDScopePrefix(string schemaName, string bankName) + { + return "{0}.{1}.".Args(schemaName, bankName); + } + + + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + if (node==null || !node.Exists) return; + + + m_CentralArea = null; + m_Areas.Clear(); + + var nodes = node.Children.Where( cn => cn.IsSameName(MDBArea.CONFIG_AREA_SECTION) && (cn.IsSameNameAttr(MDBArea.CENTRAL_AREA_NAME))); + if (!nodes.Any() || nodes.Count()>1) + throw new MDBException(StringConsts.MDB_STORE_CONFIG_MANY_CENTRAL_ERROR); + + var centralNode = nodes.First(); + m_CentralArea = new MDBCentralArea(this, centralNode); + + m_Areas.Register( m_CentralArea );//central area gets registered anyway + + //non-central - they are not required + nodes = node.Children.Where( cn => cn.IsSameName(MDBArea.CONFIG_AREA_SECTION) && (!cn.IsSameNameAttr(MDBArea.CENTRAL_AREA_NAME))); + foreach(var anode in nodes) + { + var area = new MDBPartitionedArea(this, anode); + m_Areas.Register( area ); + } + + + + } + + protected override void DoStart() + { + if (m_TargetName.IsNullOrWhiteSpace()) + throw new MDBException(StringConsts.MDB_STORE_CONFIG_NO_TARGET_NAME_ERROR); + + if (m_Areas.Count==0) + throw new MDBException(StringConsts.MDB_STORE_CONFIG_NO_AREAS_ERROR); + + + try + { + GDIDAuthorityService.CheckNameValidity(m_SchemaName); + GDIDAuthorityService.CheckNameValidity(m_BankName); + + var gdidScope = GetGDIDScopePrefix(m_SchemaName, m_BankName); + m_GDIDGenerator = new GDIDGenerator("GDIDGen({0})".Args(gdidScope), this, gdidScope, null); + if (AgniSystem.IsMetabase) + { + foreach(var ah in AgniSystem.Metabase.GDIDAuthorities) + { + m_GDIDGenerator.AuthorityHosts.Register(ah); + App.Log.Write( new NFX.Log.Message + { + Type = NFX.Log.MessageType.InfoD, + Topic = SysConsts.LOG_TOPIC_MDB, + From = GetType().FullName+".makeGDIDGen()", + Text = "Registered GDID authority host: "+ah.ToString() + }); + } + } + + } + catch(Exception error) + { + throw new MDBException(StringConsts.MDB_STORE_CONFIG_GDID_ERROR + error.ToMessageWithType()); + } + + try + { + m_Cache = new LocalCache("MDBDataStore::"+Name, this); + m_Cache.Pile = new DefaultPile(m_Cache, "MDBDataStore::Pile::"+Name); + m_Cache.Configure(null); + m_Cache.Start(); + } + catch + { + try { DisposableObject.DisposeAndNull(ref m_GDIDGenerator);} catch{} + try { DisposableObject.DisposeAndNull(ref m_Cache); } catch {} + throw; + } + } + + protected override void DoWaitForCompleteStop() + { + base.DoWaitForCompleteStop(); + DisposableObject.DisposeAndNull(ref m_GDIDGenerator); + DisposableObject.DisposeAndNull(ref m_Cache); + } + + #endregion + + } +} diff --git a/src/Agni/MDB/MDBPartitionedArea.cs b/src/Agni/MDB/MDBPartitionedArea.cs new file mode 100644 index 0000000..aed572b --- /dev/null +++ b/src/Agni/MDB/MDBPartitionedArea.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; + +namespace Agni.MDB +{ + /// + /// Represents a non-central area with partitions + /// + public sealed class MDBPartitionedArea : MDBArea + { + + internal MDBPartitionedArea(MDBDataStore store, IConfigSectionNode node) : base(store, node) + { + m_Name = node.AttrByName(Configuration.CONFIG_NAME_ATTR).ValueAsString(string.Empty).Trim(); + + if (m_Name.IsNullOrWhiteSpace()) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_NO_NAME_ERROR); + + if (m_Name.EqualsOrdIgnoreCase(CENTRAL_AREA_NAME)) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_INVALID_NAME_ERROR+"MDBPartitionedArea can not be called '{0}'".Args(CENTRAL_AREA_NAME)); + + m_Partitions = new List(); + foreach(var pnode in node.Children.Where( cn => cn.IsSameName(CONFIG_PARTITION_SECTION))) + { + var partition = new Partition(this, pnode); + m_Partitions.Add( partition ); + } + + if (m_Partitions.Count==0) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_NO_PARTITIONS_ERROR.Args(Name)); + + if (m_Partitions.Count != m_Partitions.Distinct().Count()) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_DUPLICATE_RANGES_ERROR.Args(Name)); + + m_Partitions.Sort();// sort by StartGDID + + for(var i=0; i m_Partitions; + + public override string Name{ get{ return m_Name;}} + + + /// + /// Returns partitions ordered by StartGDID + /// + public IEnumerable Partitions{ get{ return m_Partitions;}} + + public override IEnumerable AllPartitions { get { return Partitions;} } + + + public override IEnumerable AllShards + { + get + { + foreach(var p in AllPartitions) + foreach(var s in p.Shards) + yield return s; + } + } + + /// + /// Returns CRUDOperations facade connected to the appropriate database server within this named area + /// which services the shard computed from briefcase GDID + /// + public CRUDOperations PartitionedOperationsFor(GDID idBriefcase) + { + var partition = FindPartitionForBriefcase(idBriefcase); + return partition.ShardedOperationsFor(idBriefcase); + } + + + /// + /// Returns a partition that holds the data for a briefcase identified by id + /// + public Partition FindPartitionForBriefcase(GDID idBriefcase) + { + Partition partition = null; + for(var i=0; i0) break; + partition = current; + } + + if (partition==null) + throw new MDBException(StringConsts.MDB_AREA_CONFIG_PARTITION_NOT_FOUND_ERROR.Args(Name, idBriefcase)); + + return partition; + } + + /// + /// Returns an enumerations partitions that start from the specified briefcase identified by id. + /// If briefcase is null or zero then all partitions are returned + /// + public IEnumerable GetPartitionsStartingFromBriefcase(GDID? idBriefcase) + { + if (!idBriefcase.HasValue || idBriefcase.Value.IsZero) return m_Partitions; + var partition = FindPartitionForBriefcase(idBriefcase.Value); + return m_Partitions.Skip(partition.Order); + } + + public override string ToString() + { + return "Area('{0}')[{1}]".Args(this.Name, m_Partitions.Count); + } + + } + +} diff --git a/src/Agni/MDB/Readme.txt b/src/Agni/MDB/Readme.txt new file mode 100644 index 0000000..110916d --- /dev/null +++ b/src/Agni/MDB/Readme.txt @@ -0,0 +1,100 @@ +MDB - Medium Database +===================== + +MDB is for storing relational data* organized by sharding ID in "briefcases" within areas. +* - we can also store documents as easily using MongoDB. + +The data is organized in areas. Areas are Named. +The purpose of each area is to provide the best possible load accommodation for particular data type. +For example: User area holds user-related data (profile, transactions, balances) for users partitioning it on GDID. +This approach allows to use traditional query methods within one "briefcase" of data. + +Within an area, a "briefcase" is a segment of relational tables that is keyed on the same partition GDID - a "briefcase key". +It represents a logical set of data needed for business logic. +For example: User briefcase stores all user-related tables on the same RDBMS server. +This allows for quick traversal/joins of data within the briefcase identified by GDID ID. +Briefcases are not physical entities, they are logical sets of rows in various tables within the same DB server. +An area may hold MORE THAN ONE type of briefcases (as its name should reflect). + +Internally, briefcases are usually hierarchical structures of tables. For example: + - User + - Pay Accounts + - Account Authorization History + - Addresses + - Orders + - Order Line-items + - Transactions + - Balances + + + +All areas except "CENTRAL" store data in partitions that are range-based on GDID, then within a partition data is sharded among the +shard count. +The SHARD COUNT within partition is normally immutable (pre-arranged). +If the shard count needs to change within a partition, briefcase resharding within that partition has to be done (see below). + +Area "User" ++------------------+ +-----------------------------+ +-----------------------------+ ++ Partition 1 + + Partition 2 + + Partition X + ++ Range Start 0 + + Range Start 250000 + + Range Start >250000 + ++--------+---------+ +--------+----------+---------+ .... +--------+----------+---------+ ++Shard 1 | Shard 2 + +Shard 1 | Shard 2 | Shard 3 | +Shard 1 | ... | Shard N | ++--------+---------+ +--------+----------+---------+ +--------+----------+---------+ + + +Partitioning IDs are always GDID (global unique identifier), not strings; this is because +MDB uses range partitioning in all but CENTRAL area. + +CENTRAL Area DOES NOT use any partitioning. It is a special Area used for global definitions/indexing. +It still uses sub-shards. CENTRAL Area is the only area that has SHARDING KEY: object (not gdid), as it allows to lookup + by strings and other shard keys. + +When a STRING ID (i.e. user email) needs to be mapped to GDID (i.e. user briefcase GDID), the CENTRAL Area should be queried (index tables). +Most of the data in Central area is static, so it gets aggressively cached. + +All other areas use Range partitioning. Range partitioning works as follows: + All sharding IDS are monotonically-increasing GDIDs (with authority bits discarded: Era 32bit + Counter 60 bit) + Sharding GDID is requested. The system needs to locate a shard that holds that data within the area. + System looks-up the system config that maps GDID start {Era, Counter} to the partition. + + The partition is further broken-down by shards, this is needed so that write/read load of current + data does not create hotspot on one server. The Number of shards within partition is not expected to be changed + (or briefcase data rebalancing would need to take place with 3rd party tools, see below). + + + Benefits: + 1. Data does not need to be physically moved between partitions on partition addition. Once a partition has been assigned, data remains there + 2. Quick mapping of GDID_SHARDING_ID -> physical server + 3. Ability to gradually increase capacity: start business with one partition, assess the load and add more partitions when necessary + 4. Fast in-area queries - as data is physically pre-grouped in "briefcases" by GDID_SHARDING_ID (all briefcase-related data is on the same physical server) + + Drawbacks: + 1. If "older GDID" data gets stale*, the older shards experience less load + 2. Possibly uneven distribution of "newer/hotter" data goes towards the end + 3. Theoretically not 100% even distribution as some USERS(or other briefcases) may have more + data than others, 100 users on one server!=100 users on another. Because of it, MORE CAPACITY has to be reserved in partition. + Mitigation may be: scale particular server up (faster CPU+more ram/disk) + + * - keep in mind, "older" users still have new transactions comming into their shard, + as transactions/balances are co-located with user + + + Future/Data Support/Archiving tasks + ----------------------------------- + With time (after X years), some data may get deleted from the MDB. Older customer data may get archived and moved-out into a long-term storage. + Instead of adding new partitions, we can set a GDID brakepoint (one number) after which the range partitioning tables will start over - that is + the GDIDs below the brakepoint will get routed according to the first range set, after brakepoint, to another.. and so on + + How to re-shard the data whithin the partition (briefcase move) + --------------------------------------------------------------- + The business logic-aware tool(script/utility) would need to be constructed to move briefcases (all logically-grouped data) between DB servers. + It is important to note that AUTO-INC ids SHOULD NOT be used because of possible collisions, instead GDIDs need to be used throughout + all tables + + + + + + + + diff --git a/src/Agni/MDB/ShardingUtils.cs b/src/Agni/MDB/ShardingUtils.cs new file mode 100644 index 0000000..6bfeb95 --- /dev/null +++ b/src/Agni/MDB/ShardingUtils.cs @@ -0,0 +1,248 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.DataAccess.Distributed; +using NFX.DataAccess.Cache; +using NFX.IO.ErrorHandling; + + +namespace Agni.MDB +{ + /// + /// WARNING!!! Any change to this class may lead to change in sharding distribution + /// and render all existing data partitioning invalid. Use EXTREME care! + /// + public static class ShardingUtils + { + + /// + /// Takes first X chars from a trimmed string. + /// If a string is null returns null. If a string does not have enough chars the function returns what the string has. + /// WARNING! Changing this function will render all existing sharding partitioning invalid. Use extreme care! + /// + public static string TakeFirstChars(string str, int count) + { + if (str == null) return null; + str = str.Trim(); + if (str.Length <= count) return str; + return str.Substring(0, count); + } + + + /// + /// Gets sharding ID for object that can be IShardingIDProvider, string, parcel, GDID, long/ulong and int/uint. + /// WARNING! Changing this function will render all existing sharding partitioning invalid. Use extreme care! + /// + public static ulong ObjectToShardingID(object key) + { + //WARNING! Never use GetHashCode here as it is platform-dependent, but this function must be 100% deterministic + + if (key == null) return 0; + + var cnt = 0; + while (key != null && key is IShardingPointerProvider) + { + key = ((IShardingPointerProvider)key).ShardingPointer.ID; + cnt++; + if (cnt > 10) + throw new MDBException(StringConsts.MDB_POSSIBLE_SHARDING_ID_CYCLE_ERROR); + } + + if (key == null) return 0; + + if (key is IDistributedStableHashProvider) return ((IDistributedStableHashProvider)key).GetDistributedStableHash();//covers GDID + if (key is string) return StringToShardingID((string)key); + if (key is byte[]) return ByteArrayToShardingID((byte[])key); + if (key is Guid) + { + var garr = ((Guid)key).ToByteArray(); + var seg1 = garr.ReadBEUInt64(); + var seg2 = garr.ReadBEUInt64(8); + + return seg1 ^ seg2; + } + if (key is DateTime) return (ulong)((DateTime)key).ToMillisecondsSinceUnixEpochStart(); + + if (key is ulong) return ((ulong)key); + if (key is long) return ((ulong)(long)key); + + if (key is uint) return ((ulong)((uint)key) * 1566083941ul); + if (key is int) return ((ulong)((int)key) * 1566083941ul); + if (key is bool) return ((bool)key) ? 999331ul : 3ul; + + throw new MDBException(StringConsts.MDB_OBJECT_SHARDING_ID_ERROR.Args(key.GetType().FullName)); + } + + /// + /// Gets sharding ID for string, that is - computes string hash as UInt64 . + /// WARNING! Changing this function will render all existing sharding partitioning invalid. Use extreme care! + /// + public static ulong StringToShardingID(string key) + { + //WARNING! Never use GetHashCode here as it is platform-dependent, but this function must be 100% deterministic + + /* +From Microsoft on MSDN: + + Best Practices for Using Strings in the .NET Framework + + Recommendations for String Usage + + Use the String.ToUpperInvariant method instead of the String.ToLowerInvariant method when you normalize strings for comparison. + +Why? From Microsoft: + + Normalize strings to uppercase + + There is a small group of characters that when converted to lowercase cannot make a round trip. + +What is example of such a character that cannot make a round trip? + + Start: Greek Rho Symbol (U+03f1) ϱ + Uppercase: Capital Greek Rho (U+03a1) Ρ + Lowercase: Small Greek Rho (U+03c1) ρ + + ϱ , Ρ , ρ + +That is why, if your want to do case insensitive comparisons you convert the strings to uppercase, and not lowercase. + */ + + + if (key == null) return 0; + key = key.ToUpperInvariant();//DANGER!!!!!!!!!todo Dima peredelat!!! + var sl = key.Length; + if (sl == 0) return 0; + + ulong hash1 = 0; + for (int i = sl - 1; i > sl - 1 - sizeof(ulong) && i >= 0; i--)//take 8 chars from end (string suffix), for most string the + { //string tail is the most changing part (i.e. 'Alex Kozloff'/'Alex Richardson'/'System.A'/'System.B' + if (i < sl - 1) hash1 <<= 8; + var c = key[i]; + var b1 = (c & 0xff00) >> 8; + var b2 = c & 0xff; + hash1 |= (byte)(b1 ^ b2); + } + + ulong hash2 = 1566083941ul * (ulong)Adler32.ForString(key); + + return hash1 ^ hash2; + } + + /// + /// Gets sharding ID for byte[], that is - computes byte[] hash as UInt64 . + /// WARNING! Changing this function will render all existing sharding partitioning invalid. Use extreme care! + /// + public static ulong ByteArrayToShardingID(byte[] key) + { + if (key == null) return 0; + var len = key.Length; + if (len == 0) return 0; + + ulong result = 1566083941ul * (ulong)Adler32.ForBytes(key); + return result; + } + + /// + /// Returns GDID range partition calculated as the counter bit shift from the original ID. + /// This function is used to organize transactions into "batches" that otherwise would have required an unnecessary + /// lookup entity (i.e. transaction partition). With this function the partition may be derived right from the original GDID + /// + /// Original ID + /// Must be between 4..24, so the partitions are caped at 16M(2^24) entries + /// Must be between 0 and less than bit size + /// Partition ID obtained id + public static GDID MakeGDIDRangePartition(GDID id, int bitSize = 16, int bitSubShard = 4) + { + if (id.IsZero) return GDID.Zero; + + if (bitSize < 4 || bitSize > 24 || bitSubShard < 0 || bitSubShard >= bitSize) + throw new MDBException(StringConsts.ARGUMENT_ERROR + "bitSize must be [4..24] and bitSubShard must be [0..> bitSize) << bitSubShard) | (id.Counter & lowMask); + + var result = new GDID(era, 0, newCtr); + if (result.IsZero) result = new GDID(0, 0, 1); + return result; + } + } + + + + /// + /// Used to return ID data from multiple elements, i.e. multiple parcel fields so sharding framework may obtain + /// ULONG sharding key. You can not compare or equate instances (only reference comparison of data buffer) + /// + public struct CompositeShardingID : IDistributedStableHashProvider, IEnumerable + { + + public CompositeShardingID(params object[] data) + { + if (data == null || data.Length == 0) + throw new DistributedDataAccessException(StringConsts.ARGUMENT_ERROR + "CompositeShardingID.ctor(data==null|empty)"); + + m_Data = data; + + m_HashCode = 0ul; + + for (var i = 0; i < m_Data.Length; i++) + { + var elm = m_Data[i]; + + ulong ehc; + + if (elm != null) + ehc = ShardingUtils.ObjectToShardingID(elm); + else + ehc = 0xaa018055ul; + + m_HashCode <<= 1; + m_HashCode ^= ehc; + } + } + + + private ulong m_HashCode; + private object[] m_Data; + + public int Count { get { return m_Data == null ? 0 : m_Data.Length; } } + public object this[int i] { get { return m_Data == null ? null : (i >= 0 && i < m_Data.Length ? m_Data[i] : null); } } + + public ulong GetDistributedStableHash() + { + return m_HashCode; + } + + public override string ToString() + { + if (m_Data == null) return "[]"; + + var sb = new StringBuilder("CompositeShardingID["); + for (var i = 0; i < m_Data.Length; i++) + { + if (i > 0) sb.Append(", "); + + var elm = m_Data[i]; + + if (elm == null) + sb.Append(""); + else + sb.Append(elm.ToString()); + } + sb.Append(']'); + + return sb.ToString(); + } + + public IEnumerator GetEnumerator() { return m_Data != null ? ((IEnumerable)m_Data).GetEnumerator() : Enumerable.Empty().GetEnumerator(); } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return m_Data != null ? m_Data.GetEnumerator() : Enumerable.Empty().GetEnumerator(); } + } +} \ No newline at end of file diff --git a/src/Agni/Metabase/Enums.cs b/src/Agni/Metabase/Enums.cs new file mode 100644 index 0000000..322cf35 --- /dev/null +++ b/src/Agni/Metabase/Enums.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Agni.Metabase +{ + /// + /// Designates types of reachability in the named network + /// + public enum NetworkScope + { + /// + /// Everything accessible from everywhere + /// + Any, + + /// + /// Only accessible within the same Network Operation Center + /// + NOC, + + /// + /// Only accessible within the same-named group. Groups are properties of every host. Groups are defined in 'networks' root file + /// + Group, + + /// + /// Only accessible within the same named group within the same Network Operation Center + /// + NOCGroup + } +} diff --git a/src/Agni/Metabase/Metabank.AppCatalog.cs b/src/Agni/Metabase/Metabank.AppCatalog.cs new file mode 100644 index 0000000..2dcd301 --- /dev/null +++ b/src/Agni/Metabase/Metabank.AppCatalog.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX.Security; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + + + /// + /// Represents a system catalog of application + /// + public sealed class AppCatalog : SystemCatalog + { + #region CONSTS + public const string CONFIG_APPLICATIONS_SECTION = "applications"; + public const string CONFIG_APPLICATION_SECTION = "application"; + public const string CONFIG_ROLES_SECTION = "roles"; + #endregion + + #region .ctor + internal AppCatalog(Metabank bank) : base(bank, APP_CATALOG) + { + + } + + #endregion + + #region Fields + + + #endregion + + #region Properties + /// + /// Returns registry of application sections + /// + public IRegistry Applications + { + get + { + const string CACHE_KEY = "apps"; + //1. Try to get from cache + var result = Metabank.cacheGet(APP_CATALOG, CACHE_KEY) as IRegistry; + if (result!=null) return result; + + result = + Metabank.fsAccess("AppCatalog.Applications.get", Metabank.FileSystem.CombinePaths(APP_CATALOG, CONFIG_APPLICATIONS_SECTION), + (session, dir) => + { + var reg = new Registry(); + foreach(var sdir in dir.SubDirectoryNames) + { + var spath = Metabank.JoinPaths(APP_CATALOG, CONFIG_APPLICATIONS_SECTION, sdir); + var region = new SectionApplication(this, sdir, spath, session); + reg.Register(region); + } + return reg; + } + ); + + Metabank.cachePut(APP_CATALOG, CACHE_KEY, result); + return result; + } + } + + /// + /// Returns registry of role sections + /// + public IRegistry Roles + { + get + { + const string CACHE_KEY = "roles"; + //1. Try to get from cache + var result = Metabank.cacheGet(APP_CATALOG, CACHE_KEY) as IRegistry; + if (result!=null) return result; + + result = + Metabank.fsAccess("AppCatalog.Roles.get", Metabank.FileSystem.CombinePaths(APP_CATALOG, CONFIG_ROLES_SECTION), + (session, dir) => + { + var reg = new Registry(); + foreach(var sdir in dir.SubDirectoryNames) + { + var spath = Metabank.JoinPaths(APP_CATALOG, CONFIG_ROLES_SECTION, sdir); + var region = new SectionRole(this, sdir, spath, session); + reg.Register(region); + } + return reg; + } + ); + + Metabank.cachePut(APP_CATALOG, CACHE_KEY, result); + return result; + } + } + + #endregion + + #region Public + public override void Validate(ValidationContext ctx) + { + foreach(var sapp in Applications) + sapp.Validate(ctx); + + foreach(var srole in Roles) + srole.Validate(ctx); + + } + + #endregion + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.BinCatalog.cs b/src/Agni/Metabase/Metabank.BinCatalog.cs new file mode 100644 index 0000000..d1d90c4 --- /dev/null +++ b/src/Agni/Metabase/Metabank.BinCatalog.cs @@ -0,0 +1,304 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX.IO.FileSystem.Packaging; +using NFX.Security; + +using Agni.AppModel.HostGovernor; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + + + /// + /// Represents a system catalog of binary resources + /// + public sealed class BinCatalog : SystemCatalog + { + #region Inner + + /// + /// Provides information about binary package that includes: name, platform, os, version + /// + public sealed class PackageInfo + { + public const string ANY = "any"; + + internal PackageInfo(string fullName) + { + if (fullName.IsNullOrWhiteSpace()) + throw new AgniException(StringConsts.ARGUMENT_ERROR + "PackageInfo.ctor(fullName=null|empty)"); + + var segs = fullName.Split('.'); + if (segs.Length<4) + throw new AgniException(StringConsts.ARGUMENT_ERROR + "PackageInfo.ctor(!name.plat.os.ver)"); + + Name = string.Join(".", segs, 0, segs.Length-3); + Version = segs[segs.Length-1]; + OS = segs[segs.Length-2]; + Platform = segs[segs.Length-3]; + + if (Name.IsNullOrWhiteSpace() || + Platform.IsNullOrWhiteSpace() || + OS.IsNullOrWhiteSpace() || + Version.IsNullOrWhiteSpace()) + throw new AgniException(StringConsts.ARGUMENT_ERROR + "PackageInfo.ctor(!name.plat.os.ver)"); + } + + public PackageInfo(string name, string version, string platform = ANY, string os = ANY) + { + if (name.IsNullOrWhiteSpace() || version.IsNullOrWhiteSpace()) + throw new AgniException(StringConsts.ARGUMENT_ERROR + "PackageInfo.ctor(name|ver=null|empty)"); + + Name = name; + Version = version; + + if (platform.IsNullOrWhiteSpace()) platform = ANY; + if (os.IsNullOrWhiteSpace()) os = ANY; + + Platform = platform; + OS = os; + } + + /// + /// Name portion of the package. Note: Name alone DOES NOT uniquely identify package in the catalog + /// + public readonly string Name; + public readonly string Platform; + public readonly string OS; + public readonly string Version; + + public bool IsAnyOS { get { return INVSTRCMP.Equals(OS, ANY);} } + public bool IsAnyPlatform { get { return INVSTRCMP.Equals(Platform, ANY);} } + + /// + /// Returns relative specificity score - the more specific this definition is the higher is the score. + /// Used for match comparison - finding better matches taking platform and OS into consideration + /// + public int Specificity + { + get + { + var result = 0; + if (!IsAnyPlatform) result+=1000; + if (!IsAnyOS) result+=100; + return result; + } + } + + /// + /// Returns metabase directory full name that this item was constructed from + /// + public string FullName + { + get { return "{0}.{1}.{2}.{3}".Args(Name, Platform, OS, Version);} + } + + public override string ToString() + { + return FullName; + } + + public override int GetHashCode() + { + return INVSTRCMP.GetHashCode(Name) ^ INVSTRCMP.GetHashCode(Platform) ^ INVSTRCMP.GetHashCode(OS) ^ INVSTRCMP.GetHashCode(Version); + } + + public override bool Equals(object obj) + { + var other = obj as PackageInfo; + if (other==null) return false; + + return INVSTRCMP.Equals(this.Name, other.Name) && + INVSTRCMP.Equals(this.Platform, other.Platform)&& + INVSTRCMP.Equals(this.OS, other.OS)&& + INVSTRCMP.Equals(this.Version, other.Version); + } + } + + + #endregion + + #region .ctor + internal BinCatalog(Metabank bank) : base(bank, BIN_CATALOG) + { + + } + + #endregion + + #region Fields + + + #endregion + + #region Properties + /// + /// Returns packages in the catalog + /// + public IEnumerable Packages + { + get + { + const string CACHE_KEY = "pnames"; + //1. Try to get from cache + var result = Metabank.cacheGet(BIN_CATALOG, CACHE_KEY) as IEnumerable; + if (result!=null) return result; + + result = + Metabank.fsAccess("BinCatalog.Packages.get", BIN_CATALOG, + (session, dir) => + { + var list = new List(); + foreach(var sdir in dir.SubDirectoryNames) + { + list.Add( new PackageInfo( sdir ) ); + } + return list; + } + ); + + foreach(var pi in result) + { + if(!pi.IsAnyPlatform) + if (!Metabank.PlatformNames.Any(pn=> Metabank.INVSTRCMP.Equals(pn, pi.Platform))) + throw new MetabaseException(StringConsts.METABASE_BIN_PACKAGE_INVALID_PLATFORM_ERROR.Args(pi)); + + if(!pi.IsAnyOS) + if (!Metabank.OSNames.Any(on=> Metabank.INVSTRCMP.Equals(on, pi.OS))) + throw new MetabaseException(StringConsts.METABASE_BIN_PACKAGE_INVALID_OS_ERROR.Args(pi)); + } + + + Metabank.cachePut(BIN_CATALOG, CACHE_KEY, result); + return result; + } + } + + + #endregion + + #region Public + + public override void Validate(ValidationContext ctx) + { + var output = ctx.Output; + + try + { + var packages = Packages; + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, this, null, error.ToMessageWithType(), error) ); + return; + } + + try + { + Metabank.fsAccess("BinCatalog.Validate", BIN_CATALOG, + (session, dir) => + { + foreach(var sdn in dir.SubDirectoryNames) + using(var sdir = dir.GetSubDirectory(sdn)) + using(var mf = sdir.GetFile(ManifestUtils.MANIFEST_FILE_NAME)) + { + if (mf==null) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, this, null, + StringConsts.METABASE_BIN_PACKAGE_MISSING_MANIFEST_ERROR + .Args(sdn, ManifestUtils.MANIFEST_FILE_NAME), null) + ); + continue; + } + + var manifest = LaconicConfiguration.CreateFromString( mf.ReadAllText()).Root; + var computed = ManifestUtils.GeneratePackagingManifest(sdir); + + if (!ManifestUtils.HasTheSameContent(manifest, computed, oneWay: false)) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, this, null, + StringConsts.METABASE_BIN_PACKAGE_OUTDATED_MANIFEST_ERROR + .Args(sdn, ManifestUtils.MANIFEST_FILE_NAME), null) + ); + } + + return true; + } + ); + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, this, null, error.ToMessageWithType(), error) ); + return; + } + + } + + + /// + /// Checks the local version and performs local software installation on this machine if needed + /// This process is an integral part og AHGOV/HostGovernorService implementation and should not be called by developers + /// + /// True if installation was performed + internal bool CheckAndPerformLocalSoftwareInstallation(IList progress, bool force = false) + { + return + Metabank.fsAccess("BinCatalog.CheckAndPerformLocalSoftwareInstallation", BIN_CATALOG, + (session, dir) => + { + var sw = System.Diagnostics.Stopwatch.StartNew(); + + if (progress!=null) progress.Add("{0} Building install set...".Args(App.LocalizedTime)); + + var installSet = new List(); + foreach(var appPackage in HostGovernorService.Instance.AllPackages) + { + var subDir = appPackage.MatchedPackage.FullName; + var packageDir = dir.GetSubDirectory(subDir); + + if (progress!=null) + progress.Add(" + {0}".Args(appPackage.ToString())); + + if (packageDir==null) + throw new MetabaseException(StringConsts.METABASE_INSTALLATION_BIN_PACKAGE_NOT_FOUND_ERROR.Args(appPackage.Name, subDir)); + + installSet.Add(new LocalInstallation.PackageInfo(appPackage.Name, packageDir, appPackage.Path)); + } + + if (progress!=null) + { + progress.Add("Initiating local installation"); + progress.Add(" Root Path: {0}".Args(HostGovernorService.Instance.UpdatePath)); + progress.Add(" Manifest Path: {0}".Args(HostGovernorService.Instance.RunPath)); + progress.Add(" Force: {0}".Args(force)); + } + + var anew = false; + using(var install = new LocalInstallation(HostGovernorService.Instance.UpdatePath, HostGovernorService.Instance.RunPath)) + { + anew = install.CheckLocalAndInstallIfNeeded(installSet, force); + } + if (progress!=null) + { + progress.Add(" Installed anew: {0}".Args(anew)); + progress.Add("{0} Done. Duration: {1}".Args(App.LocalizedTime, sw.Elapsed)); + } + return anew; + } + ); + } + + #endregion + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.Catalogs.cs b/src/Agni/Metabase/Metabank.Catalogs.cs new file mode 100644 index 0000000..b7d1640 --- /dev/null +++ b/src/Agni/Metabase/Metabank.Catalogs.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX.Security; + + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Represents a catalog in metabase - a top-level folder in the metabase file system that contains + /// logically-grouped data. Catalog implementers contain various instances of Section-derived classes that represent pieces of metabase whole data that + /// can lazily load from the source file system. Contrary to monolithic application configurations, that load + /// at once from a single source (i.e. disk file), metadata class allows to wrap configuration and load segments of configuration + /// on a as-needed basis + /// + public abstract class Catalog : INamed + { + #region CONSTS + + #endregion + + #region .ctor + internal Catalog(Metabank bank, string name) + { + if (bank==null || name.IsNullOrWhiteSpace()) + throw new MetabaseException(StringConsts.ARGUMENT_ERROR + this.GetType().Name + ".ctor(bank==null|name==null|empty)"); + + Metabank = bank; + m_Name = name; + + Metabank.m_Catalogs.Register( this ); + } + + #endregion + + #region Fields + + public readonly Metabank Metabank; + protected readonly string m_Name; + + + #endregion + + #region Properties + + ///// + ///// References metabank instance that created this object + ///// + //public Metabank Metabank { get{ return m_Metabank;} } + + + /// + /// Returns catalog name + /// + public string Name { get{ return m_Name;}} + + + /// + /// Returns true to designate catalog instance as a system-recognized. + /// System catalogs are the ones that metabank hard-codes, i.e. "Regions" is a hard-coded catalog that contains system information. + /// Metabase can also contain user-definable catalogs in which case this property will return false + /// + public bool IsSystem { get { return this is SystemCatalog; }} + + + #endregion + + #region Public + + /// + /// Validates catalog + /// + public abstract void Validate(ValidationContext ctx); + + public override string ToString() + { + return this.Name; + } + + #endregion + + + }//Catalog + + /// + /// Denotes a system catalog + /// + public abstract class SystemCatalog : Catalog + { + protected SystemCatalog(Metabank bank, string name) : base(bank, name) + { + + } + }//SystemCatalog + + /// + /// Denotes a user catalog + /// + public abstract class UserCatalog : Catalog + { + protected UserCatalog(Metabank bank, string name) : base(bank, name) + { + + } + }//UserCatalog + + + + +}} diff --git a/src/Agni/Metabase/Metabank.RegCatalog.cs b/src/Agni/Metabase/Metabank.RegCatalog.cs new file mode 100644 index 0000000..18d38c3 --- /dev/null +++ b/src/Agni/Metabase/Metabank.RegCatalog.cs @@ -0,0 +1,439 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX.Security; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + + + /// + /// Represents a system catalog of region objects + /// + public sealed class RegCatalog : SystemCatalog + { + #region CONSTS + public const string REG_EXT = ".r"; + public const string NOC_EXT = ".noc"; + public const string ZON_EXT = ".z"; + public const string HST_EXT = ".h"; + #endregion + + #region .ctor + internal RegCatalog(Metabank bank) : base(bank, REG_CATALOG) + { + + } + + #endregion + + #region Fields + + + #endregion + + #region Properties + /// + /// Returns registry of top-level regions + /// + public IRegistry Regions + { + get + { + const string CACHE_KEY = "topregs"; + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as IRegistry; + if (result!=null) return result; + + result = + Metabank.fsAccess("RegionCatalog.Regions.get", REG_CATALOG, + (session, dir) => + { + var reg = new Registry(); + foreach(var sdir in dir.SubDirectoryNames.Where(dn=>dn.EndsWith(RegCatalog.REG_EXT))) + { + var spath = Metabank.JoinPaths(REG_CATALOG, sdir); + var region = new SectionRegion(this, null, Metabank.chopExt(sdir), spath, session); + reg.Register(region); + } + return reg; + } + ); + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + } + + /// + /// Navigates to region path and returns the corresponding section or null. + /// The path root starts at region catalog level i.e.: '/Us/East/Clev.noc/A/IV/wlarge0001' - + /// a path to a host called 'wlarge0001' located in zone A-IV in Cleveland network center etc. + /// NOTE: may omit the '.r', '.noc', and '.z' region/noc and zone designators. + /// The navigation is done using case-insensitive name comparison, however + /// the underlying file system may be case-sensitive and must be supplied the exact name + /// + public SectionRegionBase this[string path] + { + get + { + if (path==null) return null; + var segs = path.Split('/').Where(s=>s.IsNotNullOrWhiteSpace()).Select(s=>s.Trim()); + SectionRegionBase result = null; + var first = true; + foreach(var seg in (segs)) + { + if (first) + { + first = false; + if (seg.EndsWith(RegCatalog.REG_EXT)) + result = Regions[ seg.Substring(0, seg.LastIndexOf(RegCatalog.REG_EXT)) ]; + else + result = Regions[seg]; + } + else + result = result[seg]; + + if (result==null) break; + } + return result; + } + } + + /// + /// Enumerates all NOCs. This may fetch much data if enumerated to the end + /// + public IEnumerable AllNOCs + { + get + { + foreach(var region in Regions) + { + foreach(var noc in region.AllNOCs) + yield return noc; + } + } + } + + + #endregion + + #region Public + + + /// + /// Tries to navigate to region path as far as possible and returns the deepest section or null. + /// This method is needed to obtain root paths from detailed paths with wild cards. + /// Method also returns how deep it could navigate(how many path levels resolved). + /// For example: + /// '/Us/East/Clev.noc/{1}/{2}' - Clev.noc with depth=3 will be returned. + /// + public SectionRegionBase TryNavigateAsFarAsPossible(string path, out int depth) + { + depth = 0; + if (path==null) return null; + var segs = path.Split('/').Where(s=>s.IsNotNullOrWhiteSpace()).Select(s=>s.Trim()); + SectionRegionBase existing = null; + SectionRegionBase current = null; + var first = true; + foreach(var seg in (segs)) + { + if (first) + { + first = false; + if (seg.EndsWith(RegCatalog.REG_EXT)) + current = Regions[ seg.Substring(0, seg.LastIndexOf(RegCatalog.REG_EXT)) ]; + else + current = Regions[seg]; + } + else + current = current[seg]; + + if (current==null) break; + depth++; + existing = current; + } + return existing; + } + + + + /// + /// Navigates to region section or throws. NOTE: may omit the '.r' suffix + /// + public SectionRegion NavigateRegion(string path) + { + var result = this[path] as SectionRegion; + if (result==null) + throw new MetabaseException(StringConsts.METABASE_REG_CATALOG_NAV_ERROR.Args("NavigateRegion()", path ?? SysConsts.NULL)); + return result; + } + + /// + /// Navigates to NOC section or throws. NOTE: may omit the '.r' and '.noc' suffixes + /// + public SectionNOC NavigateNOC(string path) + { + var result = this[path] as SectionNOC; + if (result==null) + throw new MetabaseException(StringConsts.METABASE_REG_CATALOG_NAV_ERROR.Args("NavigateNOC()", path ?? SysConsts.NULL)); + return result; + } + + /// + /// Navigates to zone section or throws. NOTE: may omit the '.r','.noc', and '.z' suffixes + /// + public SectionZone NavigateZone(string path) + { + var result = this[path] as SectionZone; + if (result==null) + throw new MetabaseException(StringConsts.METABASE_REG_CATALOG_NAV_ERROR.Args("NavigateZone()", path ?? SysConsts.NULL)); + return result; + } + + /// + /// Navigates to host section or throws. NOTE: may omit the '.r','.noc','.z', and '.h' suffixes + /// + public SectionHost NavigateHost(string path) + { + var result = this[path] as SectionHost; + if (result==null) + throw new MetabaseException(StringConsts.METABASE_REG_CATALOG_NAV_ERROR.Args("NavigateHost()", path ?? SysConsts.NULL)); + return result; + } + + + public override void Validate(ValidationContext ctx) + { + var output = ctx.Output; + + foreach(var sreg in Regions) + sreg.Validate(ctx); + + var here = AgniSystem.HostMetabaseSection.EffectiveGeoCenter; + var allNOCs = AllNOCs.ToList(); + foreach(var noc in allNOCs) + { + if (noc.EffectiveGeoCenter.Equals(SysConsts.DEFAULT_GEO_LOCATION)) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, this, noc, + StringConsts.METABASE_NOC_DEFAULT_GEO_CENTER_WARNING.Args(noc.Name, noc.EffectiveGeoCenter)) ); + + output.Add(new MetabaseValidationMsg(MetabaseValidationMessageType.Info, this, noc, + "Distance to '{0}' is {1}km".Args(AgniSystem.HostMetabaseSection.RegionPath, + (int)here.HaversineEarthDistanceKm(noc.EffectiveGeoCenter)))); + + } + + if (allNOCs.Select(n=>n.Name).Count()!=allNOCs.Select(n=>n.Name).Distinct(StringComparer.InvariantCultureIgnoreCase).Count()) + output.Add(new MetabaseValidationMsg(MetabaseValidationMessageType.Error, this, null, + "There are duplicate NOCs defined. NOC names should be globally unique regardless of their region")); + + } + + + /// + /// Counts the number of matching region path segments that resolve to the same section in the region catalog. + /// Example: /US/East/CLE/A/I and /US/East/JFK/A/I returns 2 + /// + public int CountMatchingPathSegments(string path1, string path2) + { + if (path1.IsNullOrWhiteSpace() && path2.IsNullOrWhiteSpace()) return 1; + if (path1.IsNullOrWhiteSpace() || path2.IsNullOrWhiteSpace()) return 0; + + var sect1 = this[path1]; + var sect2 = this[path2]; + return CountMatchingPathSegments(sect1, sect2); + } + + + /// + /// Counts the number of matching region path segments that resolve to the same section in the region catalog. + /// Example: /US/East/CLE/A/I and /US/East/JFK/A/I returns 2 + /// + public int CountMatchingPathSegments(SectionRegionBase sect1, SectionRegionBase sect2) + { + if (sect1==null || sect2==null) return 0; + + var chain1 = sect1.SectionsOnPath.ToList(); + var chain2 = sect2.SectionsOnPath.ToList(); + + var i = 0;//count number of matching segments + for(; i + /// Returns the ratio of matching region path segments that resolve to the same section in the region catalog, to maximum path length. + /// Example: /US/East/CLE/A and /US/East/JFK/A returns 0.5d = 50% match + /// + public double GetMatchingPathSegmentRatio(string path1, string path2) + { + if (path1.IsNullOrWhiteSpace() && path2.IsNullOrWhiteSpace()) return 1d; + if (path1.IsNullOrWhiteSpace() || path2.IsNullOrWhiteSpace()) return 0d; + + var sect1 = this[path1]; + var sect2 = this[path2]; + if (sect1==null || sect2==null) return 0; + + var chain1 = sect1.SectionsOnPath.ToList(); + var chain2 = sect2.SectionsOnPath.ToList(); + + double max = chain1.Count > chain2.Count ? chain1.Count : chain2.Count; + + var i = 0;//count number of matching segments + for(; i m_CacheGetDistanceBetweenPaths = new ConcurrentDictionary(StringComparer.Ordinal); + + /// + /// Calculates the logical distance between two entities in the cloud. + /// Although the distance is measured in kilometers it is really a logical distance which is somewhat related to physical. + /// At first, the two paths are compared in terms of matching segment count, if paths match 100% (point to the same entity) then + /// the distance is zero, otherwise the path match ratio is prorated against the earth circumference to get distance value. + /// The second comparison is performed against physical geo centers that use haversine formula to compute the distance 'as the crow flies'. + /// The second step is necessary for the comparison of various non-matching paths. i.e.: '/world/us/east/cle...' and '/world/us/east/ny...' both yield + /// the same distance as far as paths comparison however, 'cle' is closer while compared with 'los angeles' than 'ny' + /// + /// This function does caching of results based on case-sensitive paths keys + public double GetDistanceBetweenPaths(string path1, string path2) + { + const int MAX_DISTANCE_KM = 20000;//Earth equator 40,000km /2 + + if (path1.IsNullOrWhiteSpace() || path2.IsNullOrWhiteSpace()) return double.MaxValue; + + var cacheKey = path1 + " /////// " + path2; + double result = 0; + if (m_CacheGetDistanceBetweenPaths.TryGetValue(cacheKey, out result)) return result; + + var sect1 = this[path1]; + var sect2 = this[path2]; + if (sect1!=null && sect2!=null) + { + if (sect1.IsLogicallyTheSame(sect2)) + result = 0d; + else + { + var matchRatio = GetMatchingPathSegmentRatio(path1, path2); + var matchDistanceKm = MAX_DISTANCE_KM * (1.0d - matchRatio); + + var geoDistanceKm = sect1.EffectiveGeoCenter.HaversineEarthDistanceKm(sect2.EffectiveGeoCenter); + + result = matchDistanceKm + geoDistanceKm; + } + } + else result = double.MaxValue; + + m_CacheGetDistanceBetweenPaths.TryAdd(cacheKey, result); + + return result; + } + + + /// + /// Calculates the physical distance between two entities in the cloud. + /// The distance is based on paths geo centers and maybe inaccurate if geo coordinates are not properly set + /// + public double GetGeoDistanceBetweenPaths(string path1, string path2) + { + var sect1 = this[path1]; + var sect2 = this[path2]; + if (sect1==null || sect2==null) return double.MaxValue; + + if (sect1.IsLogicallyTheSame(sect2)) return 0; + + return sect1.EffectiveGeoCenter.HaversineEarthDistanceKm(sect2.EffectiveGeoCenter); + } + + + /// + /// Tries to navigate to NOC section in the specified absolute path and returns it or null if path is invalid/does not lead to NOC. + /// Ignores anything after the NOC section (zones, hosts etc..). For example '/world/us/east/CLE.noc/a/ii/x' will return 'CLE.noc' section, + /// but '/world/us/east' will return null as there is no NOC in the path + /// + public SectionNOC GetNOCofPath(string path) + { + if (path==null) return null; + var segs = path.Split('/').Where(s=>s.IsNotNullOrWhiteSpace()); + SectionRegionBase current = null; + var first = true; + foreach(var seg in (segs)) + { + if (first) + { + first = false; + if (seg.EndsWith(RegCatalog.REG_EXT)) + current = Regions[ seg.Substring(0, seg.LastIndexOf(RegCatalog.REG_EXT)) ]; + else + current = Regions[seg]; + } + else + { + if (current==null) return null; + current = current[seg]; + } + if (current is SectionNOC) return (SectionNOC)current; + } + return null; + } + + /// + /// Returns true when all of the supplied paths are in the same NOC + /// + public bool ArePathsInSameNOC(params string[] paths) + { + if (paths==null || paths.Length==0) return false; + if (paths.Any(p=>p.IsNullOrWhiteSpace())) return false; + + if (paths.Length==1) return true; + + + var noc = GetNOCofPath(paths[0]); + if (noc==null) return false; + + return paths.All(p => noc.IsLogicallyTheSame(GetNOCofPath(p))); + } + + /// + /// Combines region paths using '/' where needed + /// + public static string JoinPathSegments(params string[] segments) + { + if (segments == null || segments.Length == 0) return string.Empty; + + var source = segments.Where(s => s.IsNotNullOrWhiteSpace()); + + if (!source.Any()) return string.Empty; + + var prefix = source.First().Trim(); + var leading = prefix.Length > 0 && prefix[0] == '/'; + + var result = string.Join("/", source.Select(s => s.Trim('/', ' ')).Where(s => s.IsNotNullOrWhiteSpace())); + + if (leading) return '/' + result; + + return result; + } + + #endregion + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.ReslvNetSvc.cs b/src/Agni/Metabase/Metabank.ReslvNetSvc.cs new file mode 100644 index 0000000..c307839 --- /dev/null +++ b/src/Agni/Metabase/Metabank.ReslvNetSvc.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Log; +using NFX.Glue; +using NFX.Environment; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + + /// + /// Holds data that describes network service peer + /// + public struct NetSvcPeer + { + public const string BASE_MATCH = "*"; + + public NetSvcPeer(string addr, string port, string group) + { + Address = addr; + Port = port; + Group = group; + } + public readonly string Address; + public readonly string Port; + public readonly string Group; + + /// + /// Overrides this value with another field by field, where '*' represents the base value, i.e. + /// if address is "123.12" gets overridden by address "*.2", then result will be "123.12.2". Same rule applies field-by-field for address,port and group + /// + /// Another peer to override this one with + /// New peer with field-by-field overridden data + public NetSvcPeer Override(NetSvcPeer other) + { + if (other.Blank) return this; + var addr = Address; + var port = Port; + var group = Group; + + if (other.Address.IsNotNullOrWhiteSpace()) + addr = other.Address.Replace(BASE_MATCH, addr).Trim(); + + if (other.Port.IsNotNullOrWhiteSpace()) + port = other.Port.Replace(BASE_MATCH, port).Trim(); + + if (other.Group.IsNotNullOrWhiteSpace()) + group = other.Group.Replace(BASE_MATCH, group).Trim(); + + return new NetSvcPeer(addr, port, group); + } + + public bool Blank + { + get + { + return Address.IsNullOrWhiteSpace()&& + Port.IsNullOrWhiteSpace()&& + Group.IsNullOrWhiteSpace(); + } + } + } + + + + + /// + /// Resolves logical service name on the specified remote host into physical Glue.Node connection string suitable + /// for making remote calls from this machine + /// + /// Metabase host name i.e. 'World/Us/East/CLE/A/I/wmed0001' of the destination to make a call to + /// Network name i.e. 'internoc' + /// Service name i.e. 'zgov' + /// Optional preferred binding name i.e. 'sync' + /// Optional metabase host name for the host that calls will be made from. If null, then local host as determined at boot will be used + /// Glue Node instance with remote address visible to the calling party + public Node ResolveNetworkService(string host, string net, string svc, string binding = null, string fromHost = null) + { + return new Node(ResolveNetworkServiceToConnectString(host, net, svc, binding, fromHost)); + } + + + /// + /// Resolves logical service name on the specified remote host into physical Glue.Node connection string suitable + /// for making remote calls from this machine + /// + /// Metabase host name i.e. 'World/Us/East/CLE/A/I/wmed0001' of the destination to make a call to + /// Network name i.e. 'internoc' + /// Service name i.e. 'zgov' + /// Optional preferred binding name i.e. 'sync' + /// Optional metabase host name for the host that calls will be made from. If null, then local host as determined at boot will be used + /// Glue Node as connection string string instance with remote address visible to the calling party + public string ResolveNetworkServiceToConnectString(string host, string net, string svc, string binding = null, string fromHost = null) + { + try + { + return resolveNetworkServiceToConnectString(host, net, svc, binding, fromHost); + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_NETWORK_SVC_RESOLVE_ERROR.Args(host, net, svc, error.ToMessageWithType()), error); + } + + } + + + private ConcurrentDictionary m_NetResolverCache = new ConcurrentDictionary(StringComparer.Ordinal); + + + private string resolveNetworkServiceToConnectString(string host, string net, string svc, string binding = null, string fromHost = null) + { + if (host.IsNullOrWhiteSpace() || + net.IsNullOrWhiteSpace() || + svc.IsNullOrWhiteSpace()) + throw new MetabaseException(StringConsts.ARGUMENT_ERROR + "Metabase.ResolveNetSvc(host|net|svc==null|empty)"); + + if (fromHost.IsNullOrWhiteSpace()) fromHost = AgniSystem.HostName; + + if (fromHost.IsNullOrWhiteSpace()) + throw new MetabaseException(StringConsts.ARGUMENT_ERROR + "Metabase.ResolveNetSvc(fromHost==null|empty & AgniSystem is not avail)"); + + var sb = new StringBuilder(); + sb.Append(host); sb.Append(';'); + sb.Append(net); sb.Append(';'); + sb.Append(svc); sb.Append(';'); + sb.Append(binding.IsNullOrWhiteSpace() ? SysConsts.NULL : binding); sb.Append(';'); + sb.Append(fromHost); + + var cacheKey = sb.ToString(); + + string result = null; + if (m_NetResolverCache.TryGetValue(cacheKey, out result)) return result; + + + var bindingNode = GetNetworkSvcBindingConfNode(net, svc, binding); + binding = bindingNode.Name; + var dfltAddress = bindingNode.AttrByName(CONFIG_ADDRESS_ATTR).Value; + var dfltPort = bindingNode.AttrByName(CONFIG_PORT_ATTR).Value; + + + var fromh = CatalogReg.NavigateHost(fromHost); + var toh = CatalogReg.NavigateHost(host); + + var scope = GetNetworkScope(net); + var nsvc = GetNetworkSvcConfNode(net, svc); + + if (scope==NetworkScope.NOC || scope==NetworkScope.NOCGroup) + if (!fromh.NOC.IsLogicallyTheSame(toh.NOC)) + throw new MetabaseException(StringConsts.METABASE_NET_SVC_RESOLVER_TARGET_NOC_INACCESSIBLE_ERROR.Args(svc, fromh.RegionPath, toh.RegionPath, net, scope)); + + + + var toPeer = new NetSvcPeer(dfltAddress, dfltPort, null); + foreach(var segment in toh.SectionsOnPath) + { + var level = segment.MatchNetworkRoute(net, svc, binding, fromh.RegionPath); + toPeer = toPeer.Override(level); + } + + + if (scope== NetworkScope.Group || scope==NetworkScope.NOCGroup) + if (toPeer.Group.IsNotNullOrWhiteSpace()) + { + var fromPeer = new NetSvcPeer(dfltAddress, dfltPort, null); + foreach(var segment in fromh.SectionsOnPath) + { + var level = segment.MatchNetworkRoute(net, svc, binding, fromh.RegionPath); + fromPeer = fromPeer.Override(level); + } + + if (!INVSTRCMP.Equals( fromPeer.Group, toPeer.Group)) + throw new MetabaseException(StringConsts.METABASE_NET_SVC_RESOLVER_TARGET_GROUP_INACCESSIBLE_ERROR.Args(svc, fromh.RegionPath, toh.RegionPath, net, scope)); + } + + if (toh.Dynamic) + { + var dynPeer = resolveDynamicHostAddress(host, net, svc, fromh, toh, toPeer); + result = "{0}://{1}:{2}".Args(binding, dynPeer.Address, dynPeer.Port); + //TODO no cache for dynamic host? or use cahce_Put... + } + else + { + result = "{0}://{1}:{2}".Args(binding, toPeer.Address, toPeer.Port); + m_NetResolverCache.TryAdd(cacheKey, result);//put in cache ONLY for non-dynamic hosts + } + + return result; + } + + private NetSvcPeer resolveDynamicHostAddress(string fullHostName, string net, string svc, SectionHost fromh, SectionHost toh, NetSvcPeer toPeer) + { + Contracts.HostInfo hinfo = null; + + SectionZone zone = toh.ParentZone; + while(zone!=null) + { + var hzgovs = zone.ZoneGovernorHosts.Where( h => !h.Dynamic );//Where for safeguard check, as dynamic host can not be zonegov, but in case someone ignores AMM error + foreach (var hzgov in hzgovs) + { + try + { + using (var cl = Contracts.ServiceClientHub.New(hzgov)) + { + cl.TimeoutMs = this.m_ResolveDynamicHostNetSvcTimeoutMs; + hinfo = cl.GetSubordinateHost(fullHostName); + break; + } + } + catch(Exception error) + { + //todo Perf counter + log(MessageType.Error, + "resolveDynamicHostAddress()", + "Error resolving net svc on dynamic host '{0}' while contacting zgov on '{1}': {2}".Args(fullHostName, hzgov.RegionPath, error.ToMessageWithType()), + error); + } + }//foreach + zone = zone.ParentZone; //loop only WITHIN the NOC + }//while + + if (hinfo == null) + throw new MetabaseException(StringConsts.METABASE_NET_SVC_RESOLVER_DYN_HOST_UNKNOWN_ERROR.Args(svc, fromh.RegionPath, toh.RegionPath, net)); + + var pattern = toPeer.Address + "*"; + foreach(var nic in hinfo.NetInfo.Adapters) + { + foreach(var addr in nic.Addresses.Where(a => a.Unicast)) + { + if (NFX.Parsing.Utils.MatchPattern(addr.Name, pattern)) + return new NetSvcPeer(addr.Name, toPeer.Port, toPeer.Group ); + } + } + + throw new MetabaseException(StringConsts.METABASE_NET_SVC_RESOLVER_DYN_HOST_NO_ADDR_MATCH_ERROR.Args(svc, fromh.RegionPath, toh.RegionPath, net, toPeer.Address)); + } + + + + + + + + + /// + /// Returns conf node for named binding per network service, or if binding name is blank then returns default network binding. + /// If net service does not have default binding specified then takes fist available binding. + /// Throws if no match could be made + /// + public IConfigSectionNode GetNetworkSvcBindingConfNode(string net, string svc, string binding = null) + { + try + { + var bindingNodes = GetNetworkSvcBindingNodes(net, svc); + if (binding.IsNullOrWhiteSpace()) + { + //get default + var nsNode = GetNetworkSvcConfNode(net, svc); + binding = nsNode.AttrByName(CONFIG_DEFAULT_BINDING_ATTR).Value; + if (binding.IsNullOrWhiteSpace()) + return bindingNodes.First(); + } + return bindingNodes.First(n=>n.IsSameName(binding)); + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_NETWORK_GET_BINDING_NODE_ERROR.Args(net, svc, binding, error.ToMessageWithType()), error); + } + } + +}} diff --git a/src/Agni/Metabase/Metabank.SecCatalog.cs b/src/Agni/Metabase/Metabank.SecCatalog.cs new file mode 100644 index 0000000..aa8b202 --- /dev/null +++ b/src/Agni/Metabase/Metabank.SecCatalog.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX.Security; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + + + /// + /// Represents a system catalog of security metadata + /// + public sealed class SecCatalog : SystemCatalog + { + #region CONSTS + + #endregion + + #region .ctor + internal SecCatalog(Metabank bank) : base(bank, SEC_CATALOG) + { + + } + + #endregion + + #region Fields + + + #endregion + + #region Properties + + + #endregion + + #region Public + + public override void Validate(ValidationContext ctx) + { + + } + + #endregion + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.SectionApplication.cs b/src/Agni/Metabase/Metabank.SectionApplication.cs new file mode 100644 index 0000000..8b3dcbb --- /dev/null +++ b/src/Agni/Metabase/Metabank.SectionApplication.cs @@ -0,0 +1,175 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.IO.FileSystem; +using NFX.Environment; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Represents metadata for Application + /// + public sealed class SectionApplication : SectionWithAnyAppConfig + { + + public sealed class AppPackage : IEquatable + { + internal AppPackage(string name, string version, string path) + { + Name = name.IsNullOrWhiteSpace() ? SysConsts.NULL : name; + Version = version.IsNullOrWhiteSpace() ? Metabank.DEFAULT_PACKAGE_VERSION : version; + Path = path ?? string.Empty; + } + + internal AppPackage(AppPackage other, BinCatalog.PackageInfo matchedPackage) + { + Name = other.Name; + Version = other.Version; + Path = other.Path; + MatchedPackage = matchedPackage; + } + + public readonly string Name; + public readonly string Version; + public readonly string Path; + + /// + /// The best matching binary package, or null + /// + public readonly BinCatalog.PackageInfo MatchedPackage; + + + public override bool Equals(object obj) + { + return this.Equals((AppPackage)obj); + } + + public bool Equals(AppPackage other) + { + return INVSTRCMP.Equals(Name, other.Name); + } + + public override int GetHashCode() + { + return INVSTRCMP.GetHashCode(Name); + } + + public override string ToString() + { + return "{0} {1} {2} {3}".Args(Name, Version, Path, MatchedPackage); + } + } + + + + internal SectionApplication(AppCatalog catalog, string name, string path, FileSystemSession session) : base(catalog, name, path, session) + { + + } + + + + public override string RootNodeName + { + get { return "application"; } + } + + /// + /// Enumerates all packages that this application is comprised of + /// + public IEnumerable Packages + { + get + { + var npackages = this.LevelConfig[CONFIG_PACKAGES_SECTION]; + + return npackages.Children.Where(c=>c.IsSameName(CONFIG_PACKAGE_SECTION)) + .Select(c => new AppPackage( c.AttrByName(CONFIG_NAME_ATTR).ValueAsString(string.Empty).Trim(), + c.AttrByName(CONFIG_VERSION_ATTR).ValueAsString(Metabank.DEFAULT_PACKAGE_VERSION).Trim(), + c.AttrByName(CONFIG_PATH_ATTR).Value) + ); + } + } + + + /// + /// Returns application executable command used for app start + /// + public string ExeFile + { + get + { + return this.LevelConfig.AttrByName(Metabank.CONFIG_EXE_FILE_ATTR).ValueAsString(string.Empty); + } + } + + /// + /// Returns application executable command arguments used for app start + /// + public string ExeArgs + { + get + { + return this.LevelConfig.AttrByName(Metabank.CONFIG_EXE_ARGS_ATTR).ValueAsString(string.Empty); + } + } + + + + public override void Validate(ValidationContext ctx) + { + base.Validate(ctx); + + if (!Packages.Any()) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, Catalog, this, StringConsts.METABASE_NO_APP_PACKAGES_WARNING.Args(Name) ) ); + + if (Packages.Count() != Packages.Distinct().Count()) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, StringConsts.METABASE_APP_PACKAGE_REDECLARED_ERROR.Args(Name) ) ); + + foreach(var ap in Packages) + { + if (ap.Name.IsNullOrEmpty()) + { + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, StringConsts.METABASE_APP_PACKAGE_BLANK_NAME_ERROR.Args(Name) ) ); + continue; + } + + var refed = Metabank.CatalogBin.Packages.Where(pi=> Metabank.INVSTRCMP.Equals(pi.Name, ap.Name)).Select(pi=>pi.Name).FirstOrDefault(); + + if (refed==null) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, StringConsts.METABASE_APP_PACKAGE_NOT_FOUND_IN_BIN_CATALOG_ERROR.Args(Name, ap.Name) ) ); + } + } + + /// + /// Returns the best suitable package binaries for this application - packages that match the specified OS + /// + /// OS to match + /// Best suitable packages + public IEnumerable MatchPackageBinaries(string os) + { + var platform = Metabank.GetOSPlatformName(os); + + var result = new List(); + foreach(var ap in Packages) + { + var match = Metabank.CatalogBin.Packages.Where(pi => Metabank.INVSTRCMP.Equals(pi.Name, ap.Name) && + Metabank.INVSTRCMP.Equals(pi.Version, ap.Version) && + (pi.IsAnyPlatform || Metabank.INVSTRCMP.Equals(pi.Platform, platform)) && + (pi.IsAnyOS || Metabank.INVSTRCMP.Equals(pi.OS, os)) + ).OrderBy(pi => -pi.Specificity).FirstOrDefault();//highest first + if (match==null) + throw new MetabaseException(StringConsts.METABASE_APP_DOES_NOT_HAVE_MATCHING_BIN_ERROR.Args(Name, ap.Name, ap.Version, os)); + + result.Add( new AppPackage(ap, match)); + } + return result; + } + + } + + +}} diff --git a/src/Agni/Metabase/Metabank.SectionHost.cs b/src/Agni/Metabase/Metabank.SectionHost.cs new file mode 100644 index 0000000..a62df48 --- /dev/null +++ b/src/Agni/Metabase/Metabank.SectionHost.cs @@ -0,0 +1,417 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.IO.FileSystem; +using NFX.Environment; + +using Agni.AppModel; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Represents metadata for Host + /// + public sealed class SectionHost : SectionRegionBase + { + + internal SectionHost(SectionZone parentZone, string name, string path, FileSystemSession session) : base(parentZone.Catalog, parentZone, name, path, session) + { + m_ParentZone = parentZone; + } + + private SectionZone m_ParentZone; + + + public override string RootNodeName + { + get { return "host"; } + } + + /// + /// Returns the parent zone that this host is in + /// + public SectionZone ParentZone { get { return m_ParentZone;} } + + /// + /// Returns the NOC that this host is under + /// + public SectionNOC NOC { get { return m_ParentZone.NOC;}} + + /// + /// Navigates to a named child region or NOC. The search is done using case-insensitive comparison, however + /// the underlying file system may be case-sensitive and must be supplied the exact name + /// + public override SectionRegionBase this[string name] + { + get + { + throw new MetabaseException(StringConsts.METABASE_INVALID_OPERTATION_ERROR + "SectionHost.navigate[{0}]".Args(name)); + } + } + + /// + /// Returns a name of the role (as defined by the attribute in host confog file) that this host has in the agni + /// + public string RoleName { get{ return this.LevelConfig.AttrByName(CONFIG_ROLE_ATTR).ValueAsString(SysConsts.NULL); } } + + /// + /// Returns a role app catalog section for the role that this host has in the agni + /// + public SectionRole Role + { + get + { + var result = Metabank.CatalogApp.Roles[RoleName]; + if (result==null) + throw new MetabaseException(StringConsts.METABASE_BAD_HOST_ROLE_ERROR.Args(RegionPath, RoleName)); + + return result; + } + } + + /// + /// Returns the name of the operating system that this host operates on. The value is required or exception is thrown + /// + public string OS + { + get + { + var result = this.LevelConfig.AttrByName(CONFIG_OS_ATTR).Value; + if (result.IsNullOrWhiteSpace()) + throw new MetabaseException(StringConsts.METABASE_HOST_MISSING_OS_ATTR_ERROR.Args(RegionPath)); + + //check that OS is registered in the list + Metabank.GetOSConfNode(result); + + return result; + } + } + + /// + /// Returns true if host is dynamic - that is: a physical representation of host gets created/torn at the runtime of cluter. + /// The Dynamic host section represents a proptotype of (possibly) many host instances spawn in the agni. + /// Every particular instance of dynamic host gets a HOST-unique signature that can be supplied to host resolution service + /// to get actual network addresses for particular host + /// + public bool Dynamic { get { return this.LevelConfig.AttrByName(CONFIG_HOST_DYNAMIC_ATTR).ValueAsBool(); } } + + /// + /// Returns true when this host has zone governor application in its role + /// + public bool IsZGov { get { return this.Role.Applications.Any(app => Metabank.INVSTRCMP.Equals(app.Name, SysConsts.APP_NAME_ZGOV)); } } + + /// + /// Returns true for ZGov nodes that are failover lock service nodes which are used when primary nodes fail + /// + public bool IsZGovLockFailover { get { return this.LevelConfig.AttrByName(CONFIG_HOST_ZGOV_LOCK_FAILOVER_ATTR).ValueAsBool(); } } + + /// + /// Returns true for Host nodes that are process host + /// + public bool IsProcessHost { get { return this.LevelConfig.AttrByName(CONFIG_HOST_PROCESS_HOST_ATTR).ValueAsBool(); } } + + /// + /// Returns the first/primary host that runs the parent Zone governor for this host. + /// Null returned for top-level hosts that do not have zone governors higher than themselves in this NOC unless transcendNOC is true + /// in which case the governor from higher-level NOC may be returned + /// + public SectionHost ParentZoneGovernorPrimaryHost(bool transcendNOC = false) + { + var myZone = m_ParentZone; + + return myZone.FindNearestParentZoneGovernors(transcendNOC: transcendNOC).FirstOrDefault(); + } + + /// + /// Returns true if this zone has direct or indirect parent zone governor above it, optionally examining higher-level NOCs. + /// + public bool HasDirectOrIndirectParentZoneGovernor(SectionHost zgovHost, bool? iAmZoneGovernor = null, bool transcendNOC = false) + { + var myZone = m_ParentZone; + + return myZone.HasDirectOrIndirectParentZoneGovernor(zgovHost, iAmZoneGovernor, transcendNOC); + } + + /// + /// Returns true if this zone has direct or indirect parent zone governor above it, optionally examining higher-level NOCs. + /// + public bool HasDirectOrIndirectParentZoneGovernor(string zgovHost, bool? iAmZoneGovernor = null, bool transcendNOC = false) + { + var myZone = m_ParentZone; + + return myZone.HasDirectOrIndirectParentZoneGovernor(zgovHost, iAmZoneGovernor, transcendNOC); + } + + + /// + /// Gets an application configuration tree for the particular named application on this host. + /// This method traverses the whole region catalog and calculates the effective configuration tree for this host - + /// the one that will get supplied into the application container process-wide. + /// + /// The traversal is done in the following order: MetabaseRoot->Role->App->[Regions]->NOC->[Zones]->Host. + /// + /// Metabase application name which should resolve in App catalog + /// Pass true to bypass the metabase cache on read so the config tree is recalculated + public IConfigSectionNode GetEffectiveAppConfig(string appName, bool latest = false) + { + IConfigSectionNode result = null; + const string CACHE_TABLE = "EffectiveAppConfigs"; + string CACHE_KEY = (this.RegionPath+"."+appName).ToLowerInvariant(); + + //1. Try to get from cache + if (!latest) + { + result = Metabank.cacheGet(CACHE_TABLE, CACHE_KEY) as IConfigSectionNode; + if (result!=null) return result; + } + + try + { + result = calculateEffectiveAppConfig(appName); + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_EFFECTIVE_APP_CONFIG_ERROR.Args(appName, RegionPath, error.ToMessageWithType()), error); + } + + Metabank.cachePut(CACHE_TABLE, CACHE_KEY, result); + return result; + } + + /// + /// Gets the information for the best suitable binary packages on this host for the named app + /// + /// Metabase application name which should resolve in App catalog + public IEnumerable GetAppPackages(string appName) + { + const string CACHE_TABLE = "AppPackages"; + string CACHE_KEY = (this.RegionPath+"."+appName).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(CACHE_TABLE, CACHE_KEY) as IEnumerable; + if (result!=null) return result; + + try + { + result = this.calculateAppPackages(appName); + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_APP_PACKAGES_ERROR.Args(appName, RegionPath, error.ToMessageWithType()), error); + } + + Metabank.cachePut(CACHE_TABLE, CACHE_KEY, result); + return result; + } + + public override void Validate(ValidationContext ctx) + { + var output = ctx.Output; + + base.Validate(ctx); + + try + { + var os = OS; + var role = Role; + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, error.ToMessageWithType(), error) ); + return; + } + + + foreach(var app in Role.AppNames) + try + { + var appConfig = GetEffectiveAppConfig(app); + if (Metabank.INVSTRCMP.Equals(app, SysConsts.APP_NAME_GDIDA)) + { + var authorityIDs = appConfig[Identification.GDIDAuthorityService.CONFIG_GDID_AUTHORITY_SECTION] + .AttrByName( Identification.GDIDAuthorityService.CONFIG_AUTHORITY_IDS_ATTR ).ValueAsByteArray(); + + if (authorityIDs==null || authorityIDs.Length<1) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, + Catalog, + this, + "The app container is improperly configured for '{0}' application. No GDID authorities specified".Args(SysConsts.APP_NAME_GDIDA) )); + continue; + } + + var stateKey = "{0}::AuthorityRoles".Args(this.GetType().FullName); + var authIDs = ctx.StateAs>(stateKey); + if (authIDs==null) + { + authIDs = new HashSet(); + ctx.State[stateKey] = authIDs; + } + + foreach(var aid in authorityIDs) + { + if (aid> NFX.DataAccess.Distributed.GDID.AUTHORITY_MAX) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, + Catalog, + this, + "The app container is improperly configured for '{0}' application. GDID Authority id is over the limit: {1:X1}".Args(SysConsts.APP_NAME_GDIDA, aid)) ); + + if (!authIDs.Add(aid)) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, + Catalog, + this, + "The app container is improperly configured for '{0}' application. Duplicate GDID authority: {1:X1}".Args(SysConsts.APP_NAME_GDIDA, aid)) ); + } + } + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, error.ToMessageWithType(), error) ); + } + + foreach(var app in Role.AppNames) + try + { + GetAppPackages(app); + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, error.ToMessageWithType(), error) ); + } + + if (this.Dynamic) + { + + if (!this.ParentZone.ZoneGovernorHosts.Any()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, Catalog, this, + "Host is dynamic, but there are no zone governors in this zone") ); + + if (this.ParentZoneGovernorPrimaryHost()==null) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + "Host is dynamic, but there are no zone governors in either this zone or any parent zones above in the same NOC. Dynamic hosts must operate under zone governors in the same NOC that they are in") ); + + var violations = this.Role.AppNames.Intersect(SysConsts.APP_NAMES_FORBIDDEN_ON_DYNAMIC_HOSTS, Metabank.INVSTRCMP); + if (violations.Any() ) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + "Host is dynamic, but the assigned role '{0}' contains system process(es): {1} that can not run on dynamic hosts" + .Args( this.RoleName, violations.Aggregate(string.Empty, (a, v) => a += "'{0}', ".Args(v) )) + )); + + }//dynamic + + if (this.LevelConfig.AttrByName(CONFIG_HOST_ZGOV_LOCK_FAILOVER_ATTR).Exists && !this.IsZGov) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + "This host has '{0}' attribute defined, however it is not a zone governor host" + .Args(CONFIG_HOST_ZGOV_LOCK_FAILOVER_ATTR) ) + ); + + } + + + + + + + #region .pvt + + private struct configLevel + { + public configLevel(object from, IConfigSectionNode node) { From = from; Node = node;} + public readonly object From; + public readonly IConfigSectionNode Node; + } + + //20150403 DKh added support for include pragmas + private ConfigSectionNode calculateEffectiveAppConfig(string appName) + { + var pragmasDisabled = Metabank.RootConfig.AttrByName(Metabank.CONFIG_APP_CONFIG_INCLUDE_PRAGMAS_DISABLED_ATTR).ValueAsBool(false); + var includePragma = Metabank.RootConfig.AttrByName(Metabank.CONFIG_APP_CONFIG_INCLUDE_PRAGMA_ATTR).Value; + + + var conf = new MemoryConfiguration(); + conf.CreateFromNode( Metabank.RootAppConfig ); + + var result = conf.Root; + if (!pragmasDisabled) + BootConfLoader.ProcessAllExistingIncludes(result, includePragma, "root"); + + var derivation = new List(); + + #region build derivation chain -------------------------------------------------------------------------- + //Role level + var role = this.Role; + + if (!role.AppNames.Any(an => INVSTRCMP.Equals(an, appName))) + throw new MetabaseException(StringConsts.METABASE_HOST_ROLE_APP_MISMATCH_ERROR.Args(appName, RoleName)); + + derivation.Add( new configLevel(role, role.AnyAppConfig) ); + + //App level + var app = Metabank.CatalogApp.Applications[appName]; + if (app==null) + throw new MetabaseException(StringConsts.METABASE_BAD_HOST_APP_ERROR.Args(appName)); + + derivation.Add(new configLevel(app, app.AnyAppConfig)); + + //Regional level + var parents = this.ParentSectionsOnPath; + foreach(var item in parents) + { + derivation.Add( new configLevel(item, item.AnyAppConfig) ); + derivation.Add( new configLevel(item, item.GetAppConfig(appName)) ); + } + + //This Level + derivation.Add( new configLevel(this, this.AnyAppConfig) ); + derivation.Add( new configLevel(this, this.GetAppConfig(appName)) ); + #endregion + + foreach(var clevel in derivation.Where(cl => cl.Node!=null)) + { + if (!pragmasDisabled) + BootConfLoader.ProcessAllExistingIncludes((ConfigSectionNode)clevel.Node, includePragma, clevel.From.ToString()); + + try + { + result.OverrideBy(clevel.Node); + } + catch (Exception error) + { + throw new MetabaseException(StringConsts.METABASE_EFFECTIVE_APP_CONFIG_OVERRIDE_ERROR.Args(clevel.From.ToString(), error.ToMessageWithType()), error); + } + } + + //OS Include + var include = result[CONFIG_OS_APP_CONFIG_INCLUDE_SECTION]; + if (include.Exists) + { + var osInclude = Metabank.GetOSConfNode(this.OS)[CONFIG_APP_CONFIG_SECTION] as ConfigSectionNode; + if (osInclude.Exists) + include.Configuration.Include(include, osInclude); + } + + result.ResetModified(); + return result; + } + + private IEnumerable calculateAppPackages(string appName) + { + var role = this.Role; + if (!role.AppNames.Any(an => INVSTRCMP.Equals(an, appName))) + throw new MetabaseException(StringConsts.METABASE_HOST_ROLE_APP_MISMATCH_ERROR.Args(appName, RoleName)); + var app = Metabank.CatalogApp.Applications[appName]; + if (app==null) + throw new MetabaseException(StringConsts.METABASE_BAD_HOST_APP_ERROR.Args(appName)); + + return app.MatchPackageBinaries(this.OS); + } + #endregion + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.SectionNOC.cs b/src/Agni/Metabase/Metabank.SectionNOC.cs new file mode 100644 index 0000000..12dfa27 --- /dev/null +++ b/src/Agni/Metabase/Metabank.SectionNOC.cs @@ -0,0 +1,182 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.IO.FileSystem; +using NFX.Environment; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Represents metadata for Network Operations Center (NOC). A NOC name must be globally-unique regardless of it's parent region + /// + public sealed class SectionNOC : SectionRegionBase + { + internal SectionNOC(RegCatalog catalog, SectionRegion parentRegion, string name, string path, FileSystemSession session) : base(catalog, parentRegion, name, path, session) + { + m_ParentRegion = parentRegion; + } + + private SectionRegion m_ParentRegion; + + + public override string RootNodeName + { + get { return "noc"; } + } + + /// + /// Returns parent region for this NOC + /// + public SectionRegion ParentRegion { get { return m_ParentRegion;} } + + + /// + /// Returns Zone section in parent NOC that this whole NOC is logically (not phisically) under, or null if + /// this NOC does not have any NOC above it. + /// This property is used to detect ZoneGovernor procesees which are higher in hierarchy than any zone in this NOC. + /// This is important for failover to parent. + /// The path must point to an existing zone in a NOC which is higher in hierarchy than this one and + /// it must be of the same root, that is - have at least one common region with this one + /// + public SectionZone ParentNOCZone + { + get + { + var path = this.ParentNOCZonePath; + if (path.IsNullOrWhiteSpace()) return null; + var zone = Catalog[path] as SectionZone; + if (zone==null) + throw new MetabaseException(StringConsts.METABASE_REG_NOC_PARENT_NOC_ZONE_ERROR.Args(Name, path)); + + if (Catalog.CountMatchingPathSegments(zone, this)<1) + throw new MetabaseException(StringConsts.METABASE_REG_NOC_PARENT_NOC_ZONE_NO_ROOT_ERROR.Args(Name, path)); + + if (zone.NOC.SectionsOnPath.Count()>=this.SectionsOnPath.Count()) + throw new MetabaseException(StringConsts.METABASE_REG_NOC_PARENT_NOC_ZONE_LEVEL_ERROR.Args(Name, path)); + + return zone; + } + } + + + /// + /// Returns Zone section path in parent NOC that this whole NOC is logically (not phisically) under, or null if + /// this NOC does not have any NOC above it. + /// This property is used to detect ZoneGovernor procesees which are higher in hierarchy than any zone in this NOC. + /// This is important for failover to parent. + /// The path must point to an existing zone in a NOC which is higher in hierarchy than this one and + /// it must be of the same root, that is - have at least one common region with this one + /// + public string ParentNOCZonePath { get{ return LevelConfig.AttrByName(CONFIG_PARENT_NOC_ZONE_ATTR).Value;}} + + + /// + /// Returns names of child zones + /// + public IEnumerable ZoneNames + { + get + { + string CACHE_KEY = ("NOC.zn"+Path).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as IEnumerable; + if (result!=null) return result; + + result = + Metabank.fsAccess("NOC[{0}].ZoneNames.get".Args(m_Name), Path, + (session, dir) => dir.SubDirectoryNames + .Where(dn=>dn.EndsWith(RegCatalog.ZON_EXT)) + .Select(dn=>Metabank.chopExt(dn)) + .ToList() + ); + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + } + + + /// + /// Navigates to a named child zone. The search is done using case-insensitive comparison, however + /// the underlying file system may be case-sensitive and must be supplied the exact name + /// + public override SectionRegionBase this[string name] + { + get + { + if (name.IsNullOrWhiteSpace()) return null; + + name = name.Trim(); + + if (name.EndsWith(RegCatalog.ZON_EXT)) + name = name.Substring(0, name.LastIndexOf(RegCatalog.ZON_EXT)); + + if (ZoneNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetZone(name); + + return null; + } + } + + + + /// + /// Gets zone by name + /// + public SectionZone GetZone(string name) + { + string CACHE_KEY = ("NOC.z"+Path+name).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as SectionZone; + if (result!=null) return result; + + var rdir = "{0}{1}".Args(name, RegCatalog.ZON_EXT); + var rpath = Metabank.JoinPaths(Path, rdir); + result = + Metabank.fsAccess("NOC[{0}].GetZone({1})".Args(m_Name, name), rpath, + (session, dir) => new SectionZone(this, null, name, rpath, session) + ); + + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + + public override void Validate(ValidationContext ctx) + { + var output = ctx.Output; + + base.Validate(ctx); + + SectionZone pnz = null; + try + { + pnz = this.ParentNOCZone;//throws + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + "Invalid '{0}' specified: {1}".Args(CONFIG_PARENT_NOC_ZONE_ATTR, error.ToMessageWithType()))); + } + + if (pnz==null && ParentSectionsOnPath.Count()>1) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, Catalog, this, + "This NOC is not top-level but it does not have any '{0}' specified".Args(CONFIG_PARENT_NOC_ZONE_ATTR))); + + if (pnz!=null && !pnz.ZoneGovernorHosts.Any()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + "ParenNOCZone '{0}' pointed to by this NOC does not have any hosts that run zone governor application".Args(pnz.RegionPath))); + + foreach(var szone in this.ZoneNames) + this.GetZone(szone).Validate(ctx); + } + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.SectionRegion.cs b/src/Agni/Metabase/Metabank.SectionRegion.cs new file mode 100644 index 0000000..66eefcd --- /dev/null +++ b/src/Agni/Metabase/Metabank.SectionRegion.cs @@ -0,0 +1,492 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.IO.FileSystem; +using NFX.Environment; +using NFX.Geometry; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + + /// + /// Base class for all sections that are in region catalog + /// + public abstract class SectionRegionBase : SectionWithNamedAppConfigs + { + internal SectionRegionBase(RegCatalog catalog, SectionRegionBase parentSection, string name, string path, FileSystemSession session) : base(catalog, name, path, session) + { + ParentSection = parentSection; + + var gc = this.LevelConfig.AttrByName(Metabank.CONFIG_GEO_CENTER_ATTR).Value; + + if (gc.IsNotNullOrWhiteSpace()) + try + { + GeoCenter = new LatLng(gc); + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_REG_GEO_CENTER_ERROR.Args(GetType().Name, name, gc, error.ToMessageWithType())); + } + } + + /// + /// A section which is a parent of this one + /// + public readonly SectionRegionBase ParentSection; + + + /// + /// Returns geo center coordinates for this level or null. Use EffectiveGeoCenter to get coordinates defaulted from parent if not set on this level + /// + public readonly LatLng? GeoCenter; + + + /// + /// Returns the effective geo center of this entity, that is - if the coordinates are not specified on this level then + /// coordinates of parent are taken + /// + public LatLng EffectiveGeoCenter + { + get + { + if (this.GeoCenter.HasValue) return this.GeoCenter.Value; + if (ParentSection==null) + return SysConsts.DEFAULT_GEO_LOCATION;//very default + + return ParentSection.EffectiveGeoCenter; + } + } + + /// + /// Returns region path + /// + public virtual string RegionPath + { + get + { + return "{0}/{1}".Args( ParentSection==null ? string.Empty : ParentSection.RegionPath, Name); + } + } + + /// + /// Enumerates parent sections on the path to this section in the order of their descent - from root down into the region tree + /// + public IEnumerable ParentSectionsOnPath + { + get + { + var parents = new List(); + var level = this.ParentSection; + while(level!=null) + { + parents.Add(level); + level = level.ParentSection; + } + parents.Reverse();// zone:noc:region -> region:noc:zone + return parents; + } + } + + /// + /// Enumerates parent sections on the path to this section in the order of their descent - from root down into the region tree including this section. + /// See also ParentSectionsOnPath which only returns parent sections without this one + /// + public IEnumerable SectionsOnPath + { + get + { + var chain = ParentSectionsOnPath.ToList(); + chain.Add(this); + return chain; + } + } + + /// + /// Provides short mnemonic description of this section type (i.e. "reg", "noc", "zone", "host") + /// + public string SectionMnemonicType + { + get + { // do not localize! + return this is Metabank.SectionRegion ? SysConsts.REGION_MNEMONIC_REGION : + this is Metabank.SectionNOC ? SysConsts.REGION_MNEMONIC_NOC : + this is Metabank.SectionZone ? SysConsts.REGION_MNEMONIC_ZONE : SysConsts.REGION_MNEMONIC_HOST; + } + } + + /// + /// Navigates to a named child section + /// + public abstract SectionRegionBase this[string name] { get;} + + public override string ToString() + { + return "{0}({1})".Args(GetType().Name, this.RegionPath); + } + + /// + /// Returns true if another instance represents logically the same regional entity: has the same type and path, regardless of path extensions ('.r','.noc',...). + /// Use this as a test as Equals() is not overriden by this class and does instance-based comparison by default + /// + public bool IsLogicallyTheSame(SectionRegionBase other) + { + if (other==null) return false; + if (GetType()!=other.GetType())return false; + if (!this.RegionPath.IsSameRegionPath(other.RegionPath)) return false; + return true; + } + + + /// + /// Matches the requested parameters to the most appropriate network route specified in the 'networks' level config subsection + /// + /// Network + /// Service + /// Binding name + /// Calling party - region path to host i.e. '/US/East/CLE/A/I/wmed0001' + /// Best matched NetSvcPeer descriptor which may be blank. Check NetSvcPeer.Blank + public NetSvcPeer MatchNetworkRoute(string net, string svc, string binding, string from) + { + var routing = this.LevelConfig[CONFIG_NETWORK_ROUTING_SECTION]; + if (!routing.Exists) + return new NetSvcPeer(); + + var routeNodes = routing.Children.Where(n => n.IsSameName(CONFIG_NETWORK_ROUTING_ROUTE_SECTION)); + + var match = routeNodes.Where(n => + ( n.AttrByName(CONFIG_NETWORK_ROUTING_NETWORK_ATTR).Value.IsNullOrWhiteSpace() || + Metabank.INVSTRCMP.Equals(n.AttrByName(CONFIG_NETWORK_ROUTING_NETWORK_ATTR).Value, net) + ) && + ( n.AttrByName(CONFIG_NETWORK_ROUTING_SERVICE_ATTR).Value.IsNullOrWhiteSpace() || + Metabank.INVSTRCMP.Equals(n.AttrByName(CONFIG_NETWORK_ROUTING_SERVICE_ATTR).Value, svc) + ) && + ( n.AttrByName(CONFIG_NETWORK_ROUTING_BINDING_ATTR).Value.IsNullOrWhiteSpace() || + Metabank.INVSTRCMP.Equals(n.AttrByName(CONFIG_NETWORK_ROUTING_BINDING_ATTR).Value, binding) + ) && + ( n.AttrByName(CONFIG_NETWORK_ROUTING_FROM_ATTR).Value.IsNullOrWhiteSpace() || + matchRegPath(n.AttrByName(CONFIG_NETWORK_ROUTING_FROM_ATTR).Value, from)>0 + ) + ).Select(n => new + { + ToAddress = n.AttrByName(CONFIG_NETWORK_ROUTING_TO_ADDRESS_ATTR).Value, + ToPort = n.AttrByName(CONFIG_NETWORK_ROUTING_TO_PORT_ATTR) .Value, + ToGroup = n.AttrByName(CONFIG_NETWORK_ROUTING_TO_GROUP_ATTR) .Value, + Score = + (n.AttrByName(CONFIG_NETWORK_ROUTING_NETWORK_ATTR).Value.IsNullOrWhiteSpace() ? 0 : 1000000000) + + (n.AttrByName(CONFIG_NETWORK_ROUTING_SERVICE_ATTR).Value.IsNullOrWhiteSpace() ? 0 : 1000000) + + (n.AttrByName(CONFIG_NETWORK_ROUTING_BINDING_ATTR).Value.IsNullOrWhiteSpace() ? 0 : 1000) + + (n.AttrByName(CONFIG_NETWORK_ROUTING_FROM_ATTR) .Value.IsNullOrWhiteSpace() ? 0 : + 1+matchRegPath(n.AttrByName(CONFIG_NETWORK_ROUTING_FROM_ATTR).Value, from)) + } + ).OrderBy(m => -m.Score).FirstOrDefault(); + if (match==null) + return new NetSvcPeer(); + + return new NetSvcPeer(match.ToAddress, match.ToPort, match.ToGroup); + } + + public override void Validate(ValidationContext ctx) + { + base.Validate(ctx); + + validateRouting(ctx); + } + + + private int matchRegPath(string path, string pattern) + { + return Metabank.CatalogReg.CountMatchingPathSegments(path, pattern); + } + + private void validateRouting(ValidationContext ctx) + { + var routing = this.LevelConfig[CONFIG_NETWORK_ROUTING_SECTION]; + if (!routing.Exists) return; + var routeNodes = routing.Children.Where(n => n.IsSameName(CONFIG_NETWORK_ROUTING_ROUTE_SECTION)); + + foreach(var rnode in routeNodes) + { + foreach(var anode in rnode.Attributes) + if (!anode.IsSameName(CONFIG_NETWORK_ROUTING_NETWORK_ATTR) && + !anode.IsSameName(CONFIG_NETWORK_ROUTING_SERVICE_ATTR) && + !anode.IsSameName(CONFIG_NETWORK_ROUTING_BINDING_ATTR) && + !anode.IsSameName(CONFIG_NETWORK_ROUTING_FROM_ATTR) && + !anode.IsSameName(CONFIG_NETWORK_ROUTING_TO_ADDRESS_ATTR) && + !anode.IsSameName(CONFIG_NETWORK_ROUTING_TO_PORT_ATTR) && + !anode.IsSameName(CONFIG_NETWORK_ROUTING_TO_GROUP_ATTR) + ) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, Catalog, this, + StringConsts.METABASE_NETWORK_REGION_ROUTING_ATTR_UNRECOGNIZED_WARNING.Args(RegionPath, anode.Name) ) ); + + + + + var net = rnode.AttrByName( CONFIG_NETWORK_ROUTING_NETWORK_ATTR ).Value; + var svc = rnode.AttrByName( CONFIG_NETWORK_ROUTING_SERVICE_ATTR ).Value; + var binding = rnode.AttrByName( CONFIG_NETWORK_ROUTING_BINDING_ATTR ).Value; + var from = rnode.AttrByName( CONFIG_NETWORK_ROUTING_FROM_ATTR ).Value; + + var toaddr = rnode.AttrByName( CONFIG_NETWORK_ROUTING_TO_ADDRESS_ATTR ).Value; + var toport = rnode.AttrByName( CONFIG_NETWORK_ROUTING_TO_PORT_ATTR ).Value; + var togroup = rnode.AttrByName( CONFIG_NETWORK_ROUTING_TO_GROUP_ATTR ).Value; + + if (toaddr.IsNullOrWhiteSpace() && + toport.IsNullOrWhiteSpace() && + togroup.IsNullOrWhiteSpace()) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, StringConsts.METABASE_NETWORK_REGION_ROUTING_EMPTY_ROUTE_ASSIGNMENT_ERROR.Args(RegionPath) ) ); + + if (net.IsNotNullOrEmpty()) + { + try + { + var netNode = Metabank.GetNetworkConfNode(net); + } + catch(Exception error) + { + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, error.ToMessageWithType() ) ); + } + + if (svc.IsNotNullOrWhiteSpace()) + { + try + { + var svcNode = Metabank.GetNetworkSvcConfNode(net, svc); + } + catch(Exception error) + { + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, error.ToMessageWithType() ) ); + } + + if (binding.IsNotNullOrWhiteSpace()) + try + { + var svcNode = Metabank.GetNetworkSvcBindingConfNode(net, svc, binding); + } + catch(Exception error) + { + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, error.ToMessageWithType() ) ); + } + + } + } + + if (from.IsNotNullOrWhiteSpace()) + { + var reg = Metabank.CatalogReg[from]; + if (reg==null) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, StringConsts.METABASE_NETWORK_REGION_ROUTING_FROM_PATH_ERROR.Args(RegionPath, from) ) ); + } + + + } + } + + } + + + + + /// + /// Represents metadata for Region + /// + public sealed class SectionRegion : SectionRegionBase + { + internal SectionRegion(RegCatalog catalog, SectionRegion parentRegion, string name, string path, FileSystemSession session) : base(catalog, parentRegion, name, path, session) + { + m_ParentRegion = parentRegion; + } + + private SectionRegion m_ParentRegion; + + + public override string RootNodeName + { + get { return "region"; } + } + + /// + /// Returns parent region for this region or null if this region is top-level + /// + public SectionRegion ParentRegion { get { return m_ParentRegion;} } + + + /// + /// Returns names of child regions + /// + public IEnumerable SubRegionNames + { + get + { + string CACHE_KEY = ("Reg.srn"+Path).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as IEnumerable; + if (result!=null) return result; + + result = + Metabank.fsAccess("Region[{0}].SubRegionNames.get".Args(m_Name), Path, + (session, dir) => dir.SubDirectoryNames + .Where(dn=>dn.EndsWith(RegCatalog.REG_EXT)) + .Select(dn=>Metabank.chopExt(dn)) + .ToList() + ); + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + } + + + /// + /// Returns all NOC sections that this region and all sub-regions contain. + /// This property may fetch much data for large cloud if enumerated to the end + /// + public IEnumerable AllNOCs + { + get + { + foreach(var nn in this.NOCNames) + yield return this.GetNOC(nn); + + foreach(var srn in this.SubRegionNames) + { + var sr = this.GetSubRegion(srn); + foreach(var srnoc in sr.AllNOCs) + yield return srnoc; + } + } + } + + + /// + /// Gets names of network op centers in this region + /// + public IEnumerable NOCNames + { + get + { + string CACHE_KEY = ("Reg.nn"+Path).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as IEnumerable; + if (result!=null) return result; + + result = + Metabank.fsAccess("Region[{0}].NOCNames.get".Args(m_Name), Path, + (session, dir) => dir.SubDirectoryNames + .Where(dn=>dn.EndsWith(RegCatalog.NOC_EXT)) + .Select(dn=>Metabank.chopExt(dn)) + .ToList() + ); + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + } + + /// + /// Navigates to a named child region or NOC. The search is done using case-insensitive comparison, however + /// the underlying file system may be case-sensitive and must be supplied the exact name + /// + public override SectionRegionBase this[string name] + { + get + { + if (name.IsNullOrWhiteSpace()) return null; + + name = name.Trim(); + + if (name.EndsWith(RegCatalog.REG_EXT)) + { + name = name.Substring(0, name.LastIndexOf(RegCatalog.REG_EXT)); + if (SubRegionNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetSubRegion(name); + return null; + } + + + if (name.EndsWith(RegCatalog.NOC_EXT)) + { + name = name.Substring(0, name.LastIndexOf(RegCatalog.NOC_EXT)); + if (NOCNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetNOC(name); + return null; + } + + + if (SubRegionNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetSubRegion(name); + + if (NOCNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetNOC(name); + + return null; + } + } + + + /// + /// Gets sub-region by name + /// + public SectionRegion GetSubRegion(string name) + { + string CACHE_KEY = ("Reg.sr"+Path+name).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as SectionRegion; + if (result!=null) return result; + + var rdir = "{0}{1}".Args(name, RegCatalog.REG_EXT); + var rpath = Metabank.JoinPaths(Path, rdir); + result = + Metabank.fsAccess("Region[{0}].GetSubRegion({1})".Args(m_Name, name), rpath, + (session, dir) => new SectionRegion(Catalog, this, name, rpath, session) + ); + + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + + /// + /// Gets child network operation center by name + /// + public SectionNOC GetNOC(string name) + { + string CACHE_KEY = ("Reg.cn"+Path+name).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as SectionNOC; + if (result!=null) return result; + + var rdir = "{0}{1}".Args(name, RegCatalog.NOC_EXT); + var rpath = Metabank.JoinPaths(Path, rdir); + result = + Metabank.fsAccess("Region[{0}].GetNOC({1})".Args(m_Name, name), rpath, + (session, dir) => new SectionNOC(Catalog, this, name, rpath, session) + ); + + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + + public override void Validate(ValidationContext ctx) + { + base.Validate(ctx); + + foreach(var sreg in this.SubRegionNames) + this.GetSubRegion(sreg).Validate(ctx); + + foreach(var snoc in this.NOCNames) + this.GetNOC(snoc).Validate(ctx); + } + + + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.SectionRole.cs b/src/Agni/Metabase/Metabank.SectionRole.cs new file mode 100644 index 0000000..827341c --- /dev/null +++ b/src/Agni/Metabase/Metabank.SectionRole.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.IO.FileSystem; +using NFX.Environment; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Represents metadata for Application Role + /// + public sealed class SectionRole : SectionWithAnyAppConfig + { + + public struct AppInfo + { + internal AppInfo(string name, int? autoRun, string exeFile, string exeArgs) + { + Name = name; + AutoRun = autoRun; + ExeFile = exeFile; + ExeArgs = exeArgs; + } + + /// + /// Metabase application name + /// + public readonly string Name; + + /// + /// Specifies the relative order of start if application should be auto-started by host governor, null otherwise + /// + public readonly int? AutoRun; + + /// + /// Specifies the execurtable file + /// + public readonly string ExeFile; + + /// + /// Specifies the execurtable file arguments + /// + public readonly string ExeArgs; + + + public override string ToString() + { + return "AppInfo('{0}', autos: {1}, exe: '{2}'/'{3}')".Args(Name, AutoRun, ExeFile, ExeArgs); + } + + } + + + + + internal SectionRole(AppCatalog catalog, string name, string path, FileSystemSession session) : base(catalog, name, path, session) + { + + } + + public override string RootNodeName + { + get { return "role"; } + } + + /// + /// Enumerates all application names that this role has + /// + public IEnumerable AppNames + { + get { return Applications.Select(ai=>ai.Name); } + } + + /// + /// Enumerates all application infos declared for this role + /// + public IEnumerable Applications + { + get + { + return LevelConfig.Children + .Where(n=>n.IsSameName(AppCatalog.CONFIG_APPLICATION_SECTION)) + .Select(n=> + new AppInfo( n.AttrByName(Metabank.CONFIG_NAME_ATTR).ValueAsString(string.Empty).Trim(), + n.AttrByName(Metabank.CONFIG_AUTO_RUN_ATTR).ValueAsNullableInt(), + n.AttrByName(Metabank.CONFIG_EXE_FILE_ATTR).Value, + n.AttrByName(Metabank.CONFIG_EXE_ARGS_ATTR).Value + )); + } + } + + public override void Validate(ValidationContext ctx) + { + base.Validate(ctx); + + foreach(var appInfo in Applications) + try + { + var appName = appInfo.Name; + var app = Metabank.CatalogApp.Applications[appName]; + if (app==null) + { + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + StringConsts.METABASE_VALIDATION_ROLE_APP_ERROR.Args(Name, appName)) ); + continue; + } + + if (appInfo.ExeFile.IsNullOrWhiteSpace() && app.ExeFile.IsNullOrWhiteSpace()) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + StringConsts.METABASE_VALIDATION_ROLE_APP_EXE_MISSING_ERROR.Args(Name, appName, Metabank.CONFIG_EXE_FILE_ATTR)) ); + } + catch(Exception error) + { + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, error.ToMessageWithType(), error) ); + } + } + + + } + + +}} diff --git a/src/Agni/Metabase/Metabank.SectionZone.cs b/src/Agni/Metabase/Metabank.SectionZone.cs new file mode 100644 index 0000000..bf6a287 --- /dev/null +++ b/src/Agni/Metabase/Metabank.SectionZone.cs @@ -0,0 +1,410 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.IO.FileSystem; +using NFX.Environment; + + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Represents metadata for Region + /// + public sealed class SectionZone : SectionRegionBase + { + internal SectionZone(SectionNOC noc, SectionZone parentZone, string name, string path, FileSystemSession session) + : base(noc.Catalog, (SectionRegionBase)parentZone ?? (SectionRegionBase)noc, name, path, session) + { + m_NOC = noc; + m_ParentZone = parentZone; + } + + private SectionNOC m_NOC; + private SectionZone m_ParentZone; + private Dictionary> m_ProcessorMap; + + public override string RootNodeName + { + get { return "zone"; } + } + + /// + /// Returns the NOC that this zone is under + /// + public SectionNOC NOC { get { return m_NOC;} } + + /// + /// Returns parent zone for this zone or null if this zone is top-level under NOC + /// + public SectionZone ParentZone { get { return m_ParentZone;} } + + + /// + /// Returns region path to this zone + /// + public override string RegionPath + { + get + { + var parent = (m_ParentZone==null) ? (SectionRegionBase)m_NOC : (SectionRegionBase)m_ParentZone; + return "{0}/{1}".Args( parent.RegionPath, Name); + } + } + + /// + /// Returns names of child zones + /// + public IEnumerable SubZoneNames + { + get + { + string CACHE_KEY = ("ZN.szn"+Path).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as IEnumerable; + if (result!=null) return result; + + result = + Metabank.fsAccess("Zone[{0}].SubZoneNames.get".Args(m_Name), Path, + (session, dir) => dir.SubDirectoryNames + .Where(dn=>dn.EndsWith(RegCatalog.ZON_EXT)) + .Select(dn=>Metabank.chopExt(dn)) + .ToList() + ); + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + } + + + /// + /// Gets names of hosts + /// + public IEnumerable HostNames + { + get + { + string CACHE_KEY = ("ZN.hns"+Path).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as IEnumerable; + if (result!=null) return result; + + result = + Metabank.fsAccess("Zone[{0}].HostNames.get".Args(m_Name), Path, + (session, dir) => dir.SubDirectoryNames + .Where(dn=>dn.EndsWith(RegCatalog.HST_EXT)) + .Select(dn=>Metabank.chopExt(dn)) + .ToList() + ); + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + } + + + /// + /// Navigates to a named child zone or host. The search is done using case-insensitive comparison, however + /// the underlying file system may be case-sensitive and must be supplied the exact name + /// + public override SectionRegionBase this[string name] + { + get + { + if (name.IsNullOrWhiteSpace()) return null; + + name = name.Trim(); + + //ignore the dynamic host name suffix (if any) at the end + var ic = name.LastIndexOf(HOST_DYNAMIC_SUFFIX_SEPARATOR); + if (ic>0) name = name.Substring(0, ic).Trim(); + + + + if (name.EndsWith(RegCatalog.ZON_EXT)) + { + name = name.Substring(0, name.LastIndexOf(RegCatalog.ZON_EXT)); + if (SubZoneNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetSubZone(name); + return null; + } + + + if (name.EndsWith(RegCatalog.HST_EXT)) + { + name = name.Substring(0, name.LastIndexOf(RegCatalog.HST_EXT)); + if (HostNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetHost(name); + return null; + } + + + if (SubZoneNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetSubZone(name); + + if (HostNames.Any(srn=>INVSTRCMP.Equals(srn, name))) + return GetHost(name); + + return null; + } + } + + /// + /// Returns the first host that has Zone governor in its role, or null if there are no governors in the zone + /// + public SectionHost ZoneGovernorPrimaryHost + { + get { return this.ZoneGovernorHosts.FirstOrDefault(); } + } + + /// + /// Returns zone governor host/s in order of their names, primary host returned first. + /// Returns empty enumeration if there are no zgov hosts + /// + public IEnumerable ZoneGovernorHosts + { + get + { + return this.HostNames + .Select(hn => this.GetHost(hn)) + .Where(host => host.IsZGov) + .OrderBy( host => host.Name); + } + } + + /// + /// Returns a mapping of ProcessorID to process controller host pair {Primary, Secondary} + /// + public IDictionary> ProcessorMap + { + get + { + if (m_ProcessorMap == null) + { + var hostNodes = LevelConfig[CONFIG_PROCESSOR_SET_SECTION].Children.Where(cn => cn.IsSameName(CONFIG_PROCESSOR_HOST_SECTION)); + + m_ProcessorMap = new Dictionary>(hostNodes.Count()); + + foreach (var hn in hostNodes) + { + var id = hn.AttrByName(CONFIG_PROCESSOR_HOST_ID_ATTR).ValueAsNullableInt(null); + if (!id.HasValue) + throw new MetabaseException(StringConsts.METABASE_PROCESSOR_SET_MISSING_ATTRIBUTE_ERROR.Args(CONFIG_PROCESSOR_HOST_ID_ATTR)); + + var primaryHostPath = RegCatalog.JoinPathSegments(RegionPath, hn.AttrByName(CONFIG_PROCESSOR_HOST_PRIMARY_PATH_ATTR).Value); + var primaryHost = Catalog.NavigateHost(primaryHostPath); + if (!primaryHost.IsProcessHost) + throw new MetabaseException(StringConsts.METABASE_PROCESSOR_SET_HOST_IS_NOT_PROCESSOR_HOST_ERROR.Args(id.Value)); + + var seondaryHostPath = RegCatalog.JoinPathSegments(RegionPath, hn.AttrByName(CONFIG_PROCESSOR_HOST_SECONDARY_PATH_ATTR).Value); + var secondaryHost = Catalog.NavigateHost(seondaryHostPath); + if (!secondaryHost.IsProcessHost) + throw new MetabaseException(StringConsts.METABASE_PROCESSOR_SET_HOST_IS_NOT_PROCESSOR_HOST_ERROR.Args(id.Value)); + + if (m_ProcessorMap.ContainsKey(id.Value)) + throw new MetabaseException(StringConsts.METABASE_PROCESSOR_SET_DUPLICATE_ATTRIBUTE_ERROR.Args(CONFIG_PROCESSOR_HOST_ID_ATTR)); + m_ProcessorMap.Add(id.Value, new[] { primaryHost, secondaryHost }); + } + } + return m_ProcessorMap; + } + } + + /// + /// Maps sharding ID to processor ID. This method is used to map process unqiue mutex id, or GDID to processor ID - which denotes a + /// set of actual executor hosts + /// + public int MapShardingKeyToProcessorID(string shardingKey) + { + var ids = ProcessorMap.Keys.ToArray(); + if (ids.Length == 0) throw new MetabaseException("TODO: MapShardingKeyToProcessorID(ProcessorMap is empty)"); + var hash = MDB.ShardingUtils.StringToShardingID(shardingKey); + var idx = hash % (ulong)ids.Length; + return ids[idx]; + } + + /// + /// Tries to map processor id to {Primary, Secondary} host pair or null if there is no mapping + /// + public IEnumerable TryGetProcessorHostsByID(int id) + { + var dict = ProcessorMap; + IEnumerable result; + if (dict.TryGetValue(id, out result)) return result; + return null; + } + + /// + /// Maps map processor id to {Primary, Secondary} host pair or throws + /// + public IEnumerable GetProcessorHostsByID(int id) + { + var hosts = TryGetProcessorHostsByID(id); + + if (hosts == null) + throw new MetabaseException(StringConsts.METABASE_ZONE_COULD_NOT_FIND_PROCESSOR_HOST_ERROR.Args(RegionPath, id)); + + return hosts; + } + + /// + /// Returns true if this zone has direct or indirect parent zone governor above it, optionally examining higher-level NOCs. + /// If iAmZoneGovernor is true then this zone is skipped as the zone gov of this zone gov is not itself. + /// + public bool HasDirectOrIndirectParentZoneGovernor(string zgovHost, bool? iAmZoneGovernor = null, bool transcendNOC = false) + { + if (zgovHost.IsNullOrWhiteSpace()) return false; + return FindNearestParentZoneGovernors(iAmZoneGovernor, (sh) => AgniExtensions.IsSameRegionPath( sh.RegionPath, zgovHost), transcendNOC).Any(); + } + + /// + /// Returns true if this zone has direct or indirect parent zone governor above it, optionally examining higher-level NOCs. + /// If iAmZoneGovernor is true then this zone is skipped as the zone gov of this zone gov is not itself. + /// + public bool HasDirectOrIndirectParentZoneGovernor(SectionHost zgovHost, bool? iAmZoneGovernor = null, bool transcendNOC = false) + { + if (zgovHost==null) return false; + return FindNearestParentZoneGovernors(iAmZoneGovernor, (sh) => sh.IsLogicallyTheSame( zgovHost ), transcendNOC).Any(); + } + + + /// + /// Tries to find hosts that run zone governors in this zone or parent zone chain, optionally looking in the higher-level NOCs. + /// If iAmZoneGovernor is true then this zone is skipped as the zone gov of this zone gov is not itself. + /// Pass filter lambda to filter-out-unneeded hosts in the chain. Returns empty enumeration for no matches + /// + public IEnumerable FindNearestParentZoneGovernors(bool? iAmZoneGovernor = null, Func filter = null, bool transcendNOC = false) + { + if (!iAmZoneGovernor.HasValue) + iAmZoneGovernor = AppModel.BootConfLoader.SystemApplicationType == AppModel.SystemApplicationType.ZoneGovernor; + + var zone = this; + + if (iAmZoneGovernor.Value) + { + zone = zone.ParentZone; + } + + while(zone!=null) + { + var zgovs = zone.ZoneGovernorHosts; + if (filter==null ? zgovs.Any() : zgovs.Any(filter)) return zgovs; + zone = zone.ParentZone; + } + + if (!transcendNOC) return Enumerable.Empty(); //not found in this NOC + + SectionNOC noc = this.NOC; + + while(true) + { + zone = noc.ParentNOCZone; + if (zone==null) return Enumerable.Empty();//no parent NOC or no NOC higher (already at the top) + + noc = zone.NOC; + + while(zone!=null) + { + var zgovs = zone.ZoneGovernorHosts; + if (filter==null ? zgovs.Any() : zgovs.Any(filter)) return zgovs; + zone = zone.ParentZone; + } + } + } + + /// + /// Gets sub-zone by name + /// + public SectionZone GetSubZone(string name) + { + string CACHE_KEY = ("ZN.sz"+Path+name).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as SectionZone; + if (result!=null) return result; + + var rdir = "{0}{1}".Args(name, RegCatalog.ZON_EXT); + var rpath = Metabank.JoinPaths(Path, rdir); + result = + Metabank.fsAccess("Zone[{0}].GetSubZone({1})".Args(m_Name, name), rpath, + (session, dir) => new SectionZone(m_NOC, this, name, rpath, session) + ); + + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + + /// + /// Gets host by name + /// + public SectionHost GetHost(string name) + { + string CACHE_KEY = ("ZN.h"+Path+name).ToLowerInvariant(); + //1. Try to get from cache + var result = Metabank.cacheGet(REG_CATALOG, CACHE_KEY) as SectionHost; + if (result!=null) return result; + + var rdir = "{0}{1}".Args(name, RegCatalog.HST_EXT); + var rpath = Metabank.JoinPaths(Path, rdir); + result = + Metabank.fsAccess("Zone[{0}].GetHost({1})".Args(m_Name, name), rpath, + (session, dir) => new SectionHost(this, name, rpath, session) + ); + + + Metabank.cachePut(REG_CATALOG, CACHE_KEY, result); + return result; + } + + + public override void Validate(ValidationContext ctx) + { + var output = ctx.Output; + + base.Validate(ctx); + + foreach(var szone in this.SubZoneNames) + this.GetSubZone(szone).Validate(ctx); + + foreach(var shost in this.HostNames) + this.GetHost(shost).Validate(ctx); + + var processorMap = ProcessorMap; + + //check Zone governor locking + var zgovs = this + .ZoneGovernorHosts + .OrderBy( host => host.Name) + .ToArray(); + + if (zgovs.Length>0) + { + + var primaryLockZgovs = this + .ZoneGovernorHosts.Where(host => !host.IsZGovLockFailover) + .OrderBy( host => host.Name) + .ToArray(); + + if (primaryLockZgovs.Length<1) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, "No primary ZGov locking nodes defined") ); + + var failoverLockZgovs = this + .ZoneGovernorHosts.Where(host => host.IsZGovLockFailover) + .OrderBy( host => host.Name) + .ToArray(); + + if (failoverLockZgovs.Length>0 && primaryLockZgovs.Length!=failoverLockZgovs.Length) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, "The number of primary ZGov locking nodes does not equal to the number of failover nodes") ); + } + } + + + + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.Sections.cs b/src/Agni/Metabase/Metabank.Sections.cs new file mode 100644 index 0000000..c34603b --- /dev/null +++ b/src/Agni/Metabase/Metabank.Sections.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; +using NFX.IO.FileSystem; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Base class for metadata section stored in metabank catalogs. Section represents a piece of metabase catalog that + /// can lazily load from the source file system. Contrary to monolithic application configurations, that load + /// at once from a single source (i.e. disk file), metadata class allows to wrap configuration and load segments of configuration + /// on a as-needed basis + /// + public abstract class Section : INamed + { + protected Section(Catalog catalog, string name, string path, FileSystemSession session) + { + if (name.IsNullOrWhiteSpace() || catalog==null || path.IsNullOrWhiteSpace()) + throw new MetabaseException(StringConsts.ARGUMENT_ERROR + GetType().Name+".ctor(catalog|name|path==null|empty"); + Catalog = catalog; + Metabank = catalog.Metabank; + m_Name = name; + Path = Metabank.JoinPaths("", path);//ensure root path symbol i.e. + + m_LevelConfig = Metabank.getConfigFromFile(session, Metabank.JoinPaths(path, Metabank.CONFIG_SECTION_LEVEL_FILE)).Root; + + if (!m_LevelConfig.IsSameName(RootNodeName)) + throw new MetabaseException(StringConsts.METABASE_METADATA_CTOR_1_ERROR.Args(GetType().Name, RootNodeName, m_LevelConfig.Name)); + + var cn = m_LevelConfig.AttrByName(Metabank.CONFIG_NAME_ATTR); + if ( !cn.Exists || !m_LevelConfig.IsSameNameAttr(name)) + throw new MetabaseException(StringConsts.METABASE_METADATA_CTOR_2_ERROR.Args(GetType().Name, name, cn.ValueAsString(SysConsts.UNKNOWN_ENTITY))); + + if (!name.IsValidName()) + throw new MetabaseException(StringConsts.METABASE_METADATA_CTOR_3_ERROR.Args(GetType().Name, name, path)); + + Metabank.includeCommonConfig(m_LevelConfig); + m_LevelConfig.ResetModified(); + } + + public readonly Metabank Metabank; + public readonly Catalog Catalog; + + /// + /// File system path to this section with section name + /// + public readonly string Path; + + protected readonly string m_Name; + private ConfigSectionNode m_LevelConfig; + + /// + /// Section name + /// + public string Name { get { return m_Name; }} + + /// + /// Section description + /// + public string Description { get { return m_LevelConfig.AttrByName(Metabank.CONFIG_DESCRIPTION_ATTR).ValueAsString(string.Empty); }} + + + /// + /// Returns metabase config for this section/level + /// + public IConfigSectionNode LevelConfig { get { return m_LevelConfig;} } + + /// + /// Returns the name of root node in section level file + /// + public abstract string RootNodeName { get;} + + + /// + /// Validates metabase section by checking all of it contents for consistency + /// + public abstract void Validate(ValidationContext ctx); + + public override string ToString() + { + return "{0}({1})".Args(GetType().Name, this.Name); + } + + } + + /// + /// Metadata section that also has application-level configuration file for any application + /// + public abstract class SectionWithAnyAppConfig : Section + { + protected SectionWithAnyAppConfig(Catalog catalog, string name, string path, FileSystemSession session) : base(catalog, name, path, session) + { + m_AnyAppConfig = Metabank.getConfigFromFile(session, Metabank.JoinPaths(path, Metabank.CONFIG_SECTION_LEVEL_ANY_APP_FILE), require: false).Root; + } + + private IConfigSectionNode m_AnyAppConfig; + + /// + /// Any application(the one that does not specify a particular app name) config for this level or null + /// + public IConfigSectionNode AnyAppConfig { get { return m_AnyAppConfig;} } + + public override void Validate(ValidationContext ctx) + { + if (m_AnyAppConfig!=null) + if (!m_AnyAppConfig.IsSameName(Metabank.RootAppConfig)) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + StringConsts.METABASE_VALIDATION_APP_CONFIG_ROOT_MISMTACH_ERROR.Args(m_AnyAppConfig.Name)) ); + } + + } + + + /// + /// Metadata section that also has application-level configuration files for any and named applications + /// + public abstract class SectionWithNamedAppConfigs : SectionWithAnyAppConfig + { + protected SectionWithNamedAppConfigs(Catalog catalog, string name, string path, FileSystemSession session) : base(catalog, name, path, session) + { + m_AppConfigs = + Metabank.fsAccess("SectionWithAppConfig.ctor", path, + (fss, dir) => + { + var result = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + foreach(var fn in dir.FileNames.Where(fn=>fn.StartsWith(Metabank.CONFIG_SECTION_LEVEL_APP_FILE_PREFIX))) + { + var ipr = fn.IndexOf(Metabank.CONFIG_SECTION_LEVEL_APP_FILE_PREFIX); + var si = ipr + Metabank.CONFIG_SECTION_LEVEL_APP_FILE_PREFIX.Length; + var isfx = fn.IndexOf(Metabank.CONFIG_SECTION_LEVEL_APP_FILE_SUFFIX); + + if (ipr!=0 || isfx<=si) continue; + var appName = fn.Substring(si, isfx-si); + + var app = this.Metabank.CatalogApp.Applications[appName]; + if (app==null) + throw new MetabaseException(StringConsts.METABASE_APP_CONFIG_APP_DOESNT_EXIST_ERROR.Args(path, fn, appName)); + + var config = Metabank.getConfigFromExistingFile(session, Metabank.JoinPaths(dir.Path, fn)).Root; + result.Add(appName, config); + } + return result; + } + ); + + + } + + private Dictionary m_AppConfigs; + + /// + /// Gets an application configuration file for the particular named application or null + /// + public IConfigSectionNode GetAppConfig(string appName) + { + IConfigSectionNode result; + if (m_AppConfigs.TryGetValue(appName, out result)) return result; + return null; + } + + public override void Validate(ValidationContext ctx) + { + base.Validate(ctx); + foreach(var name in m_AppConfigs.Keys) + { + var root = m_AppConfigs[name]; + if (root!=null) + if (!root.IsSameName(Metabank.RootAppConfig)) + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, Catalog, this, + StringConsts.METABASE_VALIDATION_APP_CONFIG_ROOT_MISMTACH_ERROR.Args(name+"."+root.Name)) ); + } + } + + } + + + public abstract class Section : Section where TCatalog : Catalog + { + protected Section(TCatalog catalog, string name, string path, FileSystemSession session) + :base(catalog, name, path, session) + { + } + + + public new TCatalog Catalog { get { return (TCatalog)base.Catalog;}} + } + + /// + /// Metadata section that also has application-level configuration file for any application + /// + public abstract class SectionWithAnyAppConfig : SectionWithAnyAppConfig where TCatalog : Catalog + { + protected SectionWithAnyAppConfig(TCatalog catalog, string name, string path, FileSystemSession session) + :base(catalog, name, path, session) + { + } + + public new TCatalog Catalog { get { return (TCatalog)base.Catalog;}} + } + + /// + /// Metadata section that also has application-level configuration files for any and named applications + /// + public abstract class SectionWithNamedAppConfigs : SectionWithNamedAppConfigs where TCatalog : Catalog + { + protected SectionWithNamedAppConfigs(TCatalog catalog, string name, string path, FileSystemSession session) + :base(catalog, name, path, session) + { + } + + public new TCatalog Catalog { get { return (TCatalog)base.Catalog;}} + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.Validation.cs b/src/Agni/Metabase/Metabank.Validation.cs new file mode 100644 index 0000000..ef95453 --- /dev/null +++ b/src/Agni/Metabase/Metabank.Validation.cs @@ -0,0 +1,231 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Glue; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + + + private void validateMetabase(ValidationContext ctx) + { + if (validate_Contracts(ctx)) validate_GDIDAuthorities(ctx); + validate_Platforms(ctx); + validate_Networks(ctx); + validate_Catalogs(ctx); + } + + + private bool validate_Contracts(ValidationContext ctx) + { + var output = ctx.Output; + + try + { + var instance = Contracts.ServiceClientHub.Instance; + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_CONTRACTS_SERVICE_HUB_ERROR+error.ToMessageWithType())); + return false; + } + + foreach(var mapping in Contracts.ServiceClientHub.Instance.CachedMap) + { + if (mapping.Local.Service.IsNullOrWhiteSpace()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, null, null, " Contract mapping '{0}' does not specify local service name".Args(mapping) )); + + if (mapping.Global.Service.IsNullOrWhiteSpace()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, null, null, " Contract mapping '{0}' does not specify global service name".Args(mapping) )); + } + + return true; + } + + + private void validate_GDIDAuthorities(ValidationContext ctx) + { + var output = ctx.Output; + + var authorities = GDIDAuthorities; + if (!authorities.Any()) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Warning, null, null, StringConsts.METABASE_GDID_AUTHORITIES_NONE_DEFINED_WARNING) ); + return; + } + + if (authorities.Count() != authorities.Distinct().Count()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_GDID_AUTHORITIES_DUPLICATION_ERROR) ); + + + foreach(var authority in authorities) + { + var host = CatalogReg[authority.Name] as SectionHost; + if (host==null) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_GDID_AUTHORITY_BAD_HOST_ERROR.Args(authority.Name)) ); + continue; + } + + if (!host.Role.AppNames.Any(n=>INVSTRCMP.Equals(n, SysConsts.APP_NAME_GDIDA))) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_GDID_AUTHORITY_HOST_NOT_AGDIDA_ERROR.Args(authority.Name, host.Role)) ); + + try + { + Contracts.ServiceClientHub.TestSetupOf(host.RegionPath); + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_GDID_AUTHORITY_SVC_RESOLUTION_ERROR.Args(authority.Name, error.ToMessageWithType())) ); + } + } + } + + + private void validate_Platforms(ValidationContext ctx) + { + var output = ctx.Output; + + try//PLATFORMS + { + foreach(var pn in PlatformConfNodes) + { + var name = pn.AttrByName(CONFIG_NAME_ATTR).Value; + if (name.IsNullOrWhiteSpace()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NAME_ATTR_UNDEFINED_ERROR + "platforms" ) ); + + if (!name.IsValidName()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_ENTITY_NAME_INVALID_ERROR.Args("platforms", name) ) ); + } + + + //Duplicate platform name + if (PlatformNames.Distinct(INVSTRCMP).Count()!=PlatformNames.Count()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_PLATFORM_NAME_DUPLICATION_ERROR ) ); + + //Platform/OS names + foreach(var os in OSNames) + { + if (!os.IsValidName()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_ENTITY_NAME_INVALID_ERROR.Args("OS", os) ) ); + + try + { + if (INVSTRCMP.Equals(os,BinCatalog.PackageInfo.ANY)) + throw new MetabaseException(StringConsts.METABASE_PLATFORM_OS_RESERVED_NAME_ERROR.Args(os)); + GetOSConfNode(os); + } + catch(Exception error) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, error.ToMessageWithType(), error) ); + } + } + } + catch(Exception general) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, general.ToMessageWithType(), general) ); + } + } + + + + private void validate_Networks(ValidationContext ctx) + { + var output = ctx.Output; + + try + { + foreach(var node in NetworkConfNodes) + { + var name = node.AttrByName(CONFIG_NAME_ATTR).Value; + if (name.IsNullOrWhiteSpace()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NAME_ATTR_UNDEFINED_ERROR + "networks" ) ); + + if (!name.IsValidName()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_ENTITY_NAME_INVALID_ERROR.Args("networks", name) ) ); + } + + + //Duplicate network name + if (NetworkNames.Distinct(INVSTRCMP).Count()!=NetworkNames.Count()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NETWORK_NAME_DUPLICATION_ERROR ) ); + + //Duplicate service name under network or binding under network/service + foreach(var net in NetworkNames) + { + if (GetNetworkSvcNames(net).Distinct(INVSTRCMP).Count()!=GetNetworkSvcNames(net).Count()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NETWORK_SVC_NAME_DUPLICATION_ERROR.Args(net) ) ); + + if (GetNetworkGroupNames(net).Distinct(INVSTRCMP).Count()!=GetNetworkGroupNames(net).Count()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NETWORK_GRP_NAME_DUPLICATION_ERROR.Args(net) ) ); + + var services = GetNetworkSvcNodes(net); + + if (services.Count()==0) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NETWORK_NO_SVC_ERROR.Args(net) ) ); + foreach(var svcNode in services) + { + var svc = svcNode.AttrByName(CONFIG_NAME_ATTR).Value; + if (svc.IsNullOrWhiteSpace()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NAME_ATTR_UNDEFINED_ERROR + "network: '{0}' service".Args(net) ) ); + + if (!svc.IsValidName()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_ENTITY_NAME_INVALID_ERROR.Args("network: '{0}' service".Args(net), svc) ) ); + + + if (GetNetworkSvcBindingNames(net, svc).Distinct(INVSTRCMP).Count()!=GetNetworkSvcBindingNames(net, svc).Count()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NETWORK_SVC_BINDING_NAME_DUPLICATION_ERROR.Args(net, svc) ) ); + + + var bindings = GetNetworkSvcBindingNodes(net,svc); + if (bindings.Count()==0) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NETWORK_NO_SVC_BINDINGS_ERROR.Args(net, svc) ) ); + + var db = svcNode.AttrByName(CONFIG_DEFAULT_BINDING_ATTR).Value; + if (db.IsNotNullOrWhiteSpace()) + if (!bindings.Any(n=>n.IsSameName(db))) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, + StringConsts.METABASE_NETWORK_SVC_DEFAULT_BINDING_NAME_ERROR.Args(net, svc, db) ) ); + } + + var groups = GetNetworkGroupNodes(net); + foreach(var node in groups) + { + var grpName = node.AttrByName(CONFIG_NAME_ATTR).Value; + if (grpName.IsNullOrWhiteSpace()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_NAME_ATTR_UNDEFINED_ERROR + "network: '{0}' group".Args(net) ) ); + + if (!grpName.IsValidName()) + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_ENTITY_NAME_INVALID_ERROR.Args("network: '{0}' group".Args(net), grpName) ) ); + + } + } + } + catch(Exception general) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, general.ToMessageWithType(), general) ); + } + } + + + + private void validate_Catalogs(ValidationContext ctx) + { + foreach(var catalog in Catalogs) + try + { + catalog.Validate(ctx); + } + catch(Exception error) + { + ctx.Output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, error.ToMessageWithType(), error) ); + } + } + + + + +}} diff --git a/src/Agni/Metabase/Metabank.ValidationContext.cs b/src/Agni/Metabase/Metabank.ValidationContext.cs new file mode 100644 index 0000000..4ed622a --- /dev/null +++ b/src/Agni/Metabase/Metabank.ValidationContext.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Glue; + +namespace Agni.Metabase{ public sealed partial class Metabank{ + + /// + /// Kepos information during validation scan + /// + public class ValidationContext + { + + public ValidationContext(IList output) + { + Output = output; + State = new Dictionary( StringComparer.InvariantCultureIgnoreCase ); + } + + + /// + /// The output target (i.e. EventedList) + /// + public readonly IList Output; + + /// + /// Dictionary of variables that may be needed to retain state during validation + /// + public readonly Dictionary State; + + public T StateAs(string key) where T : class + { + object value; + if (State.TryGetValue(key, out value)) + return value as T; + + return null; + } + } + + + +}} diff --git a/src/Agni/Metabase/Metabank.cs b/src/Agni/Metabase/Metabank.cs new file mode 100644 index 0000000..329a2a6 --- /dev/null +++ b/src/Agni/Metabase/Metabank.cs @@ -0,0 +1,1020 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; + +using NFX; +using NFX.Log; +using NFX.ApplicationModel; +using NFX.DataAccess.Cache; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX.IO.FileSystem.Local; +using NFX.Security; +using NFX.Instrumentation; + + + +namespace Agni.Metabase +{ + + /// + /// Provides interface to METABASE - a metadata bank for the Agni. + /// Metabase is organized as a hierarchical directory structure with various files such as: + /// * configuration files that describe structure, policies, settings, hosts etc. + /// * configs files for particular components (i.e.: application, region, zone, db bank etc.) + /// * binary packages - resources necessary to distribute and run code across agni (assemblies, images, static data files etc.) + /// Metabase is usually stored in a version-controlled system, such as SVN or GIT so the whole distribution + /// is versioned so any change can be rolled-back. This class is thread safe for using its instance methods + /// + /// + /// Metabase is a special kind of database that contains meta information about the whole system. + /// The information is logically organized into catalogs that are further broken down into sections. + /// Catalog implementers contain various instances of Section classes that represent pieces of metabase whole data that + /// can lazily load from the source file system. Contrary to monolithic application configurations, that load + /// at once from a single source (i.e. disk file), metadata class allows to wrap configuration and load segments of configuration + /// on a as-needed basis. + /// + /// The following table describes the root metabase structure: + /// + /// /app - application catalog, contains Application and Roles + /// /applications + /// /app1 + /// .. + /// /appX + /// /roles + /// /role1 + /// .. + /// /roleX + /// /bin - contains binary packages thatget installed on appropriate hosts + /// /package-name1.platform.os.version + /// .. + /// /package-nameX.platform.os.version + /// /reg - regional catalog, organizes hierarcies of regions, NOC(facilities), zones and hosts + /// /region1 + /// /sub-region1 + /// /noc1 + /// /zone1 + /// /subzone1 + /// /host1 + /// .. + /// .. + /// .. + /// .. + /// .. + /// .. + /// /regionX + /// + /// + /// + public sealed partial class Metabank : ApplicationComponent, IInstrumentable + { + #region CONSTS + public const string CONFIG_APP_CONFIG_INCLUDE_PRAGMA_ATTR = "app-config-include-pragma"; + public const string CONFIG_APP_CONFIG_INCLUDE_PRAGMAS_DISABLED_ATTR = "app-config-include-pragmas-disabled"; + + public const string CONFIG_COMMON_FILE = "$.common"; + public const string CONFIG_SECTION_LEVEL_FILE = "$"; + public const string CONFIG_SECTION_LEVEL_ANY_APP_FILE = "$.app"; + public const string CONFIG_PLATFORMS_FILE = "platforms"; + public const string CONFIG_NETWORKS_FILE = "networks"; + public const string CONFIG_CONTRACTS_FILE = "contracts"; + + public const string CONFIG_SECTION_LEVEL_APP_FILE_PREFIX = "$."; + public const string CONFIG_SECTION_LEVEL_APP_FILE_SUFFIX = ".app."; + + + public const string APP_CATALOG = "app"; + public const string BIN_CATALOG = "bin"; + public const string REG_CATALOG = "reg"; + public const string SEC_CATALOG = "sec"; + + public const string CONFIG_NAME_ATTR = "name"; + public const string CONFIG_DESCRIPTION_ATTR = "description"; + public const string CONFIG_OFFLINE_ATTR = "offline"; + public const string CONFIG_ROLE_ATTR = "role"; + public const string CONFIG_PATH_ATTR = "path"; + public const string CONFIG_GEO_CENTER_ATTR = "geo-center"; + public const string CONFIG_VERSION_ATTR = "version"; + public const string CONFIG_AUTO_RUN_ATTR = "auto-run"; + public const string CONFIG_EXE_FILE_ATTR = "exe-file"; + public const string CONFIG_EXE_ARGS_ATTR = "exe-args"; + + public const string CONFIG_APP_CONFIG_SECTION = "app-config"; + public const string CONFIG_OS_APP_CONFIG_INCLUDE_SECTION = "include-os-app-config"; + + public const string CONFIG_OS_ATTR = "os"; + + public const string CONFIG_PLATFORM_SECTION = "platform"; + public const string CONFIG_OS_SECTION = "os"; + + public const string CONFIG_PACKAGES_SECTION = "packages"; + public const string CONFIG_PACKAGE_SECTION = "package"; + + public const string DEFAULT_PACKAGE_VERSION = "head"; + + public const string CONFIG_GDID_SECTION = "gdid"; + + public const string CONFIG_NETWORK_SECTION = "network"; + + public const string CONFIG_NETWORK_ROUTING_SECTION = "network-routing"; + public const string CONFIG_NETWORK_ROUTING_ROUTE_SECTION = "route"; + public const string CONFIG_NETWORK_ROUTING_NETWORK_ATTR = "network"; + public const string CONFIG_NETWORK_ROUTING_FROM_ATTR = "from"; + public const string CONFIG_NETWORK_ROUTING_SERVICE_ATTR = "service"; + public const string CONFIG_NETWORK_ROUTING_BINDING_ATTR = "binding"; + public const string CONFIG_NETWORK_ROUTING_TO_ADDRESS_ATTR = "to-address"; + public const string CONFIG_NETWORK_ROUTING_TO_PORT_ATTR = "to-port"; + public const string CONFIG_NETWORK_ROUTING_TO_GROUP_ATTR = "to-group"; + public const string CONFIG_NETWORK_ROUTING_HOST_ATTR = "host"; + + public const char HOST_DYNAMIC_SUFFIX_SEPARATOR = '~';//cross-check with IsValidName() which should include this value from valid names + + public const string CONFIG_HOST_DYNAMIC_ATTR = "dynamic"; + + public const string CONFIG_HOST_ZGOV_LOCK_FAILOVER_ATTR = "zgov-lock-failover"; + public const string CONFIG_HOST_PROCESS_HOST_ATTR = "process-host"; + + + public const string CONFIG_SERVICE_SECTION = "service"; + public const string CONFIG_GROUP_SECTION = "group"; + public const string CONFIG_DEFAULT_BINDING_ATTR = "default-binding"; + public const string CONFIG_BINDINGS_SECTION = "bindings"; + public const string CONFIG_SCOPE_ATTR = "scope"; + public const string CONFIG_ADDRESS_ATTR = "address"; + public const string CONFIG_PORT_ATTR = "port"; + + public const string CONFIG_TARGET_SUFFIX_ATTR = "target-suffix"; + + public const string CONFIG_PARENT_NOC_ZONE_ATTR = "parent-noc-zone"; + + + public const string CONFIG_HOST_SET_BUILDER_SECTION = "host-set-builder"; + public const string CONFIG_HOST_SET_SECTION = "host-set"; + public const string CONFIG_HOST_SET_HOST_SECTION = "host"; + + public const string CONFIG_PROCESSOR_SET_SECTION = "processor-set"; + public const string CONFIG_PROCESSOR_HOST_SECTION = "host"; + public const string CONFIG_PROCESSOR_HOST_ID_ATTR = "id"; + public const string CONFIG_PROCESSOR_HOST_PRIMARY_PATH_ATTR = "primary-path"; + public const string CONFIG_PROCESSOR_HOST_SECONDARY_PATH_ATTR = "secondary-path"; + + public const int DEFAULT_RESOLVE_DYNAMIC_HOST_NET_SVC_TIMEOUT_MS = 2000; + public const int MIN_RESOLVE_DYNAMIC_HOST_NET_SVC_TIMEOUT_MS = 1000; + + #endregion + + #region .ctor + + /// + /// Opens metabase from the specified file system instance at the specified metabase root path + /// + /// An instance of a file system that stores the metabase files + /// File system connection params + /// A path to the directory within the file system that contains metabase root data + public Metabank(IFileSystem fileSystem, FileSystemSessionConnectParams fsSessionParams, string rootPath) : base() + { + if (fileSystem is LocalFileSystem && fsSessionParams==null) + fsSessionParams = new FileSystemSessionConnectParams(); + + if (fileSystem==null || fsSessionParams==null) + throw new MetabaseException(StringConsts.ARGUMENT_ERROR+"Metabank.ctor(fileSystem|fsSessionParams==null)"); + + using(var session = ctorFS(fileSystem, fsSessionParams, rootPath)) + { + m_CommonLevelConfig = getConfigFromFile(session, CONFIG_COMMON_FILE).Root; + + m_RootConfig = getConfigFromFile(session, CONFIG_SECTION_LEVEL_FILE).Root; + includeCommonConfig(m_RootConfig); + m_RootConfig.ResetModified(); + + m_RootAppConfig = getConfigFromFile(session, CONFIG_SECTION_LEVEL_ANY_APP_FILE).Root; + m_PlatformConfig = getConfigFromFile(session, CONFIG_PLATFORMS_FILE).Root; + m_NetworkConfig = getConfigFromFile(session, CONFIG_NETWORKS_FILE).Root; + m_ContractConfig = getConfigFromFile(session, CONFIG_CONTRACTS_FILE).Root; + } + m_Catalogs = new Registry(); + var cacheStore = new CacheStore("AC.Metabank"); + //No app available - nowhere to configure: //cacheStore.Configure(null); + /* + cacheStore.TableOptions.Register( new TableOptions(APP_CATALOG, 37, 3) ); + cacheStore.TableOptions.Register( new TableOptions(BIN_CATALOG, 37, 7) ); + cacheStore.TableOptions.Register( new TableOptions(REG_CATALOG, 37, 17) ); + superceeded by the cacheStore.DefaultTableOptions below: + */ + cacheStore.DefaultTableOptions = new TableOptions("*", 37, 17); + //reg catalog needs more space + cacheStore.TableOptions.Register( new TableOptions(REG_CATALOG, 571, 37) ); + + cacheStore.InstrumentationEnabled = false; + + m_Cache = new ComplexKeyHashingStrategy(cacheStore); + + new AppCatalog( this ); + new BinCatalog( this ); + new SecCatalog( this ); + new RegCatalog( this ); + + ConfigAttribute.Apply(this, m_RootConfig); + } + + //tests and sets FS connection params + private FileSystemSession ctorFS(IFileSystem fileSystem, FileSystemSessionConnectParams fsSessionParams, string rootPath) + { + FileSystemSession session = null; + //Test FS connection + try + { + session = fileSystem.StartSession(fsSessionParams); + if (fsSessionParams.Version==null) + fsSessionParams.Version = session.LatestVersion; + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_FS_CONNECTION_ERROR.Args(fileSystem.GetType().FullName, + fileSystem.Name, + fsSessionParams.ToString(), + error.ToMessageWithType() + ), error); + } + + m_FS = fileSystem; + m_FSSessionConnectParams = fsSessionParams; + m_FSRootPath = rootPath ?? string.Empty; + + return session; + } + + protected override void Destructor() + { + m_Active = false; + + if (m_FSSessionCacheThread!=null) + { + m_FSSessionCacheThreadWaiter.Set(); + m_FSSessionCacheThread.Join(); + m_FSSessionCacheThreadWaiter.Close(); + + m_FSSessionCacheThread = null; + m_FSSessionCacheThreadWaiter = null; + } + + if (m_Cache!=null) + if (m_Cache.Store!=null) + m_Cache.Store.Dispose(); + + base.Destructor(); + } + + #endregion + + #region Fields + + //Invariant cluture ignore case comparer + private static readonly StringComparer INVSTRCMP = StringComparer.InvariantCultureIgnoreCase; + + private bool m_Active = true; + private IFileSystem m_FS; + private FileSystemSessionConnectParams m_FSSessionConnectParams; + private string m_FSRootPath; + + private bool m_InstrumentationEnabled; + + private ComplexKeyHashingStrategy m_Cache; + + + private ConfigSectionNode m_CommonLevelConfig; + private ConfigSectionNode m_RootConfig; + private ConfigSectionNode m_RootAppConfig; + private ConfigSectionNode m_PlatformConfig; + private ConfigSectionNode m_NetworkConfig; + private ConfigSectionNode m_ContractConfig; + + private Registry m_Catalogs; + + private int m_ResolveDynamicHostNetSvcTimeoutMs = DEFAULT_RESOLVE_DYNAMIC_HOST_NET_SVC_TIMEOUT_MS; + + #endregion + + #region Properties + + + public override string ComponentCommonName { get { return "metabase"; }} + + /// + /// Returns false as soon as destruction starts + /// + public bool Active{get{return m_Active;}} + + /// + /// Returns file system that this bank is read from + /// + public IFileSystem FileSystem { get{ return m_FS;} } + + /// + /// Returns parameters for file system session establishment + /// + public FileSystemSessionConnectParams FileSystemSessionConnectParams { get{ return m_FSSessionConnectParams;}} + + /// + /// Returns root path of the metabase data in the file system + /// + public string FileSystemRootPath { get{ return m_FSRootPath;} } + + /// + /// Controls instrumentation availability + /// + [Config] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_METABASE, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public bool InstrumentationEnabled + { + get { return m_InstrumentationEnabled; } + set + { + m_Cache.Store.InstrumentationEnabled = value; + m_InstrumentationEnabled = value; + } + } + + [Config(Default = DEFAULT_RESOLVE_DYNAMIC_HOST_NET_SVC_TIMEOUT_MS)] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_METABASE)] + public int ResolveDynamicHostNetSvcTimeoutMs + { + get { return m_ResolveDynamicHostNetSvcTimeoutMs;} + set + { + m_ResolveDynamicHostNetSvcTimeoutMs = value < MIN_RESOLVE_DYNAMIC_HOST_NET_SVC_TIMEOUT_MS ? + DEFAULT_RESOLVE_DYNAMIC_HOST_NET_SVC_TIMEOUT_MS : value; + } + } + + + /// + /// Returns common config for all levels, the one that gets included in all level configs of metabase + /// + public IConfigSectionNode CommonLevelConfig { get{ return m_CommonLevelConfig;}} + + + /// + /// Returns root-level config for the whole meatabase + /// + public IConfigSectionNode RootConfig { get{ return m_RootConfig;}} + + /// + /// Returns root-level config for all applications that this meatabase defines + /// + public IConfigSectionNode RootAppConfig { get{ return m_RootAppConfig;}} + + + /// + /// Returns all catalogs that this metabank has initialized + /// + public IRegistry Catalogs { get{ return m_Catalogs;}} + + + + /// + /// Returns application catalog instance + /// + public AppCatalog CatalogApp { get{ return (AppCatalog)m_Catalogs[APP_CATALOG];}} + + /// + /// Returns bin catalog instance + /// + public BinCatalog CatalogBin { get{ return (BinCatalog)m_Catalogs[BIN_CATALOG];}} + + + /// + /// Returns region catalog instance + /// + public RegCatalog CatalogReg { get{ return (RegCatalog)m_Catalogs[REG_CATALOG];}} + + /// + /// Returns security catalog instance + /// + public SecCatalog CatalogSec { get{ return (SecCatalog)m_Catalogs[SEC_CATALOG];}} + + /// + /// Returns the list of platform nodes declared in platforms file + /// + public IEnumerable PlatformConfNodes + { + get{ return m_PlatformConfig.Children.Where(cn=>cn.IsSameName(CONFIG_PLATFORM_SECTION));} + } + + + /// + /// Returns the list of platform names declared in platforms file + /// + public IEnumerable PlatformNames + { + get { return PlatformConfNodes.Select(n => n.AttrByName(CONFIG_NAME_ATTR).Value); } + } + + /// + /// Returns the list of operating system nodes declared in platforms file + /// + public IEnumerable OSConfNodes + { + get + { + foreach(var pnode in PlatformConfNodes) + foreach(var osNode in pnode.Children.Where(cn=>cn.IsSameName(CONFIG_OS_SECTION))) + yield return osNode; + } + } + + /// + /// Returns the list of operating system names declared in platforms file + /// + public IEnumerable OSNames + { + get { return OSConfNodes.Select(n => n.AttrByName(CONFIG_NAME_ATTR).Value); } + } + + + + /// + /// Returns the list of network nodes declared in networks file + /// + public IEnumerable NetworkConfNodes + { + get{ return m_NetworkConfig.Children.Where(cn=>cn.IsSameName(CONFIG_NETWORK_SECTION));} + } + + /// + /// Returns the list of network names declared in networks file + /// + public IEnumerable NetworkNames + { + get { return NetworkConfNodes.Select(n => n.AttrByName(CONFIG_NAME_ATTR).Value); } + } + + /// + /// Returns metabase root section for service client hub from contracts.acmb + /// + public IConfigSectionNode ServiceClientHubConfNode + { + get { return m_ContractConfig[Contracts.ServiceClientHub.CONFIG_SERVICE_CLIENT_HUB_SECTION];} + } + + + /// + /// Returns enumeration of Global Distributed ID generation Authorities declared in root metabase level + /// + public IEnumerable GDIDAuthorities + { + get + { + return Identification.GDIDGenerator.AuthorityHost.FromConfNode(m_RootConfig[CONFIG_GDID_SECTION]); + } + } + + + /// + /// Helper method to dump the status of metabase cache + /// + public void DumpCacheStatus(StringBuilder to) + { + if (m_Cache==null) return; + var cache = m_Cache.Store; + if (cache==null) return; + + to.AppendLine("Table definitions for cache store: " + cache.Name); + to.AppendLine("Table Buckets RPPage Locks Empt.Size"); + to.AppendLine("---------------------------------------------------------------------------"); + foreach(var tbl in cache.Tables) + to.AppendLine( + "{0,-26} {1,-9} {2,-7} {3,-6} {4,-10}".Args(tbl.Name, tbl.BucketCount, tbl.RecPerPage, tbl.LockCount, tbl.BucketCount*IntPtr.Size) + ); + to.AppendLine(); + to.AppendLine("Table current status for cache store: " + cache.Name); + to.AppendLine("Table Count Pages Hits Misses"); + to.AppendLine("---------------------------------------------------------------------------"); + foreach(var tbl in cache.Tables) + to.AppendLine( + "{0,-26} {1,-9} {2,-7} {3,-6} {4,-10}".Args(tbl.Name, tbl.Count, tbl.PageCount, tbl.StatComplexHitCount, tbl.StatComplexMissCount) + ); + + } + + + /// + /// Returns named parameters that can be used to control this component + /// + public IEnumerable> ExternalParameters{ get { return ExternalParameterAttribute.GetParameters(this); } } + + /// + /// Returns named parameters that can be used to control this component + /// + public IEnumerable> ExternalParametersForGroups(params string[] groups) + { + return ExternalParameterAttribute.GetParameters(this, groups); + } + + + #endregion + + + #region Public + + /// + /// Validates metabase by checking all of it contents for consistency - may take time for large metabases. + /// This method is not expected to be called by business applications, only by tools + /// + /// + /// As of Jul 2015, Validation is purposely convoluted as procedural code (validate methods) + /// instead of doing injectable rules etc. This is done for Practical simplicity of the design + /// + /// + /// A list where output such as erros and warnings is redirected. + /// May use NFX.Collections.EventedList for receiving notifications upon list addition + /// + public void Validate(IList output) + { + var context = new ValidationContext(output); + validate(context); + } + + private void validate(ValidationContext ctx) + { + var output = ctx.Output; + + var fromHost = AppModel.BootConfLoader.HostName; + + if (fromHost.IsNullOrWhiteSpace()) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_VALIDATION_WRONG_HOST_ERROR.Args("null|empty")) ); + return; + } + + var host = CatalogReg[fromHost] as SectionHost; + if (host==null) + { + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Error, null, null, StringConsts.METABASE_VALIDATION_WRONG_HOST_ERROR.Args(fromHost)) ); + output.Add( new MetabaseValidationMsg(MetabaseValidationMessageType.Info, null, null, StringConsts.METABASE_VALIDATION_WRONG_HOST_INFO.Args(AppModel.BootConfLoader.ENV_VAR_HOST_NAME)) ); + } + + validateMetabase(ctx); + } + + /// + /// Joins paths per injected file system + /// + public string JoinPaths(string first, params string[] other) + { + return m_FS.CombinePaths(first, other); + } + + /// + /// Returns config node for the specified OS, or throws if this OS is not declared in platforms root file or it is declared more than once + /// + public IConfigSectionNode GetOSConfNode(string osName) + { + var list = OSConfNodes.Where(n=>n.IsSameNameAttr(osName)).ToList(); + + if (list.Count==0) + throw new MetabaseException(StringConsts.METABASE_PLATFORMS_OS_NOT_DEFINED_ERROR.Args(osName)); + + if (list.Count>1) + throw new MetabaseException(StringConsts.METABASE_PLATFORMS_OS_DUPLICATION_ERROR.Args(osName)); + + return list[0]; + } + + /// + /// Gets platform config node per named OS + /// + public IConfigSectionNode GetOSPlatformNode(string osName) + { + return GetOSConfNode(osName).Parent; + } + + /// + /// Gets Platform name per named OS + /// + public string GetOSPlatformName(string osName) + { + return GetOSPlatformNode(osName).AttrByName(CONFIG_NAME_ATTR).Value; + } + + + + /// + /// Returns conf node for the network or throws if network with requested name was not found + /// + public IConfigSectionNode GetNetworkConfNode(string netName) + { + var net = NetworkConfNodes.FirstOrDefault(n=>n.IsSameNameAttr(netName)); + if (net==null) + throw new MetabaseException(StringConsts.METABASE_NAMED_NETWORK_NOT_FOUND_ERROR.Args(netName ?? SysConsts.NULL)); + return net; + } + + /// + /// Returns true when the network with the specified name exists in metabase netoworks definition + /// + public bool NetworkExists(string netName) + { + var net = NetworkConfNodes.FirstOrDefault(n=>n.IsSameNameAttr(netName)); + return net!=null; + } + + /// + /// Returns description for the network or throws if network with requested name was not found + /// + public string GetNetworkDescription(string netName) + { + return GetNetworkConfNode(netName).AttrByName(CONFIG_DESCRIPTION_ATTR).Value; + } + + /// + /// Returns network scope for the network or throws if network with requested name was not found + /// + public NetworkScope GetNetworkScope(string netName) + { + return GetNetworkConfNode(netName).AttrByName(CONFIG_SCOPE_ATTR).ValueAsEnum(NetworkScope.Any); + } + + + /// + /// Returns a list of config nodes for services for the named network + /// + public IEnumerable GetNetworkSvcNodes(string netName) + { + try + { + var netNode = NetworkConfNodes.Single(n=>n.IsSameNameAttr(netName)); + return netNode.Children.Where(cn=>cn.IsSameName(CONFIG_SERVICE_SECTION)); + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_NETWORK_CONFIG_ERROR+"GetNetworkSvcNodes({0}): {1}".Args(netName, error.ToMessageWithType(), error)); + } + } + + /// + /// Returns a list of service names for the named network + /// + public IEnumerable GetNetworkSvcNames(string netName) + { + return GetNetworkSvcNodes(netName).Select(n=>n.AttrByName(CONFIG_NAME_ATTR).Value); + } + + /// + /// Returns a list of config nodes for groups for the named network + /// + public IEnumerable GetNetworkGroupNodes(string netName) + { + try + { + var netNode = NetworkConfNodes.Single(n=>n.IsSameNameAttr(netName)); + return netNode.Children.Where(cn=>cn.IsSameName(CONFIG_GROUP_SECTION)); + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_NETWORK_CONFIG_ERROR+"GetNetworkGroupNodes({0}): {1}".Args(netName, error.ToMessageWithType(), error)); + } + } + + /// + /// Returns a list of group names for the named network + /// + public IEnumerable GetNetworkGroupNames(string netName) + { + return GetNetworkGroupNodes(netName).Select(n=>n.AttrByName(CONFIG_NAME_ATTR).Value); + } + + + + + /// + /// Returns a list of config nodes for service bindings for the named service in the named network + /// + public IEnumerable GetNetworkSvcBindingNodes(string netName, string svcName) + { + try + { + var svcNodes = GetNetworkSvcNodes(netName).Single(n=>n.IsSameNameAttr(svcName)); + return svcNodes[CONFIG_BINDINGS_SECTION].Children; + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_NETWORK_CONFIG_ERROR+"GetNetworkSvcBindingNodes({0},{1}): {2}".Args(netName, svcName, error.ToMessageWithType(), error)); + } + } + + /// + /// Returns a list of service binding names for the named service in the named network + /// + public IEnumerable GetNetworkSvcBindingNames(string netName, string svcName) + { + return GetNetworkSvcBindingNodes(netName, svcName).Select(n=>n.Name); + } + + + /// + /// Returns conf node for the network service or throws if network service with requested name was not found + /// + public IConfigSectionNode GetNetworkSvcConfNode(string netName, string svcName) + { + var net = this.GetNetworkSvcNodes(netName).FirstOrDefault(n=>n.IsSameNameAttr(svcName)); + if (net==null) + throw new MetabaseException(StringConsts.METABASE_NAMED_NETWORK_SVC_NOT_FOUND_ERROR.Args(netName ?? SysConsts.NULL, svcName ?? SysConsts.NULL)); + return net; + } + + /// + /// Returns conf node for the network group or throws if network group with requested name was not found + /// + public IConfigSectionNode GetNetworkGroupConfNode(string netName, string groupName) + { + var net = this.GetNetworkGroupNodes(netName).FirstOrDefault(n=>n.IsSameNameAttr(groupName)); + if (net==null) + throw new MetabaseException(StringConsts.METABASE_NAMED_NETWORK_GRP_NOT_FOUND_ERROR.Args(netName ?? SysConsts.NULL, groupName ?? SysConsts.NULL)); + return net; + } + + /// + /// Returns description for the network service or throws if network with requested name was not found + /// + public string GetNetworkSvcDescription(string netName, string svcName) + { + return GetNetworkSvcConfNode(netName, svcName).AttrByName(CONFIG_DESCRIPTION_ATTR).Value; + } + + /// + /// Returns description for the network group or throws if network with requested name was not found + /// + public string GetNetworkGroupDescription(string netName, string groupName) + { + return GetNetworkGroupConfNode(netName, groupName).AttrByName(CONFIG_DESCRIPTION_ATTR).Value; + } + + + /// + /// Gets external parameter value returning true if parameter was found + /// + public bool ExternalGetParameter(string name, out object value, params string[] groups) + { + return ExternalParameterAttribute.GetParameter(this, name, out value, groups); + } + + /// + /// Sets external parameter value returning true if parameter was found and set + /// + public bool ExternalSetParameter(string name, object value, params string[] groups) + { + return ExternalParameterAttribute.SetParameter(this, name, value, groups); + } + + + + + #endregion + + #region .pvt .impl + + ///reads supported configuration file taking file path relative to file system root + private Configuration getConfigFromFile(FileSystemSession session, string path, bool require = true) + { + try + { + var fnwe = m_FS.CombinePaths(m_FSRootPath, path); + foreach(var fmt in Configuration.AllSupportedFormats) + { + var fn = "{0}.{1}".Args(fnwe, fmt); // filename. + var file = session[fn] as FileSystemFile; + if (file==null) continue; + using(file) + { + var text = file.ReadAllText(); + return Configuration.ProviderLoadFromString(text, fmt); + } + } + if (require) + throw new MetabaseException(StringConsts.METABASE_FILE_NOT_FOUND_ERROR.Args(path)); + else + { + var result = new MemoryConfiguration(); + + result.Create(SysConsts.DEFAULT_APP_CONFIG_ROOT); + result.Root.ResetModified(); + return result; + } + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_CONFIG_LOAD_ERROR.Args(path ?? SysConsts.UNKNOWN_ENTITY, error.ToMessageWithType()), error); + } + } + + ///reads supported configuration file taking file path is absolute + private Configuration getConfigFromExistingFile(FileSystemSession session, string path) + { + try + { + var file = session[path] as FileSystemFile; + + if (file==null) throw new MetabaseException(StringConsts.METABASE_FILE_NOT_FOUND_EXACT_ERROR.Args(path)); + + using(file) + { + var text = file.ReadAllText(); + var fmt = chopNameLeaveExt(path); + + return Configuration.ProviderLoadFromString(text, fmt); + } + } + catch(Exception error) + { + throw new MetabaseException(StringConsts.METABASE_CONFIG_LOAD_ERROR.Args(path ?? SysConsts.UNKNOWN_ENTITY, error.ToMessageWithType()), error); + } + } + + + internal Configuration GetConfigFromExistingFile(string path) + { + var session = obtainSession(); + try + { + var rootedPath = m_FS.CombinePaths(m_FSRootPath, path); + return getConfigFromExistingFile(session, rootedPath); + } + finally + { + Monitor.Exit(session); + } + } + + + private void cachePut(string entity, string key, object item) + { + m_Cache.Put(entity, key, item); + } + + private object cacheGet(string entity, string key) + { + return m_Cache.Get(entity, key); + } + + + private Dictionary m_FSSessionCache = new Dictionary(NFX.ReferenceEqualityComparer.Instance); + private Thread m_FSSessionCacheThread; + private AutoResetEvent m_FSSessionCacheThreadWaiter; + + private class _fss + { + public FileSystemSession Session; + public DateTime LastAccess; + } + + private FileSystemSession obtainSession() + { + EnsureObjectNotDisposed(); + var callerThread = Thread.CurrentThread; + _fss session; + + lock(m_FSSessionCache) + { + if (m_FSSessionCacheThread==null) + { + m_FSSessionCacheThread = new Thread( + (object objBank)=> + { + var bank = objBank as Metabank; + while(bank.Active) + { + m_FSSessionCacheThreadWaiter.WaitOne(1000); + try + { + lock(m_FSSessionCache) + { + var now = DateTime.UtcNow; + var allKvp = m_FSSessionCache.ToList(); + foreach(var kvp in allKvp) + if (Monitor.TryEnter(kvp.Value.Session)) + try + { + if ((now - kvp.Value.LastAccess).TotalSeconds > 30)//how long to keep fsSession open + { + try{ kvp.Value.Session.Dispose();} + finally{m_FSSessionCache.Remove(kvp.Key);} + } + } + finally + { + Monitor.Exit(kvp.Value.Session); + } + } + } + catch(Exception error) + { + log(MessageType.Error, + "obtainSession(timedLoop)", + "Thread '{0}' leaked: {1}".Args(m_FSSessionCacheThread.Name, error.ToMessageWithType()), + error); + } + + }//while + } + ); + m_FSSessionCacheThread.IsBackground = false; + m_FSSessionCacheThread.Name = "Metabank FS Session Cache Thread"; + m_FSSessionCacheThreadWaiter = new AutoResetEvent(false); + m_FSSessionCacheThread.Start(this); + }//if thread==null + + if (m_FSSessionCache.TryGetValue(callerThread, out session)) + { + session.LastAccess = DateTime.UtcNow; + } + else + { + session = new _fss{ Session = m_FS.StartSession(m_FSSessionConnectParams), LastAccess = DateTime.UtcNow}; + m_FSSessionCache.Add(callerThread, session); + } + + Monitor.Enter(session.Session); + return session.Session; + } + + + } + + /// + /// Internal function that facilitates guarded acceess to the file system. Developers - do not use + /// + internal T fsAccess(string operationName, string path, Func body) + { + var session = obtainSession(); + try + { + var catPath = m_FS.CombinePaths(m_FSRootPath, path); + var catDir = session[catPath] as FileSystemDirectory; + + if (catDir==null) throw new MetabaseException(StringConsts.METABASE_PATH_NOT_FOUND_ERROR.Args(operationName, catPath)); + + T result; + try + { + result = body(session, catDir); + } + finally + { + //close all file and directory handles + var sessionItems = session.Items.ToList(); + foreach(var item in sessionItems) + item.Dispose(); + } + return result; + } + finally + { + Monitor.Exit(session); + } + } + + + private string chopExt(string name) + { + var idx = name.LastIndexOf('.'); + if (idx<=0) return name; + return name.Substring(0, idx); + } + + private string chopNameLeaveExt(string name) + { + var idx = name.LastIndexOf('.'); + if (idx<=0 || idx==name.Length-1) return string.Empty; + return name.Substring(idx+1); + } + + + private void includeCommonConfig(ConfigSectionNode levelRoot) + { + var placeholder = levelRoot.AddChildNode(Guid.NewGuid().ToString()); + placeholder.Configuration.Include(placeholder, m_CommonLevelConfig); + } + + private void log(MessageType type, string from, string text, Exception error = null) + { + App.Log.Write( + new Message + { + Topic = SysConsts.LOG_TOPIC_METABASE, + Type = type, + From = "Metabank."+from, + Text = text, + Exception = error + } + ); + } + + + #endregion + + } + +} diff --git a/src/Agni/Metabase/MetabankFileConfigNodeProvider.cs b/src/Agni/Metabase/MetabankFileConfigNodeProvider.cs new file mode 100644 index 0000000..e5049fe --- /dev/null +++ b/src/Agni/Metabase/MetabankFileConfigNodeProvider.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.IO.FileSystem; +using NFX; + +namespace Agni.Metabase +{ + /// + /// Provides shortcut access to mounted metabank file system + /// + public sealed class MetabankFileConfigNodeProvider : ApplicationComponent, IConfigNodeProvider + { + public MetabankFileConfigNodeProvider() : base(null) + { + m_Metabank = AgniSystem.Metabase; + } + + private Metabank m_Metabank; + + [Config] + public string File { get; set; } + + public void Configure(IConfigSectionNode node) + { + ConfigAttribute.Apply(this, node); + } + + public ConfigSectionNode ProvideConfigNode(object context = null) + { + if (File.IsNullOrWhiteSpace()) return null; + + return m_Metabank.GetConfigFromExistingFile(File).Root; + } + } +} diff --git a/src/Agni/Metabase/MetabaseException.cs b/src/Agni/Metabase/MetabaseException.cs new file mode 100644 index 0000000..eabd292 --- /dev/null +++ b/src/Agni/Metabase/MetabaseException.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.Metabase +{ + /// + /// Thrown to indicate metabase-related problems + /// + [Serializable] + public class MetabaseException : AgniException + { + public MetabaseException() : base() { } + public MetabaseException(string message) : base(message) { } + public MetabaseException(string message, Exception inner) : base(message, inner) { } + protected MetabaseException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Metabase/MetabaseValidationMsg.cs b/src/Agni/Metabase/MetabaseValidationMsg.cs new file mode 100644 index 0000000..089d834 --- /dev/null +++ b/src/Agni/Metabase/MetabaseValidationMsg.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; + +namespace Agni.Metabase +{ + + public enum MetabaseValidationMessageType + { + Info = 0, + Warning, + Error + } + + /// + /// Denotes a message that is generated by Metabank.Validate method + /// + public sealed class MetabaseValidationMsg + { + public MetabaseValidationMsg(MetabaseValidationMessageType type, + Metabank.Catalog catalog, + Metabank.Section section, + string message, + Exception exception = null) + { + this.Type = type; + this.Catalog = catalog; + this.Section = section; + this.Message = message; + this.Exception = exception; + } + + public readonly MetabaseValidationMessageType Type; + public readonly Metabank.Catalog Catalog; + public readonly Metabank.Section Section; + public readonly string Message; + public readonly Exception Exception; + + public override string ToString() + { + return ToString(true); + } + + public string ToString(bool type) + { + var sb = new StringBuilder(); + if (type) + { + sb.Append(Type); + sb.Append(" "); + } + if (Catalog!=null) + { + sb.Append("Catalog: "); + sb.Append(Catalog); + sb.Append(" "); + } + + if (Section!=null) + { + sb.Append("Section: "); + sb.Append(Section); + sb.Append(" "); + } + + if (Message!=null) + { + sb.Append(Message); + sb.Append(" "); + } + + if (Exception!=null) + { + if (this.Message.IndexOf(Exception.Message)<0) + sb.Append(Exception.ToMessageWithType()); + } + + return sb.ToString(); + } + + } +} diff --git a/src/Agni/Properties/AssemblyInfo.cs b/src/Agni/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..b15f85b --- /dev/null +++ b/src/Agni/Properties/AssemblyInfo.cs @@ -0,0 +1,8 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + + +[assembly: InternalsVisibleTo("amm")] +[assembly: InternalsVisibleTo("Agni.UTest")] +[assembly: InternalsVisibleTo("WinFormsTest")] \ No newline at end of file diff --git a/src/Agni/Security/AgniAuthenticationToken.cs b/src/Agni/Security/AgniAuthenticationToken.cs new file mode 100644 index 0000000..6a13a00 --- /dev/null +++ b/src/Agni/Security/AgniAuthenticationToken.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Security; +using NFX.Serialization.JSON; + +namespace Agni.Security +{ + /// + /// Represents AuthenticationToken in textual form that can be stored. + /// SecurityManager implementations in Agni are expected to use string token.Data + /// + public static class AgniAuthenticationTokenSerializer + { + public static string Serialize(AuthenticationToken token) + { + var data = token.Data; + if (data != null) + { + if (!(data is string)) + throw new SecurityException("AgniAuthenticationTokenSerializer can not serialize unexpected data '{0}'. Token.Data must be of 'string' type".Args(data.GetType().FullName)); + } + + return new { r = token.Realm, d = data }.ToJSON(JSONWritingOptions.CompactASCII); + } + + public static AuthenticationToken Deserialize(string token) + { + try + { + var dataMap = JSONReader.DeserializeDataObject(token) as JSONDataMap; + var realm = dataMap["r"].AsString(); + var data = dataMap["d"].AsString(); + + return new AuthenticationToken(realm, data); + } + catch (Exception error) + { + throw new SecurityException("AgniAuthenticationTokenSerializer can not deserialize unexpected data", error); + } + } + } +} diff --git a/src/Agni/Security/Permissions/Admin/AppRemoteTerminalPermission.cs b/src/Agni/Security/Permissions/Admin/AppRemoteTerminalPermission.cs new file mode 100644 index 0000000..402e110 --- /dev/null +++ b/src/Agni/Security/Permissions/Admin/AppRemoteTerminalPermission.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Security; + +namespace Agni.Security.Permissions.Admin +{ + /// + /// Controls whether users can access remote terminal of application context + /// + public sealed class AppRemoteTerminalPermission : TypedPermission + { + public AppRemoteTerminalPermission() : base(NFX.Security.AccessLevel.VIEW) { } + + public override string Description + { + get { return StringConsts.PERMISSION_DESCRIPTION_AppRemoteTerminalPermission; } + } + } +} diff --git a/src/Agni/Security/Permissions/Admin/RemoteTerminalOperatorPermission.cs b/src/Agni/Security/Permissions/Admin/RemoteTerminalOperatorPermission.cs new file mode 100644 index 0000000..5d54fd2 --- /dev/null +++ b/src/Agni/Security/Permissions/Admin/RemoteTerminalOperatorPermission.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Security; + +namespace Agni.Security.Permissions.Admin +{ + /// + /// Controls whether users can access remote terminals + /// + public sealed class RemoteTerminalOperatorPermission : TypedPermission + { + public RemoteTerminalOperatorPermission() : base(NFX.Security.AccessLevel.VIEW) { } + + public override string Description + { + get { return StringConsts.PERMISSION_DESCRIPTION_RemoteTerminalOperatorPermission; } + } + } +} diff --git a/src/Agni/StringConsts.cs b/src/Agni/StringConsts.cs new file mode 100644 index 0000000..c0c62f2 --- /dev/null +++ b/src/Agni/StringConsts.cs @@ -0,0 +1,443 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Agni +{ + internal static class StringConsts + { + public const string APP_LOADER_ALREADY_LOADED_ERROR = "Application loader has already loaded"; + public const string APP_LOADER_ERROR = "Application loader exception: "; + + + public const string COMPONENT_NAME_EMPTY_ERROR = "Component name can not be empty"; + + public const string ARGUMENT_ERROR = "Argument error: "; + + public const string GLUE_BINDING_UNSUPPORTED_FUNCTION_ERROR = "Glue binding `{0}` supports only `{1}`. `{2}` is unsupported"; + public const string GLUE_BINDING_RESPONSE_ERROR = "Glue binding `{0}` response error: {1}"; + public const string GLUE_BINDING_REQUEST_ERROR = "Glue binding `{0}` request error: {1}"; + + public const string INTERNAL_IMPLEMENTATION_ERROR = "Error in internal implementation: "; + + public const string HOST_NAME_EMPTY_ERROR = "Host name can not be empty"; + + public const string METABASE_APP_NAME_ASSIGNMENT_ERROR = "Metabase application name must be assigned a non-null non-empty string only once at process entry point"; + + public const string BOOT_LOCAL_CONFIGURATION_ROOT_DOES_NOT_EXIST_ERROR = "Application boot - local configuration root '{0}' does not exist"; + + public const string RT_CMDLET_ACTIVATION_ERROR = + "Internal error. The cmdlet activation returned for '{0}'"; + + public const string RT_CMDLET_DONTKNOW_ERROR = + "Server: do not know how to handle cmdlet '{0}'. Use 'help' to see cmdlet list"; + + + public const string PERMISSION_DESCRIPTION_RemoteTerminalOperatorPermission = + "Controls whether users can access remote terminals"; + + public const string PERMISSION_DESCRIPTION_AppRemoteTerminalPermission = + "Controls whether users can access remote terminal of application context"; + + public const string GDIDGEN_ALL_AUTHORITIES_FAILED_ERROR = "GDIDGenerator failed to obtain GDIDBlock from any authority. Tried: \n"; + + public const string GDIDGEN_SET_TESTING_ERROR = "GDIDGenerator can not set TestingAuthorityGlueNode as the block was already allocated"; + + public const string GDIDAUTH_AUTHORITY_ASSIGNMENT_WARNING = + "AUTHORITY = {0}. This is a warning because an extra care should be taken with AUTHORITY assignment"; + + + public const string GDIDAUTH_INSTANCE_NOT_ALLOCATED_ERROR = "GDIDAuthorityService instance is not allocated"; + public const string GDIDAUTH_INSTANCE_ALREADY_ALLOCATED_ERROR = "GDIDAuthorityService instance is already allocated"; + public const string GDIDAUTH_INSTANCE_NOT_RUNNING_ERROR = "GDIDAuthorityService instance is not running or shutting down"; + public const string GDIDAUTH_LOCATIONS_CONFIG_ERROR = "GDIDAuthorityServiceBase locations configuration error: "; + public const string GDIDAUTH_LOCATIONS_READ_FAILURE_ERROR = "GDIDAuthorityServiceBase was not able to read sequence value from any persistence locations. Exception(s): "; + public const string GDIDAUTH_LOCATION_PERSISTENCE_FAILURE_ERROR = "GDIDAuthorityServiceBase was not able to persist sequence increment in any persistence locations. Exception(s): "; + public const string GDIDAUTH_DISK_PATH_TOO_LONG_ERROR = "GDIDAuthorityService can not save sequence, path is too long. Use shorter sequence and scope names. Path: {0}"; + public const string GDIDAUTH_NAME_INVALID_CHARS_ERROR = "GDIDAuthorityService can not use the supplied name '{0}' as it contains invalid chars. Scope and sequence names, may only contain alphanumeric or ['-','.','_'] chars and may only start from and end with either a latin letter or a digit"; + public const string GDIDAUTH_NAME_INVALID_LEN_ERROR = "GDIDAuthorityService can not use the supplied name '{0}' as it is either null/blank or longer than the limit of {1}"; + public const string GDIDAUTH_IDS_INVALID_AUTHORITY_VALUE_ERROR = "GDIDAuthorityService.AuthorityIDs set to ivalid value of '{0}'. An array of at least one element having all of its element values between 0..0x0f is required"; + public const string GDIDAUTH_ID_DATA_PARSING_ERROR = "GDIDAuthorityService::_id parsing error of '{0}'. Inner: {1}"; + public const string GDIDAUTH_ERA_EXHAUSTED_ERROR = "GDIDAuthorityService CATASTROPHIC FAILURE scope '{0}', sequence '{1}'. The era is exhausted. No more generation possible"; + + public const string GDIDAUTH_ERA_EXHAUSTED_ALERT = "GDIDAuthorityService CRITICAL ALERT scope '{0}', sequence '{1}'. The era is about to be exhausted"; + + public const string GDIDAUTH_ERA_PROMOTED_WARNING = "GDIDAuthorityService Era in scope '{0}', sequence '{1}' has been promoted to {2}"; + + public const string HOST_SET_BUILDER_CONFIG_FIND_ERROR = + "HostSet Builder '{0}' could not find config section for a named set '{1}' in any of the region zones starting at '{2}'; search parent: {3}, transcend NOC: {4}"; + + public const string HOST_SET_DYNAMIC_HOST_NOT_SUPPORTED_ERROR = + "HostSet '{0}' declares dynamic host '{1}' which is not supported"; + + + public const string METABASE_FS_CONNECTION_ERROR = "Metabase file system {0}('{1}','{2}') connection error: {3}"; + + public const string METABASE_NOT_AVAILABLE_ERROR = + "Metabase is needed by '{0}' but it is not available as AgniSystem.Metabase==null. The AgniApplication is not allocated yet or BootConfLoader.LoadForTest() was not called"; + + public const string METABASE_INVALID_OPERTATION_ERROR = "Invalid metabase operation: "; + public const string METABASE_CONFIG_LOAD_ERROR = "Metabase config file '{0}' load error: {1}"; + public const string METABASE_FILE_NOT_FOUND_ERROR = "Could not find file '{0}' in any of the supported config formats"; + public const string METABASE_FILE_NOT_FOUND_EXACT_ERROR = "Could not find file by exact name '{0}'"; + + public const string METABASE_PLATFORMS_OS_NOT_DEFINED_ERROR = "Metabase platforms file does not contain the definition for '{0}' operating system"; + public const string METABASE_PLATFORMS_OS_DUPLICATION_ERROR = "Metabase platforms file defines '{0}' operating system more than once"; + + + + public const string METABASE_PATH_NOT_FOUND_ERROR = "Metabase operation '{0}' could not find path '{1}'"; + + public const string METABASE_METADATA_CTOR_1_ERROR = + "Metadata '{0}' could not be created. Level configuration root name should be called '{1}' but is called '{2}' instead"; + + public const string METABASE_METADATA_CTOR_2_ERROR = + "Metadata '{0}' could not be created. A 'name' attribute must be declared on the root config node level having its value equal to '{1}', however it is equal to '{2}'"; + + public const string METABASE_METADATA_CTOR_3_ERROR = + "Metadata '{0}' could not be created. The name '{1}' is invalid agni entity name. See .IsValidName() function. Path: {2}"; + + + public const string METABASE_STRUCTURE_NOTEXISTS_ERROR = "Metadabase structure error: '{0}' does not exist "; + + public const string METABASE_APP_CONFIG_APP_DOESNT_EXIST_ERROR = + "Metadabase section '{0}' contains application config file '{1}' that references non-existing application '{2}'"; + + public const string METABASE_BAD_HOST_ROLE_ERROR = + "The host '{0}' specifies the role name '{1}' which does not resolve to an existing role in the app catalog"; + + public const string METABASE_HOST_MISSING_OS_ATTR_ERROR = + "The host '{0}' does not specify its 'os' attribute"; + + public const string METABASE_BAD_HOST_APP_ERROR = + "App name '{0}' does not resolve to an existing app in the app catalog"; + + public const string METABASE_HOST_ROLE_APP_MISMATCH_ERROR = + "App name '{0}' is not a part of agni role '{1}' that this host has"; + + public const string METABASE_VALIDATION_ROLE_APP_ERROR = + "Role '{0}' declares app '{1}' which was not found in app catalog"; + + public const string METABASE_VALIDATION_ROLE_APP_EXE_MISSING_ERROR = + "Role '{0}' declares app '{1}' which does not specify any executable command on either application or role level. Add '{2}' attribute to app or role level"; + + public const string METABASE_NOC_DEFAULT_GEO_CENTER_WARNING = + "NOC '{0}' uses the default geo center: '{1}'"; + + public const string METABASE_EFFECTIVE_APP_CONFIG_OVERRIDE_ERROR = + "Override error at '{0}': {1}"; + + public const string METABASE_EFFECTIVE_APP_CONFIG_ERROR = + "Error calculating the effective config for app name '{0}' at host '{1}': {2}"; + + public const string METABASE_APP_PACKAGES_ERROR = + "Error calculating packages for app name '{0}' at host '{1}': {2}"; + + public const string METABASE_VALIDATION_APP_CONFIG_ROOT_MISMTACH_ERROR = + "App config root name of '{0}' does not match the very metabase root"; + + public const string METABASE_PLATFORM_OS_RESERVED_NAME_ERROR = + "Metabase platform file declares a '{0}' operating system which is a reserved name"; + + public const string METABASE_NAME_ATTR_UNDEFINED_ERROR = + "Metabase entity must have a name attribute with a non-blank value: "; + + public const string METABASE_ENTITY_NAME_INVALID_ERROR = + "Metabase entity {0} has an invalid name '{1}'. See .IsValidName() func"; + + public const string METABASE_PLATFORM_NAME_DUPLICATION_ERROR = + "Metabase platforms file declares some platform(s) more than once"; + + public const string METABASE_NETWORK_NAME_DUPLICATION_ERROR = + "Metabase networks file declares some network(s) more than once"; + + public const string METABASE_NETWORK_GET_BINDING_NODE_ERROR = + "Metabase could not get binding conf node for network '{0}' service '{1}' binding '{2}'. Service has to have at least one binding. Error: {3}"; + + public const string METABASE_NETWORK_SVC_NAME_DUPLICATION_ERROR = + "Metabase networks file declares a network '{0}' that has some service(s) listed more than once"; + + public const string METABASE_NETWORK_GRP_NAME_DUPLICATION_ERROR = + "Metabase networks file declares a network '{0}' that has some group(s) listed more than once"; + + public const string METABASE_NETWORK_NO_SVC_ERROR = + "Metabase networks file declares a network '{0}' that has no services listed"; + + public const string METABASE_NETWORK_NO_SVC_BINDINGS_ERROR = + "Metabase networks file declares a network '{0}' service'{1}' that has no bindings listed"; + + public const string METABASE_NETWORK_SVC_BINDING_NAME_DUPLICATION_ERROR = + "Metabase networks file declares a network '{0}' service '{1}' that has some binding(s) listed more than once"; + + public const string METABASE_NETWORK_CONFIG_ERROR = + "Metabase networks file contains errors:"; + + public const string METABASE_NETWORK_SVC_RESOLVE_ERROR = + "Can not resolve host '{0}' network '{1}' service '{2}'. Error: {3}"; + + public const string METABASE_NETWORK_REGION_ROUTING_EMPTY_ROUTE_ASSIGNMENT_ERROR = + "Region level config at '{0}' defines a networking route without any meaningful 'to-address/port/group' assignments"; + + public const string METABASE_NETWORK_REGION_ROUTING_ATTR_UNRECOGNIZED_WARNING = + "Region level config at '{0}' defines a networking route with an unknown attribute '{1}'. It is neither a route pattern filter nor a 'to-*' resolver attribute"; + + public const string METABASE_NETWORK_REGION_ROUTING_FROM_PATH_ERROR = + "Region level config at '{0}' defines a netwotrking route with filter 'from='{1}'' that does not resolve to any entity in region catalog"; + + public const string METABASE_NAMED_NETWORK_NOT_FOUND_ERROR = "Metabase networks file does not define network '{0}'"; + + public const string METABASE_NAMED_NETWORK_SVC_NOT_FOUND_ERROR = "Metabase networks file does not define network '{0}' service '{1}'"; + + public const string METABASE_NAMED_NETWORK_GRP_NOT_FOUND_ERROR = "Metabase networks file does not define network '{0}' group '{1}'"; + + public const string METABASE_NETWORK_SVC_DEFAULT_BINDING_NAME_ERROR = + "Metabase networks file declares a network '{0}' service '{1}' that references default binding '{2}' that is not declared"; + + public const string METABASE_BIN_PACKAGE_INVALID_PLATFORM_ERROR = + "Bin package '{0}' references platform that is not known to the metabase"; + + public const string METABASE_BIN_PACKAGE_INVALID_OS_ERROR = + "Bin package '{0}' references operating system that is not known to the metabase"; + + public const string METABASE_BIN_PACKAGE_MISSING_MANIFEST_ERROR = + "Bin package '{0}' is missing a manifest file '{1}'"; + + public const string METABASE_BIN_PACKAGE_OUTDATED_MANIFEST_ERROR = + "Bin package '{0}' contains an outdated manifest file '{1}'. Regenerate package manifest using AMM tool with '/gbm' switch"; + + public const string METABASE_REG_CATALOG_NAV_ERROR = + "Error navigating region catalog. Operation: '{0}'. Path: '{1}' does not resolve to expected target"; + + public const string METABASE_REG_NOC_PARENT_NOC_ZONE_ERROR = + "NOC '{0}' specifies ParentNOCZonePath of '{1}' which does not resolve to existing zone"; + + public const string METABASE_REG_NOC_PARENT_NOC_ZONE_NO_ROOT_ERROR = + "NOC '{0}' specifies ParentNOCZonePath of '{1}' which has no common root with this NOC"; + + public const string METABASE_REG_NOC_PARENT_NOC_ZONE_LEVEL_ERROR = + "NOC '{0}' specifies ParentNOCZonePath of '{1}' which must be higher in agni hierarchy than this NOC, but it is not"; + + public const string METABASE_REG_GEO_CENTER_ERROR = + "Error in region catalog section {0}('{1}').$'geo-center' = '{2}'. Error: {3}"; + + public const string METABASE_NET_SVC_RESOLVER_TARGET_NOC_INACCESSIBLE_ERROR = + "Can not resolve target service '{0}' from host '{1}' to host '{2}' on network '{3}' scope '{4}'. Destination inaccessible because parties are in different NOCs"; + + public const string METABASE_NET_SVC_RESOLVER_TARGET_GROUP_INACCESSIBLE_ERROR = + "Can not resolve target service '{0}' from host '{1}' to host '{2}' on network '{3}' scope '{4}'. Destination inaccessible because parties are in different groups"; + + public const string METABASE_NET_SVC_RESOLVER_DYN_HOST_UNKNOWN_ERROR = + "Can not resolve target service '{0}' from host '{1}' to host '{2}' on network '{3}'. Dynamic destination host is not known"; + + public const string METABASE_NET_SVC_RESOLVER_DYN_HOST_NO_ADDR_MATCH_ERROR = + "Can not resolve target service '{0}' from host '{1}' to host '{2}' on network '{3}'. No dynamic host adapter address matches required pattern: '{4}'"; + + public const string METABASE_GDID_AUTHORITIES_NONE_DEFINED_WARNING = + "Metabase root level does not define any GDID authorities with valid host|name attributes"; + + public const string METABASE_GDID_AUTHORITIES_DUPLICATION_ERROR = + "Metabase root level defines some GDID authority/ies more than once"; + + public const string METABASE_GDID_AUTHORITY_BAD_HOST_ERROR = + "Metabase root level defines GDID authority host '{0}' that does not resolve in regional catalog"; + + public const string METABASE_GDID_AUTHORITY_BAD_NETWORK_ERROR = + "Metabase root level defines GDID authority host '{0}' that references network '{1}' that is not known"; + + public const string METABASE_GDID_AUTHORITY_HOST_NOT_AGDIDA_ERROR = + "Metabase root level defines GDID authority host '{0}' that does not have AGDIDA application in its role '{1}'"; + + public const string METABASE_GDID_AUTHORITY_SVC_RESOLUTION_ERROR = + "Metabase root level defines GDID authority host '{0}' which causes service resolution error: {1}"; + + public const string METABASE_VALIDATION_WRONG_HOST_ERROR = + "Metabase validation is to be performed as if from host '{0}' which does not resolve in region catalog"; + + public const string METABASE_VALIDATION_WRONG_HOST_INFO = + "If using AMM tool, you may have used wrong name under /from|/host switch or your {0} environment var is misconfigured"; + + + public const string METABASE_CONTRACTS_SERVICE_HUB_ERROR = + "Metabase ServiceContractHub could not be initialized: "; + + + public const string METABASE_NO_APP_PACKAGES_WARNING = + "Metabase declares an application '{0}' with no packages"; + + public const string METABASE_APP_PACKAGE_REDECLARED_ERROR = + "Metabase declares an application '{0}' with some package/s declared more than once"; + + public const string METABASE_INSTALLATION_BIN_PACKAGE_NOT_FOUND_ERROR = + "Package installation tries to install a package '{0}' in metabase directory '{1}' which is not found in BIN catalog"; + + public const string METABASE_APP_PACKAGE_BLANK_NAME_ERROR = + "Metabase declares an application '{0}' which lists some package/s without a name"; + + public const string METABASE_APP_PACKAGE_NOT_FOUND_IN_BIN_CATALOG_ERROR = + "Metabase declares an application '{0}' with a references to the package '{1}' which is not declared for any platform/os in binary catalog"; + + public const string METABASE_APP_DOES_NOT_HAVE_MATCHING_BIN_ERROR = + "Metabase declares an application '{0}' which needs a package '{1}' having version '{2}' that does not have a matching bin resource per supplied OS '{3}'"; + + public const string METABASE_PROCESSOR_SET_MISSING_ATTRIBUTE_ERROR = + "Zone processor set host is missing '{0}'"; + + public const string METABASE_PROCESSOR_SET_DUPLICATE_ATTRIBUTE_ERROR = + "Zone processor set host is duplicate '{0}'"; + + public const string METABASE_PROCESSOR_SET_HOST_IS_NOT_PROCESSOR_HOST_ERROR = + "Zone processor set host with id '{0}' is not processor host"; + + public const string METABASE_ZONE_COULD_NOT_FIND_PROCESSOR_HOST_ERROR = + "Zone '{0}' could not find processor host by id '{1}'"; + + public const string APPL_CMD_STOPPING_INFO = + "Application container is stopping as the result of the app terminal command received from session '{0}' connected on '{1}' as '{2}'"; + + public const string AHGOV_INSTANCE_ALREADY_ALLOCATED_ERROR = "HostGovernorService instance is already allocated"; + + public const string AHGOV_INSTANCE_NOT_ALLOCATED_ERROR = "HostGovernorService instance is not allocated"; + + public const string AHGOV_APP_PROCESS_CRASHED_AT_STARTUP_ERROR = + "Application '{0}' process '{1}' with args '{2}' crashed while startup, see its logs"; + + public const string AHGOV_APP_PROCESS_STD_OUT_NULL_ERROR = + "Application '{0}' process '{1}' with args '{2}' standard output is null"; + + public const string AHGOV_APP_PROCESS_NO_SUCCESS_AT_STARTUP_ERROR = + "Application '{0}' process '{1}' with args '{2}' did not return success code 'OK.', see its logs"; + + public const string AHGOV_ARD_UPDATE_PROBLEM_ERROR = + "AHGOV respawned by ARD after an update problem. Check ARD logs. 'UPD' and/or 'RUN' folders may be locked by some other foreign process"; + + + public const string AZGOV_INSTANCE_NOT_ALLOCATED_ERROR = "ZoneGovernorService instance is not allocated"; + public const string AZGOV_INSTANCE_ALREADY_ALLOCATED_ERROR = "ZoneGovernorService instance is already allocated"; + + public const string AZGOV_REGISTER_SUBORDINATE_HOST_ERROR = "ZoneGovernorService can not register subordinate host '{0}' because of the error: {1}"; + public const string AZGOV_REGISTER_SUBORDINATE_HOST_PARENT_ERROR = "This zone governor '{0}' is not a direct or indirect parent of the host '{1}' in this NOC"; + + public const string INSTR_SEND_TELEMETRY_TOP_LOST_ERROR = "Could not send instrumentation telemetry up the zone gov agni chain as the very root reached. Error: {0}"; + + public const string LOG_SEND_TOP_LOST_ERROR = "Could not send log up the zone gov agni chain as the very root reached. Error: {0}"; + + + public const string UNIT_NAME_TIME = "times"; + + + public const string AGNI_SVC_CLIENT_HUB_SINGLETON_CTOR_ERROR = "Error while making singleton instance of service client hub implementation '{0}'. Error: {1}"; + public const string AGNI_SVC_CLIENT_HUB_MAPPING_ERROR = "Service hub error mapping a client for '{0}' service: {1}"; + public const string AGNI_SVC_CLIENT_HUB_MAKE_INSTANCE_ERROR = "Service hub error making a client instance for contract mapping '{0}'. Activation error: {1}"; + public const string AGNI_SVC_CLIENT_HUB_NET_RESOLVE_ERROR = "Service hub error resolving service node for contract mapping '{0}'. Resolver error: {1}"; + public const string AGNI_SVC_CLIENT_HUB_CALL_RETRY_FAILED_ERROR = "Service hub error calling '{0}' after {1} retries tried"; + public const string AGNI_SVC_CLIENT_HUB_SETUP_INSTANCE_ERROR = "Service hub error from setup a client instance for contract mapping '{0}'. Setup error: {1}"; + public const string AGNI_SVC_CLIENT_HUB_RETRY_CALL_HOST_ERROR = "Service hub error calling '{0}' service on '{1}'"; + public const string AGNI_SVC_CLIENT_MAPPING_CTOR_ERROR = "Service hub ContractMapping.ctor(' {0} ') error: {1}"; + + + + public const string LOCK_SESSION_PATH_ERROR = "LockSession can not be created at the path '{0}'. Error: {1}"; + public const string LOCK_SESSION_ZGOV_SETUP_ERROR = "Invalid metabase setup. Locking failover host count is different from primary host count in the zone '{0}'"; + public const string LOCK_SESSION_NOT_ACTIVE_ERROR = "LockSession '{0}' / '{1}' is not present in the list of active sessions"; + public const string LOCK_SESSION_PATH_LEVEL_NO_ZGOVS_ERROR = "Lock session can not be established at the level identified by path '{0}' as there are no zgov nodes in the hierarchy at the level or above to service the request"; + + + public const string KDB_TABLE_IS_NULL_OR_EMPTY_ERROR = "KDB operation'{0}' error: Table name is null or white space"; + public const string KDB_TABLE_CHARACTER_ERROR = "KDB operation'{0}' error: Table name contains invalid character '{1}'"; + public const string KDB_TABLE_MAX_LEN_ERROR = "KDB operation'{0}' error: Table name of {1} characters exeeds max len of {2}"; + public const string KDB_KEY_MAX_LEN_ERROR = "KDB operation'{0}' error: Key name of {1} characters exeeds max len of {2}"; + public const string KDB_KEY_IS_NULL_OR_EMPTY_ERROR = "KDB operation'{0}' error: Key is null or empty"; + + + + public const string MDB_AREA_CONFIG_PARTITION_NOT_FOUND_ERROR = + "MDB area '{0}' configuration has no partition slot that fits the briefcase GDID: '{1}'"; + + public const string MDB_AREA_CONFIG_NO_NAME_ERROR = + "MDB area is defined without a name in config. Name must be explicitly defined"; + + public const string MDB_AREA_CONFIG_NO_DATASTORE_ERROR = + "MDB area '{0}' is defined without a data store"; + + public const string MDB_AREA_CONFIG_INVALID_NAME_ERROR = + "MDB area has an invalid name defined"; + + public const string MDB_AREA_CONFIG_NO_PARTITIONS_ERROR = + "MDB area '{0}' config has no partitions defined'"; + + public const string MDB_AREA_CONFIG_DUPLICATE_RANGES_ERROR = + "MDB area '{0}' config contains duplicate ranges'"; + + public const string MDB_AREA_CONFIG_PARTITION_GDID_ERROR = + "MDB area '{0}' config contains partition with wrong GDID '{1}'. Reason: {2}'"; + + public const string MDB_AREA_CONFIG_NO_PARTITION_SHARDS_ERROR = + "MDB area '{0}' config has no shards defined for partition '{1}'"; + + public const string MDB_AREA_CONFIG_DUPLICATE_PARTITION_SHARDS_ERROR = + "MDB area '{0}' config has duplicate shard order/number defined for partition '{1}'"; + + public const string MDB_AREA_CONFIG_SHARD_CSTR_ERROR = + "MDB area '{0}' config partition '{1}' shard connect string '{2}' is missing"; + + public const string MDB_STORE_CONFIG_NO_TARGET_NAME_ERROR = + "MDB store requires non-blank target name to be assigned in config"; + + public const string MDB_STORE_CONFIG_NO_AREAS_ERROR = + "MDB store config missing any areas"; + + public const string MDB_STORE_CONFIG_GDID_ERROR = + "MDB store config has problems starting with GDIDGenerator: "; + + public const string MDB_PARTITIONED_AREA_MISSING_ERROR = + "MDB area '{0}' does not exist or is not a partitioned area"; + + public const string MDB_STORE_CONFIG_MANY_CENTRAL_ERROR = + "MDBStore config specifies more than one central area"; + + public const string MDB_OBJECT_SHARDING_ID_ERROR = + "Can not obtain sharding ID from object of type '{0}'"; + + public const string MDB_POSSIBLE_SHARDING_ID_CYCLE_ERROR = + "Can not obtain sharding ID because of possible cycle in IShardingProvider.ShardingID graph"; + + public const string PM_HOSTSET_CONFIG_MISSING_NAME_ERROR = "ProcessManager hostset config is missing the 'name' attribute"; + public const string PM_HOSTSET_CONFIG_DUPLICATE_NAME_ERROR = "ProceeManager hostset config already contains HostSet named '{0}'"; + public const string PM_HOSTSET_CONFIG_PATH_MISSING_ERROR = "ProceeManager hostset config 'path' is missing for HostSet named '{0}'"; + + public const string CONFIGURATION_INCLUDE_PRAGMA_DEPTH_ERROR = "Include pragma recursive depths exceeded: {0}"; + public const string CONFIGURATION_INCLUDE_PRAGMA_ERROR = "Include error at '{0}': {1}"; + + public const string TODO_QUEUE_NOT_FOUND_ERROR = "Todo queue '{0}' not found"; + public const string TODO_QUEUE_ENQUEUE_DIFFERENT_ERROR = "Can not enqueue todos from different queues in one enqueue call"; + public const string TODO_FRAME_SER_NOT_SUPPORTED_ERROR = "TodoFrame of '{0}' serializer not supported: {1}"; + public const string TODO_FRAME_DESER_NOT_SUPPORTED_ERROR = "TodoFrame of '{0}' deserializer not supported: {1}"; + public const string TODO_FRAME_SER_ERROR = "TodoFrame serialization error of '{0}': {1}"; + public const string TODO_FRAME_DESER_ERROR = "TodoFrame deserialization error of '{0}': {1}"; + + public const string TODO_CORRELATED_MERGE_ERROR = "CorrelatedTodo '{0}'.Merge('{1}') leaked: {2}"; + + public const string TODO_ENQUEUE_TX_BODY_ERROR = "Error executing enqueue in '{0}' transaction body: {1}"; + + public const string LOG_ARCHIVE_PUT_TX_BODY_ERROR = "Error executing put in '{0}' transaction body: {1}"; + public const string LOG_ARCHIVE_MESSAGE_NOT_FOUND_ERROR = "Log message with id {0} not found in log archive"; + + public const string TELEMETRY_ARCHIVE_PUT_TX_BODY_ERROR = "Error executing put in '{0}' transaction body: {1}"; + + public const string PROCESS_FRAME_SER_NOT_SUPPORTED_ERROR = "ProcessFrame of '{0}' serializer not supported: {1}"; + public const string PROCESS_FRAME_DESER_NOT_SUPPORTED_ERROR = "ProcessFrame of '{0}' deserializer not supported: {1}"; + public const string PROCESS_FRAME_SER_ERROR = "ProcessFrame serialization error of '{0}': {1}"; + public const string PROCESS_FRAME_DESER_ERROR = "ProcessFrame deserialization error of '{0}': {1}"; + + public const string SIGNAL_FRAME_SER_NOT_SUPPORTED_ERROR = "SignalFrame of '{0}' serializer not supported: {1}"; + public const string SIGNAL_FRAME_DESER_NOT_SUPPORTED_ERROR = "SignalFrame of '{0}' deserializer not supported: {1}"; + public const string SIGNAL_FRAME_SER_ERROR = "SignalFrame serialization error of '{0}': {1}"; + public const string SIGNAL_FRAME_DESER_ERROR = "SignalFrame deserialization error of '{0}': {1}"; + public const string PID_PARSE_ERROR = "String value '{0}' can not be parsed as PID"; + + + public const string WM_SERVICE_NO_CHANNELS_ERROR = "{0} service start error - no channels configured"; + public const string WM_SERVICE_DUPLICATE_CHANNEL_ERROR = "{0} service config error - duplicate channel name '{0}'"; + } +} diff --git a/src/Agni/SysConsts.cs b/src/Agni/SysConsts.cs new file mode 100644 index 0000000..07b8984 --- /dev/null +++ b/src/Agni/SysConsts.cs @@ -0,0 +1,262 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Reflection; + +using NFX; +using NFX.Environment; +using NFX.Geometry; + +namespace Agni +{ + + /// + /// Defines system constants. Do not localize! + /// + public static class SysConsts + { + /// + /// Defines maximum allowed clock drift in the cluster system. + /// Certain algorithms depend on this value for ordering and avoidance of locking logic. + /// The injected time source has to guarantee that all clocks on all machines in the system + /// differ by at most this number of milliseconds. + /// + public const int MAX_ALLOWED_CLOCK_DRIFT_MS = 2000; + + /// + /// Provides a safe margin that various cluster algorithms have to consider to avoid locking + /// and other ordering optimizations + /// + public const int CLOCK_DRIFT_SAFE_MARGIN_MS = 2 * MAX_ALLOWED_CLOCK_DRIFT_MS; + + + + public const string UNKNOWN_ENTITY = ""; + public const string NULL = ""; + + public const string GDID_NS_DYNAMIC_HOST = "SYS-AgniDynamicHost"; + public const string GDID_NAME_DYNAMIC_HOST = "DYNHOST"; + + public const string GDID_NS_WORKER = "SYS-AgniWORKER"; + public const string GDID_NAME_WORKER_TODO = "TODO"; + public const string GDID_NAME_WORKER_PROCESS = "PROCESS"; + public const string GDID_NAME_WORKER_SIGNAL = "SIGNAL"; + + public const string GDID_NS_MESSAGING = "SYS-AgniMESSAGING"; + public const string GDID_NAME_MESSAGING_MESSAGE = "MESSAGE"; + + /// + /// The name of application config root section that agni-related components should nest under. + /// This is needed not to cause an collision in inner section names with possible existing root-level non-agni names + /// + public const string APPLICATION_CONFIG_ROOT_SECTION = "agni"; + + public const string EXT_PARAM_GROUP_METABASE = "metabase"; + public const string EXT_PARAM_GROUP_WORKER = "worker"; + + public const string LOG_TOPIC_METABASE = "mtbs"; + public const string LOG_TOPIC_ID_GEN = "idgen"; + public const string LOG_TOPIC_APP_MANAGEMENT = "AppMgmt"; + public const string LOG_TOPIC_ZONE_MANAGEMENT = "ZnMgmt"; + public const string LOG_TOPIC_LOCKING = "lck"; + public const string LOG_TOPIC_INSTRUMENTATION = "instr"; + public const string LOG_TOPIC_WWW = "www"; + public const string LOG_TOPIC_SVC = "svc"; + public const string LOG_TOPIC_LOCALIZATION = "loclz"; + public const string LOG_TOPIC_MDB = "mdb"; + public const string LOG_TOPIC_KDB = "kdb"; + public const string LOG_TOPIC_HOST_SET = "hostSet"; + public const string LOG_TOPIC_WORKER = "wrkr"; + public const string LOG_TOPIC_WMSG = "wmsg"; + + + public static readonly LatLng DEFAULT_GEO_LOCATION = new LatLng("41.4996374,-81.6936649", "Cleveland, OH, USA"); + + /// + /// Supplied in command line from ARD to signify that AHGOV was launched by ARD + /// + public const string ARD_PARENT_CMD_PARAM = "ARDPARENT"; + + /// + /// Supplied in command line from ARD to signify that AHGOV was launched by ARD and last update failed + /// + public const string ARD_UPDATE_PROBLEM_CMD_PARAM = "UPDATEPROBLEM"; + + public const string HGOV_ARD_DIR = "ard"; + public const string HGOV_RUN_NETF_DIR = "run-netf"; + public const string HGOV_RUN_CORE_DIR = "run-core"; + public const string HGOV_UPDATE_DIR = "upd"; + public const string HGOV_UPDATE_FINISHED_FILE = "update.finished"; + public const string HGOV_UPDATE_FINISHED_FILE_OK_CONTENT = "OK."; + + + public static readonly string[] APP_NAMES_FORBIDDEN_ON_DYNAMIC_HOSTS = { SysConsts.APP_NAME_GDIDA, SysConsts.APP_NAME_ZGOV }; + + public const string APP_NAME_HGOV = "ahgov"; + public const string APP_NAME_ZGOV = "azgov"; + public const string APP_NAME_GDIDA = "agdida"; + public const string APP_NAME_WS = "aws"; + public const string APP_NAME_SH = "ash"; + public const string APP_NAME_PH = "aph"; + public const string APP_NAME_LOG = "alog"; + + public const string DEFAULT_WORLD_REGION_PATH = "/World.r"; + public const string DEFAULT_WORLD_GLOBAL_NOC_PATH = DEFAULT_WORLD_REGION_PATH + "/Global.noc"; + public const string DEFAULT_WORLD_GLOBAL_ZONE_PATH = DEFAULT_WORLD_GLOBAL_NOC_PATH + "/Default.z"; + + public const string DEFAULT_APP_CONFIG_ROOT = "application"; + + public const int REMOTE_TERMINAL_TIMEOUT_MS = 10 * //min + 60 * //sec + 1000; //msec + + //Default Bindings + public const string APTERM_BINDING = "apterm"; + public const string SYNC_BINDING = "sync"; + public const string ASYNC_BINDING = "async"; + + public const string DEFAULT_BINDING = ASYNC_BINDING; + + public const int APP_TERMINAL_PORT_OFFSET = 16; + + public const int DEFAULT_ZONE_GOV_WEB_PORT = 8081; + public const int DEFAULT_ZONE_GOV_SVC_SYNC_PORT = 2000; + public const int DEFAULT_ZONE_GOV_SVC_ASYNC_PORT = 2001; + public const int DEFAULT_ZONE_GOV_APPTERM_PORT = DEFAULT_ZONE_GOV_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + + public const int DEFAULT_HOST_GOV_WEB_PORT = 8082; + public const int DEFAULT_HOST_GOV_SVC_SYNC_PORT = 3000; + public const int DEFAULT_HOST_GOV_SVC_ASYNC_PORT = 3001; + public const int DEFAULT_HOST_GOV_APPTERM_PORT = DEFAULT_HOST_GOV_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + + public const int DEFAULT_GDID_AUTH_WEB_PORT = 8083; + public const int DEFAULT_GDID_AUTH_SVC_SYNC_PORT = 4000; + public const int DEFAULT_GDID_AUTH_SVC_ASYNC_PORT = 4001; + public const int DEFAULT_GDID_AUTH_APPTERM_PORT = DEFAULT_GDID_AUTH_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + + public const int DEFAULT_ASH_WEB_PORT = 8084; + public const int DEFAULT_ASH_APPTERM_PORT = 5000 + APP_TERMINAL_PORT_OFFSET; + public const int DEFAULT_ASH_SVC_BASE_SYNC_PORT = 6000; + public const int DEFAULT_ASH_SVC_BASE_ASYNC_PORT = 7000; + + public const int DEFAULT_AWS_WEB_PORT = 8085; + public const int DEFAULT_AWS_SVC_SYNC_PORT = 8000; + public const int DEFAULT_AWS_SVC_ASYNC_PORT = 8001; + public const int DEFAULT_AWS_APPTERM_PORT = DEFAULT_AWS_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + + public const int DEFAULT_PH_WEB_PORT = 8086; + public const int DEFAULT_PH_SVC_SYNC_PORT = 9000; + public const int DEFAULT_PH_SVC_ASYNC_PORT = 9001; + public const int DEFAULT_PH_APPTERM_PORT = DEFAULT_PH_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + + public const int DEFAULT_LOG_WEB_PORT = 8087; + public const int DEFAULT_LOG_SVC_SYNC_PORT = 10000; + public const int DEFAULT_LOG_SVC_ASYNC_PORT = 10001; + public const int DEFAULT_LOG_APPTERM_PORT = DEFAULT_LOG_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + + public const int DEFAULT_TELEMETRY_WEB_PORT = 8088; + public const int DEFAULT_TELEMETRY_SVC_SYNC_PORT = 11000; + public const int DEFAULT_TELEMETRY_SVC_ASYNC_PORT = 11001; + public const int DEFAULT_TELEMETRY_APPTERM_PORT = DEFAULT_TELEMETRY_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + + public const int DEFAULT_WEB_MESSAGE_SYSTEM_WEB_PORT = 8089; + public const int DEFAULT_WEB_MESSAGE_SYSTEM_SVC_SYNC_PORT = 12000; + public const int DEFAULT_WEB_MESSAGE_SYSTEM_SVC_ASYNC_PORT = 12001; + public const int DEFAULT_WEB_MESSAGE_SYSTEM_SVC_APPTERM_PORT = DEFAULT_WEB_MESSAGE_SYSTEM_SVC_SYNC_PORT + APP_TERMINAL_PORT_OFFSET; + //Network Service Name Resolution + + public const string NETWORK_UTESTING = "utesting"; //used for local loop unit testing + public const string NETWORK_INTERNOC = "internoc"; + public const string NETWORK_NOCGOV = "nocgov"; + public const string NETWORK_DBSHARD = "dbshard"; + + public const string NETWORK_SVC_GDID_AUTHORITY = "gdida"; + public const string NETWORK_SVC_WEB_MANAGER_PREFIX = "webman-"; + public const string NETWORK_SVC_ZGOV_WEB_MANAGER = NETWORK_SVC_WEB_MANAGER_PREFIX + APP_NAME_ZGOV; + public const string NETWORK_SVC_HGOV_WEB_MANAGER = NETWORK_SVC_WEB_MANAGER_PREFIX + APP_NAME_HGOV; + public const string NETWORK_SVC_ZGOVTELEMETRY = "zgovtelemetry"; + public const string NETWORK_SVC_ZGOVLOG = "zgovlog"; + public const string NETWORK_SVC_ZGOVHOSTREG = "zgovhostreg"; + public const string NETWORK_SVC_ZGOVHOSTREPL = "zgovhostrepl"; + public const string NETWORK_SVC_ZGOVLOCKER = "zgovlocker"; + public const string NETWORK_SVC_HGOV = "hgov"; + public const string NETWORK_SVC_HGOVPINGER = "hgovpinger"; + public const string NETWORK_SVC_PROCESSCONTROLLER = "processcontroller"; + public const string NETWORK_SVC_LOGRECEIVER = "logreceiver"; + public const string NETWORK_SVC_TELEMETRYRECEIVER = "telemetryreceiver"; + public const string NETWORK_SVC_TESTER = "tester"; + public const string NETWORK_SVC_WEB_MESSAGE_SYSTEM = "webmessagesystem"; + + public const string REGION_MNEMONIC_REGION = "reg"; + public const string REGION_MNEMONIC_NOC = "noc"; + public const string REGION_MNEMONIC_ZONE = "zone"; + public const string REGION_MNEMONIC_HOST = "host"; + + + public static readonly HashSet NAME_INVALID_CHARS = new HashSet + { + (char)0, + (char)0x0d, + (char)0x0a, + Metabase.Metabank.HOST_DYNAMIC_SUFFIX_SEPARATOR, + '@', '#', ',' , ';' , ':' , '%', '&', + '/' , '\\' , '\'' , '"' , '|' , + '*' , '?', + '<' , '>', + '[' , ']', + '{' , '}', + '(' , ')', + }; + + public const string CONFIG_ENABLED_ATTR = "enabled"; + } + + /// + /// Resolves environment variables using values declared in code in static class SysConstants. + /// The name must be prepended with "SysConsts." prefix, otherwise the OS resolver is used + /// + public sealed class SystemVarResolver : IEnvironmentVariableResolver + { + public const string PREFIX = "SysConsts."; + + private static SystemVarResolver s_Instance = new SystemVarResolver(); + + private SystemVarResolver() + { + + } + + public static void Bind() + { + Configuration.ProcesswideEnvironmentVarResolver = SystemVarResolver.s_Instance; + } + + /// + /// Returns a singleton class instance + /// + public static SystemVarResolver Instance { get { return s_Instance; } } + + + public bool ResolveEnvironmentVariable(string name, out string value) + { + value = null; + if (name.StartsWith(PREFIX)) + { + var sname = name.Substring(PREFIX.Length); + var tp = typeof(SysConsts); + var fld = tp.GetField(sname, BindingFlags.Public | BindingFlags.IgnoreCase | BindingFlags.Static); + if (fld != null) + { + var val = fld.GetValue(null); + if (val != null) + { + value = val.ToString(); + return true; + } + } + } + return false; + } + } +} diff --git a/src/Agni/TerminalUtils.cs b/src/Agni/TerminalUtils.cs new file mode 100644 index 0000000..45dc461 --- /dev/null +++ b/src/Agni/TerminalUtils.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; + +using NFX; +using NFX.IO; +using NFX.Glue; +using NFX.Environment; + +namespace Agni +{ + /// + /// Provides helper methods for working with/implementing IRemoteTerminal + /// + public static class TerminalUtils + { + /// + /// Parses command string into config node + /// + public static Configuration ParseCommand(string command, IEnvironmentVariableResolver resolver = null) + { + var cfg = LaconicConfiguration.CreateFromString(command); + cfg.EnvironmentVarResolver = resolver; + return cfg; + } + + public static void ShowRemoteException(RemoteException exception) + { + var remote = exception.Remote; + + ConsoleUtils.Error("Remote error " + exception.ToMessageWithType()); + ShowRemoteExceptionData(remote, 2); + } + + public static void ShowRemoteExceptionData(WrappedExceptionData data, int lvl) + { + var ind = "".PadLeft(lvl); + + Console.WriteLine(ind + "Application: " + data.ApplicationName); + Console.WriteLine(ind + "Code: {0} Source: {1}".Args(data.Code, data.Source)); + Console.WriteLine(ind + "Type: " + data.TypeName); + Console.WriteLine(ind + "Message: " + data.Message); + Console.WriteLine(ind + "Stack: " + data.StackTrace); + if (data.InnerException != null) + { + Console.WriteLine(ind + "Inner exception: "); + ShowRemoteExceptionData(data.InnerException, lvl + 2); + } + } + + + /// + /// Allows to execute an otherwise-blocking Console.ReadLine() call with the ability to abort the call gracefully. + /// This class can be constructed only once per process + /// + public class AbortableLineReader + { + private static object s_Lock = new object(); + private static volatile AbortableLineReader s_Instance; + + /// + /// This .ctor can be called only once per process + /// + public AbortableLineReader() + { + lock (s_Lock) + { + if (s_Instance != null) + throw new AgniException("{0}.ctor(already called)".Args(GetType().FullName)); + s_Instance = this; + + + m_Thread = new Thread(spin, 16 * 1024);//stack size enough to call Console.ReadLine() with all of its dependencies + m_Thread.IsBackground = true; + m_Thread.Name = GetType().FullName; + m_Thread.Start(); + + Console.CancelKeyPress += (_, e) => { Abort("CTRL+C\r\n"); e.Cancel = true;}; + } + } + + private bool m_Aborted; + private string m_Line; + private Thread m_Thread; + + public bool Aborted { get { return m_Aborted; } } + + /// + /// Returns non-null when stdin supplied newline string + /// + public string Line { get { return m_Line; } } + + public void Abort(string line = null) + { + if (m_Aborted) return; + m_Line = line; + m_Thread.Abort(); + m_Aborted = true; + } + + private void spin() + { + try + { + m_Line = Console.ReadLine(); + } + catch (ThreadAbortException) + { + + } + } + } + + } +} diff --git a/src/Agni/Tools/agm/Help.txt b/src/Agni/Tools/agm/Help.txt new file mode 100644 index 0000000..11a7f43 --- /dev/null +++ b/src/Agni/Tools/agm/Help.txt @@ -0,0 +1,28 @@ + + + Usage: + + agm auth_node scope sequence [block_size][/h | /? | /help] + [/s | /silent] + + auth_node - Authority glue node (i.e. "async://127.0.0.1:2300") + scope - Sequence scope name (namespace) + sequence - Name of sequence + block_size - Howe many values to reserve at once + + + Options: + + /h | /help | /? - displays help message + /s | /silent - suppresses logo and other info messages + /j | /json - output JSON results + /array - delimit data as array + + + Examples: + + + agm "async://192.168.1.123:2300" DEV patient 25 +Get next 25 'patient' sequence values in the scope 'DEV' + + \ No newline at end of file diff --git a/src/Agni/Tools/agm/ProgramBody.cs b/src/Agni/Tools/agm/ProgramBody.cs new file mode 100644 index 0000000..800759a --- /dev/null +++ b/src/Agni/Tools/agm/ProgramBody.cs @@ -0,0 +1,115 @@ +using System; + +using NFX; +using NFX.IO; +using NFX.Environment; +using NFX.ApplicationModel; +using NFX.Serialization.JSON; + +using Agni.Identification; + +namespace Agni.Tools.agm +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + run(args); + } + catch (Exception error) + { + ConsoleUtils.Error(error.ToMessageWithType()); + ConsoleUtils.Info("Exception details:"); + Console.WriteLine(error.ToString()); + + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + using (var app = new ServiceBaseApplication(args, null)) + { + var silent = app.CommandArgs["s", "silent"].Exists; + if (!silent) + { + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + + ConsoleUtils.Info("Build information:"); + Console.WriteLine(" NFX: " + BuildInformation.ForFramework); + Console.WriteLine(" Agni: " + new BuildInformation(typeof(Agni.AgniSystem).Assembly)); + Console.WriteLine(" Tool: " + new BuildInformation(typeof(agm.ProgramBody).Assembly)); + } + + if (app.CommandArgs["?", "h", "help"].Exists) + { + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Help.txt")); + return; + } + + var authority = app.CommandArgs.AttrByIndex(0).Value; + var connectToAuthority = authority.ToResolvedServiceNode(false).ConnectString; + var scope = app.CommandArgs.AttrByIndex(1).Value; + var seq = app.CommandArgs.AttrByIndex(2).Value; + var bsize = app.CommandArgs.AttrByIndex(3).ValueAsInt(1); + + if (!silent) + { + ConsoleUtils.Info("Authority: " + authority); + ConsoleUtils.Info("Connect to: " + connectToAuthority); + ConsoleUtils.Info("Scope: " + scope); + ConsoleUtils.Info("Sequence: " + seq); + ConsoleUtils.Info("Block Size: " + bsize); + } + + + + var w = System.Diagnostics.Stopwatch.StartNew(); + + var generator = new GDIDGenerator(); + generator.AuthorityHosts.Register(new GDIDGenerator.AuthorityHost(connectToAuthority)); + + + var json = app.CommandArgs["j", "json"].Exists; + var arr = app.CommandArgs["array"].Exists; + + + if (arr) Console.WriteLine("["); + + for (var i = 0; i < bsize; i++) + { + var gdid = generator.GenerateOneGDID(scope, seq, bsize - i, noLWM: true); +////System.Threading.Thread.Sleep(5); +//Console.ReadKey(); + string line; + + if (json) + line = new + { + Era = gdid.Era, + ID = gdid.ID, + Authority = gdid.Authority, + Counter = gdid.Counter + }.ToJSON(JSONWritingOptions.Compact); + else + line = "{0}: {1}".Args(i, gdid); + + + Console.Write(line); + Console.WriteLine(arr && i != bsize - 1 ? ", " : " "); + } + + if (arr) Console.WriteLine("]"); + + if (!silent) + { + Console.WriteLine(); + ConsoleUtils.Info("Run time: " + w.Elapsed.ToString()); + } + }//using APP + + } + } +} diff --git a/src/Agni/Tools/agm/Welcome.txt b/src/Agni/Tools/agm/Welcome.txt new file mode 100644 index 0000000..e69cdbd --- /dev/null +++ b/src/Agni/Tools/agm/Welcome.txt @@ -0,0 +1,6 @@ + +Agni GDID Manager +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/Tools/amm/Help.txt b/src/Agni/Tools/amm/Help.txt new file mode 100644 index 0000000..beb096d --- /dev/null +++ b/src/Agni/Tools/amm/Help.txt @@ -0,0 +1,44 @@ + + + Usage: + + amm metabase_path [/h | /? | /help] + [/s | /silent] + [/host | /from host] + [/config config_file] + [/gbm] + + metabase_path - Local metabase path (i.e. "C:\mbase") + + + Options: + + /h | /help | /? - displays help message + /s | /silent - suppresses logo and other info messages + /gbm - (re)/generate binary package manifests + /host | /from host - specifies the host that operation + is performed from (THIS host). If omitted then AGNI_HOST_NAME environment + variable is queried instead. + host is full host path i.e.: "all/us/east/cle/a/i/machine001" + /config - specifies alternate configuration. This option is + usefull when multiple tool instances need to run and they may conflict + while writing to the common logger location. + Inject an alternate config file to divert log output to a different place or + use a NOP loggger/destination. + An alternate config file "nolog.laconf" is provided for NOP-logger config + + + Examples: + + + amm "c:\ac\mbase\dev" +Checks the local metabase at the specified path for errors. Assumes that +AGNI_HOST_NAME local environment var holds local host path + + + amm "c:\ac\mbase\dev" /from "us/east/cle/A/II/wmed001" +Checks the local metabase at the specified path for errors as if calling from +the specified host + + + \ No newline at end of file diff --git a/src/Agni/Tools/amm/ProgramBody.cs b/src/Agni/Tools/amm/ProgramBody.cs new file mode 100644 index 0000000..9bfd5c9 --- /dev/null +++ b/src/Agni/Tools/amm/ProgramBody.cs @@ -0,0 +1,156 @@ +using System; +using System.IO; + +using NFX; +using NFX.IO; +using NFX.IO.FileSystem; +using NFX.IO.FileSystem.Local; +using NFX.IO.FileSystem.Packaging; +using NFX.Collections; +using NFX.Environment; +using NFX.ApplicationModel; + +using Agni.Metabase; +using Agni.AppModel; + + +namespace Agni.Tools.amm +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + run(args); + } + catch (Exception error) + { + ConsoleUtils.Error(error.ToMessageWithType()); + ConsoleUtils.Info("Exception details:"); + Console.WriteLine(error.ToString()); + + Environment.ExitCode = -1; + } + } + + + static void run(string[] args) + { + using (var app = new ServiceBaseApplication(args, null)) + { + var silent = app.CommandArgs["s", "silent"].Exists; + if (!silent) + { + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + + ConsoleUtils.Info("Build information:"); + Console.WriteLine(" NFX: " + BuildInformation.ForFramework); + Console.WriteLine(" Agni: " + new BuildInformation(typeof(Agni.AgniSystem).Assembly)); + Console.WriteLine(" Tool: " + new BuildInformation(typeof(amm.ProgramBody).Assembly)); + } + + if (app.CommandArgs["?", "h", "help"].Exists) + { + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Help.txt")); + return; + } + + + var mbPath = app.CommandArgs + .AttrByIndex(0) + .ValueAsString(System.Environment.GetEnvironmentVariable(BootConfLoader.ENV_VAR_METABASE_FS_ROOT)); + + if (!Directory.Exists(mbPath)) + throw new Exception("Specified metabase path not found"); + + var fromHost = app.CommandArgs["host", "from"].AttrByIndex(0).Value; + if (fromHost.IsNullOrWhiteSpace()) + fromHost = System.Environment.GetEnvironmentVariable(BootConfLoader.ENV_VAR_HOST_NAME); + + if (!silent) + { + ConsoleUtils.Info("Metabase path: " + mbPath); + ConsoleUtils.Info("Host (this machine): " + fromHost); + } + + var w = System.Diagnostics.Stopwatch.StartNew(); + + using (var fs = new LocalFileSystem("amm")) + using (var mb = new Metabank(fs, new FileSystemSessionConnectParams(), mbPath)) + { + using (BootConfLoader.LoadForTest(SystemApplicationType.Tool, mb, fromHost)) + { + if (app.CommandArgs["gbm"].Exists) + generateManifests(mb, silent); + else + validate(mb, silent); + } + } + + if (!silent) + { + Console.WriteLine(); + ConsoleUtils.Info("Run time: " + w.Elapsed.ToString()); + } + }//using APP + + } + + private static void validate(Metabank mb, bool silent) + { + var output = new EventedList(); + + var ni = 0; + var nw = 0; + var ne = 0; + + output.GetReadOnlyEvent = (_) => false; + output.ChangeEvent = + (_, ct, p, i, item) => + { + if (p != EventPhase.After) return; + switch (item.Type) + { + case MetabaseValidationMessageType.Info: ni++; ConsoleUtils.Info(item.ToString(false), ni); break; + case MetabaseValidationMessageType.Warning: nw++; ConsoleUtils.Warning(item.ToString(false), nw); break; + case MetabaseValidationMessageType.Error: ne++; ConsoleUtils.Error(item.ToString(false), ne); break; + } + Console.WriteLine(); + }; + mb.Validate(output); + + if (!silent) + Console.WriteLine("--------------------------------------------"); + + Console.WriteLine("Infos: {0} Warnings: {1} Errors: {2}".Args(ni, nw, ne)); + + } + + private static void generateManifests(Metabank mb, bool silent) + { + mb.fsAccess("amm.GenerateManifest()", Metabank.BIN_CATALOG, + (session, dir) => + { + foreach (var sdn in dir.SubDirectoryNames) + using (var sdir = dir.GetSubDirectory(sdn)) + { + var computed = ManifestUtils.GeneratePackagingManifest(sdir); + var computedContent = computed.Configuration.ToLaconicString(); + + if (!silent) + ConsoleUtils.WriteMarkupContent( + "Pckg: {0,-42} Mnfst.Sz: {1}\n".Args("'{0}'".Args(sdn), computedContent.Length)); + + using (var file = sdir.CreateFile(ManifestUtils.MANIFEST_FILE_NAME)) + { + file.WriteAllText(computedContent); + } + } + + return true; + } + ); + } + } +} diff --git a/src/Agni/Tools/amm/Welcome.txt b/src/Agni/Tools/amm/Welcome.txt new file mode 100644 index 0000000..c0dc76b --- /dev/null +++ b/src/Agni/Tools/amm/Welcome.txt @@ -0,0 +1,6 @@ + +Agni Metabase Manager +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/Tools/ascon/Help.txt b/src/Agni/Tools/ascon/Help.txt new file mode 100644 index 0000000..baf3346 --- /dev/null +++ b/src/Agni/Tools/ascon/Help.txt @@ -0,0 +1,53 @@ + + + Usage: + + ascon glue_node [/h | /? | /help] + [/s | /silent] + [/config config_file] + [/c | /cred id=user_id [pwd=user_password]] + [/f | /file command_file] + [/t | /txt command_text] + + node - Glue node specification in the form [biding://]host[:service] + + + Options: + + /h | /help | /? - displays help message + /s | /silent - suppresses logo and other info messages + /c | /cred - specified user credentials + /f | /file - reads and executes command text from command_file parameter + /t | /txt - reads and executes command from command_text parameter + /config - specifies alternate configuration. This option is + usefull when multiple tool instances need to run and they may conflict + while writing to the common logger location. + Inject an alternate config file to divert log output to a different place or + use a NOP loggger/destination. + An alternate config file "nolog.laconf" is provided for NOP-logger config + + Note: + If both "/txt" and "/file" switches are both specified at the same time then + command specified with the "/txt" switch is executed first + + + + Examples: + + + ascon "async://192.168.3.18:7700" /cred id="alex" +Connects to server by IP using "async" binding supplying "alex" for user name + + + ascon /c id="mary" pwd="suMMer45" /t who /s +Connects to local host governor (default Glue node) supplying "mary" for user +name along with plain-text password. Immediately executes "who" command. +Suppresses all logo and info messages + + + ascon /c id="jake" /config nolog.laconf +Connects to local host governor (default Glue node) supplying "jake" for user +name and uses "nolog" config so many instances of the tool will not conflict +while writing to log + + \ No newline at end of file diff --git a/src/Agni/Tools/ascon/ProgramBody.cs b/src/Agni/Tools/ascon/ProgramBody.cs new file mode 100644 index 0000000..c392b56 --- /dev/null +++ b/src/Agni/Tools/ascon/ProgramBody.cs @@ -0,0 +1,206 @@ +using System; + +using NFX; +using NFX.IO; +using NFX.Environment; +using NFX.ApplicationModel; +using NFX.Security; +using NFX.Glue; +using NFX.Glue.Protocol; + +using Agni.Clients; + +namespace Agni.Tools.ascon +{ + public static class ProgramBody + { + public static void Main(string[] args) + { + try + { + run(args); + } + catch (Exception error) + { + ConsoleUtils.Error(error.ToMessageWithType()); + ConsoleUtils.Info("Exception details:"); + Console.WriteLine(error.ToString()); + + if (error is RemoteException) + TerminalUtils.ShowRemoteException((RemoteException)error); + + Environment.ExitCode = -1; + } + } + + static void run(string[] args) + { + using (var app = new ServiceBaseApplication(args, null)) + { + var silent = app.CommandArgs["s", "silent"].Exists; + if (!silent) + { + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Welcome.txt")); + + ConsoleUtils.Info("Build information:"); + Console.WriteLine(" NFX: " + BuildInformation.ForFramework); + Console.WriteLine(" Agni: " + new BuildInformation(typeof(Agni.AgniSystem).Assembly)); + Console.WriteLine(" Tool: " + new BuildInformation(typeof(ascon.ProgramBody).Assembly)); + } + + if (app.CommandArgs["?", "h", "help"].Exists) + { + ConsoleUtils.WriteMarkupContent(typeof(ProgramBody).GetText("Help.txt")); + return; + } + + + var cred = app.CommandArgs["c", "cred"]; + var user = cred.AttrByName("id").Value; + var pwd = cred.AttrByName("pwd").Value; + + if (user.IsNullOrWhiteSpace()) + { + if (!silent) Console.Write("User ID: "); + user = Console.ReadLine(); + } + else + if (!silent) ConsoleUtils.Info("User ID: " + user); + + if (pwd.IsNullOrWhiteSpace()) + { + if (!silent) Console.Write("Password: "); + pwd = ConsoleUtils.ReadPassword('*'); + Console.WriteLine(); + } + else + if (!silent) ConsoleUtils.Info("Password: "); + + + var node = app.CommandArgs.AttrByIndex(0).ValueAsString("{0}://127.0.0.1:{1}".Args(SysConsts.APTERM_BINDING, + SysConsts.DEFAULT_HOST_GOV_APPTERM_PORT)); + + if (new Node(node).Binding.IsNullOrWhiteSpace()) + node = "{0}://{1}".Args(SysConsts.APTERM_BINDING, node); + + if (new Node(node).Service.IsNullOrWhiteSpace()) + node = "{0}:{1}".Args(node, SysConsts.DEFAULT_HOST_GOV_APPTERM_PORT); + + var file = app.CommandArgs["f", "file"].AttrByIndex(0).Value; + + if (file.IsNotNullOrWhiteSpace()) + { + if (!System.IO.File.Exists(file)) + throw new AgniException("File not found:" + file); + if (!silent) ConsoleUtils.Info("Reading from file: " + file); + file = System.IO.File.ReadAllText(file); + if (!silent) ConsoleUtils.Info("Command text: " + file); + } + + var txt = app.CommandArgs["t", "txt"].AttrByIndex(0).Value; + + if (txt.IsNotNullOrWhiteSpace()) + { + if (!silent) ConsoleUtils.Info("Verbatim command text: " + txt); + } + + var credentials = new IDPasswordCredentials(user, pwd); + + + using (var client = new RemoteTerminal(node.ToResolvedServiceNode(true))) + { + client.Headers.Add(new AuthenticationHeader(credentials)); + + var hinfo = client.Connect("{0}@{1}".Args(user, System.Environment.MachineName)); + if (!silent) + { + var c = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Magenta; + Console.WriteLine("Connected. Use ';' at line end to submit statement, 'exit;' to disconnect"); + Console.WriteLine("Type 'help;' for edification or ' /?;' for command-specific help"); + Console.ForegroundColor = ConsoleColor.Cyan; + Console.WriteLine(hinfo.WelcomeMsg); + Console.ForegroundColor = c; + } + + if (txt.IsNotNullOrWhiteSpace() || file.IsNotNullOrWhiteSpace()) + { + try + { + if (txt.IsNotNullOrWhiteSpace()) write(client.Execute(txt)); + if (file.IsNotNullOrWhiteSpace()) write(client.Execute(file)); + } + catch (RemoteException remoteError) + { + TerminalUtils.ShowRemoteException(remoteError); + Environment.ExitCode = -1; + } + } + else + { + while (true) + { + if (!silent) + { + var c = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.White; + Console.Write("{0}@{1}@{2}>".Args(hinfo.TerminalName, hinfo.AppName, hinfo.Host)); + Console.ForegroundColor = c; + } + var command = ""; + + while (true) + { + var ln = Console.ReadLine(); + command += ln; + if (ln.EndsWith(";")) break; + if (!silent) + { + var c = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.White; + Console.Write(">"); + Console.ForegroundColor = c; + } + } + + command = command.Remove(command.Length - 1, 1); + + if (command == "exit") break; + + string response = null; + + try + { + response = client.Execute(command); + } + catch (RemoteException remoteError) + { + TerminalUtils.ShowRemoteException(remoteError); + continue; + } + write(response); + } + } + + var disconnectMessage = client.Disconnect(); + if (!silent) + write(disconnectMessage); + + } + } + + }//run + + private static void write(string content) + { + if (content == null) return; + if (content.StartsWith(Agni.AppModel.Terminal.AppRemoteTerminal.MARKUP_PRAGMA)) + { + content = content.Remove(0, Agni.AppModel.Terminal.AppRemoteTerminal.MARKUP_PRAGMA.Length); + ConsoleUtils.WriteMarkupContent(content); + } + else + Console.WriteLine(content); + } + } +} diff --git a/src/Agni/Tools/ascon/Welcome.txt b/src/Agni/Tools/ascon/Welcome.txt new file mode 100644 index 0000000..dd1afd8 --- /dev/null +++ b/src/Agni/Tools/ascon/Welcome.txt @@ -0,0 +1,6 @@ + +Agni Server Console +Copyright (c) 2016-2018 Agnicore Inc. +Version 5.0 / Jan 21 2018 + + \ No newline at end of file diff --git a/src/Agni/WebManager/AWMException.cs b/src/Agni/WebManager/AWMException.cs new file mode 100644 index 0000000..3dbd543 --- /dev/null +++ b/src/Agni/WebManager/AWMException.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.WebManager +{ + /// + /// Base exception thrown by the WebManager site + /// + [Serializable] + public class AWMException : AgniException + { + public AWMException(int code, string message) : this(message, null, code, null, null) { } + public AWMException(string message) : this(message, null, 0, null, null) { } + public AWMException(string message, Exception inner) : this(message, inner, 0, null, null) { } + public AWMException(string message, Exception inner, int code, string sender, string topic) : base(message, inner, code, sender, topic) { } + protected AWMException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/WebManager/AWMSessionFilter.cs b/src/Agni/WebManager/AWMSessionFilter.cs new file mode 100644 index 0000000..6e7a608 --- /dev/null +++ b/src/Agni/WebManager/AWMSessionFilter.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Security.Cryptography; + + +using NFX; +using NFX.Environment; +using NFX.Web; +using NFX.Wave; +using NFX.Wave.Filters; +using NFX.Serialization.JSON; + + +namespace Agni.WebManager +{ + /// + /// Provides session management for AWM-specific sessions + /// + public sealed class AWMSessionFilter : SessionFilter + { + #region .ctor + public AWMSessionFilter(WorkDispatcher dispatcher, string name, int order) : base(dispatcher, name, order) {} + public AWMSessionFilter(WorkDispatcher dispatcher, IConfigSectionNode confNode): base(dispatcher, confNode) {ctor(confNode);} + public AWMSessionFilter(WorkHandler handler, string name, int order) : base(handler, name, order) {} + public AWMSessionFilter(WorkHandler handler, IConfigSectionNode confNode): base(handler, confNode) {ctor(confNode);} + + private void ctor(IConfigSectionNode confNode) + { + ConfigAttribute.Apply(this, confNode); + } + + #endregion + + + protected override WaveSession MakeNewSessionInstance(WorkContext work) + { + return new AWMWebSession(Guid.NewGuid()); + } + + } +} diff --git a/src/Agni/WebManager/AWMStockContentSiteHandler.cs b/src/Agni/WebManager/AWMStockContentSiteHandler.cs new file mode 100644 index 0000000..4ec5e6d --- /dev/null +++ b/src/Agni/WebManager/AWMStockContentSiteHandler.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Wave; +using NFX.Wave.Handlers; +using NFX.Environment; + +namespace Agni.WebManager +{ + /// + /// This handler serves the embedded content of Agni Web Manager site + /// + public class AWMStockContentSiteHandler : StockContentSiteHandler + { + public AWMStockContentSiteHandler(WorkDispatcher dispatcher, string name, int order, WorkMatch match) + : base(dispatcher, name, order, match){} + + + public AWMStockContentSiteHandler(WorkDispatcher dispatcher, IConfigSectionNode confNode) + : base(dispatcher, confNode) {} + + + public override string RootResourcePath + { + get { return "Agni.WebManager.Site"; } + } + } +} diff --git a/src/Agni/WebManager/AWMWebSession.cs b/src/Agni/WebManager/AWMWebSession.cs new file mode 100644 index 0000000..78539c8 --- /dev/null +++ b/src/Agni/WebManager/AWMWebSession.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.ApplicationModel; +using NFX.Security; +using NFX.Wave; + +namespace Agni.WebManager +{ + /// + /// Represents AWM-specific web session on WAVE server + /// + [Serializable] + public class AWMWebSession : WaveSession + { + protected AWMWebSession() : base(){} //used by serializer + public AWMWebSession(Guid id) : base(id) {} + + + /// + /// Returns language code for session - defaulted from geo-location + /// + public override string LanguageISOCode + { + get + { + string lang = null; + + if (GeoEntity!=null && GeoEntity.Location.HasValue) + { + var country = GeoEntity.CountryISOName; + if (country.IsNotNullOrWhiteSpace()) + lang = Localizer.CountryISOCodeToLanguageISOCode(country); + } + + if (lang.IsNullOrWhiteSpace()) + lang = Localizer.ISO_LANG_ENGLISH; + + return lang; + } + } + } +} diff --git a/src/Agni/WebManager/Controllers/AWMController.cs b/src/Agni/WebManager/Controllers/AWMController.cs new file mode 100644 index 0000000..8cf108c --- /dev/null +++ b/src/Agni/WebManager/Controllers/AWMController.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Wave; +using NFX.Wave.MVC; + +namespace Agni.WebManager.Controllers +{ + /// + /// Provides base for AWM controllers + /// + public abstract class AWMController : Controller + { + } +} diff --git a/src/Agni/WebManager/Controllers/Home.cs b/src/Agni/WebManager/Controllers/Home.cs new file mode 100644 index 0000000..8187196 --- /dev/null +++ b/src/Agni/WebManager/Controllers/Home.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Wave; +using NFX.Security; +using NFX.Wave.MVC; +using NFX.DataAccess.Distributed; + +namespace Agni.WebManager.Controllers +{ + + public sealed class Home : AWMController + { + + [Action] + public object Index() + { + return Localizer.MakePage(); + } + + [Action] + public object Console() + { + return Localizer.MakePage(); + } + + [Action] + public object Login(string id, string pwd) + { + WorkContext.NeedsSession(); + var session = WorkContext.Session; + + + if (session==null || id.IsNullOrWhiteSpace()) + return Localizer.MakePage(""); + + + if (session.User.Status==UserStatus.Invalid) + { + var cred = new IDPasswordCredentials(id, pwd); + var user = App.SecurityManager.Authenticate(cred); + if (user.Status==UserStatus.Invalid) + return Localizer.MakePage("Invalid login"); + + WorkContext.Session.User = user; + } + return new Redirect("/"); + } + + [Action] + public object Logout() + { + var session = WorkContext.Session; + if (session!=null) session.User = null; + + return Localizer.MakePage(""); + } + + [Action] + public object TheSystem() + { + return Localizer.MakePage(); + } + + [Action] + public object Instrumentation() + { + return Localizer.MakePage(); + } + + [Action] + public object ProcessManager(string zone) + { + return Localizer.MakePage(zone); + } + + [Action("instrumentation-charts", 20)] + public object InstrumentationCharts() + { + return Localizer.MakePage(); + } + + [Action("instrumentation-logs", 30)] + public object InstrumentationLogs() + { + return Localizer.MakePage(); + } + } +} diff --git a/src/Agni/WebManager/Controllers/Instrumentation.cs b/src/Agni/WebManager/Controllers/Instrumentation.cs new file mode 100644 index 0000000..44f9ed3 --- /dev/null +++ b/src/Agni/WebManager/Controllers/Instrumentation.cs @@ -0,0 +1,528 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +using NFX; +using NFX.ApplicationModel; +using NFX.Web; +using NFX.Wave.MVC; +using NFX.Serialization.JSON; +using NFX.Instrumentation; +using NFX.DataAccess.CRUD; +using NFX.Log; + +using Agni.Metabase; +using Agni.AppModel; +using Agni.AppModel.ZoneGovernor; + +namespace Agni.WebManager.Controllers +{ + public class Instrumentation: AWMController + { + #region Consts + + private const int MAX_COUNT = 100 * 1024; + private const int MIN_PROCESSING_INTERVAL_MS = 3000; + + #endregion + + #region Nested classes + public class DatumRequest : TypedRow + { + [Field] public NSDescr[] Namespaces { get; set; } + [Field] public DateTime ToUTC { get; set; } + public class NSDescr: TypedRow { [Field] public string NS { get; set; } [Field] public SRCDescr[] Sources { get; set; } } + public class SRCDescr : TypedRow { [Field] public string SRC{ get; set; } [Field] public DateTime FromUTC { get; set; } } + } + #endregion + + /// + /// Navigates/redirects request to governor process (hgov/zgov) at host with metabase path "metabasePath" + /// + [Action] + public object NavigateToHost(string metabasePath) + { + var host = AgniSystem.Metabase.CatalogReg[metabasePath] as Metabank.SectionHost; + if (host != null) + { + string hostURL; + try + { + hostURL = AgniSystem.Metabase.ResolveNetworkServiceToConnectString( + metabasePath, + SysConsts.NETWORK_INTERNOC, + host.IsZGov ? SysConsts.NETWORK_SVC_ZGOV_WEB_MANAGER : SysConsts.NETWORK_SVC_HGOV_WEB_MANAGER); + return new Redirect(hostURL); + } + catch (Exception ex) + { + log(MessageType.Error, ".NavigateToHost(metabasePath='{0}')".Args(metabasePath), ex.ToMessageWithType(), ex); + } + } + + WorkContext.Response.StatusCode = WebConsts.STATUS_404; + WorkContext.Response.StatusDescription = WebConsts.STATUS_404_DESCRIPTION; + return "Couldn't navigate to host/zone '{0}'".Args(metabasePath); + } + + [Action] + public object LoadLogMessages(DateTime? from = null, int? maxCnt = null, MessageType? fromType = null, bool forZone = false) + { + //var buf = new Message[] { + // new Message() { Type = MessageType.Debug, Text="test111111111111111111", TimeStamp=DateTime.Now}, + // new Message() { Type = MessageType.Trace, Text="test111111111111111111"}, + // new Message() { Type = MessageType.Info, Text="test111111111111111111"}, + // new Message() { Type = MessageType.Warning, Text="test111111111111111111"}, + // new Message() { Type = MessageType.Error, Text="test111111111111111111"}, + // new Message() { Type = MessageType.Emergency, Text="test111111111111111111"} + //}; + + var log = App.Log as LogServiceBase; + if (log == null) return NFX.Wave.SysConsts.JSON_RESULT_ERROR; + + from = (from == null) ? log.LocalizedTime.AddSeconds(-600) : log.UniversalTimeToLocalizedTime(from.Value.ToUniversalTime()); + + IEnumerable buf; + if (forZone && + AgniSystem.SystemApplicationType == SystemApplicationType.ZoneGovernor && + ZoneGovernorService.IsZoneGovernor) + buf = ZoneGovernorService.Instance.GetSubordinateInstrumentationLogBuffer(true); + else + buf = log.GetInstrumentationBuffer(true); + + if (fromType.HasValue) buf = buf.Where(m => m.Type >= fromType); + + buf = buf.Where(m => m.TimeStamp >= from); + + if (!maxCnt.HasValue || maxCnt > MAX_COUNT || maxCnt < 0) maxCnt = MAX_COUNT; + + buf = buf.Take(maxCnt.Value); + + var msgObjects = buf.Select(m => new + { + id = m.Guid.ToString(), + rid = (m.RelatedTo != Guid.Empty) ? m.RelatedTo.ToString() : null, + host = m.Host, + ts = m.TimeStamp, + tp = m.Type, + topic = m.Topic, + from = m.From, + src = m.Source, + txt = m.Text + }); + + var to = log.LocalizedTimeToUniversalTime(log.LocalizedTime); + + return new + { + OK = true, + fromType = fromType, + bufSize = log.InstrumentationBufferSize, + to = to, + buf = msgObjects.ToArray() + }; + } + + [Action] + public object SetComponentParameter(ulong sid, string key, string val) + { + var cmp = ApplicationComponent.GetAppComponentBySID(sid); + if (cmp == null) return new { OK = false, err = "Component (SID={0}) couldn't be found".Args(sid)}; + + var parameterized = cmp as IExternallyParameterized; + if (parameterized == null) return new { OK = false, err = "Component (SID={0}) doesn't implement IExternallyParameterized".Args(sid) }; + + if (!parameterized.ExternalSetParameter(key, val)) + return new + { + OK = false, + err = "Parameter \"{0}\" of component (SID={1}) couldn't be set to value \"{2}\"".Args(key, sid, val) + }; + + object setVal; + parameterized.ExternalGetParameter(key, out setVal); + + var valStr = setVal.AsString(); + if (valStr.IsNotNullOrWhiteSpace() && valStr.StartsWith(NFX.CoreConsts.EXT_PARAM_CONTENT_LACONIC)) + { + valStr = valStr.Substring(NFX.CoreConsts.EXT_PARAM_CONTENT_LACONIC.Length); + return new { OK = true, val = valStr.AsLaconicConfig().ToJSONDataMap()}; + } + else + { + return new { OK = true, val = setVal}; + } + } + + [Action] + public object LoadComponentParamGroups() + { + var groups = + ApplicationComponent.AllComponents + .OfType() + .SelectMany(c => (c.ExternalParameters ?? Enumerable.Empty>()) + .SelectMany(p => + { + var tcomp = c.GetType(); + var prop = tcomp.GetProperty(p.Key); + if (prop!=null) + return prop.GetCustomAttributes(); + else + return Enumerable.Empty(); + } + )) + .SelectMany(a => a.Groups ?? Enumerable.Empty()) + .Distinct() + .OrderBy(g => g) + .ToArray(); + + return new { OK = true, groups = groups }; + } + + [Action] + public object LoadComponentTree(string group = null) + { + var res = new JSONDataMap(); + + var all = ApplicationComponent.AllComponents; + + var rootArr = new JSONDataArray(); + + foreach(var cmp in all.Where(c => c.ComponentDirector==null)) + rootArr.Add(getComponentTreeMap(all, cmp, 0, group)); + + res["root"] = rootArr; + + var otherArr = new JSONDataArray(); + + foreach(var cmp in all.Where(c => c.ComponentDirector!=null && !(c is ApplicationComponent))) + rootArr.Add(getComponentTreeMap(all, cmp, 0)); + + res["other"] = otherArr; + + return new {OK=true, tree=res}; + } + + private JSONDataMap getComponentTreeMap(IEnumerable all, ApplicationComponent cmp, int level, string group = null) + { + var cmpTreeMap = new JSONDataMap(); + + if (level>7) return cmpTreeMap;//cyclical ref + + cmpTreeMap = getComponentMap(cmp, group); + + var children = new JSONDataArray(); + + foreach (var child in all.Where(c => object.ReferenceEquals(cmp, c.ComponentDirector))) + { + var childMap = getComponentTreeMap(all, child, level + 1, group); + children.Add(childMap); + } + + if (children.Count() > 0) cmpTreeMap["children"] = children; + + return cmpTreeMap; + } + + private JSONDataMap getComponentMap(ApplicationComponent cmp, string group = null) + { + var cmpMap = new JSONDataMap(); + + var parameterized = cmp as IExternallyParameterized; + var instrumentable = cmp as IInstrumentable; + + cmpMap["instrumentable"] = instrumentable != null; + cmpMap["instrumentationEnabled"] = instrumentable != null ? instrumentable.InstrumentationEnabled : false; + cmpMap["SID"] = cmp.ComponentSID; + cmpMap["startTime"] = Agni.AppModel.Terminal.Cmdlets.Appl.fdt( cmp.ComponentStartTime ); + cmpMap["tp"] = cmp.GetType().FullName; + if (cmp.ComponentCommonName.IsNotNullOrWhiteSpace()) cmpMap["commonName"] = cmp.ComponentCommonName; + if (cmp is INamed) cmpMap["name"] = ((INamed)cmp).Name; + if (cmp.ComponentDirector != null) cmpMap["director"] = cmp.ComponentDirector.GetType().FullName; + + if (parameterized == null) return cmpMap; + + var pars = group.IsNotNullOrWhiteSpace() ? parameterized.ExternalParametersForGroups(group) : parameterized.ExternalParameters; + if (pars == null) return cmpMap; + + pars = pars.Where(p => p.Key != "InstrumentationEnabled").OrderBy(p => p.Key); + if (pars.Count() == 0) return cmpMap; + + var parameters = new List(); + + foreach(var par in pars) + { + object val; + if (!parameterized.ExternalGetParameter(par.Key, out val)) continue; + + var parameterMap = new JSONDataMap(); + + string[] plist = null; + var tp = par.Value; + if (tp == typeof(bool)) plist = new string[] { "true", "false" }; + else if (tp.IsEnum) plist = Enum.GetNames(tp); + + parameterMap["key"] = par.Key; + parameterMap["plist"] = plist; + + var valStr = val.AsString(); + if (valStr.IsNotNullOrWhiteSpace() && valStr.StartsWith(NFX.CoreConsts.EXT_PARAM_CONTENT_LACONIC)) + { + valStr = valStr.Substring(NFX.CoreConsts.EXT_PARAM_CONTENT_LACONIC.Length); + parameterMap["val"] = valStr.AsLaconicConfig().ToJSONDataMap(); +// parameterMap["val"] = @" { +// 'detailed-instrumentation': true, +// tables: +// { +// master: { name: 'tfactory', 'fields-qty': 14}, +// slave: { name: 'tdoor', 'fields-qty': 20, important: true} +// }, +// tables1: +// { +// master: { name: 'tfactory', 'fields-qty': 14}, +// slave: { name: 'tdoor', 'fields-qty': 20, important: true} +// }, +// tables2: +// { +// master: { name: 'tfactory', 'fields-qty': 14}, +// slave: { name: 'tdoor', 'fields-qty': 20, important: true} +// } +// }".JSONToDataObject(); + parameterMap["ct"] = "obj"; // content type is object (string in JSON format) + } + else + { + parameterMap["val"] = val; + parameterMap["ct"] = "reg"; // content type is regular + } + + parameters.Add(parameterMap); + } + + cmpMap["params"] = parameters; + + return cmpMap; + } + + + [Action] + public object LoadComponents(string group = null) + { + var components = ApplicationComponent.AllComponents.OrderBy(c => c.ComponentStartTime); + + var componentsList = new List(); + + foreach (var cmp in components) + { + var componentMap = new JSONDataMap(); + componentsList.Add(componentMap); + + var parameterized = cmp as IExternallyParameterized; + var instrumentable = cmp as IInstrumentable; + + componentMap["instrumentable"] = instrumentable != null; + componentMap["instrumentationEnabled"] = instrumentable != null ? instrumentable.InstrumentationEnabled : false; + componentMap["SID"] = cmp.ComponentSID; + componentMap["startTime"] = Agni.AppModel.Terminal.Cmdlets.Appl.fdt( cmp.ComponentStartTime ); + componentMap["tp"] = cmp.GetType().FullName; + if (cmp.ComponentCommonName.IsNotNullOrWhiteSpace()) componentMap["commonName"] = cmp.ComponentCommonName; + if (cmp is INamed) componentMap["name"] = ((INamed)cmp).Name; + if (cmp.ComponentDirector != null) componentMap["director"] = cmp.ComponentDirector.GetType().FullName; + + if (parameterized == null) continue; + + var pars = group.IsNotNullOrWhiteSpace() ? parameterized.ExternalParametersForGroups(group) : parameterized.ExternalParameters; + if (pars == null) continue; + + pars = pars.Where(p => p.Key != "InstrumentationEnabled").OrderBy(p => p.Key); + if (pars.Count() == 0) continue; + + var parameters = new List(); + + foreach(var par in pars) + { + object val; + if (!parameterized.ExternalGetParameter(par.Key, out val)) continue; + + var parameterMap = new JSONDataMap(); + + string[] plist = null; + var tp = par.Value; + if (tp == typeof(bool)) plist = new string[] { "true", "false" }; + else if (tp.IsEnum) plist = Enum.GetNames(tp); + + parameterMap["key"] = par.Key; + parameterMap["plist"] = plist; + parameterMap["val"] = val; + + parameters.Add(parameterMap); + } + + componentMap["params"] = parameters; + } + + return new { OK = true, components = componentsList }; + } + + public enum DataGrouping { NS = 0, Namespace = 0, Namespaces = 0, Name = 0, Names = 0, + Intf = 1, Interface = 1, Interfaces = 1, Class=1, Classes=1 } + + [Action] + public object GetTree(bool forZone = false, DataGrouping grouping = DataGrouping.Namespace) + { + var instrumentation = App.Instrumentation; + + if (forZone && + AgniSystem.SystemApplicationType == SystemApplicationType.ZoneGovernor && + ZoneGovernorService.IsZoneGovernor) + instrumentation = ZoneGovernorService.Instance.SubordinateInstrumentation; + + var data = (grouping == DataGrouping.Namespace) ? getByNamespace(instrumentation) : getByInterface(instrumentation); + + var procInterval = instrumentation.ProcessingIntervalMS; + if (procInterval < MIN_PROCESSING_INTERVAL_MS) procInterval = MIN_PROCESSING_INTERVAL_MS; + + return new { + OK = true, + grouping = grouping, + enabled = instrumentation.Enabled, + recordCount = instrumentation.RecordCount, + maxRecordCount = instrumentation.MaxRecordCount, + tree = data, + processingIntervalMS = procInterval + }; + } + + [Action("GetData", 0, "match{methods=POST}")] + public object GetData(DatumRequest request, bool forZone = false) + { + var instrumentation = App.Instrumentation; + + if (forZone && + AgniSystem.SystemApplicationType == SystemApplicationType.ZoneGovernor && + ZoneGovernorService.IsZoneGovernor) + instrumentation = ZoneGovernorService.Instance.SubordinateInstrumentation; + + var nsMap = new JSONDataMap(); + var to = request.ToUTC.ToUniversalTime(); + + var total = 0; + foreach (var ns in request.Namespaces) + { + if (total>=MAX_COUNT) break; + var srcMap = new JSONDataMap(); + + foreach (var src in ns.Sources) + { + if (total>=MAX_COUNT) break; + + // TIME REVIEW!!! Log stores msg in local format, Instrumentation - in UTC + //Console.WriteLine("from original: {0} {1}".Args(src.FromUTC, src.FromUTC.Kind)); + //Console.WriteLine("from to utc: {0} {1}".Args(src.FromUTC.ToUniversalTime(), src.FromUTC.ToUniversalTime().Kind)); + + ////var from = App.UniversalTimeToLocalizedTime(src.FromUTC.ToUniversalTime()); + + //Console.WriteLine("from UniversalTimeToLocalizedTime: {0} {1}".Args(from, from.Kind)); + //Console.WriteLine(); + + var from = src.FromUTC.ToUniversalTime(); + + var buf = instrumentation.GetBufferedResultsSince(from) + .Where(d => d.GetType().FullName == ns.NS && d.Source == src.SRC) + .Take(MAX_COUNT - total); + var arr = buf.ToArray(); + srcMap[src.SRC] = arr; + total += arr.Length; + } + nsMap[ns.NS] = srcMap; + } + + var procInterval = instrumentation.ProcessingIntervalMS; + if (procInterval < MIN_PROCESSING_INTERVAL_MS) procInterval = MIN_PROCESSING_INTERVAL_MS; + return new + { + OK = true, + to = to, + data = nsMap, + recordCount = instrumentation.RecordCount, + maxRecordCount = instrumentation.MaxRecordCount, + total = total, + truncated = total > MAX_COUNT, + processingIntervalMS = procInterval + }; + } + + private JSONDataMap getByNamespace(IInstrumentation instr) + { + var data = new JSONDataMap(); + + IEnumerable typeKeys = instr.DataTypes.OrderBy(t => t.FullName); + foreach (var tkey in typeKeys) + { + Datum datum = null; + var sourceKeys = instr.GetDatumTypeSources(tkey, out datum).OrderBy(s => s); + if (datum==null) continue; + + var tData = new JSONDataMap(); + tData["data"] = sourceKeys; + + tData["descr"] = datum.Description; + tData["unit"] = datum.ValueUnitName; + tData["error"] = datum is IErrorInstrument; + tData["gauge"] = datum is Gauge; + + data.Add(tkey.FullName, tData); + } + + return data; + } + + private JSONDataMap getByInterface(IInstrumentation instr) + { + var data = new JSONDataMap(); + + var sortedTypes = instr.DataTypes.OrderBy(t => t.FullName); + + IEnumerable intfKeys = instr.DataTypes.SelectMany( t=> Datum.GetViewGroupInterfaces(t) ).Distinct().OrderBy(ti => ti.FullName); + foreach (var ikey in intfKeys) + { + + var iData = new JSONDataMap(); + + IEnumerable typeKeys = sortedTypes.Where(t => Datum.GetViewGroupInterfaces(t).Any(i => i == ikey)); + foreach (var tkey in typeKeys) + { + Datum datum = null; + var sourceKeys = instr.GetDatumTypeSources(tkey, out datum); + if (datum==null) continue; + + var tData = new JSONDataMap(); + tData["descr"] = datum.Description; + tData["unit"] = datum.ValueUnitName; + tData["error"] = datum is IErrorInstrument; + tData["gauge"] = datum is Gauge; + tData["data"] = sourceKeys; + + iData.Add(tkey.FullName, tData); + } + + data.Add(ikey.Name, iData); + } + + return data; + } + + private static void log(MessageType tp, string from, string text, Exception error = null) + { + App.Log.Write(new Message + { + Type = tp, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = "{0}.{1}".Args(typeof(TheSystem).FullName, from), + Text = text, + Exception = error + }); + } + } +} diff --git a/src/Agni/WebManager/Controllers/ProcessManager.cs b/src/Agni/WebManager/Controllers/ProcessManager.cs new file mode 100644 index 0000000..e5f84c5 --- /dev/null +++ b/src/Agni/WebManager/Controllers/ProcessManager.cs @@ -0,0 +1,96 @@ +using System; +using System.Linq; +using System.Text; +using Agni.AppModel.Terminal; +using Agni.Security.Permissions.Admin; +using Agni.Workers; +using NFX; +using NFX.Serialization.JSON; +using NFX.Wave.MVC; + +namespace Agni.WebManager.Controllers +{ + + public sealed class ProcessManager : AWMController + { + + [Action] + public object List(string zone, string signal, DateTime? startDate, DateTime? endDate, string description) + { + try + { + ProcessStatus? status = null; + if ("Created".EqualsOrdIgnoreCase(signal)) status = ProcessStatus.Created; + else if ("Started".EqualsOrdIgnoreCase(signal)) status = ProcessStatus.Started; + else if ("Finished".EqualsOrdIgnoreCase(signal)) status = ProcessStatus.Finished; + else if ("Canceled".EqualsOrdIgnoreCase(signal)) status = ProcessStatus.Canceled; + else if ("Terminated".EqualsOrdIgnoreCase(signal)) status = ProcessStatus.Terminated; + else if ("Failed".EqualsOrdIgnoreCase(signal)) status = ProcessStatus.Failed; + + var sd = startDate != null ? new DateTime(startDate.Value.Year, startDate.Value.Month, startDate.Value.Day, 0, 0, 0) : DateTime.MinValue; + var ed = endDate != null ? new DateTime(endDate.Value.Year, endDate.Value.Month, endDate.Value.Day, 23, 59, 59) : DateTime.MaxValue; + + var result = AgniSystem.ProcessManager + .List(zone) + .Where(descriptor => (signal.IsNullOrEmpty() || signal.EqualsOrdIgnoreCase("ALL") || descriptor.Status.Equals(status) ) && + ( sd <= descriptor.Timestamp && descriptor.Timestamp <= ed ) && + (description.IsNullOrWhiteSpace() || descriptor.Description.ToUpper().Contains(description.ToUpper()) || descriptor.StatusDescription.ToUpper().Contains(description.ToUpper())) + ) + .ToJSON(); + return new {Status = "OK", Result = result}; + } + catch (Exception ex) + { + return new {Status = "Error", Exception = ex.Message}; + } + } + + [Action("SendCancel", 0, "match { methods=POST accept-json=true}")] + public object SendCancel(JSONDataMap map) + { + var pid = new PID(map); + var cancel = Signal.MakeNew(pid); + var result = AgniSystem.ProcessManager.Dispatch(cancel); + return new {Status = result.GetType().Name}; + } + + [Action("SendTerminate", 0, "match { methods=POST accept-json=true}")] + public object SendTerminate(JSONDataMap map) + { + var pid = new PID(map); + var terminate = Signal.MakeNew(pid); + var result = AgniSystem.ProcessManager.Dispatch(terminate); + return new {Status = result.GetType().Name}; + } + + [Action("SendSignal", 0, "match { methods=POST accept-json=true}")] + public object SendSignal(JSONDataMap map) + { + try + { + var _pid = map["pid"] as JSONDataMap; + var _signal = map["signal"]; + var pid = new PID(_pid); + var conf = _signal.AsLaconicConfig(); + var result = AgniSystem.ProcessManager.Dispatch(pid, conf); + return new {Status = result.GetType().Name}; + } + catch (Exception ex) + { + return new {Status = "Error", Exception = ex.Message}; + } + } + + [Action("GetDetails", 0, "match { methods=POST accept-json=true}")] + public object GetDetails(JSONDataMap pidMap) + { + var _pid = pidMap["pid"] as JSONDataMap; + var pid = new PID(_pid); + var result = AgniSystem.ProcessManager.Get(pid); + return result; + } + + } + + +} \ No newline at end of file diff --git a/src/Agni/WebManager/Controllers/PublicAPI.cs b/src/Agni/WebManager/Controllers/PublicAPI.cs new file mode 100644 index 0000000..2779f0f --- /dev/null +++ b/src/Agni/WebManager/Controllers/PublicAPI.cs @@ -0,0 +1,202 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Drawing; + +using NFX; +using NFX.Instrumentation; +using NFX.Graphics; +using NFX.Wave.MVC; +using NFX.Serialization.JSON; +using NFX.Log; +using NFX.OS.Instrumentation; + +using Agni.AppModel; +using Agni.AppModel.ZoneGovernor; + +namespace Agni.WebManager.Controllers +{ + + public sealed class PublicAPI : AWMController + { + private const int MAX_COUNT = 1024; + + [Action] + public object HostPerformance(int duration = 3000, int sample = 1000, DateTime? lastErrorSample = null) + { + if (duration < 1000) duration = 1000; + if (duration > 10000) duration = 10000; + + if (sample < 250) sample = 250; + if (sample > 2000) sample = 2000; + + + var watch = System.Diagnostics.Stopwatch.StartNew(); + + var load = new List(); + + var first = true; + while (watch.ElapsedMilliseconds < duration) + { + var cpu = NFX.OS.Computer.CurrentProcessorUsagePct; + var ram = NFX.OS.Computer.GetMemoryStatus().LoadPct; + + System.Threading.Thread.Sleep(sample); + + + var datum = new JSONDataMap{ + {"at", App.TimeSource.UTCNow}, + {"cpu",cpu}, + {"ram",ram} }; + + if (first) + { + addError(datum, "warning", App.Log.LastWarning, lastErrorSample); + addError(datum, "error", App.Log.LastError, lastErrorSample); + addError(datum, "catastrophe", App.Log.LastCatastrophe, lastErrorSample); + } + + load.Add(datum); + first = false; + } + + + return new + { + ProcessAllocated = GC.GetTotalMemory(false), + Load = load + }; + } + + [Action] + public object PerformanceImg(int width = 64, int height = 64, int lookBackSec = 300) + { + const int MIN_LOOKBACK_SEC = 10, MAX_LOOKBACK_SEC = 600; + + if (lookBackSec < MIN_LOOKBACK_SEC) lookBackSec = MIN_LOOKBACK_SEC; + else if (lookBackSec > MAX_LOOKBACK_SEC) lookBackSec = MAX_LOOKBACK_SEC; + + var utcNow = App.TimeSource.UTCNow.AddSeconds(-lookBackSec); + IInstrumentation myInstrumentation = App.Instrumentation; + + IInstrumentation zoneInstrumentation = null; + if (AgniSystem.SystemApplicationType == SystemApplicationType.ZoneGovernor && ZoneGovernorService.IsZoneGovernor) + zoneInstrumentation = ZoneGovernorService.Instance.SubordinateInstrumentation; + + Func filter = (instr) => instr.GetBufferedResultsSince(utcNow) + .Where(d => d.GetType() == typeof(CPUUsage) || d.GetType() == typeof(RAMUsage)) + .Take(MAX_COUNT) + .ToArray(); + + Datum[] myData = filter(myInstrumentation); + + Datum[] zoneData = null; + if (zoneInstrumentation != null) zoneData = filter(zoneInstrumentation); + //Datum[] zoneData = myData; // TESTING!!!!! + + var img = renderPerformanceImg(width, height, myData, zoneData); + + return new Picture(img, JpegImageFormat.Standard); + } + + private Image renderPerformanceImg(int width, int height, Datum[] myData, Datum[] zoneData) + { + return Image.Of(width, height); + + #warning ETO nado peredelat na Canvas vmesto Graphics ispolzuja NFXv5.Graphics namespace + + ////////var cpuData = myData.Where(d => d.GetType() == typeof(CPUUsage)).Cast().ToArray(); + ////////var ramData = myData.Where(d => d.GetType() == typeof(RAMUsage)).Cast().ToArray(); + + ////////const int MIN_WIDTH = 16, MIN_HEIGHT = 16; + ////////const int MAX_WIDTH = 1024, MAX_HEIGHT = 1024; + + ////////if (width < MIN_WIDTH) width = MIN_WIDTH; + ////////else if (width > MAX_WIDTH) width = MAX_WIDTH; + + ////////if (height < MIN_HEIGHT) height = MIN_HEIGHT; + ////////else if (height > MAX_HEIGHT) height = MAX_HEIGHT; + + ////////var dataWidth = cpuData.Length > ramData.Length ? cpuData.Length : ramData.Length; + ////////if (dataWidth < 10) dataWidth = 10; + + ////////var imgResult = new Bitmap(width, height); + ////////using(var imgDraw = new Bitmap(zoneData == null ? dataWidth : 2 * dataWidth, height)) + ////////{ + //////// using (var gr = Graphics.FromImage(imgDraw)) + //////// { + //////// var backColor = Color.FromArgb(40, 40, 40); + //////// gr.Clear(backColor); + + //////// drawSet(gr, 0, dataWidth, height, cpuData, ramData); + //////// if (zoneData != null) + //////// { + //////// var zoneCPUData = zoneData.Where(d => d.GetType() == typeof(CPUUsage)).Cast().ToArray(); + //////// var zoneRAMData = zoneData.Where(d => d.GetType() == typeof(RAMUsage)).Cast().ToArray(); + + //////// drawSet(gr, dataWidth, dataWidth, height, zoneCPUData, zoneRAMData); + //////// } + //////// }//using Graphics + + //////// using (var gr = Graphics.FromImage(imgResult)) + //////// { + //////// //scale the picture + //////// gr.DrawImage(imgDraw, new Rectangle(0, 0, width, height)); + //////// } + //////// }//using imgDraw + ////////return imgResult; + } + + #warning ETO nado peredelat na Canvas vmesto Graphics ispolzuja NFXv5.Graphics namespace + ////////private void drawSet(Graphics gr, float startX, int dataWidth, int height, CPUUsage[] cpuData, RAMUsage[] ramData) + ////////{ + //////// //======= CPU + //////// using (var br = new System.Drawing.Drawing2D.LinearGradientBrush(new Rectangle(0, 0, dataWidth, height), + //////// Color.FromArgb(0xFF, 0x40, 0x00), + //////// Color.FromArgb(0x20, 0xFF, 0x00), + //////// 90f)) + //////// { + //////// using (var lp = new Pen(br, 1f)) + //////// { + //////// float x = startX; + //////// for (int i = 0; i < cpuData.Length - 1; i++) + //////// { + //////// var datum = cpuData[i]; + //////// gr.DrawLine(lp, x, height, x, (int)(height - (height * (datum.Value / 100f)))); + + //////// x += 1f; + //////// } + //////// } + //////// } + + //////// //========= RAM + //////// using (var br = new System.Drawing.Drawing2D.LinearGradientBrush(new Rectangle(0, 0, dataWidth, height), + //////// Color.FromArgb(0xA0, 0xff, 0xc0, 0xff), + //////// Color.FromArgb(0x50, 0xc0, 0xc0, 0xff), + //////// 90f)) + //////// { + //////// using (var lp = new Pen(br, 1f)) + //////// { + //////// float x = startX; + //////// for (int i = 0; i < ramData.Length - 1; i++) + //////// { + //////// var datum = ramData[i]; + //////// gr.DrawLine(lp, x, height, x, (int)(height - (height * (datum.Value / 100f)))); + + //////// x += 1f; + //////// } + //////// } + //////// } + ////////} + + private void addError(JSONDataMap datum, string type, Message msg, DateTime? lastErrorSample) + { + if (msg==null) return; + if (!lastErrorSample.HasValue || (msg.TimeStamp-lastErrorSample.Value).TotalSeconds>1) + { + datum[type] = msg.TimeStamp.ToString("dd HH:mm:ss"); + datum["LastErrorSample"] = msg.TimeStamp; + } + } + } +} diff --git a/src/Agni/WebManager/Controllers/RemoteTerminal.cs b/src/Agni/WebManager/Controllers/RemoteTerminal.cs new file mode 100644 index 0000000..383a2a9 --- /dev/null +++ b/src/Agni/WebManager/Controllers/RemoteTerminal.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Wave; +using NFX.Wave.MVC; +using NFX.DataAccess.Distributed; + +using Agni.AppModel.Terminal; +using Agni.Security.Permissions.Admin; + +namespace Agni.WebManager.Controllers +{ + /// + /// Provides AppRemoteTerminal JSON API + /// + public sealed class RemoteTerminal : AWMController + { + public const string TERMINAL_SESSION_KEY = "app remote terminal instance"; + + + [Action] + [RemoteTerminalOperatorPermission] + public object Connect(string who = null) + { + WorkContext.NeedsSession(); + var terminal = WorkContext.Session[TERMINAL_SESSION_KEY] as AppRemoteTerminal; + if (terminal!=null) + return new {Status = "Already connected", WhenConnected = terminal.WhenConnected}; + + + if (who.IsNullOrWhiteSpace()) who = "{0}-{1}".Args(WorkContext.Request.UserHostAddress, WorkContext.Session.User); + terminal = AppRemoteTerminal.MakeNewTerminal(); + var info = terminal.Connect(who); + WorkContext.Session[TERMINAL_SESSION_KEY] = terminal; + return info; + } + + [Action] + public object Disconnect() + { + WorkContext.NeedsSession(); + var terminal = WorkContext.Session[TERMINAL_SESSION_KEY] as AppRemoteTerminal; + if (terminal==null) + return new {Status = "Already disconnected"}; + + var msg = terminal.Disconnect(); + terminal.Dispose(); + WorkContext.Session[TERMINAL_SESSION_KEY] = null; + + return new {Status = msg}; + } + + [Action] + [RemoteTerminalOperatorPermission] + [AppRemoteTerminalPermission] + public object Execute(string command) + { + WorkContext.NeedsSession(); + var terminal = WorkContext.Session[TERMINAL_SESSION_KEY] as AppRemoteTerminal; + if (terminal==null) + return new {Status = "Error", Msg = "Not connected"}; + + try + { + var result = terminal.Execute(command); + + var plainText = true; + if (result.IsNotNullOrWhiteSpace()) + if (result.StartsWith(Agni.AppModel.Terminal.AppRemoteTerminal.MARKUP_PRAGMA)) + { + result = result.Remove(0, Agni.AppModel.Terminal.AppRemoteTerminal.MARKUP_PRAGMA.Length); + result = NFX.IO.ConsoleUtils.WriteMarkupContentAsHTML(result); + plainText = false; + } + + return new {Status = "OK", PlainText = plainText, Result = result}; + } + catch(Exception error) + { + return new {Status = "Error", Msg = error.Message, Exception = error.ToMessageWithType()}; + } + } + + } +} diff --git a/src/Agni/WebManager/Controllers/TheSystem.cs b/src/Agni/WebManager/Controllers/TheSystem.cs new file mode 100644 index 0000000..248491c --- /dev/null +++ b/src/Agni/WebManager/Controllers/TheSystem.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using NFX; +using NFX.Log; +using NFX.Wave.MVC; +using NFX.Serialization.JSON; +using NFX.Web; + +using Agni.Metabase; + +namespace Agni.WebManager.Controllers +{ + /// + /// Provides Metabase access + /// + public sealed class TheSystem: AWMController + { + /// + /// Navigates/redirects request to process "appName" at host with metabase path "metabasePath" + /// + [Action] + public object Navigate(string metabasePath, string appName) + { + var svc = SysConsts.NETWORK_SVC_WEB_MANAGER_PREFIX + appName; + + try + { + var url = AgniSystem.Metabase.ResolveNetworkServiceToConnectString(metabasePath, + SysConsts.NETWORK_INTERNOC, + svc); + + return new Redirect(url); + } + catch + { + WorkContext.Response.StatusCode = WebConsts.STATUS_404; + WorkContext.Response.StatusDescription = WebConsts.STATUS_404_DESCRIPTION; + return "Web Manager service in application '{0}' on host '{1}' netsvc '{2}' is not available".Args(appName, metabasePath, svc); + } + } + + [Action] + public object LoadLevel(string path, bool hosts = false) + { + return LoadLevelImpl(path, hosts); + } + + internal static object LoadLevelImpl(string path, bool hosts = false) + { + IEnumerable children = null; + + Metabank.SectionRegionBase section = null; + if (path.IsNotNullOrWhiteSpace() && path != "/" && path != "\\") + { + section = AgniSystem.Metabase.CatalogReg[path]; + if (section == null) return NFX.Wave.SysConsts.JSON_RESULT_ERROR; + + if (section is Metabank.SectionRegion) + { + var region = (Metabank.SectionRegion)section; + children = region.SubRegionNames.OrderBy(r => r).Select(r => (Metabank.SectionRegionBase)region.GetSubRegion(r)) + .Concat(region.NOCNames.OrderBy(n => n).Select(n => (Metabank.SectionRegionBase)region.GetNOC(n))); + } + else if (section is Metabank.SectionNOC) + { + var noc = (Metabank.SectionNOC)section; + children = noc.ZoneNames.OrderBy(z => z).Select(z => noc.GetZone(z)); + } + else if (section is Metabank.SectionZone) + { + var zone = (Metabank.SectionZone)section; + children = zone.SubZoneNames.OrderBy(z => z).Select(z => (Metabank.SectionRegionBase)zone.GetSubZone(z)); + if (hosts) + children = children.Concat(zone.HostNames.OrderBy(h => h).Select(h => (Metabank.SectionRegionBase)zone.GetHost(h))); + } + else + return NFX.Wave.SysConsts.JSON_RESULT_ERROR; + } + else + { + children = AgniSystem.Metabase.CatalogReg.Regions; + } + + return new + { + OK=true, + path=path, + myPath=AgniSystem.HostMetabaseSection.RegionPath, + myPathSegs=AgniSystem.HostMetabaseSection.SectionsOnPath.Select(s => s.Name).ToArray(), + children=makeChildren(children) + }; + } + + internal static object makeChildren(IEnumerable children) + { + var res = new List(); + + foreach (var child in children) + { + var d = new JSONDataMap(); + + d["name"] = child.Name; + d["path"] = child.RegionPath; + d["me"] = child.IsLogicallyTheSame(AgniSystem.HostMetabaseSection); + d["tp"] = child.SectionMnemonicType; + d["descr"] = child.Description; + + var host = child as Metabank.SectionHost; + if (host != null) + { + var isZGov = host.IsZGov; + d["role"] = host.RoleName; + d["dynamic"] = host.Dynamic; + d["os"] = host.OS; + d["apps"] = host.Role.AppNames.OrderBy(a => a).ToArray(); + d["isZGov"] = isZGov; + d["myZGov"] = child.IsLogicallyTheSame(host.ParentZoneGovernorPrimaryHost()); + + string adminURL = null; + try + { + adminURL = AgniSystem.Metabase.ResolveNetworkServiceToConnectString(host.RegionPath, + SysConsts.NETWORK_INTERNOC, + isZGov ? SysConsts.NETWORK_SVC_ZGOV_WEB_MANAGER : SysConsts.NETWORK_SVC_HGOV_WEB_MANAGER); + } + catch(Exception ex) + { + log(MessageType.Error, "LoadLevel.makeLevel()", ex.ToMessageWithType(), ex); + } + + d["adminURL"] = adminURL; + } + else + { + d["geo"] = new { lat = child.EffectiveGeoCenter.Lat, lng = child.EffectiveGeoCenter.Lng }; + } + + res.Add(d); + } + + return res; + } + + private static void log(MessageType tp, string from, string text, Exception error = null) + { + App.Log.Write(new Message + { + Type = tp, + Topic = SysConsts.LOG_TOPIC_APP_MANAGEMENT, + From = "{0}.{1}".Args(typeof(TheSystem).FullName, from), + Text = text, + Exception = error + }); + } + + } +} diff --git a/src/Agni/WebManager/Controls/AWMPage.cs b/src/Agni/WebManager/Controls/AWMPage.cs new file mode 100644 index 0000000..ac7a033 --- /dev/null +++ b/src/Agni/WebManager/Controls/AWMPage.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.DataAccess.CRUD; +using NFX.Wave.Client; +using NFX.Serialization.JSON; + + +namespace Agni.WebManager.Controls +{ + /// + /// All pages in Apex derive from this one + /// + public abstract class AWMPage : AWMTemplate + { + public const string DEFAULT_DESCRIPTION = "Manage Angi"; + + public const string DEFAULT_KEYWORDS = + "Angi, Management"; + + public const string DEFAULT_VIEWPORT = "width=device-width, initial-scale=1.0, user-scalable=no"; + + + protected static readonly string FAVICON_PNG = SURI.Image("favicon.ita.196x196.png"); + protected static readonly string FAVICON_ICO = SURI.Image("favicon.ico"); + protected static readonly string BASE_CSS = SURI.Style("base.css"); + protected static readonly string MASTER_CSS = SURI.Style("master.css"); + protected static readonly string JQUERY_JS = SURI.Stock("script/jquery-2.1.4.min.js"); + protected static readonly string WV_JS = SURI.Stock("script/wv.js"); + protected static readonly string WV_GUI_JS = SURI.Stock("script/wv.gui.js"); + protected static readonly string WV_CHART_JS = SURI.Stock("script/wv.chart.svg.js"); + protected static readonly string AWM_JS = SURI.Script("awm.js"); + protected static readonly string MASTER_JS = SURI.Script("master.js"); + + + private string m_Title; + private string m_Description; + private string m_Keywords; + private string m_Viewport; + + public virtual string Title + { + get {return m_Title.IsNullOrWhiteSpace() ? (AgniSystem.MetabaseApplicationName + "@" + AgniSystem.HostName) + : m_Title;} + set {m_Title = value;} + } + + public virtual string Description + { + get {return m_Description.IsNullOrWhiteSpace() ? DEFAULT_DESCRIPTION : m_Description;} + set {m_Description = value;} + } + + public virtual string Keywords + { + get {return m_Keywords.IsNullOrWhiteSpace() ? DEFAULT_KEYWORDS : m_Keywords;} + set {m_Keywords = value;} + } + + public virtual string Viewport + { + get {return m_Viewport.IsNullOrWhiteSpace() ? DEFAULT_VIEWPORT : m_Viewport;} + set {m_Viewport = value;} + } + + /// + /// Outputs menu HREF for menu A with optional "selectedPage" class + /// + public void MenuHREF(string uri) + { + if (this is TPage) + Context.Response.Write("href='#' class='selectedPage'"); + else + { + Context.Response.Write("href='"); + Context.Response.Write(uri); + Context.Response.Write("'"); + } + } + + public string FormJSON(FormModel form, Exception validationError = null, string recID = null, string target = null) + { + var lang = Localizer.GetLanguage(); + return RecordModelGenerator.DefaultInstance.RowToRecordInitJSON(form, validationError, recID, target, lang).ToJSON(); + } + + } +} diff --git a/src/Agni/WebManager/Controls/AWMTemplate.cs b/src/Agni/WebManager/Controls/AWMTemplate.cs new file mode 100644 index 0000000..f530671 --- /dev/null +++ b/src/Agni/WebManager/Controls/AWMTemplate.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Wave; +using NFX.Wave.Templatization; + +namespace Agni.WebManager +{ + /// + /// All templates in Agni Web Manager derive from this one + /// + public abstract class AWMTemplate : WaveTemplate + { + + } + +} diff --git a/src/Agni/WebManager/Controls/PageClassification.cs b/src/Agni/WebManager/Controls/PageClassification.cs new file mode 100644 index 0000000..e7e77e2 --- /dev/null +++ b/src/Agni/WebManager/Controls/PageClassification.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Agni.WebManager.Controls +{ + interface ISitePage{} + + interface IMainPage : ISitePage{} + interface IHomePage : IMainPage{} + interface IConsolePage : IMainPage{} + interface IInstrumentationPage : IMainPage{} + interface ITheSystemPage : IMainPage{} + interface IProcessManagerPage : IMainPage{} +} diff --git a/src/Agni/WebManager/Localizer.cs b/src/Agni/WebManager/Localizer.cs new file mode 100644 index 0000000..6d03c1a --- /dev/null +++ b/src/Agni/WebManager/Localizer.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Log; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Wave; +using NFX.Serialization.JSON; +using Agni.WebManager.Controls; +using Agni.WebManager.Pages; + +namespace Agni.WebManager +{ + /// + /// Facilitates tasks working with objects of appropriate culture per user + /// + public static class Localizer + { + public const string CONFIG_LOCALIZATION_SECTION = "localization"; + public const string CONFIG_MSG_FILE_ATTR = "msg-file"; + public const string LOC_ANY_SCHEMA_KEY = "--ANY-SCHEMA--"; + public const string LOC_ANY_FIELD_KEY = "--ANY-FIELD--"; + + public const string ISO_LANG_ENGLISH = "eng"; + public const string ISO_LANG_RUSSIAN = "rus"; + public const string ISO_LANG_GERMAN = "deu"; + public const string ISO_LANG_FRENCH = "fre"; + + + public enum MoneyFormat{WithCurrencySymbol, WithoutCurrencySymbol} + public enum DateTimeFormat{ShortDate, LongDate, ShortDateTime, LongDateTime} + + static Localizer() + { + NFX.Wave.Client.RecordModelGenerator.DefaultInstance.ModelLocalization += recGeneratorLocalization; + } + + + private static IConfigSectionNode s_LocalizationData; + + private static void ensureData() + { + if (s_LocalizationData!=null) return; + + var loc = App.ConfigRoot[CONFIG_LOCALIZATION_SECTION]; + s_LocalizationData = loc; + var msgFile = loc.AttrByName(CONFIG_MSG_FILE_ATTR).Value; + if (msgFile.IsNotNullOrWhiteSpace()) + try + { + App.Log.Write( new Message{ + Type = MessageType.Info, + From = "enusreData()", + Topic = SysConsts.LOG_TOPIC_LOCALIZATION, + Text = "Configured in '/{0}/${1}' to load localization msg file '{2}'".Args(CONFIG_LOCALIZATION_SECTION, CONFIG_MSG_FILE_ATTR, msgFile), + }); + s_LocalizationData = Configuration.ProviderLoadFromFile(msgFile).Root; + } + catch(Exception error) + { + App.Log.Write( new Message{ + Type = MessageType.CatastrophicError, + From = "enusreData()", + Topic = SysConsts.LOG_TOPIC_LOCALIZATION, + Text = "Error loading localization msg file '{0}': {1}".Args(msgFile, error.ToMessageWithType()), + Exception = error + }); + } + } + + private static string recGeneratorLocalization(NFX.Wave.Client.RecordModelGenerator sender, string schema, string field, string value, string isoLang) + { + if (value.IsNullOrWhiteSpace()) return value; + + ensureData(); + if (!s_LocalizationData.Exists) return value;//nowhere to lookup + + var session = ExecutionContext.Session as AWMWebSession; + if (session==null) return value; + + isoLang = session.LanguageISOCode; + if (isoLang==ISO_LANG_ENGLISH) return value; + + + if (schema.IsNullOrWhiteSpace()) schema = LOC_ANY_SCHEMA_KEY; + if (field.IsNullOrWhiteSpace()) field = LOC_ANY_FIELD_KEY; + bool exists; + var lv = lookupValue(isoLang, schema, field, value, out exists); + + #if DEVELOPMENT + if (!exists) + { + App.Log.Write( new Message{ + Type = MessageType.InfoZ, + From = "lookup", + Topic = SysConsts.LOCALIZATION_TOPIC, + Text = "Need localization", + Parameters = (new {iso = isoLang, sch = schema, fld = field, val = value }).ToJSON() + }); + } + #endif + + return lv; + } + + private static string lookupValue(string isoLang, string schema, string field, string value, out bool exists) + { + exists = false; + + var nlang = s_LocalizationData[isoLang]; + if (!nlang.Exists) return value; + var nschema = nlang[schema, LOC_ANY_SCHEMA_KEY]; + if (!nschema.Exists) return value; + var nfield = nschema[field, LOC_ANY_FIELD_KEY]; + if (!nfield.Exists) return value; + + var nvalue = nfield.Attributes.FirstOrDefault(a=>a.Name == value);//case SENSITIVE search + if (nvalue==null) return value; + var lv = nvalue.Value; + + if (lv.IsNotNullOrWhiteSpace()) + { + exists = true; + return lv; + } + + return value; + } + + + + + public static string Money(decimal amount, MoneyFormat format = MoneyFormat.WithCurrencySymbol, AWMWebSession session = null) + { + return amount.ToString(); //todo Implement + } + + public static string DateTime(DateTime dt, DateTimeFormat format = DateTimeFormat.LongDateTime, AWMWebSession session = null) + { + return dt.ToString();//todo implement + } + + /// + /// Converts country code into language code + /// + public static string CountryISOCodeToLanguageISOCode(string countryISOCode) + { + if (countryISOCode.IsNullOrWhiteSpace()) return ISO_LANG_ENGLISH; + countryISOCode = countryISOCode.ToLowerInvariant(); + switch(countryISOCode) + { + case "ru": + case "ua": + case "by": return ISO_LANG_RUSSIAN; + + case "de": return ISO_LANG_GERMAN; + case "fr": return ISO_LANG_FRENCH; + + default: return ISO_LANG_ENGLISH; + } + + } + + private static Dictionary s_PageTypes = new Dictionary(); + + /// + /// Makes localized page instance per session + /// + public static AWMPage MakePage(params object[] ctorArgs) where TPage : AWMPage + { + return MakePage(typeof(TPage), WorkContext.Current, ctorArgs); + } + + /// + /// Makes localized page instance per session + /// + public static AWMPage MakePage(Type type, WorkContext work, object[] ctorArgs) where TPage : AWMPage + { + string tname = string.Empty; + try + { + Type localizedType = null; + + var lang = GetLanguage(work); + + if (lang!=ISO_LANG_ENGLISH) + { + var key = type.FullName+"_"+lang; + + if (!s_PageTypes.TryGetValue(key, out localizedType)) + { + tname = "{0}_{1}, Apex.Web".Args(type.FullName, lang); + localizedType = Type.GetType(tname, false); + var dict = new Dictionary(s_PageTypes); + dict[key] = localizedType; + s_PageTypes = dict;//atomic + } + } + + if (localizedType==null) localizedType = type; + + tname = type.FullName; + return (AWMPage)Activator.CreateInstance(localizedType, ctorArgs); + } + catch(Exception error) + { + throw new AWMException("Error making localized page '{0}'. Error: {1}".Args(tname, error.ToMessageWithType()), error); + } + } + + /// + /// Tries to determine work context lang and returns it or ENG + /// + public static string GetLanguage(WorkContext work = null) + { + if (work==null) work = WorkContext.Current; + if (work==null) return ISO_LANG_ENGLISH; + + string lang = null; + + var session = work.Session as AWMWebSession; + if (session!=null) + lang = session.LanguageISOCode; + else + if (work.GeoEntity!=null) + { + var country = work.GeoEntity.CountryISOName; + if (country.IsNotNullOrWhiteSpace()) + lang = CountryISOCodeToLanguageISOCode(country); + } + + if (lang.IsNullOrWhiteSpace()) lang = ISO_LANG_ENGLISH; + + return lang; + } + + + private static Dictionary s_Content = new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + {"mnuHome_eng", "Process"}, {"mnuHome_rus", "Процесс"}, + {"mnuConsole_eng", "Console"}, {"mnuSellers_rus", "Консоль"}, + {"mnuInstrumentation_eng", "Instrumentation"}, {"mnuConsumers_rus", "Инструментарий"}, + {"mnuTheSystem_eng", "The System"}, {"mnuDevelopers_rus", "Система"}, + {"mnuProcessManager_eng", "Process manager"}, {"mnuProcessManager_rus", "Управление процессами"} + }; + + + /// + /// Gets content by name + /// + public static string Content(string key, WorkContext work = null) + { + var lang = GetLanguage(); + string result; + if (s_Content.TryGetValue(key+"_"+lang, out result)) return result; + return string.Empty; + } + + + + } +} diff --git a/src/Agni/WebManager/MenuBuilder.cs b/src/Agni/WebManager/MenuBuilder.cs new file mode 100644 index 0000000..5e82974 --- /dev/null +++ b/src/Agni/WebManager/MenuBuilder.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX; +using NFX.Environment; +using Agni.AppModel; + +namespace Agni.WebManager +{ + + /// + /// Builds HTML markup for menu configured under WEB-MANAGER config + /// + internal class MenuBuilder + { + + public const string CONFIG_MENU_SECTION = "menu"; + public const string CONFIG_ITEM_SECTION = "item"; + public const string CONFIG_TEXT_ATTR = "text"; + public const string CONFIG_HREF_ATTR = "href"; + + /// + /// If there is menu configured, returns it, or empty string. + /// Path isoLang iso code for localized descriptions, or null for english + /// Menu structure + /// item + /// { + /// text="american english text" text_fr="french text" text_deu="german text" href="/item path" + /// item{...} + /// } + /// + public static string BuildMenu(string isoLang) + { + var menu = App.ConfigRoot[AgniServiceApplication.CONFIG_WEB_MANAGER_SECTION][CONFIG_MENU_SECTION]; + if (!menu.Exists) return string.Empty; + var result = new StringBuilder(); + + result.AppendLine("
    "); + buildMenuLevel(isoLang, result, menu); + result.AppendLine("
"); + + return result.ToString(); + } + + + private static void buildMenuLevel(string isoLang, StringBuilder sb, IConfigSectionNode level) + { + sb.AppendLine("
  • "); + + string text = null; + + if (isoLang.IsNotNullOrWhiteSpace()) + text = level.AttrByName(CONFIG_TEXT_ATTR+"_"+isoLang).ValueAsString(); + + if (text.IsNullOrWhiteSpace()) + text = level.AttrByName(CONFIG_TEXT_ATTR).ValueAsString(); + + if (text.IsNotNullOrWhiteSpace()) + { + var href = level.AttrByName(CONFIG_HREF_ATTR).ValueAsString(); + if (href.IsNotNullOrWhiteSpace()) + sb.AppendLine("{1}".Args(href, text)); + else + sb.AppendLine(text); + } + + + var children = level.Children.Where(cn=>cn.IsSameName(CONFIG_ITEM_SECTION)); + + if (children.Any()) + sb.AppendLine("
      "); + + foreach(var item in children) + buildMenuLevel(isoLang, sb, item); + + if (children.Any()) + sb.AppendLine("
    "); + + sb.AppendLine("
  • "); + + } + + + } +} diff --git a/src/Agni/WebManager/Pages/Console.nht b/src/Agni/WebManager/Pages/Console.nht new file mode 100644 index 0000000..ec9d181 --- /dev/null +++ b/src/Agni/WebManager/Pages/Console.nht @@ -0,0 +1,97 @@ +# + compiler + { + using{ns="NFX.Wave"} + using{ns="Agni.WebManager.Controls"} + base-class-name="Master, IConsolePage" + namespace="Agni.WebManager.Pages" + abstract="false" + summary="Process Status Page" + + } +# +#[override renderHeader()] +

    ?[AgniSystem.MetabaseApplicationName] Remote Console

    +#[override renderBody()] + + + + + + + + +
    + +   + +
    + + + + + diff --git a/src/Agni/WebManager/Pages/HeadMaster.nht b/src/Agni/WebManager/Pages/HeadMaster.nht new file mode 100644 index 0000000..ea56906 --- /dev/null +++ b/src/Agni/WebManager/Pages/HeadMaster.nht @@ -0,0 +1,55 @@ +# + compiler + { + using{ns="NFX"} + using{ns="NFX.Security"} + using{ns="Agni.WebManager.Controls"} + base-class-name="AWMPage" + namespace="Agni.WebManager.Pages" + abstract="true" + summary="Master page that contains general document head" + } +# + +#[class] + protected abstract void renderDocumentHEAD(); + protected abstract void renderDocumentBODY(); + +#[render] + + + + + + + + + + + + + + + ?[Title] + + + + + + + + + + + + + + + @[renderDocumentHEAD();] + + + + @[renderDocumentBODY();] + + + diff --git a/src/Agni/WebManager/Pages/Instrumentation.nht b/src/Agni/WebManager/Pages/Instrumentation.nht new file mode 100644 index 0000000..ae16afa --- /dev/null +++ b/src/Agni/WebManager/Pages/Instrumentation.nht @@ -0,0 +1,352 @@ +# + compiler + { + using{ns="NFX"} + using{ns="NFX.ApplicationModel"} + using{ns="NFX.Instrumentation"} + using{ns="NFX.Wave"} + using{ns="Agni.WebManager.Controls"} + using{ns="Agni.WebManager.Controllers"} + base-class-name="Master, IInstrumentationPage" + namespace="Agni.WebManager.Pages" + abstract=false + summary="Instrumentation Page" + } +# + +#[override renderDocumentHEAD()] +@[base.renderDocumentHEAD();] + + +#[override renderHeader()] +

    Instrumentation Tree Page

    + + + +#[override renderBody()] +

    Instrumentation

    +
    +   + Charts +   + Logs +    + Parameter Group + + +

    + +
    +
    + + diff --git a/src/Agni/WebManager/Pages/InstrumentationCharts.nht b/src/Agni/WebManager/Pages/InstrumentationCharts.nht new file mode 100644 index 0000000..d28c1c9 --- /dev/null +++ b/src/Agni/WebManager/Pages/InstrumentationCharts.nht @@ -0,0 +1,1191 @@ +# + compiler + { + using{ns="NFX.Wave"} + using{ns="Agni"} + using{ns="Agni.AppModel"} + using{ns="Agni.AppModel.ZoneGovernor"} + using{ns="Agni.WebManager.Controls"} + base-class-name="HeadMaster" + namespace="Agni.WebManager.Pages" + abstract=false + summary="Instrumentation Page" + } +# +#[class] + public override string Title { get { return "Instrumentation - " + AgniSystem.MetabaseApplicationName + "@" + AgniSystem.HostName;}} + +#[override renderDocumentHEAD()] + + +#[override renderDocumentBODY()] + +
    + +
    +
    + ?[AgniSystem.MetabaseApplicationName]@?[AgniSystem.HostName] +
    +
    +   + + +
    +
    +   + +
    +
    +   + +
    +
    +   + + +
    + @[if (AgniSystem.SystemApplicationType == SystemApplicationType.ZoneGovernor && ZoneGovernorService.IsZoneGovernor) {] + @[/*if (true) {*/] +
    + + +
    + @[}] +
    + +          +   Received 0 recs.  +   Service: 0 recs. 100 free +
    + +
    +   + +
    + +
    +
    + +
    +
    +
    + +
    +
    +
    + + diff --git a/src/Agni/WebManager/Pages/InstrumentationLogs.nht b/src/Agni/WebManager/Pages/InstrumentationLogs.nht new file mode 100644 index 0000000..d2bf5ca --- /dev/null +++ b/src/Agni/WebManager/Pages/InstrumentationLogs.nht @@ -0,0 +1,377 @@ +# + compiler + { + using{ns="NFX.Wave"} + using{ns="NFX.Log"} + using{ns="Agni"} + using{ns="Agni.AppModel"} + using{ns="Agni.AppModel.ZoneGovernor"} + using{ns="Agni.WebManager.Controls"} + base-class-name="HeadMaster" + namespace="Agni.WebManager.Pages" + abstract=false + summary="Instrumentation Page" + } +# +#[class] + public override string Title { get { return "Instrumentation - " + AgniSystem.MetabaseApplicationName + "@" + AgniSystem.HostName;}} +#[override renderDocumentHEAD()] + + +#[override renderDocumentBODY()] + +
    + +
    +
    + ?[AgniSystem.MetabaseApplicationName]@?[AgniSystem.HostName] +
    +
    +   + +
    +
    +   + +
    + @[if (AgniSystem.SystemApplicationType == SystemApplicationType.ZoneGovernor && ZoneGovernorService.IsZoneGovernor) {] + @[/*if (true) {*/] +
    + + +
    + @[}] +
    + +          +   Received: 0 +
    +
    +
    + +
    +
    +
    +
    + + diff --git a/src/Agni/WebManager/Pages/Login.nht b/src/Agni/WebManager/Pages/Login.nht new file mode 100644 index 0000000..519b8d8 --- /dev/null +++ b/src/Agni/WebManager/Pages/Login.nht @@ -0,0 +1,65 @@ +# + compiler + { + using{ns="NFX"} + using{ns="NFX.Wave"} + using{ns="Agni.WebManager.Controls"} + base-class-name="Master" + namespace="Agni.WebManager.Pages" + abstract="false" + summary="Process Status Page" + } +# + +#[class] + public Login(string msg) + { + m_Message = msg; + } + + private string m_Message; + +#[override renderDocumentHEAD()] +@[base.renderDocumentHEAD();] + + +#[override renderHeader()] +

    ?[AgniSystem.MetabaseApplicationName] Agni Web Manager Log-In

    + +#[override renderBody()] +@[if (m_Message.IsNotNullOrWhiteSpace()){] +
    Login result: ?[m_Message]
    +@[}] + +
    +
    +
    +
    + + + + + + + diff --git a/src/Agni/WebManager/Pages/Master.nht b/src/Agni/WebManager/Pages/Master.nht new file mode 100644 index 0000000..6677428 --- /dev/null +++ b/src/Agni/WebManager/Pages/Master.nht @@ -0,0 +1,79 @@ +# + compiler + { + using{ns="NFX"} + using{ns="NFX.Security"} + using{ns="Agni.WebManager.Controls"} + base-class-name="HeadMaster" + namespace="Agni.WebManager.Pages" + abstract="true" + summary="Master page for main pages in AWM site" + } +# + +#[class] + protected abstract void renderHeader(); + protected virtual void renderGutter() {} + protected abstract void renderBody(); + protected virtual void renderBottom() {} + +#[override renderDocumentHEAD()] + + + +#[override renderDocumentBODY()] +
    + +
    +
    +
    ?[AgniSystem.MetabaseApplicationName]
    +
    ?[AgniSystem.HostName]
    +
    +
    + +
    + +
    + @[renderHeader();] +
    + +
    +
    +
    + @[if (Session!=null && Session.User.Status!=UserStatus.Invalid){] + ?[Session.User.ToString()], logout + + @[}] +
    + @[renderGutter();] +
    +
    @[renderBody();]
    +
    + +
    + @[renderBottom();] +
    + +
    + +
    + © 2018  Agnicore, Inc. +
    + This site is served by the NFX.Wave framework +
    + Generated on ?[DateTime.Now] for ?[Context.EffectiveCallerIPEndPoint] +
    +
    diff --git a/src/Agni/WebManager/Pages/Process.nht b/src/Agni/WebManager/Pages/Process.nht new file mode 100644 index 0000000..2e687f1 --- /dev/null +++ b/src/Agni/WebManager/Pages/Process.nht @@ -0,0 +1,211 @@ +# + compiler + { + using{ns="NFX.Wave"} + using{ns="Agni.WebManager.Controls"} + base-class-name="Master, IHomePage" + namespace="Agni.WebManager.Pages" + abstract="false" + summary="Process Status Page" + } +# + +#[override renderHeader()] +

    ?[AgniSystem.MetabaseApplicationName] Process Status

    + +#[override renderBody()] +

    Host Status

    + + + +
    + + + + + +
    + + + + + + + +
    Last Update  
    CPU Load  
    RAM Load  
    Allocated  
    Max RAM  
    + +
    + + + + + + + + + + + +
    + +

    General Information

    + + + + + @[var app = AgniSystem.Application; + trow("Name", app.Name); + trow("Host", AgniSystem.HostName ); + trow("Parent Zone Governor", AgniSystem.ParentZoneGovernorPrimaryHostName ?? SysConsts.NULL ); + trow("Role", AgniSystem.HostMetabaseSection.RoleName ); + trow("Role Apps", AgniSystem.HostMetabaseSection.Role.AppNames.Aggregate(" ",(r,a)=>r+a+", ") ); + trow("Metabase App", app.MetabaseApplicationName ); + trow("Instance ID", app.InstanceID ); + trow("Start Time", app.StartTime ); + trow("Running Time", app.LocalizedTime - app.StartTime ); + trow("Type", app.GetType().FullName ); + trow("Active", app.Active); + trow("Boot Conf Root", app.BootConfigRoot ); + trow("Conf Root", app.ConfigRoot ); + trow("Data Store", app.DataStore.GetType().FullName ); + trow("Glue", app.Glue.GetType().FullName ); + trow("Instrumentation", app.Instrumentation.GetType().FullName ); + trow("Localized Time", app.LocalizedTime ); + trow("Time Location", app.TimeLocation ); + trow("Log", app.Log.GetType().FullName ); + trow("Object Store", app.ObjectStore.GetType().FullName ); + trow("Security Manager", app.SecurityManager.GetType().FullName ); + trow("Module Root", app.ModuleRoot.GetType().FullName ); + trow("TimeSource", app.TimeSource.GetType().FullName ); + ] +
    + +

    Glue Status

    + + + + #[trow(string name, object val)] + ?[name]?[(val ?? " ").ToString()] + + diff --git a/src/Agni/WebManager/Pages/ProcessManager.nht b/src/Agni/WebManager/Pages/ProcessManager.nht new file mode 100644 index 0000000..87df3ae --- /dev/null +++ b/src/Agni/WebManager/Pages/ProcessManager.nht @@ -0,0 +1,556 @@ +# + compiler + { + using{ns="NFX.Wave"} + using{ns="Agni.WebManager.Controls"} + base-class-name="HeadMaster, IProcessManagerPage" + namespace="Agni.WebManager.Pages" + abstract="false" + summary="Process Manager Page" + } +# + +#[class] +public ProcessManager(string zone) +{ + m_Zone = zone; +} + + +private string m_Zone; + +public string Zone { get {return m_Zone;}} + +public override string Title { get { return "Process manager - " + AgniSystem.MetabaseApplicationName + "@" + AgniSystem.HostName;}} + +#[override renderDocumentHEAD()] + + + +#[override renderDocumentBODY()] +
    + +
    +
    + ?[AgniSystem.MetabaseApplicationName]@?[AgniSystem.HostName] +
    +
    +   +
    +
    +   +
    +
    + +
    +
    +
    +
    +
    +
    +
    +
    +
    + + + + + +
    +
    +
    +
    +
    +
    + + + \ No newline at end of file diff --git a/src/Agni/WebManager/Pages/TheSystem.nht b/src/Agni/WebManager/Pages/TheSystem.nht new file mode 100644 index 0000000..5019cb2 --- /dev/null +++ b/src/Agni/WebManager/Pages/TheSystem.nht @@ -0,0 +1,553 @@ +# + compiler + { + using{ns="NFX"} + using{ns="NFX.Wave"} + using{ns="Agni.WebManager.Controls"} + using{ns="NFX.Serialization.JSON"} + + base-class-name="Master, ITheSystemPage" + namespace="Agni.WebManager.Pages" + abstract="false" + summary="Agni Navigation Page" + } +# + +#[override renderHeader()] +

    Agni Navigation Page

    + +#[override renderBody()] +

    Agni Navigation Map

    + + + + + + + + + +
    +
    +
     
    +
    + Processes +
    +
    +
    + +
    +
    + + diff --git a/src/Agni/WebManager/Site/fnt/Play-Bold.ttf b/src/Agni/WebManager/Site/fnt/Play-Bold.ttf new file mode 100644 index 0000000..7b9d5bf Binary files /dev/null and b/src/Agni/WebManager/Site/fnt/Play-Bold.ttf differ diff --git a/src/Agni/WebManager/Site/fnt/Play-Bold.woff b/src/Agni/WebManager/Site/fnt/Play-Bold.woff new file mode 100644 index 0000000..11e51b4 Binary files /dev/null and b/src/Agni/WebManager/Site/fnt/Play-Bold.woff differ diff --git a/src/Agni/WebManager/Site/fnt/Play-OFL.txt b/src/Agni/WebManager/Site/fnt/Play-OFL.txt new file mode 100644 index 0000000..edfa7fc --- /dev/null +++ b/src/Agni/WebManager/Site/fnt/Play-OFL.txt @@ -0,0 +1,93 @@ +Copyright (c) 2011, Jonas Hecksher, Playtypes, e-types AS (e-types.com), +with Reserved Font Name "Play", "Playtype", "Playtype Sans". +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/src/Agni/WebManager/Site/fnt/Play-Regular.ttf b/src/Agni/WebManager/Site/fnt/Play-Regular.ttf new file mode 100644 index 0000000..57ab4b1 Binary files /dev/null and b/src/Agni/WebManager/Site/fnt/Play-Regular.ttf differ diff --git a/src/Agni/WebManager/Site/fnt/Play-Regular.woff b/src/Agni/WebManager/Site/fnt/Play-Regular.woff new file mode 100644 index 0000000..f8e6b97 Binary files /dev/null and b/src/Agni/WebManager/Site/fnt/Play-Regular.woff differ diff --git a/src/Agni/WebManager/Site/img/Antenna.ERROR.gif b/src/Agni/WebManager/Site/img/Antenna.ERROR.gif new file mode 100644 index 0000000..b69d167 Binary files /dev/null and b/src/Agni/WebManager/Site/img/Antenna.ERROR.gif differ diff --git a/src/Agni/WebManager/Site/img/Antenna.OK.gif b/src/Agni/WebManager/Site/img/Antenna.OK.gif new file mode 100644 index 0000000..b6ad078 Binary files /dev/null and b/src/Agni/WebManager/Site/img/Antenna.OK.gif differ diff --git a/src/Agni/WebManager/Site/img/Cogs.64x64.png b/src/Agni/WebManager/Site/img/Cogs.64x64.png new file mode 100644 index 0000000..5642dec Binary files /dev/null and b/src/Agni/WebManager/Site/img/Cogs.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/Computer.64x64.png b/src/Agni/WebManager/Site/img/Computer.64x64.png new file mode 100644 index 0000000..346e49b Binary files /dev/null and b/src/Agni/WebManager/Site/img/Computer.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/DynamicComputer.64x64.png b/src/Agni/WebManager/Site/img/DynamicComputer.64x64.png new file mode 100644 index 0000000..0db560c Binary files /dev/null and b/src/Agni/WebManager/Site/img/DynamicComputer.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/Earth.64x64.png b/src/Agni/WebManager/Site/img/Earth.64x64.png new file mode 100644 index 0000000..91a09f4 Binary files /dev/null and b/src/Agni/WebManager/Site/img/Earth.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/Factory.64x64.png b/src/Agni/WebManager/Site/img/Factory.64x64.png new file mode 100644 index 0000000..9a7d125 Binary files /dev/null and b/src/Agni/WebManager/Site/img/Factory.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/FormElements.png b/src/Agni/WebManager/Site/img/FormElements.png new file mode 100644 index 0000000..fdb1ebb Binary files /dev/null and b/src/Agni/WebManager/Site/img/FormElements.png differ diff --git a/src/Agni/WebManager/Site/img/Terminal.64x64.png b/src/Agni/WebManager/Site/img/Terminal.64x64.png new file mode 100644 index 0000000..83041f1 Binary files /dev/null and b/src/Agni/WebManager/Site/img/Terminal.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/Zone.64x64.png b/src/Agni/WebManager/Site/img/Zone.64x64.png new file mode 100644 index 0000000..a8708fa Binary files /dev/null and b/src/Agni/WebManager/Site/img/Zone.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/awm.logo.256x256.png b/src/Agni/WebManager/Site/img/awm.logo.256x256.png new file mode 100644 index 0000000..1ee527d Binary files /dev/null and b/src/Agni/WebManager/Site/img/awm.logo.256x256.png differ diff --git a/src/Agni/WebManager/Site/img/awm.logo.64x64.png b/src/Agni/WebManager/Site/img/awm.logo.64x64.png new file mode 100644 index 0000000..59d8945 Binary files /dev/null and b/src/Agni/WebManager/Site/img/awm.logo.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/awm.logo.ita.256x256.png b/src/Agni/WebManager/Site/img/awm.logo.ita.256x256.png new file mode 100644 index 0000000..4d5ddb1 Binary files /dev/null and b/src/Agni/WebManager/Site/img/awm.logo.ita.256x256.png differ diff --git a/src/Agni/WebManager/Site/img/awm.logo.ita.64x64.png b/src/Agni/WebManager/Site/img/awm.logo.ita.64x64.png new file mode 100644 index 0000000..b8dad4e Binary files /dev/null and b/src/Agni/WebManager/Site/img/awm.logo.ita.64x64.png differ diff --git a/src/Agni/WebManager/Site/img/btnMenu.png b/src/Agni/WebManager/Site/img/btnMenu.png new file mode 100644 index 0000000..e8f9d86 Binary files /dev/null and b/src/Agni/WebManager/Site/img/btnMenu.png differ diff --git a/src/Agni/WebManager/Site/img/favicon.196x196.png b/src/Agni/WebManager/Site/img/favicon.196x196.png new file mode 100644 index 0000000..88ac711 Binary files /dev/null and b/src/Agni/WebManager/Site/img/favicon.196x196.png differ diff --git a/src/Agni/WebManager/Site/img/favicon.ita.196x196.png b/src/Agni/WebManager/Site/img/favicon.ita.196x196.png new file mode 100644 index 0000000..c7ece6b Binary files /dev/null and b/src/Agni/WebManager/Site/img/favicon.ita.196x196.png differ diff --git a/src/Agni/WebManager/Site/img/favicon.png b/src/Agni/WebManager/Site/img/favicon.png new file mode 100644 index 0000000..6f74af6 Binary files /dev/null and b/src/Agni/WebManager/Site/img/favicon.png differ diff --git a/src/Agni/WebManager/Site/img/tv.class.24x24.png b/src/Agni/WebManager/Site/img/tv.class.24x24.png new file mode 100644 index 0000000..d574f80 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.class.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.error-event.24x24.png b/src/Agni/WebManager/Site/img/tv.error-event.24x24.png new file mode 100644 index 0000000..ccc7142 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.error-event.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.error-gauge.24x24.png b/src/Agni/WebManager/Site/img/tv.error-gauge.24x24.png new file mode 100644 index 0000000..0431181 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.error-gauge.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.error-src.18x18.png b/src/Agni/WebManager/Site/img/tv.error-src.18x18.png new file mode 100644 index 0000000..41553d8 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.error-src.18x18.png differ diff --git a/src/Agni/WebManager/Site/img/tv.event.24x24.png b/src/Agni/WebManager/Site/img/tv.event.24x24.png new file mode 100644 index 0000000..88dcf47 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.event.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.gauge.24x24.png b/src/Agni/WebManager/Site/img/tv.gauge.24x24.png new file mode 100644 index 0000000..300478e Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.gauge.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.node-minus.24x24.png b/src/Agni/WebManager/Site/img/tv.node-minus.24x24.png new file mode 100644 index 0000000..a055371 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.node-minus.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.node-plus.24x24.png b/src/Agni/WebManager/Site/img/tv.node-plus.24x24.png new file mode 100644 index 0000000..ce12de6 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.node-plus.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.ns.24x24.png b/src/Agni/WebManager/Site/img/tv.ns.24x24.png new file mode 100644 index 0000000..62fe6d7 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.ns.24x24.png differ diff --git a/src/Agni/WebManager/Site/img/tv.src.18x18.png b/src/Agni/WebManager/Site/img/tv.src.18x18.png new file mode 100644 index 0000000..2a84430 Binary files /dev/null and b/src/Agni/WebManager/Site/img/tv.src.18x18.png differ diff --git a/src/Agni/WebManager/Site/scr/awm.js b/src/Agni/WebManager/Site/scr/awm.js new file mode 100644 index 0000000..4fd21b0 --- /dev/null +++ b/src/Agni/WebManager/Site/scr/awm.js @@ -0,0 +1,49 @@ + +//--------------------------------------------------------------------------------------------- +//GLOBALS +//--------------------------------------------------------------------------------------------- + + +function global_READY() +{ + //IOS embedded site - make all link open inside the same app + if (window.navigator.standalone) + { + //todo need to handle Internal vs internal links + $( document ).on("click", "a", function( event ){ + event.preventDefault(); + location.href = $( event.target ).attr("href"); + }); + } +}; + + +WAVE.LOCALIZER.rus = +{ + "--ANY-SCHEMA--": + { + "error": + { + "Field '@f@' must have a value": "Поле '@f@' должно быть заполнено", + "Field '@f@' value can not be less than '@b@'": "Значение поля '@f@' не может быть меньше чем '@b@'", + "Field '@f@' value can not be greater than '@b@'": "Значение поля '@f@' не может быть больше чем '@b@'", + "Field '@f@' value can not be longer than @b@ characters": "Длина значения поля '@f@' не может быть больше чем @b@ символов", + "Field '@f@' value can not be shorter than @b@ characters": "Длина значения поля '@f@' не может быть меньше чем @b@ символов", + "Field '@f@' value '@v@' is not allowed": "Поле '@f@' не может иметь значение '@v@'", + "Field '@f@' must be a valid e-mail address": "Неправильный е-майл адрес в поле '@f@'", + "Field '@f@' must start from letter and contain only letters or digits separated by single '.' or '-' or '_'": + "Поле '@f@' должно начинаться с буквы и содержать только буквы или цифры, разделённые одинарным '.' или '-' или '_'" + } + } + + + +};//rus + + +var READY_FUNC = null; + +$(document).ready(function() { + global_READY(); + if (READY_FUNC) READY_FUNC(); +}); \ No newline at end of file diff --git a/src/Agni/WebManager/Site/scr/master.js b/src/Agni/WebManager/Site/scr/master.js new file mode 100644 index 0000000..5ee6ee7 --- /dev/null +++ b/src/Agni/WebManager/Site/scr/master.js @@ -0,0 +1,67 @@ + var CONSTS = { + hdrTopHeight: 72, + hdrTopMinHeight: 28, + logoWidth: 100, + minLogoWidth: 40, + logoFSRegular: "2em", + logoFSThreshold: 40, + subLogoFSThreshold: 60 + }; + +var menuIsVisible = false; + +READY_FUNC = function() { + + $(window).scroll(function(){ + var pos = $(window).scrollTop(); + var h = CONSTS.hdrTopHeight - pos; + if (h input[type="text"]:disabled { + font-size: 2em; + font-weight: bold; + color: blue; + background: white; + width: 65%; + height: 40px; + padding: 0px; + padding-left: 8px; + padding-right: 8px; + letter-spacing: 6px +} +.wvPuzzleInputs > button { + font-size: 1.2em; + font-weight: bold; + width: 20%; + height: 40px; + float: right; +} + +.wvPuzzleImg { + display: block; + border-bottom-left-radius: 12px; + border-bottom-right-radius: 12px; + background: #f8f8f8; + border: 1px solid #808080; + padding: 4px; +} + +.wvPuzzleImg > img { margin-top: -20px; margin-bottom: -10px; } + +/* Dialog Boxes */ +.wvDialogBase .wvDialogContent { + background-color: #757575; + box-shadow: 6px 6px 10px #555; +} +.wvDialogBase .wvDialogHeader { + background-color: transparent; +} +.wvDialogBase .wvDialogBody { + border-color: #808080; +} diff --git a/src/Agni/WebManager/Site/stl/instrumentation.css b/src/Agni/WebManager/Site/stl/instrumentation.css new file mode 100644 index 0000000..c162dab --- /dev/null +++ b/src/Agni/WebManager/Site/stl/instrumentation.css @@ -0,0 +1,172 @@ +#dvComponents { + width: 100%; + display: flex; + flex-flow: row wrap; + align-items: flex-start; + justify-content: flex-start; +} + +.clsComponent { + background-color: #505050; + /*border: 1px solid #c0c0c0;*/ + border-radius: 8px; + box-shadow: 2px 2px 6px #808080; + color: #c0c0c0; + display: block; + font-size: 11px; + margin: 4px; + padding: 6px; +} + +.clsComponentName { + background: linear-gradient(#202020 0%, #505040 50%, #404040 51%, #202020 100%); + border: 1px solid #606060; + border-radius: 8px; + color: white; + padding: 2px; + margin-bottom: 4px; +} + +.imgInstr { width: 16px; height: 16px; vertical-align: text-top; position: relative; top: -1px; } + +.chkInstrumentation { display: inline !important; vertical-align: top; } + +.consoleSID { color: #ffffff; font-size: 14px; font-weight: 700; } +.consoleDate { color: #f0f080; font-size: 10px; } +.consoleType { color: #b0b0b0; font-size: 10px; text-shadow: 0 0 4px #000000; } +.consoleCommon { color: #ff70ff; font-size: 14px; text-shadow: 0 0 10px #ff4040; } +.consoleName { color: #50ff20; font-size: 12px; text-shadow: 0 0 8px #70c000; } +.consoleDirector { color: #40a0f0; font-size: 10px; } + +.clsComponentParams { } +.clsComponentParam { + background: linear-gradient(90deg, #404040 0%, #606060 50%, rgba(64, 64, 64, 0.1) 100%) repeat scroll 0 0 rgba(0, 0, 0, 0); + border-bottom: 1px dotted #909090; + border-radius: 6px; + box-shadow: 0 0 10px rgba(100, 200, 255, 0.2) inset; + color: #c0c0c0; + display: block; + margin-top: 0; + padding: 1px 2px; + cursor: pointer; +} + .clsComponentParam:hover { + color: white; + box-shadow: 0 0 10px rgba(255, 250, 100, 0.7) inset; + } +.consolePName { } +.consolePValue { + color: #c0b040; + font-size: 11px; + font-weight: 700; + text-shadow: 0 0 6px #202020; +} + +#cmbGroup { + -moz-appearance: toolbarbutton-dropdown; + -webkit-appearance: menulist; + background-color: #f0f0a0; + border: 1px solid #c0c0c0; + border-radius: 2px; + box-shadow: none; + font-size: 1em; + padding: 2px; + width: auto; +} + +#dvParamSheet { +} + + #dvParamSheet > select { + -moz-appearance: toolbarbutton-dropdown; + -webkit-appearance: menulist; + background-color: #f0f0a0; + border: 1px solid #c0c0c0; + border-radius: 2px; + box-shadow: none; + font-size: 1em; + padding: 1px; + width: auto; + } + + #dvParamSheet > input[type="textbox"] { + background-color: #f0f0a0; + border: 1px solid #c0c0c0; + border-radius: 2px; + box-shadow: none; + font-size: 1em; + padding: 1px; + width: auto; + } + +#dvEditorErr { + display: none; + margin: 4px; + color: #c04040; + font-size: 12px; + font-weight: 700; +} + +.wvTreeNode { + display: table; + margin-bottom: 3px; + margin-top: 3px; +} + +.wvTreeNodeRootWithChildren { + background-color: #f0f0f0; + border: 1px solid #d0d0d0; + border-radius: 8px; + margin: 3px; + margin-top: 8px; +} + +.wvTreeNodeButtonExpanded { + display: table-cell; + color: rgba(0,0,0,0); + cursor: pointer; + padding-left: 20px; + background: url('/static/site/img/tv.node-minus.24x24.png'); + background-size: 17px 17px; + background-position: 1px 6px; + background-repeat: no-repeat; +} + +.wvTreeNodeButtonCollapsed { + display: table-cell; + color: rgba(0,0,0,0); + cursor: pointer; + padding-left: 20px; + background: url('/static/site/img/tv.node-plus.24x24.png'); + background-size: 19px 19px; + background-position: 1px 5px; + background-repeat: no-repeat; +} + +.wvTreeNodeContent { + display: table-cell; +} + +.wvTreeNodeOwn { + display: block; + width: intrinsic; /* Safari/WebKit uses a non-standard name */ + width: -moz-max-content; /* Firefox/Gecko */ + width: -webkit-max-content; /* Chrome */ +} + +.wvTreeNodeOwnSelected { + display: table; + background: linear-gradient(0deg, rgba(160, 170, 255, 0.7), rgba(240, 240, 255, 0.9), rgba(160, 170, 255, 0.3)); + font-weight: bold; +} + +.wvTreeNodeChildren { display: block; height: auto; } + +.wvTreeNodeChildrenFlex { flex-wrap: wrap; } + +#dvObjectInspector { height: 300px; overflow: auto; } + + #dvObjectInspector .wvTreeNodeButtonCollapsed { + background-size: 17px 17px; + background-position: 1px 2px; + } diff --git a/src/Agni/WebManager/Site/stl/login.css b/src/Agni/WebManager/Site/stl/login.css new file mode 100644 index 0000000..5726ab9 --- /dev/null +++ b/src/Agni/WebManager/Site/stl/login.css @@ -0,0 +1,3 @@ +#frmLogin { + max-width: 350px; +} \ No newline at end of file diff --git a/src/Agni/WebManager/Site/stl/master.css b/src/Agni/WebManager/Site/stl/master.css new file mode 100644 index 0000000..a38d163 --- /dev/null +++ b/src/Agni/WebManager/Site/stl/master.css @@ -0,0 +1,335 @@ +#navProcessMenu { + margin-left: -16px; + margin-top: 8px; + background: linear-gradient(90deg, rgba(195, 240, 255, 0.8) 0%, rgba(190, 190, 190, 0)); +} + +#navProcessMenu ul { + font-family: 'Play'; + color: #404030; + font-size: 12px; + margin-left: 8px; + margin-top: 4px; + margin-bottom: 4px; +} + +#hdrTop { + width: 100%; + height: 72px; + background-color: #d50000; + box-shadow: 0px 1px 2px #d50000; + position: fixed; + z-index: 101; + top: 0px; + right: 0px; + float: none; + min-width: 380px; +} + +#divLogo { + float: left; + width: 100px; + height: 100%; + background-image: url("/static/site/img/awm.logo.ita.256x256.png"); + background-repeat: no-repeat; + background-position: center; + background-size: 60%; +} + +#divTitleBox { + float: left; + height: 100%; + display: table; +} + +#divTitleWrapper { + display: table-cell; + vertical-align: middle; +} + +#divTitle { + font-size: 2em; + font-weight: bold; + color: #fff4f0; + -webkit-text-stroke: 1px rgba(255,255,255,0.7); +} + +#divSubTitle { + font-size: 1em; + color: #ffd0d0; + max-width: calc(100vw - 160px); + overflow: hidden; +} + +#dropdown { + height: 100%; + position: relative; +} + +/* COMPUTER */ +@media screen and (min-width: 992px) { + #navTopMenu { + height: 100%; + float: right; + display: table; + border: none; + visibility: visible; + } + + #navTopMenu a { + color: white; + display: table-cell; + vertical-align: middle; + padding-left: 4px; + padding-right: 4px; + width: 100px; + text-align: center; + -webkit-text-stroke: 1px rgba(255,255,255,0.4) + } + + #btnNavMenu { + display: none; + } +} + +/* Tablet */ +@media screen and (max-width: 991px) { + #dropdown { + float: right; + } + + #navTopMenu { + visibility: hidden; + position: absolute; + right: 0; + background-color: #402020; + min-width: 200px; + border: none; + box-shadow: 0px 2px 4px #808080; + z-index: 10; + } + + #navTopMenu a { + color: white; + display: block; + padding: 16px 16px 16px 16px; + text-align: center; + -webkit-text-stroke: 1px rgba(255,255,255,0.4); + letter-spacing: 0.1ex; + } + + #btnNavMenu { + border: none; + background-color: inherit; + height: 100%; + width: 60px; + background-image: url("/static/site/img/btnMenu.png"); + background-repeat: no-repeat; + background-position: center; + background-size: 50%; + } + + #btnNavMenu:hover { background-color: #ff5540;} + #btnNavMenu:active { background-color: #ef5350;} +} + +#navTopMenu a:hover { + background-color: #F44336; +} + +.selectedPage +{ + font-weight: bold; + background-color: #ff5252; + /* border-bottom: 4px solid #ff9090; + padding-top: 4px; */ +} + +#sectPageTitle +{ + padding-top: 80px; + padding-bottom: 8px; + width: 100%; + background: linear-gradient(90deg, #b0b0b0, #ffffff); + /* border-bottom: 2px solid white; */ + text-align: right; +} + +#sectPageTitle * +{ + padding-right: 18px; +} + + +#sectContent { + width: 100%; + background-color: #f0f0f0; + display: table; +} + +#sectGutter +{ + width: 18%; + height: auto; + background-color: #f0f0f0; + padding: 10px; + display: table-cell; +} + +#sectBody { + background: linear-gradient(182deg, #e0e0e0, #ffffff 20px); + width: 100%; + height: auto; + margin-left: 25%; + /* margin-right: auto; */ + background-color: white; + padding: 40px; + display: table-cell; +} + +footer { + width: 100%; + color: white; + background-color: #212121; + text-align: center; + height: 110px; +} + +#divFooterMenu { + padding-top: 10px; +} + +#divFooterMenu a { + padding: 0 30px; +} + +#divFooterMenu a, #divFooterMenu a:visited { + color: white; +} + +#divTailContent { + font-size: 0.8em; + color: #d0d0d0; + margin-top: 20px; +} + + +/* INPUTS */ + +.RMFView + { + display: block; + /* border: 1px solid #fefeb0; */ + padding: 4px; +} + +fieldset +{ + border: 1px solid #bfbfbf; +} + +label +{ + margin: 4px; + display: inline-block; +} + +button,input[type=submit] +{ + font-family: inherit; + font-size: 1.3em; +} + + +.checkableLabel + { + display: inline-block; + cursor: pointer; + position: relative; + padding-left: 40px; + margin-right: 40px; + height: 32px; + padding-top: 5px; + } + +input[type=radio],input[type=checkbox] { display: none; } + +.checkableLabel:before +{ + background-image: url("/static/site/img/FormElements.png"); + background-size: 133px 66px; + background-repeat: no-repeat; + content: ""; + display: inline-block; + width: 32px; + height: 32px; + position: absolute; + left: 0; + top: 0; + /* border: 1px solid gray; */ +} + +/*Enabled*/ +input[type=checkbox] + .checkableLabel:before { background-position: -1px -1px; } +input[type=checkbox]:checked + .checkableLabel:before { background-position: -33px -1px; } +input[type=radio] + .checkableLabel:before { background-position: -66px -1px; } +input[type=radio]:checked + .checkableLabel:before { background-position: -98px -1px; } + +/* Disabled */ +input[type=checkbox]:disabled + .checkableLabel:before { background-position: -1px -34px; } +input[type=checkbox]:disabled:checked + .checkableLabel:before { background-position: -33px -34px; } +input[type=radio]:disabled + .checkableLabel:before { background-position: -66px -34px; } +input[type=radio]:disabled:checked + .checkableLabel:before { background-position: -98px -34px; } + + +/* select +{ + + border: 1px solid #e0e0e0; + font-family: inherit; + font-size: 1.3em; + -webkit-appearance: none + -moz-appearance: none; + background: #fefefe; + width: 180px; + color: #black; + border-radius: 5px; + padding: 5px; + box-shadow: inset 0 0 3px rgba(000,000,000, 0.5); +} */ + +input[type=text],input[type=password],input[type=date],input[type=datetime],input[type=tel],input[type=email],select +{ + border: 1px solid #e0e0e0; + font-family: inherit; + font-size: 1.3em; + -webkit-appearance: none; + -moz-appearance: none; + background: #fefefe; + width: 100%; + color: black; + border-radius: 5px; + padding: 5px; + box-shadow: inset 0 0 3px rgba(000,000,000, 0.5); +} + +input[type=text]:focus,input[type=password]:focus,input[type=date]:focus,input[type=datetime]:focus,input[type=tel]:focus,input[type=email]:focus,select:focus +{ + background-color: #ffffc0; + border: 1px solid #808080; +} + +input[type=text]:disabled,input[type=password]:disabled,input[type=date]:disabled,input[type=datetime]:disabled,input[type=tel]:disabled,input[type=email]:disabled,select:disabled +{ + background-color: #e0e0e0; + color: #b0b0b0; + border: 1px solid #808080; +} + +/* Mobile */ +@media screen and (max-width: 767px) { + #divLogo { + width: 80px; + background-size: 80%; + } +} diff --git a/src/Agni/WebManager/UriUtils.cs b/src/Agni/WebManager/UriUtils.cs new file mode 100644 index 0000000..5641386 --- /dev/null +++ b/src/Agni/WebManager/UriUtils.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace Agni.WebManager +{ + /// + /// Centralizes site URI static resource paths management + /// + public static class SURI + { + public const string STATIC = "/static/site/"; + public const string IMG = STATIC + "img/"; + public const string STL = STATIC + "stl/"; + public const string SCR = STATIC + "scr/"; + public const string STOCK = "/static/stock/site/"; + public const string STOCK_STL = STOCK + "stl/"; + + public static string Image(string path) + { + return Path.Combine(IMG, path); + } + + public static string Style(string path) + { + return Path.Combine(STL, path); + } + + public static string StockStyle(string path) + { + return Path.Combine(STOCK_STL, path); + } + + public static string Script(string path) + { + return Path.Combine(SCR, path); + } + + public static string Stock(string path) + { + return Path.Combine(STOCK, path); + } + + } + + /// + /// Centralizes main site URIs that lead to dynamic pages + /// + public static class URIS + { + public const string HOME = "/"; + public const string MVC = "/mvc"; + + public const string CONSOLE = "/console"; + + public const string INSTRUMENTATION = "/instrumentation"; + public const string INSTRUMENTATION_MVC = MVC + INSTRUMENTATION; + + public const string INSTRUMENTATION_CHARTS = "/instrumentation-charts"; + public const string INSTRUMENTATION_LOGS = "/instrumentation-logs"; + + public const string THE_SYSTEM = "/thesystem"; + public const string THE_SYSTEM_MVC = MVC + THE_SYSTEM; + + public const string PUB_API_HOST_PERFORMANCE = "/pub-api/hostperformance"; + + public const string PROCESS_MANAGER = "/processmanager"; + public const string PROCESS_MANAGER_MVC = MVC + PROCESS_MANAGER; + + } + +} diff --git a/src/Agni/WebMessaging/AgniWebMessage.cs b/src/Agni/WebMessaging/AgniWebMessage.cs new file mode 100644 index 0000000..01f014e --- /dev/null +++ b/src/Agni/WebMessaging/AgniWebMessage.cs @@ -0,0 +1,74 @@ +using System; + +using NFX.Web.Messaging; +using NFX.Serialization.Arow; +using NFX.DataAccess.Distributed; +using NFX.DataAccess.CRUD; + +namespace Agni.WebMessaging +{ + /// + /// Represents messages stored in Agni system + /// + [Serializable, Arow] + public class AgniWebMessage : Message + { + protected AgniWebMessage() { } + public AgniWebMessage(GDID? gdid = null, Guid? id = null, DateTime? utcCreateDate = null) : base(id, utcCreateDate) + { + GDID = gdid ?? AgniSystem.GDIDProvider.GenerateOneGDID(SysConsts.GDID_NS_MESSAGING, SysConsts.GDID_NAME_MESSAGING_MESSAGE); + } + + /// + /// Represents a unique ID assigned to the message in the distributed system + /// + [Field(backendName: "gdid", isArow: true)] + public GDID GDID { get; private set; } + + /// + /// Represents a status that the message is in + /// + [Field(backendName: "stat", isArow: true)] + public MsgStatus Status { get; set; } + + /// + /// Represents a public status that the message is in + /// + [Field(backendName: "pstat", isArow: true)] + public MsgPubStatus PubStatus { get; set; } + + /// + /// Provides a public status timestamp + /// + [Field(backendName: "pstts", isArow: true)] + public DateTime? PubStatusTimestamp { get; set; } + + /// + /// Provides a public status operator - who changed the status + /// + [Field(backendName: "pstop", isArow: true)] + public string PubStatusOperator { get; set; } + + /// + /// Provides a public status description + /// + [Field(backendName: "pstd", isArow: true)] + public string PubStatusDescription { get; set; } + + /// + /// Provides a comma-separated list of "folders" that the message is in. + /// When a UI agent gets the msg feed, it puts the messages in corresponding folders per user + /// or if such folder does not exist, puts it in "Other" folder + /// + [Field(backendName: "fld", isArow: true)] + public string Folders { get; set; } + + /// + /// Provides a comma-separated list of "adornments" that the message is embellished with. + /// Example: "star,heart,triangle" + /// + [Field(backendName: "adrn", isArow: true)] + public string Adornments { get; set; } + } + +} diff --git a/src/Agni/WebMessaging/AgniWebMessageSink.cs b/src/Agni/WebMessaging/AgniWebMessageSink.cs new file mode 100644 index 0000000..ecf1da7 --- /dev/null +++ b/src/Agni/WebMessaging/AgniWebMessageSink.cs @@ -0,0 +1,88 @@ +using System; + +using NFX; +using NFX.Environment; +using NFX.Web.Messaging; + +using Agni.Contracts; + +namespace Agni.WebMessaging +{ + /// + /// Dispatches instances of AgniWebMessage into the remote IWebMessageSystem + /// + public class AgniWebMessageSink : MessageSink + { + #region CONSTS + public const string CONFIG_HOST_ATTR = "host"; + + #endregion + + public AgniWebMessageSink(MessageService director) : base(director) + { + } + + private string m_HostName; + + //todo: Refactor to use hosts sets with load balancing + private IWebMessageSystemClient m_Client; + + + /// + /// Specifies the name of the host where the messages are sent + /// + [Config] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_MESSAGING)] + public string Host { get; set; } + + public override MsgChannels SupportedChannels { get { return MsgChannels.All; } } + + + protected override bool Filter(Message msg) + { + return true; + } + + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + + //throws on bad host spec + AgniSystem.Metabase.CatalogReg.NavigateHost(Host); + } + + protected override void DoWaitForCompleteStop() + { + base.DoWaitForCompleteStop(); + DisposeAndNull(ref m_Client); + } + + + protected override bool DoSendMsg(Message msg) + { + var amsg = msg as AgniWebMessage; + if (amsg == null) return false; + + try + { + ensureClient(); + m_Client.SendMessage(amsg); + } + catch (Exception error) + { + throw new WebMessagingException("{0}.DoSend: {1}".Args(GetType().Name, error.ToMessageWithType()), error); + } + return true; + } + + private void ensureClient() + { + var hn = this.Host; + if (m_Client == null && !hn.EqualsOrdIgnoreCase(m_HostName)) + { + m_Client = ServiceClientHub.New(hn); + m_HostName = hn; + } + } + } +} diff --git a/src/Agni/WebMessaging/Enums.cs b/src/Agni/WebMessaging/Enums.cs new file mode 100644 index 0000000..becbc7b --- /dev/null +++ b/src/Agni/WebMessaging/Enums.cs @@ -0,0 +1,85 @@ + +namespace Agni.WebMessaging +{ + /// + /// Denotes the status of the message + /// + public enum MsgStatus + { + /// + /// The message is new/initial status + /// + Initial = 0, + + /// + /// The message was read, but may show in the list of read messages + /// + Read, + + /// + /// The message is archived - it should not appear in regular lists + /// + Archived, + + /// + /// The message is marked for deletion and should be destroyed + /// + Deleted + } + + /// + /// Denotes types of message visibility/publication + /// + public enum MsgPubStatus + { + /// + /// Message was banned and is not shown + /// + Banned = -1, + + /// + /// Message is published and can be shown + /// + Published = 0, + + /// + /// Message is in draft mode and not published yet ( visible only to msg author) + /// + Draft = 1, + + /// + /// Message is in preview mode for the limited audience + /// + Preview = 2, + + /// + /// Message needs review by the system/moderator before it can be shown. + /// Moderator can transition this to DRAFT, BANNED or PUBLISHED + /// + NeedsReview = 3 + } + + public enum MsgChannelWriteResult + { + /// Message was not sent now and will not be delivered in future due to sending error in channel + ChannelError = -10, + + /// Message was not sent now and will not be delivered in future due to sending error + PermanentFailure = -3, + + /// The channel encoutered error and will try to resend in some time + WillRetryAfterFailure = -2, + + /// The message can not be delivered in principle (e.g. bad address) + Undeliverable = -1, + + /// The operation finished with an undetermined state + Undefined = 0, + + /// The message was sent + Success = 1, + + /// The message was routed into gateway + Gateway = 2 + } +} diff --git a/src/Agni/WebMessaging/Exceptions.cs b/src/Agni/WebMessaging/Exceptions.cs new file mode 100644 index 0000000..a014d7f --- /dev/null +++ b/src/Agni/WebMessaging/Exceptions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.WebMessaging +{ + /// + /// Thrown to indicate web messaging problems + /// + [Serializable] + public class WebMessagingException : AgniException + { + public WebMessagingException() : base() {} + public WebMessagingException(string message) : base(message) {} + public WebMessagingException(string message, Exception inner) : base(message, inner) { } + protected WebMessagingException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/WebMessaging/Mailboxes.cs b/src/Agni/WebMessaging/Mailboxes.cs new file mode 100644 index 0000000..7fd03bc --- /dev/null +++ b/src/Agni/WebMessaging/Mailboxes.cs @@ -0,0 +1,143 @@ +using System; + +using NFX; +using NFX.DataAccess.Distributed; +using NFX.Serialization.JSON; + +namespace Agni.WebMessaging +{ + + /// + /// Represents a read-only tuple of { Channel: string, gShard: GDID, gMailbox: GDID}. + /// The gMailbox is a globally-unique ID however some large systems need to prepend it with + /// gShard which allows for instant location of a concrete data store which holds gMailbox. + /// This design allows to use the same infrastructure for product reviews, in which case, the MailboxID + /// may represent a particular product (out of millions) that receives customer review messages + /// + [Serializable] + public struct MailboxID : IEquatable + { + public MailboxID(string channel, GDID gShard, GDID gMbox) { Channel = channel; G_Shard = gShard; G_Mailbox = gMbox; } + + /// + /// System-dependent Channel ID where the mailbox is stored, for example: USER, COMPANY, VENDOR etc... + /// + public readonly string Channel; + + /// + /// Sharding GDID used to instantly find the data store shard where data is kept + /// + public readonly GDID G_Shard; + + /// + ///The global unique id of a mailbox, e.g. user GDID, company GDID, product "mailbox" = G_PRODUCT + /// + public readonly GDID G_Mailbox; + + public bool Equals(MailboxID other) + { + return string.Equals(this.Channel, other.Channel, StringComparison.Ordinal) && + this.G_Shard == other.G_Shard && + this.G_Mailbox == other.G_Mailbox; + } + + public override bool Equals(object obj) + { + if (!(obj is MailboxID)) return false; + return this.Equals((MailboxID)obj); + } + + public override int GetHashCode() + { + return Channel.GetHashCode() ^ G_Shard.GetHashCode() ^ G_Mailbox.GetHashCode(); + } + + public override string ToString() + { + return "`{0}` -> Mbx[{1}@{2}]".Args(Channel, G_Mailbox, G_Shard); + } + } + + + /// + /// Represents a read-only tuple of { mailboxID: MailboxID, gMessage: GDID} + /// + [Serializable] + public struct MailboxMsgID : IEquatable + { + public MailboxMsgID(MailboxID xid, GDID gMsg) + { + MailboxID = xid; + G_Message = gMsg; + } + + public readonly MailboxID MailboxID; + public readonly GDID G_Message; + + public bool Equals(MailboxMsgID other) + { + return this.MailboxID.Equals(other.MailboxID) && this.G_Message.Equals(other.G_Message); + } + + public override bool Equals(object obj) + { + if (!(obj is MailboxMsgID)) return false; + return this.Equals((MailboxMsgID)obj); + } + + public override int GetHashCode() + { + return MailboxID.GetHashCode() ^ G_Message.GetHashCode(); + } + + public override string ToString() + { + return "MbxMsg({0} in {1})".Args(G_Message, MailboxID); + } + } + + /// + /// Provides general information about the mailbox instance fetched by MailboxID. + /// Systems derive form this class to return more details as appropriate + /// + [Serializable] + public class MailboxInfo + { + /// + /// The global unique ID of this mailbox + /// + public MailboxID ID { get; set; } + + /// + /// Primary address of this mailbox, e.g. email address + /// + public string PrimaryAddress { get; set; } + + /// + /// Localized name and description of the mailbox + /// + public NLSMap Name { get; set; } + + /// + /// When was the mailbox established + /// + public DateTime? CreateDate { get; set; } + + /// + /// Optional URL of image representing the mailbox (i.e. user image, group image etc.) + /// + public string ProfileImageURL { get; set; } + + + /// + /// Override to translate system byte types into textual representation per supplied language + /// + public virtual string GetMailboxChannelDisplayName(string isoLang) + { + return ID.Channel; + } + } + + + +} diff --git a/src/Agni/WebMessaging/Server/Channel.cs b/src/Agni/WebMessaging/Server/Channel.cs new file mode 100644 index 0000000..3b9c367 --- /dev/null +++ b/src/Agni/WebMessaging/Server/Channel.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX.ServiceModel; +using NFX.Web.Messaging; + +using Agni.Contracts; + +namespace Agni.WebMessaging.Server +{ + /// + /// Represents a messaging channel that hosts message boxes of the particular type + /// + public abstract class Channel : ServiceWithInstrumentationBase + { + protected Channel(WebMessageSystemService director) : base(director) + { + + } + + /// + /// Writes msg into mailbox identified by the particular address. + /// Reliability: this method must not leak errors, but should handle them depending on particular + /// channel implementation and message importance (i.e. we may create ToDo that will try to redeliver) + /// + public abstract MsgChannelWriteResult Write(List deliveryList, int idx, string address, AgniWebMessage msg); + + + /// + /// Returns information about a particular mailbox on this channel or null if not found + /// + public abstract MailboxInfo GetMailboxInfo(MailboxID xid); + + /// + /// Returns message headers for the specified mailbox and query or null + /// + public abstract MessageHeaders GetMessageHeaders(MailboxID xid, string query); + + /// + /// Returns message count for the specified mailbox and query + /// + public abstract int GetMessageCount(MailboxID xid, string query); + + /// + /// Fetches mailbox message by id or null if not found + /// + public abstract AgniWebMessage FetchMailboxMessage(MailboxMsgID mid); + + /// + /// Fetches an attachment for the specified message by id and attachment index or null if not found + /// + public abstract Message.Attachment FetchMailboxMessageAttachment(MailboxMsgID mid, int attachmentIndex); + + /// + /// Updates the particular mailbox message publication status along with operator and description + /// + public abstract void UpdateMailboxMessagePublication(MailboxMsgID mid, MsgPubStatus status, string oper, string description); + + /// + /// Updates the particular mailbox message status + /// + public abstract void UpdateMailboxMessageStatus(MailboxMsgID mid, MsgStatus status, string folders, string adornments); + + /// + /// Updates mailbox messages status + /// + public abstract void UpdateMailboxMessagesStatus(IEnumerable mids, MsgStatus status, string folders, string adornments); + + } +} diff --git a/src/Agni/WebMessaging/Server/WebMessageSystemService.cs b/src/Agni/WebMessaging/Server/WebMessageSystemService.cs new file mode 100644 index 0000000..46558de --- /dev/null +++ b/src/Agni/WebMessaging/Server/WebMessageSystemService.cs @@ -0,0 +1,309 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Web.Messaging; +using NFX.ServiceModel; + +using Agni.Contracts; +using NFX.Environment; + +namespace Agni.WebMessaging.Server +{ + /// + /// Glue adapter for Contracts.IWebMessageSystem + /// + public sealed class WebMessageSystemServer : Contracts.IWebMessageSystem + { + public AgniWebMessage FetchMailboxMessage(MailboxMsgID mid) + { + return WebMessageSystemService.Instance.FetchMailboxMessage(mid); + } + + public Message.Attachment FetchMailboxMessageAttachment(MailboxMsgID mid, int attachmentIndex) + { + return WebMessageSystemService.Instance.FetchMailboxMessageAttachment(mid, attachmentIndex); + } + + public MailboxInfo GetMailboxInfo(MailboxID xid) + { + return WebMessageSystemService.Instance.GetMailboxInfo(xid); + } + + public int GetMailboxMessageCount(MailboxID xid, string query) + { + return WebMessageSystemService.Instance.GetMailboxMessageCount(xid, query); + } + + public MessageHeaders GetMailboxMessageHeaders(MailboxID xid, string query) + { + return WebMessageSystemService.Instance.GetMailboxMessageHeaders(xid, query); + } + + public MsgSendInfo[] SendMessage(AgniWebMessage msg) + { + return WebMessageSystemService.Instance.SendMessage(msg); + } + + public void UpdateMailboxMessagePublication(MailboxMsgID mid, MsgPubStatus status, string oper, string description) + { + WebMessageSystemService.Instance.UpdateMailboxMessagePublication(mid, status, oper, description); + } + + public void UpdateMailboxMessageStatus(MailboxMsgID mid, MsgStatus status, string folders, string adornments) + { + WebMessageSystemService.Instance.UpdateMailboxMessageStatus(mid, status, folders, adornments); + } + + public void UpdateMailboxMessagesStatus(IEnumerable mids, MsgStatus status, string folders, string adornments) + { + WebMessageSystemService.Instance.UpdateMailboxMessagesStatus(mids, status, folders, adornments); + } + } + + + /// + /// Provides server implementation of Contracts.IWebMessageSystem + /// + public sealed class WebMessageSystemService : ServiceWithInstrumentationBase, Contracts.IWebMessageSystem + { + #region CONSTS + public const string CONFIG_CHANNEL_SECTION = "channel"; + public const string CONFIG_GATEWAY_SECTION = "gateway"; + #endregion + + #region STATIC/.ctor + private static object s_Lock = new object(); + private static volatile WebMessageSystemService s_Instance; + + internal static WebMessageSystemService Instance + { + get + { + var instance = s_Instance; + if (instance == null) + throw new WebMessagingException("{0} is not allocated".Args(typeof(WebMessageSystemService).Name)); + return instance; + } + } + + public WebMessageSystemService(object director) : base(director) + { + lock (s_Lock) + { + if (s_Instance != null) + throw new WebMessagingException("{0} is already allocated".Args(GetType().Name)); + + m_Channels = new Registry(); + m_Gateway = new MessageService(this); + + s_Instance = this; + } + } + + protected override void Destructor() + { + base.Destructor(); + deleteChannels(); + DisposeAndNull(ref m_Gateway); + s_Instance = null; + } + + #endregion + + #region Fields + + private Registry m_Channels; + private MessageService m_Gateway; + + #endregion; + + public override bool InstrumentationEnabled { get; set;} + + #region IWebMessageSystem + public MsgSendInfo[] SendMessage(AgniWebMessage msg) + { + if (msg==null) throw new WebMessagingException(StringConsts.ARGUMENT_ERROR+"{0}.SendMessage(null)".Args(GetType().Name)); + + var deliveryList = new List(); + + var idx = -1; + var matchedAll = true; + foreach (var adr in msg.AddressToBuilder.All) + { + idx++; + var channel = m_Channels[adr.ChannelName]; + if (channel==null) + { + matchedAll = false; + deliveryList.Add( new MsgSendInfo(MsgChannelWriteResult.Gateway, null, idx) ); + continue; + } + + try + { + var result = channel.Write(deliveryList, idx, adr.ChannelAddress, msg); + if (result<0) + deliveryList.Add(new MsgSendInfo(result, null, idx)); + + // TODO: instrumentation by result, etc. + } + catch (Exception error) + { + log(NFX.Log.MessageType.Critical, "sndmsg.chn.wrt", "Channel '{0}' leaked:".Args(channel.Name, error.ToMessageWithType()), error); + deliveryList.Add(new MsgSendInfo(MsgChannelWriteResult.ChannelError, null, idx)); + } + } + + //route to gateway + if (!matchedAll) + m_Gateway.SendMsg(msg); + + return deliveryList.ToArray(); + } + + public MailboxInfo GetMailboxInfo(MailboxID xid) + { + var channel = m_Channels[xid.Channel]; + if (channel==null) return null; + + return channel.GetMailboxInfo(xid); + } + + public int GetMailboxMessageCount(MailboxID xid, string query) + { + var channel = m_Channels[xid.Channel]; + if (channel==null) return 0; + + return channel.GetMessageCount(xid, query); + } + + public MessageHeaders GetMailboxMessageHeaders(MailboxID xid, string query) + { + var channel = m_Channels[xid.Channel]; + if (channel==null) return null; + + return channel.GetMessageHeaders(xid, query); + } + + public AgniWebMessage FetchMailboxMessage(MailboxMsgID mid) + { + var channel = m_Channels[mid.MailboxID.Channel]; + if (channel==null) return null; + + return channel.FetchMailboxMessage(mid); + } + + public Message.Attachment FetchMailboxMessageAttachment(MailboxMsgID mid, int attachmentIndex) + { + var channel = m_Channels[mid.MailboxID.Channel]; + if (channel==null) return null; + + return channel.FetchMailboxMessageAttachment(mid, attachmentIndex); + } + + public void UpdateMailboxMessagePublication(MailboxMsgID mid, MsgPubStatus status, string oper, string description) + { + var channel = m_Channels[mid.MailboxID.Channel]; + if (channel==null) return; + + channel.UpdateMailboxMessagePublication(mid, status, oper, description); + } + + public void UpdateMailboxMessageStatus(MailboxMsgID mid, MsgStatus status, string folders, string adornments) + { + var channel = m_Channels[mid.MailboxID.Channel]; + if (channel==null) return; + + channel.UpdateMailboxMessageStatus(mid, status, folders, adornments); + } + + public void UpdateMailboxMessagesStatus(IEnumerable mids, MsgStatus status, string folders, string adornments) + { + var channelIdsPair = mids.ToLookup(m => m.MailboxID.Channel, m => m) + .ToDictionary(p => p.Key, p => p.AsEnumerable()); + + foreach(var IdChannelPair in channelIdsPair) + { + var channel = m_Channels[IdChannelPair.Key]; + if (channel==null) continue; + channel.UpdateMailboxMessagesStatus(IdChannelPair.Value, status, folders, adornments); + } + } + + #endregion + + #region Protected + + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + deleteChannels(); + if (node==null || !node.Exists) return; + + m_Gateway.Configure(node[CONFIG_GATEWAY_SECTION]);//Configure gateway + + foreach(var cnode in node.Children.Where(cn => cn.IsSameName(CONFIG_CHANNEL_SECTION))) + { + var channel = FactoryUtils.MakeAndConfigure(cnode, args: new object[] {this}); + if (!m_Channels.Register(channel)) throw new WebMessagingException(StringConsts.WM_SERVICE_NO_CHANNELS_ERROR.Args(GetType().Name)); + + } + } + + protected override void DoStart() + { + if (m_Channels.Count==0) + throw new WebMessagingException(StringConsts.WM_SERVICE_DUPLICATE_CHANNEL_ERROR.Args(GetType().Name)); + + m_Gateway.Start(); + + m_Channels.ForEach( c => c.Start() ); + + base.DoStart(); + } + + protected override void DoSignalStop() + { + base.DoSignalStop(); + m_Gateway.SignalStop(); + m_Channels.ForEach( c => c.SignalStop() ); + } + + protected override void DoWaitForCompleteStop() + { + base.DoWaitForCompleteStop(); + + m_Channels.ForEach( c => c.WaitForCompleteStop() ); + m_Gateway.WaitForCompleteStop(); + } + + #endregion + + #region .pvt + + private void deleteChannels() + { + m_Channels.ForEach( c => c.Dispose() ); + m_Channels.Clear(); + } + + private void log(NFX.Log.MessageType tp, string from, string text, Exception error = null, Guid? related = null) + { + App.Log.Write(new NFX.Log.Message + { + Type = tp, + Topic = SysConsts.LOG_TOPIC_WMSG, + From = "{0}.{1}".Args(GetType().Name, from), + Text = text, + Exception = error, + RelatedTo = related ?? Guid.Empty + }); + } + + #endregion + } +} diff --git a/src/Agni/Workers/Attributes.cs b/src/Agni/Workers/Attributes.cs new file mode 100644 index 0000000..1983dff --- /dev/null +++ b/src/Agni/Workers/Attributes.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; + +using NFX; + +namespace Agni.Workers +{ + /// + /// Provides information about the decorated Todo type: Queue name and assignes a globally-unique immutable type id + /// + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] + public sealed class TodoQueueAttribute : GuidTypeAttribute + { + public TodoQueueAttribute(string queueName, string typeGuid) : base(typeGuid) + { + if (queueName.IsNullOrWhiteSpace()) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + GetType().FullName + ".ctor(queueName=null|empty)"); + + QueueName = queueName; + } + + /// + /// Provides the name of the Queue which will store and process the decorated type + /// + public readonly string QueueName; + } + + /// + /// Provides information about the decorated Process type: assignes a globally-unique immutable type id + /// + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] + public sealed class ProcessAttribute : GuidTypeAttribute + { + public ProcessAttribute(string typeGuid) : base(typeGuid) {} + + public string Description { get; set; } + } + + /// + /// Provides information about the decorated Signal type: assignes a globally-unique immutable type id + /// + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] + public sealed class SignalAttribute : GuidTypeAttribute + { + public SignalAttribute(string typeGuid) : base(typeGuid) { } + } +} diff --git a/src/Agni/Workers/CorrelatedTodo.cs b/src/Agni/Workers/CorrelatedTodo.cs new file mode 100644 index 0000000..a015c87 --- /dev/null +++ b/src/Agni/Workers/CorrelatedTodo.cs @@ -0,0 +1,56 @@ +using System; + +using NFX; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; + +namespace Agni.Workers +{ + /// + /// Extends Todo toallow for correlation of multiple instances on SysCorrelationKey. + /// This is used for example to aggregate various asynchronous events together. + /// Example: batch frequent email notifications into one + /// + [Serializable] + public abstract class CorrelatedTodo : Todo + { + /// + /// Denotes values returned by a call to Merge(host, todo) + /// + public enum MergeResult + { + /// + /// The Merge operation did not merge anything, so the another correlated todo should be inserted as a new one leaving the original intact + /// + None = 0, + + /// + /// The original should be left as-is and another instance should be dropped as if it never existed + /// + IgnoreAnother = 1, + + /// + /// The original should be updated with the merged content from another. Another is discarded (as it is already merged into original) + /// + Merged = 2 + } + + + protected CorrelatedTodo() { } + + /// + /// Provides the correlation key which is used for merging the CorrelatedTodo instances. + /// The key technically may be null + /// + public string SysCorrelationKey { get; set; } + + + /// + /// Executes merge operation returning what whould happen to the original and another todo. + /// This method MUST execute be VERY FAST and only contain merge logic, do not make externall IO calls - + /// all business data must already be contained in the original and another instance + /// + protected internal abstract MergeResult Merge(ITodoHost host, DateTime utcNow, CorrelatedTodo another); + } + +} diff --git a/src/Agni/Workers/Exceptions.cs b/src/Agni/Workers/Exceptions.cs new file mode 100644 index 0000000..b22868a --- /dev/null +++ b/src/Agni/Workers/Exceptions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Runtime.Serialization; + +using NFX; + +namespace Agni.Workers +{ + /// + /// Thrown to indicate workers related problems + /// + [Serializable] + public class WorkersException : AgniException + { + public WorkersException() : base() {} + public WorkersException(string message) : base(message) {} + public WorkersException(string message, Exception inner) : base(message, inner) { } + protected WorkersException(SerializationInfo info, StreamingContext context) : base(info, context) { } + } +} diff --git a/src/Agni/Workers/Intfs.cs b/src/Agni/Workers/Intfs.cs new file mode 100644 index 0000000..6359910 --- /dev/null +++ b/src/Agni/Workers/Intfs.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Instrumentation; +using NFX.Log; +using NFX.Security; +using Agni.Coordination; + +namespace Agni.Workers +{ + /// + /// Represents the Todo hosting entity + /// + public interface ITodoHost + { + /// + /// WARNING: this method should not be used by a typical business code as it enqueues messages + /// locally bypassing all routing and networking stack. This method is used for performance optimization + /// for some limited Todo instances that do not rely on sequencing and sharding and are guaranteed to have + /// a local queue capable of processing this message + /// + void LocalEnqueue(Todo todos); + + /// + /// WARNING: this method should not be used by a typical business code as it enqueues messages + /// locally bypassing all routing and networking stack. This method is used for performance optimization + /// for some limited Todo instances that do not rely on sequencing and sharding and are guaranteed to have + /// a local queue capable of processing this message + /// + void LocalEnqueue(IEnumerable todos); + + /// + /// WARNING: this method should not be used by a typical business code as it enqueues messages + /// locally bypassing all routing and networking stack. This method is used for performance optimization + /// for some limited Todo instances that do not rely on sequencing and sharding and are guaranteed to have + /// a local queue capable of processing this message + /// + Task LocalEnqueueAsync(IEnumerable todos); + + /// + /// Returns true if host is instrumented + /// + bool InstrumentationEnabled { get;} + + /// + /// Emits a local log message based on host's logging policy + /// + Guid Log(MessageType type, + Todo todo, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null); + + /// + /// Emits a local log message based on host's logging policy + /// + Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null); + } + + /// + /// Represents the Todo hosting entity + /// + public interface IProcessHost + { + /// + /// WARNING: this method should not be used by a typical business code as it enqueues messages + /// locally bypassing all routing and networking stack. This method is used for performance optimization + /// for some limited Todo instances that do not rely on sequencing and sharding and are guaranteed to have + /// a local queue capable of processing this message + /// + void LocalSpawn(Process process, AuthenticationToken? token = null); + + ResultSignal LocalDispatch(Signal signal); + + void Update(Process process, bool sysOnly = false); + + void Finalize(Process process); + + /// + /// Returns true if host is instrumented + /// + bool InstrumentationEnabled { get; } + + /// + /// Emits a local log message based on host's logging policy + /// + Guid Log(MessageType type, + Process todo, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null); + + /// + /// Emits a local log message based on host's logging policy + /// + Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null); + } + + public interface IProcessManager + { + /// + /// Allocates process identifier (PID) in the specified zone + /// + PID Allocate(string zonePath); + + /// + /// Allocates process identifier (PID) in the specified zone based on a mutualy exclusive ID (mutex). + /// Mutexes are case-insensitive + /// + PID AllocateMutex(string zonePath, string mutex); + + /// + /// Starts the process at the host specified by PID + /// + void Spawn(TProcess process) where TProcess : Process; + void Spawn(PID pid, IConfigSectionNode args, Type type = null); + void Spawn(PID pid, IConfigSectionNode args, Guid type); + + Task Async_Spawn(TProcess process) where TProcess : Process; + Task Async_Spawn(PID pid, IConfigSectionNode args, Type type = null); + Task Async_Spawn(PID pid, IConfigSectionNode args, Guid type); + + ResultSignal Dispatch(TSignal signal) where TSignal : Signal; + ResultSignal Dispatch(PID pid, IConfigSectionNode args, Type type = null); + ResultSignal Dispatch(PID pid, IConfigSectionNode args, Guid type); + + Task Async_Dispatch(TSignal signal) where TSignal : Signal; + Task Async_Dispatch(PID pid, IConfigSectionNode args, Type type = null); + Task Async_Dispatch(PID pid, IConfigSectionNode args, Guid type); + + int Enqueue(IEnumerable todos, string hostSetName, string svcName) where TTodo : Todo; + void Enqueue(TTodo todo, string hostSetName, string svcName) where TTodo : Todo; + void Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Type type = null); + void Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Guid type); + + Task Async_Enqueue(IEnumerable todos, string hostSetName, string svcName) where TTodo : Todo; + Task Async_Enqueue(TTodo todo, string hostSetName, string svcName) where TTodo : Todo; + Task Async_Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Type type = null); + Task Async_Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Guid type); + + TProcess Get(PID pid) where TProcess : Process; + + ProcessDescriptor GetDescriptor(PID pid); + + IEnumerable List(string zonePath, IConfigSectionNode filter = null); + + IGuidTypeResolver ProcessTypeResolver { get; } + IGuidTypeResolver SignalTypeResolver { get; } + IGuidTypeResolver TodoTypeResolver { get; } + + IRegistry HostSets { get; } + } + + + public interface IProcessManagerImplementation : IProcessManager, IApplicationComponent, IDisposable, IConfigurable, IInstrumentable + { + } +} diff --git a/src/Agni/Workers/PID.cs b/src/Agni/Workers/PID.cs new file mode 100644 index 0000000..696dc30 --- /dev/null +++ b/src/Agni/Workers/PID.cs @@ -0,0 +1,85 @@ +using System; + +using NFX; +using NFX.Serialization.JSON; + +namespace Agni.Workers +{ + /// + /// Represents a globally-unique process identifier. The pids are allocated of AgniSystem.ProcessManager + /// + [Serializable] + public struct PID : IEquatable + { + public static readonly PID Zero = new PID(null, -1, null, false); + + public PID(string zone, int processorID, string id, bool isUnique) { Zone = zone; ProcessorID = processorID; ID = id; IsUnique = isUnique; } + + public PID(JSONDataMap dataMap) + { + Zone = dataMap[nameof(Zone)] .AsString(); + ProcessorID = dataMap[nameof(ProcessorID)].AsInt(); + ID = dataMap[nameof(ID)] .AsString(); + IsUnique = dataMap[nameof(IsUnique)] .AsBool(); + } + + public readonly string Zone; + public readonly int ProcessorID; + public readonly string ID; + public readonly bool IsUnique; + + public override string ToString() { return "PID[{0}:{1}:{2}]{3}".Args(Zone, ProcessorID, ID, IsUnique ? string.Empty : "*"); } + public override int GetHashCode() { return ID.GetHashCode(); } + + public override bool Equals(object obj) + { + if (!(obj is PID)) return false; + return this.Equals((PID)obj); + } + + public bool Equals(PID other) + { + return this.Zone.EqualsOrdIgnoreCase(other.Zone) && + this.ProcessorID == other.ProcessorID && + this.ID.EqualsOrdIgnoreCase(other.ID) && + this.IsUnique == other.IsUnique; + } + + public static PID Parse(string str) + { + PID parsed; + if (!TryParse(str, out parsed)) + throw new WorkersException(StringConsts.PID_PARSE_ERROR.Args(str)); + return parsed; + } + + public static bool TryParse(string str, out PID? pid) + { + PID parsed; + if (TryParse(str, out parsed)) + { + pid = parsed; + return true; + } + + pid = null; + return false; + } + + public static bool TryParse(string str, out PID pid) + { + pid = Zero; + var afterZone = str.IndexOf(':'); + var zone = str.Substring(4, afterZone - 4); + var afterProcessorID = str.IndexOf(':', ++afterZone); + int processorID; + if (!int.TryParse(str.Substring(afterZone, afterProcessorID - afterZone), out processorID)) return false; + var afterID = str.IndexOf(']', ++afterProcessorID); + var id = str.Substring(afterProcessorID, afterID - afterProcessorID); + afterID++; + var isUnique = afterID == str.Length || str[afterID] != '*'; + pid = new PID(zone, processorID, id, isUnique); + return true; + } + } +} diff --git a/src/Agni/Workers/Process.cs b/src/Agni/Workers/Process.cs new file mode 100644 index 0000000..007e71c --- /dev/null +++ b/src/Agni/Workers/Process.cs @@ -0,0 +1,148 @@ +using System; + +using NFX; +using NFX.DataAccess.CRUD; +using NFX.Environment; +using NFX.Security; + +namespace Agni.Workers +{ + /// + /// Represents a context for units of work (like Todos) executing in the distributed system. + /// Processes are controlled via AgniSystem.ProcessManager implementation + /// + [Serializable] + public abstract class Process : AmorphousTypedRow + { + /// + /// Factory method that creates new Process based on provided PID + /// + public static TProcess MakeNew(PID pid) where TProcess : Process, new() { return makeDefault(new TProcess(), pid); } + + /// + /// Factory method that creates new Process based on provided Type, PID and Configuration + /// + public static Process MakeNew(Type type, PID pid, IConfigSectionNode args) { return makeDefault(FactoryUtils.MakeAndConfigure(args, type), pid); } + + private static TProcess makeDefault(TProcess process, PID pid) where TProcess : Process + { + var attr = GuidTypeAttribute.GetGuidTypeAttribute(process.GetType()); + + var descriptor = new ProcessDescriptor( + pid, + attr.Description, + App.TimeSource.UTCNow, + "{0}@{1}@{2}".Args(App.CurrentCallUser.Name, App.Name, AgniSystem.HostName)); + + process.m_SysDescriptor = descriptor; + return process; + } + + protected Process() { } + + private ProcessDescriptor m_SysDescriptor; + + /// + /// Infrustructure method, developers do not call + /// + public void ____Deserialize(ProcessDescriptor descriptor) + { m_SysDescriptor = descriptor; } + + /// + /// Globally-unique ID of the TODO + /// + public PID SysPID { get { return m_SysDescriptor.PID; } } + + /// + /// When was created + /// + public ProcessDescriptor SysDescriptor { get { return m_SysDescriptor; } } + + public override string ToString() { return "{0}('{1}')".Args(GetType().FullName, SysPID); } + + public override int GetHashCode() { return m_SysDescriptor.GetHashCode(); } + + public override bool Equals(Row other) + { + var otherProcess = other as Process; + if (otherProcess==null) return false; + return this.m_SysDescriptor.PID.ID == otherProcess.m_SysDescriptor.PID.ID; + } + + /// + /// Executes merge operation on the existing process and another instance which tries to get spawned. + /// This method MUST execute be VERY FAST and only contain merge logic, do not make externall IO calls - + /// all business data must already be contained in the original and another instance + /// + protected internal abstract void Merge(IProcessHost host, DateTime utcNow, Process another); + + protected internal virtual ResultSignal Accept(IProcessHost host, Signal signal) + { + if (signal is FinalizeSignal) + { + host.Finalize(this); + return OkSignal.Make(this); + } + if (signal is TerminateSignal) + { + var finish = signal as TerminateSignal; + UpdateStatus(host, ProcessStatus.Terminated, "Treminated!!!!", App.TimeSource.UTCNow, finish.SysAbout); + host.Update(this, true); + return OkSignal.Make(this); + } + // a pochemy net try/cath i ERROR signal???? ili eto v drugom meste + var result = DoAccept(host, signal); //Do Accept razve ne doljen byt POSLE systemnix signalov?? + + if (result != null) return result; + + // a gde ostalnie signali? Terminate etc...? + + if (signal is FinishSignal) + { + var finish = signal as FinishSignal; + UpdateStatus(host, ProcessStatus.Finished, finish.Description, App.TimeSource.UTCNow, finish.SysAbout); + host.Update(this, true); + return OkSignal.Make(this); + } + + if (signal is CancelSignal) + { + var finish = signal as CancelSignal; + UpdateStatus(host, ProcessStatus.Canceled, "Canceled!!!", App.TimeSource.UTCNow, finish.SysAbout); + host.Update(this, true); + return OkSignal.Make(this); + } + + return UnknownSignal.Make(this, signal); + } + + protected abstract ResultSignal DoAccept(IProcessHost host, Signal signal); + + protected void UpdateStatus(IProcessHost host, ProcessStatus status, string description, DateTime timestamp, string about) + { + m_SysDescriptor = new ProcessDescriptor(SysDescriptor, status, description, timestamp, about); + } + + public void ValidateAndPrepareForSpawn(string targetName) + { + DoPrepareForEnqueuePreValidate(targetName); + + var ve = this.Validate(targetName); + if (ve != null) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + "Process.ValidateAndPrepareForEnqueue(todo).validate: " + ve.ToMessageWithType(), ve); + + DoPrepareForEnqueuePostValidate(targetName); + } + + public override Exception Validate(string targetName) + { + var ve = base.Validate(targetName); + if (ve != null) return ve; + + return null; + } + + protected virtual void DoPrepareForEnqueuePreValidate(string targetName) { } + protected virtual void DoPrepareForEnqueuePostValidate(string targetName) { } + } +} diff --git a/src/Agni/Workers/ProcessDescriptor.cs b/src/Agni/Workers/ProcessDescriptor.cs new file mode 100644 index 0000000..bfb11e4 --- /dev/null +++ b/src/Agni/Workers/ProcessDescriptor.cs @@ -0,0 +1,60 @@ +using System; + +using NFX; + +namespace Agni.Workers +{ + /// + /// Provides process status information snapshot + /// + [Serializable] + public struct ProcessDescriptor + { + public ProcessDescriptor(PID pid, string description, DateTime timestamp, string about) + { + PID = pid; + Description = description; + Timestamp = timestamp; + About = about; + Status = ProcessStatus.Created; + StatusDescription = description; + StatusTimestamp = timestamp; + StatusAbout = about; + } + + public ProcessDescriptor(ProcessDescriptor processDescriptor, ProcessStatus status, string description, DateTime timestamp, string about) + { + PID = processDescriptor.PID; + Description = processDescriptor.Description; + Timestamp = processDescriptor.Timestamp; + About = processDescriptor.About; + Status = status; + StatusDescription = description; + StatusTimestamp = timestamp; + StatusAbout = about; + } + + public ProcessDescriptor(PID pid, string description, DateTime timestamp, string about, ProcessStatus status, string statusDescription, DateTime statusTimestamp, string statusAbout) + { + PID = pid; + Description = description; + Timestamp = timestamp; + About = about; + Status = status; + StatusDescription = statusDescription; + StatusTimestamp = statusTimestamp; + StatusAbout = statusAbout; + } + + public readonly PID PID; + public readonly string Description; // Caller supplied + public readonly DateTime Timestamp; + public readonly string About; // [UserName]@[AppName]@[HostName] + public readonly ProcessStatus Status; + public readonly string StatusDescription; + public readonly DateTime StatusTimestamp; + public readonly string StatusAbout; + + public override string ToString() { return "{0}:{1}:{2}({3}) - {4}".Args(Status, PID, About, Timestamp, Description); } + } +} diff --git a/src/Agni/Workers/ProcessFrame.cs b/src/Agni/Workers/ProcessFrame.cs new file mode 100644 index 0000000..18a7bf5 --- /dev/null +++ b/src/Agni/Workers/ProcessFrame.cs @@ -0,0 +1,128 @@ + +using System; +using System.Collections.Generic; + +using NFX; +using NFX.Serialization.BSON; +using NFX.Security; + +using Agni.Security; + +namespace Agni.Workers +{ + + /// + /// Denotes process statuses: Created, Started, Finished etc.. + /// + public enum ProcessStatus + { + Created, + Started, + Finished, + Canceled, + Terminated, + Failed + } + + + /// + /// Provides an efficient data vector for marshalling and storage of process. + /// This type obviates the need of extra serialization for teleportation and storage of Process instances. + /// Special-purposed Glue binding is used to teleport ProcessFrames and directly store them without + /// unnecessary intermediate serialization steps + /// + [Serializable] + public struct ProcessFrame + { + public const int SERIALIZER_DEFAULT = 0; + public const int SERIALIZER_BSON = 1; + + public Guid Type; + public ProcessDescriptor Descriptor; + + public int Serializer; + public byte[] Content; + + /// + /// Internal. Returns the original instance that was passed to .ctor + /// This allows to use this structure for dual purpose. + /// + [NonSerialized] internal readonly Process ____CtorOriginal; + + /// + /// Frames the Todo instance, pass serialize null to frame only Sys Fields without content + /// + public ProcessFrame(Process process, int? serializer = SERIALIZER_DEFAULT) + { + if (process == null) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + "ProcessFrame.ctor(process==null)"); + + ____CtorOriginal = process; + + var qa = GuidTypeAttribute.GetGuidTypeAttribute(process.GetType()); + + if (serializer.HasValue) + { + if (serializer == SERIALIZER_DEFAULT) serializer = SERIALIZER_BSON; + else + if (serializer != SERIALIZER_BSON)//we only support BSON for now + throw new WorkersException(StringConsts.PROCESS_FRAME_SER_NOT_SUPPORTED_ERROR.Args(process.GetType().Name, serializer)); + + byte[] content; + try + { + var cdoc = RowConverter.DefaultInstance.RowToBSONDocument(process, null); + content = cdoc.WriteAsBSONToNewArray(); + } + catch (Exception error) + { + throw new WorkersException(StringConsts.PROCESS_FRAME_SER_ERROR.Args(process.GetType().FullName, error.ToMessageWithType()), error); + } + + this.Serializer = serializer.Value; + this.Content = content; + } + else + { + this.Serializer = 0; + this.Content = null; + } + + this.Type = qa.TypeGuid; + this.Descriptor = process.SysDescriptor; + } + + private static Dictionary s_TypesCache = new Dictionary(StringComparer.Ordinal); + + /// + /// Materializes the Process instance represented by this frame in the scope of IGuidTypeResolver + /// + public Process Materialize(IGuidTypeResolver resolver) + { + if (____CtorOriginal!=null) return ____CtorOriginal; + + //1. Resolve type + var type = resolver.Resolve(this.Type); + + //3. Create Process instance + var result = (Process)NFX.Serialization.SerializationUtils.MakeNewObjectInstance( type ); + result.____Deserialize(Descriptor); + + //4. Deserialize content + if (Serializer!=ProcessFrame.SERIALIZER_BSON) //for now only support this serializer + throw new WorkersException(StringConsts.PROCESS_FRAME_DESER_NOT_SUPPORTED_ERROR.Args(type.Name, Serializer)); + + try + { + var docContent = BSONDocument.FromArray(Content); + RowConverter.DefaultInstance.BSONDocumentToRow(docContent, result, null); + } + catch(Exception error) + { + throw new WorkersException(StringConsts.PROCESS_FRAME_DESER_ERROR.Args(type.Name, error.ToMessageWithType()), error); + } + + return result; + } + } +} diff --git a/src/Agni/Workers/ProcessManager.cs b/src/Agni/Workers/ProcessManager.cs new file mode 100644 index 0000000..bf47093 --- /dev/null +++ b/src/Agni/Workers/ProcessManager.cs @@ -0,0 +1,146 @@ +using System; +using System.Linq; +using System.Collections.Generic; +using System.Threading.Tasks; + +using Agni.AppModel; +using Agni.Coordination; +using NFX.Environment; + +namespace Agni.Workers +{ + public sealed class ProcessManager : ProcessManagerBase + { + public ProcessManager(IAgniApplication director) : base(director) + { + } + + protected override PID DoAllocate(string zonePath, string id, bool isUnique) + { + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(zonePath); + var processorID = zone.MapShardingKeyToProcessorID(id); + + return new PID(zone.RegionPath, processorID, id, isUnique); + } + + protected override void DoSpawn(TProcess process) + { + var pid = process.SysPID; + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(pid.Zone); + var hosts = zone.GetProcessorHostsByID(pid.ProcessorID); + + Contracts.ServiceClientHub.CallWithRetry + ( + (controller) => controller.Spawn(new ProcessFrame(process)), + hosts.Select(h => h.RegionPath) + ); + } + + protected override Task Async_DoSpawn(TProcess process) + { + var pid = process.SysPID; + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(pid.Zone); + var hosts = zone.GetProcessorHostsByID(pid.ProcessorID); + + return Contracts.ServiceClientHub.CallWithRetryAsync + ( + (controller) => controller.Async_Spawn(new ProcessFrame(process)).AsTaskReturningVoid(), + hosts.Select(h => h.RegionPath) + ); + } + + protected override ResultSignal DoDispatch(TSignal signal) + { + var pid = signal.SysPID; + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(pid.Zone); + var hosts = zone.GetProcessorHostsByID(pid.ProcessorID); + + return Contracts.ServiceClientHub.CallWithRetry + ( + (controller) => controller.Dispatch(new SignalFrame(signal)), + hosts.Select(h => h.RegionPath) + ).Materialize(SignalTypeResolver) as ResultSignal; + } + + protected override Task Async_DoDispatch(TSignal signal) + { + var pid = signal.SysPID; + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(pid.Zone); + var hosts = zone.GetProcessorHostsByID(pid.ProcessorID); + + return Contracts.ServiceClientHub.CallWithRetryAsync + ( + (controller) => controller.Async_Dispatch(new SignalFrame(signal)).AsTaskReturning(), + hosts.Select(h => h.RegionPath) + ).ContinueWith((antecedent) => antecedent.Result.Materialize(SignalTypeResolver) as ResultSignal); + } + + protected override int DoEnqueue(IEnumerable todos, HostSet hs, string svcName) + { + var hostPair = hs.AssignHost(todos.First().SysShardingKey); + return Contracts.ServiceClientHub.CallWithRetry + ( + (client) => client.Enqueue(todos.Select(t => new TodoFrame(t)).ToArray()), + hostPair.Select(host => host.RegionPath), + svcName: svcName + ); + } + + protected override Task Async_DoEnqueue(IEnumerable todos, HostSet hs, string svcName) + { + var hostPair = hs.AssignHost(todos.First().SysShardingKey); + return Contracts.ServiceClientHub.CallWithRetryAsync + ( + (client) => client.Async_Enqueue(todos.Select(t => new TodoFrame(t)).ToArray()).AsTaskReturning(), + hostPair.Select(host => host.RegionPath), + svcName: svcName + ); + } + + protected override TProcess DoGet(PID pid) + { + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(pid.Zone); + var hosts = zone.GetProcessorHostsByID(pid.ProcessorID); + + var processFrame = Contracts.ServiceClientHub.CallWithRetry + ( + (controller) => controller.Get(pid), + hosts.Select(h => h.RegionPath) + ); + + // TODO Check type + return processFrame.Materialize(ProcessTypeResolver) as TProcess; + } + + protected override ProcessDescriptor DoGetDescriptor(PID pid) + { + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(pid.Zone); + var hosts = zone.GetProcessorHostsByID(pid.ProcessorID); + + return Contracts.ServiceClientHub.CallWithRetry + ( + (controller) => controller.GetDescriptor(pid), + hosts.Select(h => h.RegionPath) + ); + } + + protected override IEnumerable DoList(string zonePath, IConfigSectionNode filter) + { + var tasks = new List>>(); + + var zone = AgniSystem.Metabase.CatalogReg.NavigateZone(zonePath); + foreach (var processorID in zone.ProcessorMap.Keys) + { + var hosts = zone.GetProcessorHostsByID(processorID); + var descriptors = Contracts.ServiceClientHub.CallWithRetry> + ( + (controller) => controller.List(processorID), + hosts.Select(h => h.RegionPath) + ); + + foreach (var descriptor in descriptors) + yield return descriptor; + } + } + } +} diff --git a/src/Agni/Workers/ProcessManagerBase.cs b/src/Agni/Workers/ProcessManagerBase.cs new file mode 100644 index 0000000..cc26d72 --- /dev/null +++ b/src/Agni/Workers/ProcessManagerBase.cs @@ -0,0 +1,304 @@ +using System; +using System.Linq; +using System.Collections.Generic; + +using NFX; +using NFX.Environment; +using NFX.ServiceModel; + +using Agni.AppModel; +using Agni.Coordination; +using System.Threading.Tasks; + +namespace Agni.Workers +{ + public abstract class ProcessManagerBase : ServiceWithInstrumentationBase, IProcessManagerImplementation + { + #region CONSTS + public const string CONFIG_PROCESS_MANAGER_SECTION = "process-manager"; + public const string CONFIG_PROCESS_TYPE_RESOLVER_SECTION = "process-type-resolver"; + public const string CONFIG_SIGNAL_TYPE_RESOLVER_SECTION = "signal-type-resolver"; + public const string CONFIG_TODO_TYPE_RESOLVER_SECTION = "todo-type-resolver"; + public const string CONFIG_PATH_ATTR = "path"; + public const string CONFIG_SEARCH_PARENT_ATTR = "search-parent"; + public const string CONFIG_TRANSCEND_NOC_ATTR = "transcend-noc"; + public const string CONFIG_TYPE_GUID_ATTR = "type-guid"; + + private static readonly TimeSpan INSTRUMENTATION_INTERVAL = TimeSpan.FromMilliseconds(3700); + #endregion + + #region .ctor + public ProcessManagerBase(IAgniApplication director) : base(director) + { + m_HostSets = new Registry(false); + } + + protected override void Destructor() + { + foreach (var hs in m_HostSets) + hs.Dispose(); + + DisposableObject.DisposeAndNull(ref m_InstrumentationEvent); + base.Destructor(); + } + #endregion + + #region Fields + private IGuidTypeResolver m_ProcessTypeResolver; + private IGuidTypeResolver m_SignalTypeResolver; + private IGuidTypeResolver m_TodoTypeResolver; + private Registry m_HostSets; + + + private bool m_InstrumentationEnabled; + private NFX.Time.Event m_InstrumentationEvent; + + private NFX.Collections.NamedInterlocked m_Stats = new NFX.Collections.NamedInterlocked(); + #endregion + + #region Properties + public IGuidTypeResolver ProcessTypeResolver { get { return m_ProcessTypeResolver; } } + + public IGuidTypeResolver SignalTypeResolver { get { return m_SignalTypeResolver; } } + + public IGuidTypeResolver TodoTypeResolver { get { return m_TodoTypeResolver; } } + + /// + /// Registry of all HostSets in the hub + /// + public IRegistry HostSets { get { return m_HostSets; } } + + /// + /// Implements IInstrumentable + /// + [Config(Default = false)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_LOCKING, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled + { + get { return m_InstrumentationEnabled; } + set + { + m_InstrumentationEnabled = value; + if (m_InstrumentationEvent == null) + { + if (!value) return; + m_Stats.Clear(); + m_InstrumentationEvent = new NFX.Time.Event(App.EventTimer, null, e => AcceptManagerVisit(this, e.LocalizedTime), INSTRUMENTATION_INTERVAL); + } + else + { + if (value) return; + DisposableObject.DisposeAndNull(ref m_InstrumentationEvent); + m_Stats.Clear(); + } + } + } + #endregion + + #region Public + public PID Allocate(string zonePath) + { + if (zonePath.IsNullOrWhiteSpace()) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".AllocateMutex(zontPath=null|empty)"); + + var gdid = AgniSystem.GDIDProvider.GenerateOneGDID(SysConsts.GDID_NS_WORKER, SysConsts.GDID_NAME_WORKER_PROCESS); + return DoAllocate(zonePath, gdid.ToString(), true); + } + + /// + /// Allocates PID by mutex. Mutexes are case-insensitive + /// + public PID AllocateMutex(string zonePath, string mutex) + { + if (zonePath.IsNullOrWhiteSpace() || mutex.IsNullOrWhiteSpace()) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + GetType().Name + ".AllocateMutex((zontPath|mutex)=null|empty)"); + + mutex = mutex.ToLowerInvariant(); + + return DoAllocate(zonePath, mutex, false); + } + + public void Spawn(PID pid, IConfigSectionNode args, Guid type) { Spawn(pid, args, ProcessTypeResolver.Resolve(type)); } + + public void Spawn(PID pid, IConfigSectionNode args, Type type = null) + { + var guid = args.AttrByName(CONFIG_TYPE_GUID_ATTR).ValueAsGUID(Guid.Empty); + if (type == null && guid != Guid.Empty) + type = ProcessTypeResolver.Resolve(guid); + Spawn(Process.MakeNew(type, pid, args)); + } + + public void Spawn(TProcess process) where TProcess : Process + { + process.ValidateAndPrepareForSpawn(null); + DoSpawn(process); + } + + public Task Async_Spawn(PID pid, IConfigSectionNode args, Guid type) { return Async_Spawn(pid, args, ProcessTypeResolver.Resolve(type)); } + + public Task Async_Spawn(PID pid, IConfigSectionNode args, Type type = null) + { + var guid = args.AttrByName(CONFIG_TYPE_GUID_ATTR).ValueAsGUID(Guid.Empty); + if (type == null && guid != Guid.Empty) + type = ProcessTypeResolver.Resolve(guid); + return Async_Spawn(Process.MakeNew(type, pid, args)); + } + + public Task Async_Spawn(TProcess process) where TProcess : Process + { + process.ValidateAndPrepareForSpawn(null); + return Async_DoSpawn(process); + } + + public ResultSignal Dispatch(PID pid, IConfigSectionNode args, Guid type) { return Dispatch(pid, args, SignalTypeResolver.Resolve(type)); } + + public ResultSignal Dispatch(PID pid, IConfigSectionNode args, Type type = null) + { + var guid = args.AttrByName(CONFIG_TYPE_GUID_ATTR).ValueAsGUID(Guid.Empty); + if (type == null && guid != Guid.Empty) + type = SignalTypeResolver.Resolve(guid); + return Dispatch(Signal.MakeNew(type, pid, args)); + } + + public ResultSignal Dispatch(TSignal signal) where TSignal : Signal + { + signal.ValidateAndPrepareForDispatch(null); + return DoDispatch(signal); + } + + public Task Async_Dispatch(PID pid, IConfigSectionNode args, Guid type){ return Async_Dispatch(pid, args, SignalTypeResolver.Resolve(type)); } + + public Task Async_Dispatch(PID pid, IConfigSectionNode args, Type type = null) + { + var guid = args.AttrByName(CONFIG_TYPE_GUID_ATTR).ValueAsGUID(Guid.Empty); + if (type == null && guid != Guid.Empty) + type = SignalTypeResolver.Resolve(guid); + return Async_Dispatch(Signal.MakeNew(type, pid, args)); + } + + public Task Async_Dispatch(TSignal signal) where TSignal : Signal + { + signal.ValidateAndPrepareForDispatch(null); + return Async_DoDispatch(signal); + } + + public void Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Guid type) { Enqueue(hostSetName, svcName, args, TodoTypeResolver.Resolve(type)); } + + public void Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Type type = null) + { + var guid = args.AttrByName(CONFIG_TYPE_GUID_ATTR).ValueAsGUID(Guid.Empty); + if (type == null && guid != Guid.Empty) + type = TodoTypeResolver.Resolve(guid); + Enqueue(Todo.MakeNew(type, args), hostSetName, svcName); + } + + public void Enqueue(TTodo todo, string hostSetName, string svcName) where TTodo : Todo { Enqueue(new[] { todo }, hostSetName, svcName); } + + public int Enqueue(IEnumerable todos, string hostSetName, string svcName) where TTodo : Todo + { + if (todos == null || !todos.Any()) return 0; + + todos.ForEach(todo => todo.ValidateAndPrepareForEnqueue(null)); + + var hs = getHostSet(hostSetName); + return DoEnqueue(todos, hs, svcName); + } + + public Task Async_Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Guid type) { return Async_Enqueue(hostSetName, svcName, args, TodoTypeResolver.Resolve(type)); } + + public Task Async_Enqueue(string hostSetName, string svcName, IConfigSectionNode args, Type type = null) + { + var guid = args.AttrByName(CONFIG_TYPE_GUID_ATTR).ValueAsGUID(Guid.Empty); + if (type == null && guid != Guid.Empty) + type = TodoTypeResolver.Resolve(guid); + return Async_Enqueue(Todo.MakeNew(type, args), hostSetName, svcName); + } + + public Task Async_Enqueue(TTodo todo, string hostSetName, string svcName) where TTodo : Todo { return Async_Enqueue(new[] { todo }, hostSetName, svcName); } + + public Task Async_Enqueue(IEnumerable todos, string hostSetName, string svcName) where TTodo : Todo + { + if (todos == null || !todos.Any()) return Task.FromResult(0); + + todos.ForEach(todo => todo.ValidateAndPrepareForEnqueue(null)); + + var hs = getHostSet(hostSetName); + return Async_DoEnqueue(todos, hs, svcName); + } + + public TProcess Get(PID pid) where TProcess : Process { return DoGet(pid); } + + public ProcessDescriptor GetDescriptor(PID pid) { return DoGetDescriptor(pid); } + + public IEnumerable List(string zonePath, IConfigSectionNode filter = null) { return DoList(zonePath, filter); } + #endregion + + #region Protected + protected abstract PID DoAllocate(string zonePath, string id, bool isUnique); + protected abstract void DoSpawn(TProcess process) where TProcess : Process; + protected abstract Task Async_DoSpawn(TProcess process) where TProcess : Process; + protected abstract ResultSignal DoDispatch(TSignal signal) where TSignal : Signal; + protected abstract Task Async_DoDispatch(TSignal signal) where TSignal : Signal; + protected abstract int DoEnqueue(IEnumerable todos, HostSet hs, string svcName) where TTodo : Todo; + protected abstract Task Async_DoEnqueue(IEnumerable todos, HostSet hs, string svcName) where TTodo : Todo; + protected abstract TProcess DoGet(PID pid) where TProcess : Process; + protected abstract ProcessDescriptor DoGetDescriptor(PID pid); + protected abstract IEnumerable DoList(string zonePath, IConfigSectionNode filter); + + protected override void DoConfigure(IConfigSectionNode node) + { + + if (node == null) + node = App.ConfigRoot[CONFIG_PROCESS_MANAGER_SECTION]; + + base.DoConfigure(node); + + var nptr = node[CONFIG_PROCESS_TYPE_RESOLVER_SECTION]; + m_ProcessTypeResolver = FactoryUtils.Make(nptr, typeof(GuidTypeResolver), new[] { nptr }); + + var nstr = node[CONFIG_SIGNAL_TYPE_RESOLVER_SECTION]; + m_SignalTypeResolver = FactoryUtils.Make(nstr, typeof(GuidTypeResolver), new[] { nstr }); + + var nttr = node[CONFIG_TODO_TYPE_RESOLVER_SECTION]; + m_TodoTypeResolver = FactoryUtils.Make(nttr, typeof(GuidTypeResolver), new[] { nttr }); + + if (node == null || !node.Exists) return; + + foreach (var cn in node.Children.Where(cn => cn.IsSameName(Metabase.Metabank.CONFIG_HOST_SET_SECTION))) + { + var name = cn.AttrByName(Configuration.CONFIG_NAME_ATTR).Value; + if (name.IsNullOrWhiteSpace()) + throw new CoordinationException(StringConsts.PM_HOSTSET_CONFIG_MISSING_NAME_ERROR); + + var path = cn.AttrByName(CONFIG_PATH_ATTR).Value; + + if (path.IsNullOrWhiteSpace()) + throw new CoordinationException(StringConsts.PM_HOSTSET_CONFIG_PATH_MISSING_ERROR.Args(name)); + + var spar = cn.AttrByName(CONFIG_SEARCH_PARENT_ATTR).ValueAsBool(true); + var tNoc = cn.AttrByName(CONFIG_TRANSCEND_NOC_ATTR).ValueAsBool(false); + var hset = HostSet.FindAndBuild(name, path, spar, tNoc); + + var added = m_HostSets.Register(hset); + if (!added) + throw new CoordinationException(StringConsts.PM_HOSTSET_CONFIG_DUPLICATE_NAME_ERROR.Args(name)); + } + } + #endregion + + #region Private + private HostSet getHostSet(string hostSetName) + { + if (hostSetName == null) + throw new CoordinationException(StringConsts.ARGUMENT_ERROR + "HostSetHub.EnqueueTodo(hostSetName=null|empty)"); + + var result = m_HostSets[hostSetName]; + + if (result == null) + throw new CoordinationException(StringConsts.ARGUMENT_ERROR + "HostSetHub.EnqueueTodo(hostSetName'{0}') not found".Args(hostSetName)); + + return result; + } + #endregion + } +} diff --git a/src/Agni/Workers/Server/AgentServiceBase.cs b/src/Agni/Workers/Server/AgentServiceBase.cs new file mode 100644 index 0000000..44cc4b6 --- /dev/null +++ b/src/Agni/Workers/Server/AgentServiceBase.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.Log; +using NFX.Environment; +using NFX.ServiceModel; +using System.Threading; +using NFX.Instrumentation; + +namespace Agni.Workers.Server +{ + public abstract class AgentServiceBase : ServiceWithInstrumentationBase + { + public const int THREAD_GRANULARITY_MS = 1750; + public const int DEFAULT_INSTRUMENTATION_GRANULARITY_MS = 5000; + public const int DEFAULT_STARTUP_DELAY_SEC = 60; + + + protected AgentServiceBase(object director) : base(director) + { + LogLevel = MessageType.Error; + } + + private Thread m_Thread; + private int m_StartupDelaySec = DEFAULT_STARTUP_DELAY_SEC; + + [Config] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_WORKER, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled { get; set; } + + [Config(Default = MessageType.Error)] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_WORKER, CoreConsts.EXT_PARAM_GROUP_LOG)] + public MessageType LogLevel { get; set; } + + + [Config(Default = DEFAULT_STARTUP_DELAY_SEC)] + public int StartupDelaySec + { + get { return m_StartupDelaySec; } + set { m_StartupDelaySec = value<0 ? 0 : value; } + } + + /// + /// Overrides main thread granularity, return 0 for auto + /// + public virtual int ThreadGranularityMs { get { return 0;} } + + /// + /// Specifies how often isntrumentation gets dumped, return 0 for default + /// + [Config] + public virtual int InstrumentationGranularityMs { get; set; } + + protected override void DoStart() + { + base.DoStart(); + m_Thread = new Thread(threadSpin); + m_Thread.Name = GetType().Name; + m_Thread.Start(); + } + + protected override void DoWaitForCompleteStop() + { + m_Thread.Join(); + m_Thread = null; + + base.DoWaitForCompleteStop(); + } + + /// + /// Writes to log on behalf of worker service + /// + public Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + var logMessage = new Message + { + Topic = SysConsts.LOG_TOPIC_WORKER, + Text = message ?? string.Empty, + Type = type, + From = "{0}.{1}".Args(this.GetType().Name, from), + Exception = error, + Parameters = parameters + }; + if (relatedMessageID.HasValue) logMessage.RelatedTo = relatedMessageID.Value; + + App.Log.Write(logMessage); + + return logMessage.Guid; + } + + protected abstract void DoThreadSpin(DateTime utcNow); + protected virtual void DoDumpStats(IInstrumentation instr, DateTime utcNow) { } + protected virtual void DoResetStats(DateTime utcNow) { } + + #region Private + private void threadSpin() + { + try + { + var ie = false; + var startTime = App.TimeSource.UTCNow; + var prevInstr = ComponentStartTime; + while (Running) + { + var granMs = ThreadGranularityMs; + if (granMs>0) + Thread.Sleep(granMs); + else + Thread.Sleep(THREAD_GRANULARITY_MS + ExternalRandomGenerator.Instance.NextScaledRandomInteger(0, THREAD_GRANULARITY_MS / 5)); + + var utcNow = App.TimeSource.UTCNow; + if ((utcNow - startTime).TotalSeconds < m_StartupDelaySec) continue; + try + { + DoThreadSpin(utcNow); + } + catch (Exception error) + { + Log(MessageType.CatastrophicError, "threadSpin().DoThreadSpin", error.ToMessageWithType(), error); + } + + try + { + utcNow = App.TimeSource.UTCNow; + + var igms = InstrumentationGranularityMs; + if (igms<=0) igms = DEFAULT_INSTRUMENTATION_GRANULARITY_MS; + + if (ie != InstrumentationEnabled) + { + ie = InstrumentationEnabled; + if (ie) resetStats(utcNow); + } + + if (ie && (utcNow - prevInstr).TotalMilliseconds >= igms) + { + dumpStats(utcNow); + prevInstr = utcNow; + } + } + catch (Exception error) + { + Log(MessageType.CatastrophicError, "threadSpin().dumpStats", error.ToMessageWithType(), error); + } + } + } + catch (Exception error) + { + Log(MessageType.CatastrophicError, "threadSpin", error.ToMessageWithType(), error); + } + } + + private void dumpStats(DateTime utcNow) + { + var i = App.Instrumentation; + if (!i.Enabled || this.InstrumentationEnabled) return; + DoDumpStats(i, utcNow); + } + + private void resetStats(DateTime utcNow) + { + DoResetStats(utcNow); + } + #endregion + } +} diff --git a/src/Agni/Workers/Server/ProcessControllerService.cs b/src/Agni/Workers/Server/ProcessControllerService.cs new file mode 100644 index 0000000..9198e03 --- /dev/null +++ b/src/Agni/Workers/Server/ProcessControllerService.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.ServiceModel; +using NFX.Log; +using NFX.Environment; +using System.Threading; +using NFX.Security; +using Agni.Security; + +namespace Agni.Workers.Server +{ + public class ProcessControllerServer : Contracts.IProcessController + { + public void Spawn(ProcessFrame frame) + { + ProcessControllerService.Instance.Spawn(frame); + } + + public ProcessFrame Get(PID pid) + { + return ProcessControllerService.Instance.Get(pid); + } + + public ProcessDescriptor GetDescriptor(PID pid) + { + return ProcessControllerService.Instance.GetDescriptor(pid); + } + + public SignalFrame Dispatch(SignalFrame signal) + { + return ProcessControllerService.Instance.Dispatch(signal); + } + + public IEnumerable List(int processorID) + { + return ProcessControllerService.Instance.List(processorID); + } + } + + public class ProcessControllerService : ServiceWithInstrumentationBase, Contracts.IProcessController, IProcessHost + { + #region CONSTS + public const string CONFIG_PROCESS_CONTROLLER_SECTION = "process-controller"; + public const string CONFIG_PROCESS_STORE_SECTION = "process-store"; + + public const MessageType DEFAULT_LOG_LEVEL = MessageType.Warning; + #endregion + + #region STATIC/.ctor + + private static object s_Lock = new object(); + private static volatile ProcessControllerService s_Instance; + + internal static ProcessControllerService Instance + { + get + { + var instance = s_Instance; + if (instance == null) throw new WorkersException("{0} is not allocated".Args(typeof(ProcessControllerService).FullName)); + return instance; + } + } + + public ProcessControllerService() : this(null) { } + + public ProcessControllerService(object director) : base(director) + { + LogLevel = MessageType.Error; + + lock (s_Lock) + { + if (s_Instance != null) + throw new WorkersException("{0} is already allocated".Args(typeof(ProcessControllerService).FullName)); + + s_Instance = this; + } + } + + protected override void Destructor() + { + base.Destructor(); + DisposeAndNull(ref m_ProcessStore); + s_Instance = null; + } + + #endregion + + private ProcessStore m_ProcessStore; + + [Config] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_WORKER, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public override bool InstrumentationEnabled { get; set; } + + [Config(Default = DEFAULT_LOG_LEVEL)] + [ExternalParameter(SysConsts.EXT_PARAM_GROUP_WORKER, CoreConsts.EXT_PARAM_GROUP_LOG)] + public MessageType LogLevel { get; set; } + + protected override void DoConfigure(IConfigSectionNode node) + { + if (node == null) + node = App.ConfigRoot[CONFIG_PROCESS_CONTROLLER_SECTION]; + + base.DoConfigure(node); + + if (node == null) return; + + DisposeAndNull(ref m_ProcessStore); + var queueStoreNode = node[CONFIG_PROCESS_STORE_SECTION]; + if (queueStoreNode.Exists) + m_ProcessStore = FactoryUtils.Make(queueStoreNode, args: new object[] { this, queueStoreNode }); + } + + protected override void DoStart() + { + if (m_ProcessStore == null) + throw new WorkersException("{0} does not have process store injected".Args(GetType().Name)); + + if (m_ProcessStore is IService) ((IService)m_ProcessStore).Start(); + base.DoStart(); + } + + protected override void DoSignalStop() + { + if (m_ProcessStore is IService) ((IService)m_ProcessStore).SignalStop(); + base.DoSignalStop(); + } + + protected override void DoWaitForCompleteStop() + { + if (m_ProcessStore is IService) ((IService)m_ProcessStore).WaitForCompleteStop(); + base.DoWaitForCompleteStop(); + } + + void IProcessHost.LocalSpawn(Process process, AuthenticationToken? token) { Spawn(new ProcessFrame(process)); } + + public void Spawn(ProcessFrame frame) + { + CheckServiceActive(); + + var tx = m_ProcessStore.BeginTransaction(); + try + { + spawnCore(frame, tx); + m_ProcessStore.CommitTransaction(tx); + } + catch (Exception error) + { + m_ProcessStore.RollbackTransaction(tx); + + Log(MessageType.CatastrophicError, "Spawn", error.ToMessageWithType(), error); + + // TODO: fix exception + throw new WorkersException(StringConsts.TODO_ENQUEUE_TX_BODY_ERROR.Args(error.ToMessageWithType()), error); + } + } + + public ProcessFrame Get(PID pid) + { + return m_ProcessStore.GetByPID(pid); + } + + public ProcessDescriptor GetDescriptor(PID pid) + { + ProcessFrame processFrame = m_ProcessStore.GetByPID(pid); + return processFrame.Descriptor; + } + + ResultSignal IProcessHost.LocalDispatch(Signal signal) { return dispatch(new SignalFrame(signal)); } + + public SignalFrame Dispatch(SignalFrame signalFrame) { return new SignalFrame(dispatch(signalFrame)); } + + private ResultSignal dispatch(SignalFrame signalFrame) + { + lockProcess(signalFrame.PID.ID); + try + { + ProcessFrame processFrame = m_ProcessStore.GetByPID(signalFrame.PID); + + var process = processFrame.Materialize(AgniSystem.ProcessManager.ProcessTypeResolver); + var signal = signalFrame.Materialize(AgniSystem.ProcessManager.SignalTypeResolver); + if (process == null || signal == null)//safeguard + throw new WorkersException("TODO"); + + return process.Accept(this, signal); + } + finally + { + releaseProcess(signalFrame.PID.ID); + } + } + + void IProcessHost.Update(Process process, bool sysOnly) { Update(process, sysOnly); } + + public void Update(Process process, bool sysOnly) { Update(new ProcessFrame(process, sysOnly ? (int?)null : ProcessFrame.SERIALIZER_BSON), sysOnly); } + + public void Update(ProcessFrame frame, bool sysOnly) + { + CheckServiceActive(); + + var tx = m_ProcessStore.BeginTransaction(); + try + { + update(frame, sysOnly, tx); + m_ProcessStore.CommitTransaction(tx); + } + catch (Exception error) + { + m_ProcessStore.RollbackTransaction(tx); + + Log(MessageType.CatastrophicError, "Update", error.ToMessageWithType(), error); + + // TODO fix exception + throw new WorkersException(StringConsts.TODO_ENQUEUE_TX_BODY_ERROR.Args(error.ToMessageWithType()), error); + } + } + + void IProcessHost.Finalize(Process process) { Finalize(process); } + + public void Finalize(Process process) { Finalize(new ProcessFrame(process)); } + + public void Finalize(ProcessFrame frame) + { + CheckServiceActive(); + + var tx = m_ProcessStore.BeginTransaction(); + try + { + delete(frame, tx); + m_ProcessStore.CommitTransaction(tx); + } + catch (Exception error) + { + m_ProcessStore.RollbackTransaction(tx); + + Log(MessageType.CatastrophicError, "Finalize", error.ToMessageWithType(), error); + + // TODO fix exception + throw new WorkersException(StringConsts.TODO_ENQUEUE_TX_BODY_ERROR.Args(error.ToMessageWithType()), error); + } + } + + public IEnumerable List(int processorID) + { + CheckServiceActive(); + + return m_ProcessStore.List(processorID); + } + + /// + /// Writes to log on behalf of worker service + /// + public Guid Log(MessageType type, + Process process, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + return Log(type, "{0}.{1}".Args(process.GetType().FullName, from), message, error, relatedMessageID, parameters); + } + + /// + /// Writes to log on behalf of worker service + /// + public Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + var logMessage = new Message + { + Topic = SysConsts.LOG_TOPIC_WORKER, + Text = message ?? string.Empty, + Type = type, + From = "{0}.{1}".Args(this.GetType().Name, from), + Exception = error, + Parameters = parameters + }; + if (relatedMessageID.HasValue) logMessage.RelatedTo = relatedMessageID.Value; + + App.Log.Write(logMessage); + + return logMessage.Guid; + } + + + private void spawnCore(ProcessFrame frame, object tx) + { + var pid = frame.Descriptor.PID; + if (pid.IsUnique) + { + put(frame, tx); + return; + } + + lockProcess(pid.ID); + try + { + var utcNow = App.TimeSource.UTCNow; + ProcessFrame existing; + if (!m_ProcessStore.TryGetByPID(pid, out existing)) + { + put(frame, tx); + return; + } + + var processExisting = existing.Materialize(AgniSystem.ProcessManager.ProcessTypeResolver); + var processAnother = frame.Materialize(AgniSystem.ProcessManager.ProcessTypeResolver); + if (processExisting == null || processAnother == null)//safeguard + { + put(frame, tx); + return; + } + + try + { + processExisting.Merge(this, utcNow, processAnother); + } + catch (Exception error) + { + // TODO : fix exception + throw new WorkersException(StringConsts.TODO_CORRELATED_MERGE_ERROR.Args(processExisting, processAnother, error.ToMessageWithType()), error); + } + } + finally + { + releaseProcess(pid.ID); + } + } + + private void put(ProcessFrame frame, object transaction = null) + { + try + { + frame.Descriptor = new ProcessDescriptor(frame.Descriptor, ProcessStatus.Started, "Started", App.TimeSource.UTCNow, "@{0}@{1}".Args(App.Name, AgniSystem.HostName)); + m_ProcessStore.Put(frame, transaction); + } + catch (Exception e) + { + Log(MessageType.Critical, "put", "{0} Leaked: {1}".Args(frame, e.ToMessageWithType()), e); + throw; + } + } + + private void update(ProcessFrame frame, bool sysOnly, object transaction = null) + { + try + { + m_ProcessStore.Update(frame, sysOnly, transaction); + } + catch (Exception e) + { + Log(MessageType.Critical, "update", "{0} Leaked: {1}".Args(frame, e.ToMessageWithType()), e); + throw; + } + } + + private void delete(ProcessFrame frame, object transaction = null) + { + try + { + m_ProcessStore.Delete(frame, transaction); + } + catch (Exception e) + { + Log(MessageType.Critical, "delete", "{0} Leaked: {1}".Args(frame, e.ToMessageWithType()), e); + throw; + } + } + + private class _lck { public Thread Thread; public int Count; } + private Dictionary m_Locker = new Dictionary(StringComparer.Ordinal); + + private void lockProcess(string key) + { + if (key == null) key = string.Empty; + + var ct = Thread.CurrentThread; + + uint spinCount = 0; + while (true) + { + lock (m_Locker) + { + _lck lck; + if (!m_Locker.TryGetValue(key, out lck)) + { + m_Locker.Add(key, new _lck { Thread = ct, Count = 1 } ); + return; + } + + if (lck.Thread == ct)//if already acquired by this thread + { + lck.Count++; + return; + } + } + + if (spinCount < 100) Thread.SpinWait(500); + else Thread.Yield(); + + unchecked { spinCount++; } + } + } + + private bool releaseProcess(string key) + { + if (key == null) key = string.Empty; + + lock (m_Locker) + { + _lck lck; + if (!m_Locker.TryGetValue(key, out lck)) return false; + + if (Thread.CurrentThread != lck.Thread) return false;//not locked by this thread + + lck.Count--; + + if (lck.Count == 0) + m_Locker.Remove(key); + + return true; + } + } + } +} diff --git a/src/Agni/Workers/Server/ProcessStore.cs b/src/Agni/Workers/Server/ProcessStore.cs new file mode 100644 index 0000000..ae62996 --- /dev/null +++ b/src/Agni/Workers/Server/ProcessStore.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; + +using NFX; +using NFX.ApplicationModel; +using NFX.Environment; +using NFX.Log; +using NFX.Security; + +namespace Agni.Workers.Server +{ + /// + /// Represents a base for entities that store queue data + /// + public abstract class ProcessStore : ApplicationComponent + { + protected ProcessStore(ProcessControllerService director, IConfigSectionNode node) : base(director) + { + ConfigAttribute.Apply(this, node); + } + + /// + /// References service that this store is under + /// + public ProcessControllerService ProcessController { get { return (ProcessControllerService)ComponentDirector;} } + + public abstract IEnumerable List(int processorID); + + public abstract bool TryGetByPID(PID pid, out ProcessFrame frame); + + public virtual ProcessFrame GetByPID(PID pid) + { + ProcessFrame result; + if (!TryGetByPID(pid, out result)) + throw new WorkersException("TODO"); + return result; + } + + public abstract void Put(ProcessFrame frame, object transaction); + + public abstract void Update(ProcessFrame frame, bool sysOnly, object transaction); + + public abstract void Delete(ProcessFrame frame, object transaction); + + public abstract object BeginTransaction(); + + public abstract void CommitTransaction(object transaction); + + public abstract void RollbackTransaction(object transaction); + + protected Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < ProcessController.LogLevel) return Guid.Empty; + + var logMessage = new Message + { + Topic = SysConsts.LOG_TOPIC_WORKER, + Text = message ?? string.Empty, + Type = type, + From = "{0}.{1}".Args(this.GetType().Name, from), + Exception = error, + Parameters = parameters + }; + if (relatedMessageID.HasValue) logMessage.RelatedTo = relatedMessageID.Value; + + App.Log.Write(logMessage); + + return logMessage.Guid; + } + } +} diff --git a/src/Agni/Workers/Server/Queue/Instrumentation/Gauges.cs b/src/Agni/Workers/Server/Queue/Instrumentation/Gauges.cs new file mode 100644 index 0000000..a9c7dda --- /dev/null +++ b/src/Agni/Workers/Server/Queue/Instrumentation/Gauges.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using NFX.Instrumentation; +using NFX.Serialization.BSON; + +namespace Agni.Workers.Server.Queue.Instrumentation +{ + /// + /// Provides base for TodoQueue long gauges + /// + [Serializable] + public abstract class TodoQueueLongGauge : LongGauge, IQueueInstrument, IWorkerInstrument + { + protected TodoQueueLongGauge(string src, long value) : base(src, value) { } + } + + /// + /// How many times Enqueue(Todo[]) was called + /// + [Serializable] + [BSONSerializable("8F0D962E-9944-416E-8B8B-9CDE6A729284")] + public class EnqueueCalls : TodoQueueLongGauge + { + public EnqueueCalls(long value) : base(null, value) { } + + public override string Description { get { return "How many times Enqueue(Todo[]) was called"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_CALL; } } + + protected override Datum MakeAggregateInstance() { return new EnqueueCalls(0); } + } + + /// + /// How many Todo message instances got enqueued + /// + [Serializable] + [BSONSerializable("54D5B7C2-E0F4-4236-9CE4-E5A0111526A3")] + public class EnqueueTodoCount : TodoQueueLongGauge + { + public EnqueueTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo message instances got enqueued"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new EnqueueTodoCount(this.Source, 0); } + } + + + /// + /// How many times the thread spun + /// + [Serializable] + [BSONSerializable("A39A8D75-25DE-4047-AC62-E7D483A07DFE")] + public class QueueThreadSpins : TodoQueueLongGauge + { + public QueueThreadSpins(long value) : base(null, value) { } + + public override string Description { get { return "How many times the thread spun"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new QueueThreadSpins(0); } + } + + /// + /// How many times queue slice was processed + /// + [Serializable] + [BSONSerializable("9E9F5A61-35E6-40DC-AF37-BFF8CA2AB034")] + public class ProcessOneQueueCount : TodoQueueLongGauge + { + public ProcessOneQueueCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many times queue slice was processed"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_TIME; } } + + protected override Datum MakeAggregateInstance() { return new ProcessOneQueueCount(this.Source, 0); } + } + + /// + /// How many Todo instances got merged + /// + [Serializable] + [BSONSerializable("C2DBB2EB-A9AA-49FD-B434-891837AD1E12")] + public class MergedTodoCount : TodoQueueLongGauge + { + public MergedTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo instances got merged"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new MergedTodoCount(this.Source, 0); } + } + + /// + /// How many Todo messages got fetched from store + /// + [Serializable] + [BSONSerializable("162AEBA6-F5B0-4A76-9DF7-F57C43D3D6FB")] + public class FetchedTodoCount : TodoQueueLongGauge + { + public FetchedTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo messages got fetched from store"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new FetchedTodoCount(this.Source, 0); } + } + + /// + /// How many Todo messages got processed + /// + [Serializable] + [BSONSerializable("B88C36FC-E0EA-4BA9-BA6D-4AFB139DEDD4")] + public class ProcessedTodoCount : TodoQueueLongGauge + { + public ProcessedTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo messages got processed"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new ProcessedTodoCount(this.Source, 0); } + } + + /// + /// How many Todo got put into the store + /// + [Serializable] + [BSONSerializable("559D9D5C-7671-4F4F-8A30-189DAA44D2B0")] + public class PutTodoCount : TodoQueueLongGauge + { + public PutTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo got put into the store"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new PutTodoCount(this.Source, 0); } + } + + + /// + /// How many Todo instances were submitted more than once + /// + [Serializable] + [BSONSerializable("53BA67B9-115E-4CDF-9B58-00854C252AEF")] + public class TodoDuplicationCount : TodoQueueLongGauge + { + public TodoDuplicationCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo instances were submitted more than once"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new TodoDuplicationCount(this.Source, 0); } + } + + /// + /// How many Todo got updated in the store + /// + [Serializable] + [BSONSerializable("1EBC5BEB-51D1-40FA-AD26-BBD9DAB52FDF")] + public class UpdateTodoCount : TodoQueueLongGauge + { + public UpdateTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo got updated in the store"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new UpdateTodoCount(this.Source, 0); } + } + + /// + /// How many Todo were completed regardless of OK or Error + /// + [Serializable] + [BSONSerializable("1CCA58EB-34B3-4698-85F5-0ED664F78A98")] + public class CompletedTodoCount : TodoQueueLongGauge + { + public CompletedTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo were completed regardless of OK or Error"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new CompletedTodoCount(this.Source, 0); } + } + + /// + /// How many Todo were completed OK + /// + [Serializable] + [BSONSerializable("790E14AC-BD70-4268-BDAB-5E56B1E2DC39")] + public class CompletedOkTodoCount : TodoQueueLongGauge + { + public CompletedOkTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo were completed OK"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new CompletedOkTodoCount(this.Source, 0); } + } + + /// + /// How many Todo were completed with Error + /// + [Serializable] + [BSONSerializable("04D233B2-CA69-45A6-9FEC-6E7B9875222D")] + public class CompletedErrorTodoCount : TodoQueueLongGauge, IErrorInstrument + { + public CompletedErrorTodoCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many Todo were completed with Error"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_MESSAGE; } } + + protected override Datum MakeAggregateInstance() { return new CompletedErrorTodoCount(this.Source, 0); } + } + + + /// + /// How many times queue processing error happened + /// + [Serializable] + [BSONSerializable("1FA8D0D3-577B-46C1-A86D-864CDBC7EBBA")] + public class QueueOperationErrorCount : TodoQueueLongGauge, IErrorInstrument + { + public QueueOperationErrorCount(string queue, long value) : base(queue, value) { } + + public override string Description { get { return "How many times queue processing error happened"; } } + + public override string ValueUnitName { get { return NFX.CoreConsts.UNIT_NAME_ERROR; } } + + protected override Datum MakeAggregateInstance() { return new QueueOperationErrorCount(this.Source, 0); } + } +} diff --git a/src/Agni/Workers/Server/Queue/TodoQueue.cs b/src/Agni/Workers/Server/Queue/TodoQueue.cs new file mode 100644 index 0000000..144e1bb --- /dev/null +++ b/src/Agni/Workers/Server/Queue/TodoQueue.cs @@ -0,0 +1,261 @@ +using System; +using System.Collections.Generic; +using System.Threading; + +using NFX; +using NFX.Environment; +using NFX.ApplicationModel; +using NFX.Instrumentation; + +namespace Agni.Workers.Server.Queue +{ + /// + /// Represents a Named Queue instance + /// + public sealed class TodoQueue : ApplicationComponent, INamed, IInstrumentable + { + /// + /// Denotes mode of execution: sequential, parallel or ParallelByKey + /// + public enum ExecuteMode + { + Sequential = 0, + Parallel, + ParallelByKey + } + + /// + /// Defines how duplicate todo submissions are handled by the queue. + /// When a client sends a Todo into server, a server may accept the todo but client may not get the confirmation message, + /// in which case client may re-submit the same todo (as identified by the SysID GDID) more than once. + /// This leads to duplication of work and may be unacceptable for some applications (e.g. financial double-posting). + /// Duplication detection will respond with OK acknowledgement for the Todo that have already been accepted. + /// + public enum DuplicationHandlingMode + { + /// + /// The server is not checking for duplicate Todo instance submission. + /// This is the fastest mode but leads to possible re-submission of the same Todo instance + /// + NotDetected = -10, + + /// + /// This is the default mode. + /// The server is going to check whether a Todo with the same SysID was already submitted recently. + /// The recency (time window size) is defined by the server configuration and is usually configured + /// to represent a typical timeouts (60 sec = 3 typical 20 sec Glue calls) for re-submission. + /// This detection mode is usually done in-RAM and provides the best balance between performance and protection guarantees. + /// Note: this mode DOES NOT detect a case when this server was down and traffic was routed to a different server in a hostset. + /// For more protection use HostsetDetection mode + /// + HostFastDetection = 0, + + /// + /// The server is going to check whether a Todo with the same SysID was already submitted and is in the queue. + /// This detection mode is usually done in-store and provides the best host-level protection but takes more time to execute + /// a store check request. This mode does not detect todos which have executed already and not in either queue or recent buffer + /// + HostAccurateDetection = 10, + + + /// + /// The server will check itself then others in a host set for possible Todo resubmission. This mode provides the best protection + /// at the expense of performance as cross-checking introduces extra network traffic. + /// + HostsetDetection = 20 + } + + + + #region CONSTS + + public const int DEFAULT_BATCH_SIZE = 32; + + public const int MAX_BATCH_SIZE = 1024; + + public const int DEFAULT_ACQUIRE_TIMEOUT_SEC = 7 * 60; + public const int MIN_ACQUIRE_TIMEOUT_SEC = 60; + + #endregion + + #region .ctor + + internal TodoQueue(TodoQueueService director, IConfigSectionNode node) : base(director) + { + ConfigAttribute.Apply(this, node); + + if (Name.IsNullOrWhiteSpace()) + throw new WorkersException(GetType().Name + ".ctor($name=null|empty)"); + } + + #endregion + + #region Fields + + #pragma warning disable CS0649 + [Config] + private string m_Name; + + private int m_BatchSize = DEFAULT_BATCH_SIZE; + + + [Config] + private DuplicationHandlingMode m_DuplicationHandling; + + + [Config] + private ExecuteMode m_Mode; + #pragma warning restore CS0649 + + private int m_AcquireTimeoutSec = DEFAULT_ACQUIRE_TIMEOUT_SEC; + + private object m_AcquireLock = new object(); + private DateTime? m_AcquireDateUTC; + + #endregion + + #region Properties + + /// + /// Unique queue name + /// + public string Name { get { return m_Name;} } + + /// + /// References service that this queue is under + /// + public TodoQueueService QueueService { get { return (TodoQueueService)ComponentDirector;} } + + [Config(Default=false)] + [ExternalParameter(CoreConsts.EXT_PARAM_GROUP_DATA, CoreConsts.EXT_PARAM_GROUP_INSTRUMENTATION)] + public bool InstrumentationEnabled { get; set; } + + + /// + /// Specifies the size of one processing quantum in todo instance count + /// + [Config(Default = DEFAULT_BATCH_SIZE)] + public int BatchSize + { + get {return m_BatchSize;} + private set + { + m_BatchSize = value<1 ? 1 : value > MAX_BATCH_SIZE ? MAX_BATCH_SIZE : value; + } + } + + /// + /// Specifies the timeout for queue acquisition. + /// The queue is acquired every time a processing quantum starts. + /// If processing stalls for some reason (i.e. long sequential todo) then + /// queue is acquired anyway after timeout expires. + /// The timeout must work in conjunction with BtachSize to ensure that during normal operation it never happens, because + /// sequential processing order is not guaranteed when queue times-out + /// + [Config(Default = DEFAULT_ACQUIRE_TIMEOUT_SEC)] + public int AcquireTimeoutSec + { + get {return m_AcquireTimeoutSec;} + private set + { + m_AcquireTimeoutSec = value + /// Denotes mode of execution: sequential, parallel or ParallelByKey + /// + public ExecuteMode Mode { get { return m_Mode;} } + + /// + /// True when queue is acquired by processing quanta + /// + public bool Acquired { get { return AcquireDateUTC.HasValue; } } + + /// + /// When was queue acquired for the last time - used for timeout + /// + public DateTime? AcquireDateUTC + { + get + { + lock(m_AcquireLock) return m_AcquireDateUTC; + } + } + + /// + /// Specifies how this queue handles duplicate Todo submissions + /// + public DuplicationHandlingMode DuplicationHandling { get { return m_DuplicationHandling; } } + + #endregion + + #region Public/Internal + + + /// + /// Returns named parameters that can be used to control this component + /// + public IEnumerable> ExternalParameters{ get { return ExternalParameterAttribute.GetParameters(this); } } + + /// + /// Returns named parameters that can be used to control this component + /// + public IEnumerable> ExternalParametersForGroups(params string[] groups) + { + return ExternalParameterAttribute.GetParameters(this, groups); + } + + /// + /// Gets external parameter value returning true if parameter was found + /// + public bool ExternalGetParameter(string name, out object value, params string[] groups) + { + return ExternalParameterAttribute.GetParameter(this, name, out value, groups); + } + + /// + /// Sets external parameter value returning true if parameter was found and set + /// + public bool ExternalSetParameter(string name, object value, params string[] groups) + { + return ExternalParameterAttribute.SetParameter(this, name, value, groups); + } + + internal bool CanBeAcquired(DateTime utcNow) + { + lock(m_AcquireLock) + { + if (m_AcquireDateUTC.HasValue && (utcNow - m_AcquireDateUTC.Value).TotalSeconds < m_AcquireTimeoutSec) return false; + return true; + } + } + + internal bool TryAcquire(DateTime utcNow) + { + lock(m_AcquireLock) + { + if (m_AcquireDateUTC.HasValue && (utcNow - m_AcquireDateUTC.Value).TotalSeconds < m_AcquireTimeoutSec) return false; + m_AcquireDateUTC = utcNow; + return true; + } + } + + internal bool Release() + { + lock(m_AcquireLock) + { + var result = m_AcquireDateUTC.HasValue; + m_AcquireDateUTC = null; + return result; + } + } + + + public override string ToString() + { + return "Queue('{0}')".Args(Name); + } + #endregion + } +} diff --git a/src/Agni/Workers/Server/Queue/TodoQueueService.Enqueue.cs b/src/Agni/Workers/Server/Queue/TodoQueueService.Enqueue.cs new file mode 100644 index 0000000..0f714f4 --- /dev/null +++ b/src/Agni/Workers/Server/Queue/TodoQueueService.Enqueue.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using NFX; +using NFX.Collections; +using NFX.Environment; +using NFX.Log; +using NFX.ServiceModel; + +using Agni.Contracts; +using System.Threading.Tasks; + +namespace Agni.Workers.Server.Queue { public sealed partial class TodoQueueService{ + + + + private void enqueue(TodoQueue queue, TodoFrame[] todos) + { + var tx = m_QueueStore.BeginTransaction(queue); + try + { + enqueueCore(queue, todos, tx); + m_QueueStore.CommitTransaction(queue, tx); + } + catch(Exception error) + { + m_QueueStore.RollbackTransaction(queue, tx); + + var from = "enqueue('{0}')".Args(queue.Name); + Log(MessageType.CatastrophicError, from, error.ToMessageWithType(), error); + if (InstrumentationEnabled) + { + m_stat_QueueOperationErrorCount.IncrementLong(ALL); + m_stat_QueueOperationErrorCount.IncrementLong(from); + } + + throw new WorkersException(StringConsts.TODO_ENQUEUE_TX_BODY_ERROR.Args(queue.Name, error.ToMessageWithType()), error); + } + } + + private void enqueueCore(TodoQueue queue, TodoFrame[] todos, object tx) + { + foreach(var todo in todos) + { + if (!todo.Assigned) + continue; + + //20171114 If the todo was already submitted, then do nothing + var isdup = checkDups(queue, todo); + if (isdup) + { + if (InstrumentationEnabled) + { + var from = "enqueue-dup('{0}')".Args(queue.Name); + m_stat_TodoDuplicationCount.IncrementLong(ALL); + m_stat_TodoDuplicationCount.IncrementLong(from); + } + continue; + } + + + if (todo.CorrelationKey==null)//regular todos just add + { + put(queue, todo, tx, true); + continue; + } + + //Correlated ------------------------------------------------------------- + var utcNow = App.TimeSource.UTCNow;//warning locking depends on this accurate date + var utcCorrelateSD = lockCorrelatedEnqueue(todo.CorrelationKey, utcNow); + try + { + var existing = m_QueueStore.FetchLatestCorrelated(queue, todo.CorrelationKey, utcCorrelateSD); + if (!existing.Assigned) //no existing with same correlation, just add + { + put(queue, todo, tx, true); + continue; + } + + var todoExisting = existing.Materialize( AgniSystem.ProcessManager.TodoTypeResolver ) as CorrelatedTodo; + var todoAnother = todo.Materialize( AgniSystem.ProcessManager.TodoTypeResolver ) as CorrelatedTodo; + if (todoExisting==null || todoAnother==null)//safeguard + { + put(queue, todo, tx, true); + continue; + } + + CorrelatedTodo.MergeResult mergeResult; + try + { + mergeResult = todoExisting.Merge( this, utcNow, todoAnother ); + } + catch(Exception error) + { + throw new WorkersException(StringConsts.TODO_CORRELATED_MERGE_ERROR.Args(todoExisting, todoAnother, error.ToMessageWithType()), error); + } + + if (InstrumentationEnabled) + { + m_stat_MergedTodoCount.IncrementLong(ALL); + m_stat_MergedTodoCount.IncrementLong(queue.Name); + m_stat_MergedTodoCount.IncrementLong(queue.Name+":"+mergeResult); + } + + if (mergeResult == CorrelatedTodo.MergeResult.Merged) + { + update(queue, todoExisting, sysOnly: false, leak: true, transaction: tx); + continue; + } else if (mergeResult == CorrelatedTodo.MergeResult.IgnoreAnother) + { + continue;//do nothing, just drop another as-if never existed + } + + //Else, merge == CorrelatedTodo.MergeResult.None + put(queue, todo, tx, true); + } + finally + { + releaseCorrelatedEnqueue(todo.CorrelationKey); + } + }//foreach + } + + + private bool checkDups(TodoQueue queue, TodoFrame todo) + { + var mode = queue.DuplicationHandling; + + if (mode==TodoQueue.DuplicationHandlingMode.NotDetected) return false;//never a duplicate + + var result = checkDupsInMemory(todo); + + if (result || mode==TodoQueue.DuplicationHandlingMode.HostFastDetection) return result; + + result = checkDupsInQueueStore(queue, todo); + + if (result || mode==TodoQueue.DuplicationHandlingMode.HostAccurateDetection) return result; + + result = checkDupsInHostset(queue, todo); + + return result; + } + + private bool checkDupsInMemory(TodoFrame todo) + { + return !m_Duplicates.Put(todo.ID); + } + + private bool checkDupsInQueueStore(TodoQueue queue, TodoFrame todo) + { + return false; // TODO implement + } + + private bool checkDupsInHostset(TodoQueue queue, TodoFrame todo) + { + return false; // TODO implement + } + + +}} diff --git a/src/Agni/Workers/Server/Queue/TodoQueueService.Lifecycle.cs b/src/Agni/Workers/Server/Queue/TodoQueueService.Lifecycle.cs new file mode 100644 index 0000000..a4c5504 --- /dev/null +++ b/src/Agni/Workers/Server/Queue/TodoQueueService.Lifecycle.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +using NFX; +using NFX.Environment; +using NFX.Log; +using NFX.ServiceModel; +using NFX.Instrumentation; + +using System.Threading.Tasks; + +namespace Agni.Workers.Server.Queue { public sealed partial class TodoQueueService{ + + /// + /// Writes to log on behalf of worker service + /// + public Guid Log(MessageType type, + Todo todo, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + if (type < LogLevel) return Guid.Empty; + + return Log(type, "{0}.{1}".Args(todo.GetType().FullName, from), message, error, relatedMessageID, parameters); + } + + protected override void DoConfigure(IConfigSectionNode node) + { + base.DoConfigure(node); + + if (node == null) return; + + DisposeAndNull(ref m_QueueStore); + var queueStoreNode = node[CONFIG_QUEUE_STORE_SECTION]; + if (queueStoreNode.Exists) + m_QueueStore = FactoryUtils.Make(queueStoreNode, args: new object[] { this, queueStoreNode }); + + m_Queues.Clear(); + foreach (var queueNode in node.Children.Where(n => n.IsSameName(CONFIG_QUEUE_SECTION))) + m_Queues.Register(new TodoQueue(this, queueNode)); + } + + protected override void DoStart() + { + m_Skip = 0; + m_CorrelationLocker.Clear(); + + if (m_QueueStore == null) + throw new WorkersException("{0} does not have queue store injected".Args(GetType().Name)); + + if (m_Queues.Count == 0) + throw new WorkersException("{0} does not have any queues injected".Args(GetType().Name)); + + if (m_QueueStore is IService) ((IService)m_QueueStore).Start(); + base.DoStart(); + } + + protected override void DoSignalStop() + { + if (m_QueueStore is IService) ((IService)m_QueueStore).SignalStop(); + base.DoSignalStop(); + } + + protected override void DoWaitForCompleteStop() + { + if (m_QueueStore is IService) ((IService)m_QueueStore).WaitForCompleteStop(); + base.DoWaitForCompleteStop(); + m_CorrelationLocker.Clear(); + m_Duplicates.Clear(); + } + + private int m_Skip; + protected override void DoThreadSpin(DateTime utcNow) + { + try + { + if (InstrumentationEnabled) Interlocked.Increment(ref m_stat_QueueThreadSpins); + + var cpuCount = Environment.ProcessorCount; + var maxWorkers = cpuCount / 2; + if (maxWorkers < 2) maxWorkers = 2; + + if (m_Skip >= m_Queues.Count) m_Skip = 0; + var working = m_Queues.Count( q => !q.CanBeAcquired(utcNow)); + foreach(var queue in m_Queues.Skip(m_Skip)) + { + if (working>=maxWorkers) break; + + m_Skip++; + if (!queue.TryAcquire(utcNow)) continue; + + working++; + try + { + Task.Factory.StartNew( q => processOneQueue(q, utcNow), queue); + } + catch + { + queue.Release(); + throw; + } + }//foreach + } + catch (Exception error) + { + Log(MessageType.CatastrophicError, "DoThreadSpin()", error.ToMessageWithType(), error); + } + } + + + protected override void DoDumpStats(IInstrumentation instr, DateTime utcNow) + { + instr.Record( new Instrumentation.EnqueueCalls( Interlocked.Exchange(ref m_stat_EnqueueCalls, 0) ) ); + m_stat_EnqueueTodoCount.SnapshotAllLongsInto(0); + instr.Record( new Instrumentation.QueueThreadSpins( Interlocked.Exchange(ref m_stat_QueueThreadSpins, 0) ) ); + m_stat_ProcessOneQueueCount.SnapshotAllLongsInto(0, instr); + m_stat_MergedTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_FetchedTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_ProcessedTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_PutTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_UpdateTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_CompletedTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_CompletedOkTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_CompletedErrorTodoCount.SnapshotAllLongsInto(0, instr); + m_stat_QueueOperationErrorCount.SnapshotAllLongsInto(0, instr); + m_stat_TodoDuplicationCount.SnapshotAllLongsInto(0, instr); + } + + protected override void DoResetStats(DateTime utcNow) + { + Interlocked.Exchange(ref m_stat_EnqueueCalls, 0); + m_stat_EnqueueTodoCount.Clear(); + Interlocked.Exchange(ref m_stat_QueueThreadSpins, 0); + m_stat_ProcessOneQueueCount .Clear(); + m_stat_MergedTodoCount .Clear(); + m_stat_FetchedTodoCount .Clear(); + m_stat_ProcessedTodoCount .Clear(); + m_stat_PutTodoCount .Clear(); + m_stat_UpdateTodoCount .Clear(); + m_stat_CompletedTodoCount .Clear(); + m_stat_CompletedOkTodoCount .Clear(); + m_stat_CompletedErrorTodoCount .Clear(); + m_stat_QueueOperationErrorCount .Clear(); + m_stat_TodoDuplicationCount .Clear(); + } + + + } +} diff --git a/src/Agni/Workers/Server/Queue/TodoQueueService.Processing.cs b/src/Agni/Workers/Server/Queue/TodoQueueService.Processing.cs new file mode 100644 index 0000000..0ca261d --- /dev/null +++ b/src/Agni/Workers/Server/Queue/TodoQueueService.Processing.cs @@ -0,0 +1,285 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using NFX; +using NFX.Log; + +using System.Threading.Tasks; + +namespace Agni.Workers.Server.Queue{public sealed partial class TodoQueueService{ + + private void processOneQueue(object queue, DateTime utcNow) + { + var q = (TodoQueue)queue; + try + { + if (InstrumentationEnabled) + { + m_stat_ProcessOneQueueCount.IncrementLong(ALL); + m_stat_ProcessOneQueueCount.IncrementLong(q.Name); + } + + processOneQueueCore(q, utcNow); + } + catch (Exception error) + { + var from = "processOneQueueCore('{0}')".Args(q.Name); + Log(MessageType.CatastrophicError, from, error.ToMessageWithType(), error); + if (InstrumentationEnabled) + { + m_stat_QueueOperationErrorCount.IncrementLong(ALL); + m_stat_QueueOperationErrorCount.IncrementLong(from); + } + } + finally + { + q.Release(); + } + } + + private void processOneQueueCore(TodoQueue queue, DateTime utcNow) + { + var fetched = m_QueueStore.Fetch(queue, utcNow); + try + { + var batch = fetched.Where( todo => + { + if (todo.CorrelationKey==null) return true; + return tryLockCorrelatedProcessing(todo.CorrelationKey, todo.StartDate); + }).ToList();//must materialize not to double lock + + if (InstrumentationEnabled) + { + var fetchCount = fetched.Count(); + m_stat_FetchedTodoCount.AddLong(ALL, fetchCount); + m_stat_FetchedTodoCount.AddLong(queue.Name, fetchCount); + } + + processOneQueueBatch(queue, batch, utcNow); + + + if (InstrumentationEnabled) + { + m_stat_ProcessedTodoCount.AddLong(ALL, batch.Count); + m_stat_ProcessedTodoCount.AddLong(queue.Name, batch.Count); + } + } + finally + { + fetched.ForEach( todo => + { + if (todo.CorrelationKey==null) return; + releaseCorrelatedProcessing(todo.CorrelationKey); + }); + } + } + + private void processOneQueueBatch(TodoQueue queue, IEnumerable batch, DateTime utcNow) + { + if (queue.Mode == TodoQueue.ExecuteMode.Sequential) + { + batch.OrderBy(t => t.StartDate).ForEach(todo => executeOne(queue, todo, utcNow)); + } + else if (queue.Mode == TodoQueue.ExecuteMode.Parallel) + { + Parallel.ForEach(batch.OrderBy(t => t.StartDate), todo => executeOne(queue, todo, utcNow)); + } + else//ParallelByKey + { + var tasks = new List(); + + var parallelTodos = batch.Where(t => t.ParallelKey == null).ToArray(); + if (parallelTodos.Length > 0) + tasks.Add(Task.Factory.StartNew(ts => Parallel.ForEach(((IEnumerable)ts).OrderBy(t => t.StartDate), todo => executeOne(queue, todo, utcNow)), parallelTodos)); + + List todos = null; + string parallelKey = null; + foreach (var todo in batch.Where(t => t.ParallelKey != null).OrderBy(t => t.ParallelKey)) + { + if (parallelKey != todo.ParallelKey) + { + if (todos != null) + tasks.Add(Task.Factory.StartNew(ts => ((IEnumerable)ts).OrderBy(t => t.StartDate).ForEach(t => executeOne(queue, t, utcNow)), todos)); + todos = new List(); + parallelKey = todo.ParallelKey; + } + todos.Add(todo); + } + + if (todos != null) + tasks.Add(Task.Factory.StartNew(ts => ((IEnumerable)ts).OrderBy(t => t.StartDate).ForEach(t => executeOne(queue, t, utcNow)), todos)); + + Task.WaitAll(tasks.ToArray()); + } + } + + + private void executeOne(TodoQueue queue, TodoFrame todoFrame, DateTime utcNow)//must not leak + { + try + { + if (!Running) return; + + Todo todo; + try + { + todo = todoFrame.Materialize( AgniSystem.ProcessManager.TodoTypeResolver ); + } + catch(Exception me) + { + var from = "executeOne('{0}').Materialize".Args(queue.Name); + Log(MessageType.Critical, from, "Frame materialization: "+me.ToMessageWithType(), me); + if (InstrumentationEnabled) + { + m_stat_QueueOperationErrorCount.IncrementLong(ALL); + m_stat_QueueOperationErrorCount.IncrementLong(from); + } + throw; + } + + var wasState = todo.SysState; + while (todo.SysState != Todo.ExecuteState.Complete) + { + var nextState = todo.Execute(this, utcNow); + + if (nextState == Todo.ExecuteState.ReexecuteUpdatedAfterError) + { + todo.SysTries++; + var ms = todo.RetryAfterErrorInMs(utcNow); + if (ms > 0) todo.SysStartDate = todo.SysStartDate.AddMilliseconds(ms); + update(queue, todo, false); + return; + } + + if (nextState == Todo.ExecuteState.ReexecuteAfterError) + { + todo.SysTries++; + var ms = todo.RetryAfterErrorInMs(utcNow); + if (ms > 0) todo.SysStartDate = todo.SysStartDate.AddMilliseconds(ms); + if (ms >= 0 || todo.SysState != wasState) update(queue, todo, true); + return; + } + + if (nextState == Todo.ExecuteState.ReexecuteUpdated) + { + update(queue, todo, false); + return; + } + + if (nextState == Todo.ExecuteState.ReexecuteSysUpdated) + { + update(queue, todo, true); + return; + } + + if (nextState == Todo.ExecuteState.Reexecute) + { + if (todo.SysState != wasState) update(queue, todo, true); + return; + } + + todo.SysTries = 0; + todo.SysState = nextState; + } + + complete(queue, todoFrame, null); + } + catch (Exception error) + { + complete(queue, todoFrame, error); + } + } + + private void complete(TodoQueue queue, TodoFrame todo, Exception error, object transaction = null) + { + try + { + m_QueueStore.Complete(queue, todo, error, transaction); + + if (error!=null) + { + Log(MessageType.Error, "complete('{0}')".Args(queue.Name), "Completed with error: " + error.ToMessageWithType(), error); + } + + if (InstrumentationEnabled) + { + m_stat_CompletedTodoCount.IncrementLong(ALL); + m_stat_CompletedTodoCount.IncrementLong(queue.Name); + + if (error!=null) + { + m_stat_CompletedErrorTodoCount.IncrementLong(ALL); + m_stat_CompletedErrorTodoCount.IncrementLong(queue.Name); + } + else + { + m_stat_CompletedOkTodoCount.IncrementLong(ALL); + m_stat_CompletedOkTodoCount.IncrementLong(queue.Name); + } + } + } + catch(Exception e) + { + var from = "complete('{0}')".Args(queue.Name); + Log(MessageType.Critical, from, "{0} Leaked: {1}".Args(todo, e.ToMessageWithType()), e); + if (InstrumentationEnabled) + { + m_stat_QueueOperationErrorCount.IncrementLong(ALL); + m_stat_QueueOperationErrorCount.IncrementLong(from); + } + } + } + + private void update(TodoQueue queue, Todo todo, bool sysOnly, object transaction = null, bool leak = false) + { + try + { + m_QueueStore.Update(queue, todo, sysOnly, transaction); + if (InstrumentationEnabled) + { + m_stat_UpdateTodoCount.IncrementLong(ALL); + m_stat_UpdateTodoCount.IncrementLong(queue.Name); + if (sysOnly) + m_stat_UpdateTodoCount.IncrementLong(queue.Name+"-sys"); + } + } + catch(Exception e) + { + var from = "update('{0}')".Args(queue.Name); + Log(MessageType.Critical, from, "{0} Leaked: {1}".Args(todo, e.ToMessageWithType()), e); + if (InstrumentationEnabled) + { + m_stat_QueueOperationErrorCount.IncrementLong(ALL); + m_stat_QueueOperationErrorCount.IncrementLong(from); + } + if (leak) throw; + } + } + + private void put(TodoQueue queue, TodoFrame todo, object transaction = null, bool leak = false) + { + try + { + m_QueueStore.Put(queue, todo, transaction); + if (InstrumentationEnabled) + { + m_stat_PutTodoCount.IncrementLong(ALL); + m_stat_PutTodoCount.IncrementLong(queue.Name); + } + } + catch(Exception e) + { + var from = "put('{0}')".Args(queue.Name); + Log(MessageType.Critical, from, "{0} Leaked: {1}".Args(todo, e.ToMessageWithType()), e); + if (InstrumentationEnabled) + { + m_stat_QueueOperationErrorCount.IncrementLong(ALL); + m_stat_QueueOperationErrorCount.IncrementLong(from); + } + if (leak) throw; + } + } + + +}} diff --git a/src/Agni/Workers/Server/Queue/TodoQueueService.cs b/src/Agni/Workers/Server/Queue/TodoQueueService.cs new file mode 100644 index 0000000..a60b986 --- /dev/null +++ b/src/Agni/Workers/Server/Queue/TodoQueueService.cs @@ -0,0 +1,340 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +using NFX; +using NFX.Collections; +using NFX.Instrumentation; +using NFX.ServiceModel; + +using Agni.Contracts; +using System.Threading.Tasks; +using NFX.Log; + +namespace Agni.Workers.Server.Queue +{ + /// + /// Glue trampoline for TodoQueueService + /// + public sealed class TodoQueueServer : ITodoQueue + { + public int Enqueue(TodoFrame[] todos) + { + return TodoQueueService.Instance.Enqueue(todos); + } + } + + /// + /// Service that enqueues todos + /// + public sealed partial class TodoQueueService : AgentServiceBase, ITodoQueue, ITodoHost + { + + #region CONSTS + public const string CONFIG_QUEUE_SECTION = "queue"; + public const string CONFIG_QUEUE_STORE_SECTION = "queue-store"; + public const string CONFIG_TYPE_RESOLVER_SECTION = "type-resolver"; + + public const int FULL_BATCH_SIZE = 1024; + + public const string ALL ="*"; + #endregion + + #region STATIC/.ctor + + private static object s_Lock = new object(); + private static volatile TodoQueueService s_Instance; + + internal static TodoQueueService Instance + { + get + { + var instance = s_Instance; + if (instance==null) throw new WorkersException("{0} is not allocated".Args(typeof(TodoQueueService).FullName)); + return instance; + } + } + + public TodoQueueService() : this(null) {} + + public TodoQueueService(object director) : base(director) + { + m_Queues = new Registry(); + + lock(s_Lock) + { + if (s_Instance!=null) + throw new WorkersException("{0} is already allocated".Args(typeof(TodoQueueService).FullName)); + + s_Instance = this; + } + } + + protected override void Destructor() + { + base.Destructor(); + DisposeAndNull(ref m_QueueStore); + lock(s_Lock) + s_Instance = null; + } + + #endregion + + #region Fields + + private TodoQueueStore m_QueueStore; + private Registry m_Queues; + + private CappedSet m_Duplicates = new CappedSet(); + + private int m_stat_EnqueueCalls; + private NamedInterlocked m_stat_EnqueueTodoCount = new NamedInterlocked(); + private int m_stat_QueueThreadSpins; + private NamedInterlocked m_stat_ProcessOneQueueCount = new NamedInterlocked(); + + private NamedInterlocked m_stat_MergedTodoCount = new NamedInterlocked(); + private NamedInterlocked m_stat_FetchedTodoCount = new NamedInterlocked(); + private NamedInterlocked m_stat_ProcessedTodoCount = new NamedInterlocked(); + + + private NamedInterlocked m_stat_PutTodoCount = new NamedInterlocked(); + private NamedInterlocked m_stat_UpdateTodoCount = new NamedInterlocked(); + private NamedInterlocked m_stat_CompletedTodoCount = new NamedInterlocked(); + private NamedInterlocked m_stat_CompletedOkTodoCount = new NamedInterlocked(); + private NamedInterlocked m_stat_CompletedErrorTodoCount = new NamedInterlocked(); + + private NamedInterlocked m_stat_QueueOperationErrorCount = new NamedInterlocked(); + + private NamedInterlocked m_stat_TodoDuplicationCount = new NamedInterlocked(); + + #endregion + + #region Properties + public override int ThreadGranularityMs { get { return 25; } } + #endregion + + #region Public + + void ITodoHost.LocalEnqueue(Todo todo) + { + this.Enqueue(new[] { todo }); + } + + void ITodoHost.LocalEnqueue(IEnumerable todos) + { + this.Enqueue(todos); + } + + Task ITodoHost.LocalEnqueueAsync(IEnumerable todos) + { + CheckServiceActive(); + return Task.Factory.StartNew(() => this.Enqueue(todos) ); + } + + public int Enqueue(IEnumerable todos) + { + if (todos==null || !todos.Any()) return FULL_BATCH_SIZE; + + todos.ForEach( todo => check(todo)); + + return this.Enqueue( todos.Select( t => new TodoFrame(t) ).ToArray() ); + } + + public int Enqueue(TodoFrame[] todos) + { + CheckServiceActive(); + if (todos == null || todos.Length == 0) + return FULL_BATCH_SIZE; + + TodoQueueAttribute attr = null; + foreach (var todo in todos) + { + if (!todo.Assigned) + continue; + + var todoAttr = GuidTypeAttribute.GetGuidTypeAttribute(todo.Type, AgniSystem.ProcessManager.TodoTypeResolver); + + if (attr==null) attr = todoAttr; + + if (attr.QueueName != todoAttr.QueueName) + throw new WorkersException(StringConsts.TODO_QUEUE_ENQUEUE_DIFFERENT_ERROR); + } + + var queue = m_Queues[attr.QueueName]; + if (queue == null) + throw new WorkersException(StringConsts.TODO_QUEUE_NOT_FOUND_ERROR.Args(attr.QueueName)); + + + if (InstrumentationEnabled) + { + Interlocked.Increment(ref m_stat_EnqueueCalls); + m_stat_EnqueueTodoCount.IncrementLong(ALL); + m_stat_EnqueueTodoCount.IncrementLong(queue.Name); + } + + enqueue(queue, todos); + + return m_QueueStore.GetInboundCapacity(queue); + } + + #endregion + + #region .pvt + private void check(Todo todo) + { + if (todo == null) throw new WorkersException(StringConsts.ARGUMENT_ERROR + "Todo.ValidateAndPrepareForEnqueue(todo==null)"); + + todo.ValidateAndPrepareForEnqueue(null); + } + + private class _lck{ public Thread TEnqueu; public DateTime Date; public int IsEnqueue; public int IsProc; } + private Dictionary m_CorrelationLocker = new Dictionary(StringComparer.Ordinal); + + +/* + * Locking works as follows: + * --------------------------- + * All locking is done per CorrelationKey (different CorrelationKey do not inter-lock at all) + * Whithin the same key: + * + * Any existing Enqueue lock blocks another Enqueue until existing is released + * Any existing Enqueue lock returns FALSE for another Process until all Enqueue released + * + * Any existing Processing lock yields next date to Enqueue (+1 sec) + * Any Exisiting Processing lock shifts next Date if another Processing lock is further advanced in time + * + * + * Enqueue lock is reentrant for the same thread. + * Processing lock is reentrant regardless of thread ownership + * + */ + + + + /// + /// Returns the point in time AFTER which the enqueue operation may fetch correlated todos + /// + private DateTime lockCorrelatedEnqueue(string key, DateTime sd) + { + if (key==null) key = string.Empty; + + var ct = Thread.CurrentThread; + + uint spinCount = 0; + while(true) + { + lock(m_CorrelationLocker) + { + _lck lck; + if (!m_CorrelationLocker.TryGetValue(key, out lck)) + { + lck = new _lck { TEnqueu = ct, Date = sd, IsEnqueue = 1, IsProc = 0 }; + m_CorrelationLocker.Add(key, lck); + return sd; + } + + if (lck.IsEnqueue==0) + { + lck.IsEnqueue = 1; + lck.TEnqueu = ct; + return lck.Date.AddSeconds(1); + } + + if (lck.TEnqueu==ct)//if already acquired by this thread + { + lck.IsEnqueue++; + if (sd + /// Tries to lock the correlated todo instance with the specified scheduled date, true if locked, false, otherwise. + /// Takes lock IF not enqueue, false otherwise + /// + private bool tryLockCorrelatedProcessing(string key, DateTime sd) + { + if (key==null) key = string.Empty; + + lock(m_CorrelationLocker) + { + _lck lck; + if (!m_CorrelationLocker.TryGetValue(key, out lck)) + { + lck = new _lck { Date = sd, IsEnqueue = 0, IsProc = 1 }; + m_CorrelationLocker.Add(key, lck); + return true; + } + + if (lck.IsEnqueue==0) + { + lck.IsProc++; + if (sd>lck.Date) lck.Date = sd; + return true; + } + + return false;//lock is enqueue type + } + } + + private bool releaseCorrelatedEnqueue(string key) + { + if (key==null) key = string.Empty; + + lock(m_CorrelationLocker) + { + _lck lck; + if (!m_CorrelationLocker.TryGetValue(key, out lck)) return false; + + if (lck.IsEnqueue==0) return false; + + if (Thread.CurrentThread!=lck.TEnqueu) return false;//not locked by this thread + + lck.IsEnqueue--; + + if (lck.IsEnqueue>0) return true; + + lck.TEnqueu = null;//release thread + + if (lck.IsProc==0) + m_CorrelationLocker.Remove(key); + + return true; + } + } + + private bool releaseCorrelatedProcessing(string key) + { + if (key==null) key = string.Empty; + + lock(m_CorrelationLocker) + { + _lck lck; + if (!m_CorrelationLocker.TryGetValue(key, out lck)) return false; + + if (lck.IsProc==0) return false; + + lck.IsProc--; + + if (lck.IsProc>0) return true; + + if (lck.IsEnqueue==0) + m_CorrelationLocker.Remove(key); + + return true; + } + } + + #endregion + + } +} diff --git a/src/Agni/Workers/Server/Queue/TodoQueueStore.cs b/src/Agni/Workers/Server/Queue/TodoQueueStore.cs new file mode 100644 index 0000000..1615a86 --- /dev/null +++ b/src/Agni/Workers/Server/Queue/TodoQueueStore.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; + +using NFX; +using NFX.Log; +using NFX.ApplicationModel; +using NFX.Environment; + +namespace Agni.Workers.Server.Queue +{ + /// + /// Represents a base for entities that store queue data + /// + public abstract class TodoQueueStore : ApplicationComponent + { + protected TodoQueueStore(TodoQueueService director, IConfigSectionNode node) : base(director) + { + ConfigAttribute.Apply(this, node); + } + + public abstract IEnumerable Fetch(TodoQueue queue, DateTime utcNow); + public abstract TodoFrame FetchLatestCorrelated(TodoQueue queue, string correlationKey, DateTime utcStartingFrom); + + + /// + /// References service that this store is under + /// + public TodoQueueService QueueService { get { return (TodoQueueService)ComponentDirector;} } + + + public abstract object BeginTransaction(TodoQueue queue); + + public abstract void CommitTransaction(TodoQueue queue, object transaction); + + public abstract void RollbackTransaction(TodoQueue queue, object transaction); + + public abstract void Put(TodoQueue queue, TodoFrame todo, object transaction); + + public abstract void Update(TodoQueue queue, Todo todo, bool sysOnly, object transaction); + + public abstract void Complete(TodoQueue queue, TodoFrame todo, Exception error = null, object transaction = null); + + public abstract int GetInboundCapacity(TodoQueue queue); + + protected Guid Log(MessageType type, + string from, + string message, + Exception error = null, + Guid? relatedMessageID = null, + string parameters = null) + { + return QueueService.Log(type, "{0}.{1}".Args(GetType().Name, from), message, error, relatedMessageID, parameters); + } + } +} diff --git a/src/Agni/Workers/Signal.cs b/src/Agni/Workers/Signal.cs new file mode 100644 index 0000000..6bc4dc9 --- /dev/null +++ b/src/Agni/Workers/Signal.cs @@ -0,0 +1,124 @@ +using System; + +using NFX; +using NFX.Environment; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; + +namespace Agni.Workers +{ + [Serializable] + public abstract class Signal : AmorphousTypedRow + { + /// + /// Factory method that creates new Signal based on provided PID + /// + public static TSignal MakeNew(PID pid) where TSignal : Signal, new() { return makeDefault(new TSignal(), pid); } + + /// + /// Factory method that creates new Signal based on provided Type, PID and Configuration + /// + public static Signal MakeNew(Type type, PID pid, IConfigSectionNode args) { return makeDefault(FactoryUtils.MakeAndConfigure(args, type), pid); } + + private static TSignal makeDefault(TSignal signal, PID pid) where TSignal : Signal + { + signal.m_SysID = AgniSystem.GDIDProvider.GenerateOneGDID(SysConsts.GDID_NS_WORKER, SysConsts.GDID_NAME_WORKER_SIGNAL); + signal.m_SysPID = pid; + signal.m_SysTimestampUTC = App.TimeSource.UTCNow; + signal.m_SysAbout = "{0}@{1}@{2}".Args(App.CurrentCallUser.Name, App.Name, AgniSystem.HostName); + return signal; + } + + protected Signal() { } + + private GDID m_SysID; + private PID m_SysPID; + private DateTime m_SysTimestampUTC; + private string m_SysAbout; + + public void ____Deserialize(GDID id, PID pid, DateTime ts, string about) + { m_SysID = id; m_SysPID = pid; m_SysTimestampUTC = ts; m_SysAbout = about; } + + /// + /// Globally-unique ID of the Signal + /// + public GDID SysID { get { return m_SysID; } } + + /// + /// Globally-unique ID of the Process + /// + public PID SysPID { get { return m_SysPID; } } + + /// + /// When was created + /// + public DateTime SysCreateTimeStampUTC { get { return m_SysTimestampUTC; } } + + /// + /// Who is creator + /// + public string SysAbout { get { return m_SysAbout; } } + + /// + /// Type Guid + /// + public Guid SysTypeGuid { get { return GuidTypeAttribute.GetGuidTypeAttribute(GetType()).TypeGuid; } } + + public override string ToString() { return "{0}('{1}')".Args(GetType().Name, SysPID); } + + public override int GetHashCode() { return m_SysID.GetHashCode(); } + + public override bool Equals(Row other) + { + var otherSignal = other as Signal; + if (otherSignal == null) return false; + return this.m_SysID == otherSignal.m_SysID; + } + + public void ValidateAndPrepareForDispatch(string targetName) + { + DoPrepareForEnqueuePreValidate(targetName); + + var ve = this.Validate(targetName); + if (ve != null) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + "Signal.ValidateAndPrepareForEnqueue(todo).validate: " + ve.ToMessageWithType(), ve); + + DoPrepareForEnqueuePostValidate(targetName); + } + + public override Exception Validate(string targetName) + { + var ve = base.Validate(targetName); + if (ve != null) return ve; + + if (SysID.IsZero) + return new CRUDFieldValidationException(this, "SysID", "SysID.IsZero, use MakeNew<>() to make new instances"); + + return null; + } + + protected virtual void DoPrepareForEnqueuePreValidate(string targetName) { } + protected virtual void DoPrepareForEnqueuePostValidate(string targetName) { } + } + + [Serializable] + public abstract class ResultSignal : Signal + { + /// + /// Factory method that creates new Result Signals assigning them new GDID + /// + public static TSignal MakeNew(Process process) where TSignal : ResultSignal, new() + { + var result = Signal.MakeNew(process.SysPID); + result.m_SysDescriptor = process.SysDescriptor; + return result; + } + + private ProcessDescriptor m_SysDescriptor; + + public ProcessDescriptor SysDescriptor { get { return m_SysDescriptor; } } + + public void ____Deserialize(ProcessDescriptor descriptor) + { m_SysDescriptor = descriptor; } + } +} diff --git a/src/Agni/Workers/SignalFrame.cs b/src/Agni/Workers/SignalFrame.cs new file mode 100644 index 0000000..18d21af --- /dev/null +++ b/src/Agni/Workers/SignalFrame.cs @@ -0,0 +1,114 @@ +using System; + +using NFX; +using NFX.DataAccess.Distributed; +using NFX.Serialization.BSON; + +namespace Agni.Workers +{ + [Serializable] + public struct SignalFrame + { + public const int SERIALIZER_DEFAULT = 0; + public const int SERIALIZER_BSON = 1; + + public GDID ID; + public PID PID; + + public Guid Type; + public DateTime Timestamp; + public string About; + + public ProcessDescriptor? Descriptor; + + public int Serializer; + public byte[] Content; + + /// + /// Internal. Returns the original instance that was passed to .ctor + /// This allows to use this structure for dual purpose. + /// + [NonSerialized] + internal readonly Signal ____CtorOriginal; + + public SignalFrame(Signal signal, int? serializer = SERIALIZER_DEFAULT) + { + if (signal == null) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + "SignalFrame.ctor(signal==null)"); + + ____CtorOriginal = signal; + + var qa = GuidTypeAttribute.GetGuidTypeAttribute(signal.GetType()); + + if (serializer.HasValue) + { + if (serializer == SERIALIZER_DEFAULT) serializer = SERIALIZER_BSON; + else + if (serializer != SERIALIZER_BSON)//we only support BSON for now + throw new WorkersException(StringConsts.SIGNAL_FRAME_SER_NOT_SUPPORTED_ERROR.Args(signal.GetType().Name, serializer)); + + byte[] content; + try + { + var cdoc = RowConverter.DefaultInstance.RowToBSONDocument(signal, null); + content = cdoc.WriteAsBSONToNewArray(); + } + catch (Exception error) + { + throw new WorkersException(StringConsts.SIGNAL_FRAME_SER_ERROR.Args(signal.GetType().FullName, error.ToMessageWithType()), error); + } + + this.Serializer = serializer.Value; + this.Content = content; + } + else + { + this.Serializer = 0; + this.Content = null; + } + + this.ID = signal.SysID; + this.PID = signal.SysPID; + this.Type = qa.TypeGuid; + this.Timestamp = signal.SysCreateTimeStampUTC; + this.About = signal.SysAbout; + var resultSignal = signal as ResultSignal; + this.Descriptor = resultSignal != null ? resultSignal.SysDescriptor : (ProcessDescriptor?)null; + } + + public Signal Materialize(IGuidTypeResolver resolver) + { + if (____CtorOriginal != null) return ____CtorOriginal; + + //1. Resolve type + var type = resolver.Resolve(this.Type); + + //3. Create signal instance + var signal = (Signal)NFX.Serialization.SerializationUtils.MakeNewObjectInstance(type); + signal.____Deserialize(ID, PID, Timestamp, About); + var resultSignal = signal as ResultSignal; + if (resultSignal != null) + { + if (!Descriptor.HasValue) + throw new WorkersException("TODO"); + resultSignal.____Deserialize(Descriptor.Value); + } + + //4. Deserialize content + if (Serializer != SignalFrame.SERIALIZER_BSON) //for now only support this serializer + throw new WorkersException(StringConsts.SIGNAL_FRAME_DESER_NOT_SUPPORTED_ERROR.Args(type.Name, Serializer)); + + try + { + var docContent = BSONDocument.FromArray(Content); + RowConverter.DefaultInstance.BSONDocumentToRow(docContent, signal, null); + } + catch (Exception error) + { + throw new WorkersException(StringConsts.SIGNAL_FRAME_DESER_ERROR.Args(type.Name, error.ToMessageWithType()), error); + } + + return signal; + } + } +} diff --git a/src/Agni/Workers/Signals.cs b/src/Agni/Workers/Signals.cs new file mode 100644 index 0000000..a31cf22 --- /dev/null +++ b/src/Agni/Workers/Signals.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using NFX; +using NFX.DataAccess.CRUD; +using NFX.Environment; + +namespace Agni.Workers +{ + [Signal("0626D816-FCF6-40BD-8652-274CAEEA3E63")] + public sealed class OkSignal : ResultSignal + { + public static OkSignal Make(Process process) + { + var ok = MakeNew(process); + return ok; + } + } + [Signal("C3FFB82B-5C9D-4683-A252-1FF6C857F5ED")] + public sealed class UnknownSignal : ResultSignal + { + public static UnknownSignal Make(Process process, Signal signal) + { + var unknown = MakeNew(process); + unknown.SignalType = signal.SysTypeGuid; + return unknown; + } + + [Config][Field(backendName: "guid", required: true)] public Guid? SignalType { get; set; } + } + + [Signal("DE276340-8075-490C-9FAF-F9240480902A")] public sealed class CancelSignal : Signal { } + [Signal("5ED96CF6-763E-4567-AB98-E705F5501264")] public sealed class TerminateSignal : Signal { } + [Signal("D0E26B05-CDC0-4A2A-8CFD-7F3D0A39C8C4")] public sealed class FinishSignal : Signal + { + public static ResultSignal Dispatch(PID pid, string description = null) + { + var finish = MakeNew(pid); + finish.Description = description; + return AgniSystem.ProcessManager.Dispatch(finish); + } + + [Config][Field(backendName: "d")] public string Description { get; set; } + } + [Signal("27DE1FF6-98FE-418D-8243-EE991510D84E")] + public sealed class FinalizeSignal : Signal + { + public static ResultSignal Dispatch(PID pid) + { + var finalize = MakeNew(pid); + return AgniSystem.ProcessManager.Dispatch(finalize); + } + } +} diff --git a/src/Agni/Workers/Todo.cs b/src/Agni/Workers/Todo.cs new file mode 100644 index 0000000..27706fd --- /dev/null +++ b/src/Agni/Workers/Todo.cs @@ -0,0 +1,200 @@ +using System; + +using NFX; +using NFX.DataAccess.CRUD; +using NFX.DataAccess.Distributed; +using NFX.Environment; + +namespace Agni.Workers +{ + /// + /// Represents a unit of abstract work that is dispatched to a remote worker in an asynchronous fashion. + /// Todos are essentially a form of a queueable asynchronous one-way command object (Execute() does not return business object). + /// Todos are dequeued in the order of submission and SysStartDate constraint, processed sequentially or in-parallel depending on a SysParallelKey + /// + [Serializable] + public abstract class Todo : AmorphousTypedRow, IDistributedStableHashProvider + { + /// + /// Denotes states of Todo execution state machine + /// + public struct ExecuteState + { + /// + /// The todo will reexecute depending on ReexecuteAfterErrorInMs() having updated in-place + /// + public static readonly ExecuteState ReexecuteUpdatedAfterError = new ExecuteState(-6, true); + + /// + /// The todo will reexecute depending on ReexecuteAfterErrorInMs() having sys fields updated in-place + /// + public static readonly ExecuteState ReexecuteAfterError = new ExecuteState(-5, true); + + /// + /// The todo must be updated in-place and reexecuted when due + /// + public static readonly ExecuteState ReexecuteUpdated = new ExecuteState(-4, true); + + /// + /// The todo sys fields must be updated in-place and reexecuted when due + /// + public static readonly ExecuteState ReexecuteSysUpdated = new ExecuteState(-3, true); + + /// + /// The todo must be re-executed as-is again + /// + public static readonly ExecuteState Reexecute = new ExecuteState(-2, true); + + /// + /// The execution is completed and todo should be discarded from the queue + /// + public static readonly ExecuteState Complete = new ExecuteState(-1, true); + + /// + /// Initial state + /// + public static readonly ExecuteState Initial = new ExecuteState(0, true); + + internal ExecuteState(int state, bool sys) { State = state; } + public ExecuteState(int state) + { + if (state <= 0) throw new WorkersException(StringConsts.ARGUMENT_ERROR + "state < 0"); + State = state; + } + + public readonly int State; + + public static bool operator ==(ExecuteState a, ExecuteState b) { return a.State == b.State; } + public static bool operator !=(ExecuteState a, ExecuteState b) { return a.State != b.State; } + + public override int GetHashCode() { return State; } + public override bool Equals(object obj) { return this == (ExecuteState)obj; } + + public override string ToString() { return "ExecuteState({0})".Args(State); } + } + + /// + /// Factory method that creates new Todos assigning them new GDID + /// + public static TTodo MakeNew() where TTodo : Todo, new() { return makeDefault(new TTodo()); } + + /// + /// Factory method that creates new Todos from Type and Configuration assigning them new GDID + /// + public static Todo MakeNew(Type type, IConfigSectionNode args) { return makeDefault(FactoryUtils.MakeAndConfigure(args, type)); } + + private static TTodo makeDefault(TTodo todo) where TTodo : Todo + { + //warning: Todo IDs must be cross-type unique (should not depend on queue) + todo.m_SysID = AgniSystem.GDIDProvider.GenerateOneGDID(SysConsts.GDID_NS_WORKER, SysConsts.GDID_NAME_WORKER_TODO); + todo.m_SysCreateTimestampUTC = App.TimeSource.UTCNow; + return todo; + } + + protected Todo() { } + + private GDID m_SysID; + private DateTime m_SysCreateTimestampUTC; + /// + /// Infrustructure method, developers do not call + /// + public void ____Deserialize(GDID id, DateTime ts) { m_SysID = id; m_SysCreateTimestampUTC = ts;} + + /// + /// Globally-unique ID of the TODO + /// + public GDID SysID { get { return m_SysID; } } + + /// + /// When was created + /// + public DateTime SysCreateTimeStampUTC { get { return m_SysCreateTimestampUTC; } } + + /// + /// Provides the sharding key which is used for dispatching items into HostSets + /// + public string SysShardingKey { get; set; } + + /// + /// Provides the key which is used for parallel processing: items with the same key + /// get executed sequentially + /// + public string SysParallelKey { get; set;} + + /// + /// Provides relative processing priority of processing + /// + public int SysPriority { get; set; } + + /// + /// When set, tells the system when (UTC) should the item be considered for processing + /// + public DateTime SysStartDate { get; set; } + + /// + /// Provides current state machine execution state + /// + public ExecuteState SysState { get; internal set; } + + /// + /// Provides current machine execution retry state + /// + public int SysTries { get; internal set; } + + /// + /// Executes the todo. Override to perform actual logic. + /// You have to handle all exceptions, otherwise the leaked exception will + /// complete the todo with error. Return the result that describes whether the item completed or should be reexecuted again. + /// Keep in mind: Todos are not designed to execute long-running(tens+ of seconds) processes, launch other async workers instead + /// + protected internal abstract ExecuteState Execute(ITodoHost host, DateTime utcBatchNow); + + /// + /// Invoked to determine when should the next reexecution takes place after an error. + /// Throw exception if your buisiness case has exhausted all allowed retries as judjed by SysTries. + /// Return -1 to indicate the immediate execution without consideration of SysTries (default) + /// + protected internal virtual int RetryAfterErrorInMs(DateTime utcBatchNow) { return -1; } + + public override string ToString() { return "{0}('{1}')".Args(GetType().FullName, SysID); } + + public override int GetHashCode() { return m_SysID.GetHashCode(); } + + public override bool Equals(Row other) + { + var otodo = other as Todo; + if (otodo==null) return false; + return this.m_SysID == otodo.m_SysID; + } + + public ulong GetDistributedStableHash() + { + return m_SysID.GetDistributedStableHash(); + } + + public void ValidateAndPrepareForEnqueue(string targetName) + { + DoPrepareForEnqueuePreValidate(targetName); + + var ve = this.Validate(targetName); + if (ve != null) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + "Todo.ValidateAndPrepareForEnqueue(todo).validate: " + ve.ToMessageWithType(), ve); + + DoPrepareForEnqueuePostValidate(targetName); + } + + public override Exception Validate(string targetName) + { + var ve = base.Validate(targetName); + if (ve != null) return ve; + + if (SysID.IsZero) + return new CRUDFieldValidationException(this, "SysID", "SysID.IsZero, use NewTodo<>() to make new instances"); + + return null; + } + + protected virtual void DoPrepareForEnqueuePreValidate(string targetName) { } + protected virtual void DoPrepareForEnqueuePostValidate(string targetName) { } + } +} diff --git a/src/Agni/Workers/TodoFrame.cs b/src/Agni/Workers/TodoFrame.cs new file mode 100644 index 0000000..4bce01e --- /dev/null +++ b/src/Agni/Workers/TodoFrame.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; + +using NFX; +using NFX.DataAccess.Distributed; +using NFX.Serialization.BSON; + +namespace Agni.Workers +{ + /// + /// Provides an efficient data vector for marshalling and storage of todo's in queues. + /// This type obviates the need of extra serialization for teleportation and storage of Todo instances. + /// Special-purposed Glue binding is used to teleport TodoFrames and directly store them in queue without + /// unnecessary intermediate serialization steps + /// + [Serializable] + public struct TodoFrame + { + public const int SERIALIZER_DEFAULT = 0; + public const int SERIALIZER_BSON = 1; + + public GDID ID; + public Guid Type; + public DateTime CreateTimestampUTC; + public string ShardingKey; + public string ParallelKey; + public int Priority; + public DateTime StartDate; + public string CorrelationKey; + + public int State; + public int Tries; + + public int Serializer; + public byte[] Content; + + + + public bool Assigned { get { return !ID.IsZero; } } + + + /// + /// Internal. Returns the original instance that was passed to .ctor + /// This allows to use this structure for dual purpose. + /// + [NonSerialized] internal readonly Todo ____CtorOriginal; + + /// + /// Frames the Todo instance, pass serialize null to frame only Sys Fields without content + /// + public TodoFrame(Todo todo, int? serializer = SERIALIZER_DEFAULT) + { + if (todo == null) + throw new WorkersException(StringConsts.ARGUMENT_ERROR + "TodoFrame.ctor(todo==null)"); + + ____CtorOriginal = todo; + + var qa = GuidTypeAttribute.GetGuidTypeAttribute(todo.GetType()); + + if (serializer.HasValue) + { + if (serializer == SERIALIZER_DEFAULT) serializer = SERIALIZER_BSON; + else + if (serializer != SERIALIZER_BSON)//we only support BSON for now + throw new WorkersException(StringConsts.TODO_FRAME_SER_NOT_SUPPORTED_ERROR.Args(todo.GetType().Name, serializer)); + + byte[] content; + try + { + var cdoc = RowConverter.DefaultInstance.RowToBSONDocument(todo, null); + content = cdoc.WriteAsBSONToNewArray(); + } + catch (Exception error) + { + throw new WorkersException(StringConsts.TODO_FRAME_SER_ERROR.Args(todo.GetType().FullName, error.ToMessageWithType()), error); + } + + this.Serializer = serializer.Value; + this.Content = content; + } + else + { + this.Serializer = 0; + this.Content = null; + } + + var t = todo.GetType(); + + this.ID = todo.SysID; + this.Type = qa.TypeGuid; + this.CreateTimestampUTC = todo.SysCreateTimeStampUTC; + this.ShardingKey = todo.SysShardingKey; + this.ParallelKey = todo.SysParallelKey; + this.Priority = todo.SysPriority; + this.StartDate = todo.SysStartDate < todo.SysCreateTimeStampUTC ? todo.SysCreateTimeStampUTC : todo.SysStartDate; + + var ct = todo as CorrelatedTodo; + this.CorrelationKey = ct!=null ? ct.SysCorrelationKey : null; + + this.State = todo.SysState.State; + this.Tries = todo.SysTries; + } + + private static Dictionary s_TypesCache = new Dictionary(StringComparer.Ordinal); + + /// + /// Materializes the Todo instance represented by this frame in the scope of IGuidTypeResolver + /// + public Todo Materialize(IGuidTypeResolver resolver) + { + if (____CtorOriginal!=null) return ____CtorOriginal; + + //1. Resolve type + var type = resolver.Resolve(this.Type); + + //3. Create TODO instance + var result = (Todo)NFX.Serialization.SerializationUtils.MakeNewObjectInstance( type ); + result.____Deserialize( ID, CreateTimestampUTC ); + + result.SysShardingKey = ShardingKey; + result.SysParallelKey = ParallelKey; + result.SysPriority = Priority; + result.SysStartDate = StartDate; + + var ct = result as CorrelatedTodo; + if (ct!=null) + ct.SysCorrelationKey = CorrelationKey; + + result.SysTries = Tries; + result.SysState = new Todo.ExecuteState(State, true); + + //4. Deserialize content + if (Serializer!=TodoFrame.SERIALIZER_BSON) //for now only support this serializer + throw new WorkersException(StringConsts.TODO_FRAME_DESER_NOT_SUPPORTED_ERROR.Args(type.Name, Serializer)); + + try + { + var docContent = BSONDocument.FromArray(Content); + RowConverter.DefaultInstance.BSONDocumentToRow(docContent, result, null); + } + catch(Exception error) + { + throw new WorkersException(StringConsts.TODO_FRAME_DESER_ERROR.Args(type.Name, error.ToMessageWithType()), error); + } + + return result; + } + + + } +} diff --git a/src/Agni/agdida.laconf b/src/Agni/agdida.laconf new file mode 100644 index 0000000..ed25478 --- /dev/null +++ b/src/Agni/agdida.laconf @@ -0,0 +1,10 @@ +boot +{ + _include { name=agni file=$(~AGNI_BOOT_CONF_FILE) } + + log + { + name="BootLogger" reliable="true" + destination { type="NFX.Log.Destinations.CSVFileDestination, NFX" name="agdida.BootLog" path="$(/agni/log-root)" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } +} \ No newline at end of file diff --git a/src/Agni/agm.laconf b/src/Agni/agm.laconf new file mode 100644 index 0000000..16cdcb4 --- /dev/null +++ b/src/Agni/agm.laconf @@ -0,0 +1,25 @@ +nfx +{ + app-name="agm" + log-csv="NFX.Log.Destinations.CSVFileDestination, NFX" + log-debug="NFX.Log.Destinations.DebugDestination, NFX" + debug-default-action="LogAndThrow" + trace-disable=true + + log + { + name="Logger" + destination { type=$(/$log-csv) name="agm" path="" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } + + glue + { + client-log-level="Error" + + bindings + { + binding { name="async" type="NFX.Glue.Native.MpxBinding" } + binding { name="sync" type="NFX.Glue.Native.SyncBinding" } + } + } +}//nfx \ No newline at end of file diff --git a/src/Agni/ahgov.laconf b/src/Agni/ahgov.laconf new file mode 100644 index 0000000..21463ae --- /dev/null +++ b/src/Agni/ahgov.laconf @@ -0,0 +1,10 @@ +boot +{ + _include { name=agni file=$(~AGNI_BOOT_CONF_FILE) } + + log + { + name="BootLogger" reliable="true" + destination { type="NFX.Log.Destinations.CSVFileDestination, NFX" name="ahgov.BootLog" path="$(/agni/log-root)" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } +} \ No newline at end of file diff --git a/src/Agni/aph.laconf b/src/Agni/aph.laconf new file mode 100644 index 0000000..4527ba0 --- /dev/null +++ b/src/Agni/aph.laconf @@ -0,0 +1,10 @@ +boot +{ + _include { name=agni file=$(~AGNI_BOOT_CONF_FILE) } + + log + { + name="BootLogger" reliable="true" + destination { type="NFX.Log.Destinations.CSVFileDestination, NFX" name="aph.BootLog" path="$(/agni/log-root)" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } +} \ No newline at end of file diff --git a/src/Agni/ascon-nolog.laconf b/src/Agni/ascon-nolog.laconf new file mode 100644 index 0000000..0bd5f59 --- /dev/null +++ b/src/Agni/ascon-nolog.laconf @@ -0,0 +1,21 @@ +nfx +{ + log-root=$(~AGNI_DISK_ROOT) + app-name="ascon" + log-csv="NFX.Log.Destinations.CSVFileDestination, NFX" + log-debug="NFX.Log.Destinations.DebugDestination, NFX" + debug-default-action="LogAndThrow" + trace-disable=true + + glue + { + client-log-level="Error" + + bindings + { + binding { name="apterm" type="Agni.Glue.AppTermBinding, Agni" } + binding { name="async" type="NFX.Glue.Native.MpxBinding" } + binding { name="sync" type="NFX.Glue.Native.SyncBinding" } + } + } +}//nfx \ No newline at end of file diff --git a/src/Agni/ascon.laconf b/src/Agni/ascon.laconf new file mode 100644 index 0000000..b92bc44 --- /dev/null +++ b/src/Agni/ascon.laconf @@ -0,0 +1,26 @@ +nfx +{ + app-name="ascon" + log-csv="NFX.Log.Destinations.CSVFileDestination, NFX" + log-debug="NFX.Log.Destinations.DebugDestination, NFX" + debug-default-action="LogAndThrow" + trace-disable=true + + log + { + name="Logger" + destination { type=$(/$log-csv) name="ascon" path="" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } + + glue + { + client-log-level="Error" + + bindings + { + binding { name="apterm" type="Agni.Glue.AppTermBinding, Agni" } + binding { name="async" type="NFX.Glue.Native.MpxBinding" } + binding { name="sync" type="NFX.Glue.Native.SyncBinding" } + } + } +}//nfx \ No newline at end of file diff --git a/src/Agni/ash.laconf b/src/Agni/ash.laconf new file mode 100644 index 0000000..45daea3 --- /dev/null +++ b/src/Agni/ash.laconf @@ -0,0 +1,10 @@ +boot +{ + _include { name=agni file=$(~AGNI_BOOT_CONF_FILE) } + + log + { + name="BootLogger" reliable="true" + destination { type="NFX.Log.Destinations.CSVFileDestination, NFX" name="ash.BootLog" path="$(/agni/log-root)" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } +} \ No newline at end of file diff --git a/src/Agni/aws.laconf b/src/Agni/aws.laconf new file mode 100644 index 0000000..77cbe18 --- /dev/null +++ b/src/Agni/aws.laconf @@ -0,0 +1,10 @@ +boot +{ + _include { name=agni file=$(~AGNI_BOOT_CONF_FILE) } + + log + { + name="BootLogger" reliable="true" + destination { type="NFX.Log.Destinations.CSVFileDestination, NFX" name="aws.BootLog" path="$(/agni/log-root)" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } +} \ No newline at end of file diff --git a/src/Agni/azgov.laconf b/src/Agni/azgov.laconf new file mode 100644 index 0000000..dc34a56 --- /dev/null +++ b/src/Agni/azgov.laconf @@ -0,0 +1,10 @@ +boot +{ + _include { name=agni file=$(~AGNI_BOOT_CONF_FILE) } + + log + { + name="BootLogger" reliable="true" + destination { type="NFX.Log.Destinations.CSVFileDestination, NFX" name="azgov.BootLog" path="$(/agni/log-root)" file-name="{0:yyyyMMdd}-$($name).csv.log" } + } +} \ No newline at end of file diff --git a/src/Agni/post-build b/src/Agni/post-build new file mode 100644 index 0000000..202ffb6 --- /dev/null +++ b/src/Agni/post-build @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +SCRIPT=`realpath -s $0` +SCRIPTPATH=`dirname $SCRIPT` + +SOLUTION_DIR=$1 +PROJECT_DIR=${SCRIPTPATH}/ diff --git a/src/Agni/post-build.cmd b/src/Agni/post-build.cmd new file mode 100644 index 0000000..4046e73 --- /dev/null +++ b/src/Agni/post-build.cmd @@ -0,0 +1,2 @@ +set SOLUTION_DIR=%1 +set PROJECT_DIR=%~dp0 diff --git a/src/Agni/pre-build b/src/Agni/pre-build new file mode 100644 index 0000000..9944b75 --- /dev/null +++ b/src/Agni/pre-build @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +SCRIPT=`realpath -s $0` +SCRIPTPATH=`dirname $SCRIPT` + +SOLUTION_DIR=$1 +PROJECT_DIR=${SCRIPTPATH}/ + +cp --force "${SOLUTION_DIR}lib/nfx/TelemetryViewer.exe" "TelemetryViewer.exe" +cp --force "${SOLUTION_DIR}lib/nfx/TelemetryViewer.laconf" "TelemetryViewer.laconf" +cp --force "${SOLUTION_DIR}lib/nfx/ntc.exe" "ntc.exe" +cp --force "${SOLUTION_DIR}lib/nfx/gluec.exe" "gluec.exe" +cp --force "${SOLUTION_DIR}lib/nfx/rsc.exe" "rsc.exe" +cp --force "${SOLUTION_DIR}lib/nfx/buildinfo.exe" "buildinfo.exe" +cp --force "${SOLUTION_DIR}lib/nfx/phash.exe" "phash.exe" +cp --force "${SOLUTION_DIR}lib/nfx/NFX.dll" "NFX.dll" +cp --force "${SOLUTION_DIR}lib/nfx/NFX.MongoDB.dll" "NFX.MongoDB.dll" +cp --force "${SOLUTION_DIR}lib/nfx/MySql.Data.dll" "MySql.Data.dll" +cp --force "${SOLUTION_DIR}lib/nfx/NFX.MySQL.dll" "NFX.MySQL.dll" + +mono buildinfo.exe > "${PROJECT_DIR}BUILD_INFO.txt" +mono ntc.exe "${PROJECT_DIR}/WebManager/Pages/*.nht" -r -ext ".auto.cs" -src diff --git a/src/Agni/pre-build.cmd b/src/Agni/pre-build.cmd new file mode 100644 index 0000000..8b55014 --- /dev/null +++ b/src/Agni/pre-build.cmd @@ -0,0 +1,8 @@ +set SOLUTION_DIR=%1 +set CONFIG=%2 +set PROJECT_DIR=%~dp0 +set TOOL_DIR=%SOLUTION_DIR%\lib\nfx\run-netf\ + + +"%TOOL_DIR%buildinfo" > "%PROJECT_DIR%BUILD_INFO.txt" +"%TOOL_DIR%ntc" "%PROJECT_DIR%WebManager\Pages\*.nht" -sub -r -ext ".auto.cs" -src