From e9fa0300cefb195667675a9f39a6004bd0a2f41c Mon Sep 17 00:00:00 2001 From: Lee Wexler Date: Fri, 5 Apr 2024 15:18:37 -0400 Subject: [PATCH] Multi-instance Support (#311) --- CHANGELOG.md | 67 ++++++ build.gradle | 13 +- gradle.properties | 9 +- .../io/xh/hoist/BaseController.groovy | 17 +- ...onnectionPoolMonitorAdminController.groovy | 36 --- .../hoist/admin/EhCacheAdminController.groovy | 64 ------ .../xh/hoist/admin/EnvAdminController.groovy | 26 --- .../admin/LogViewerAdminController.groovy | 108 --------- .../admin/MemoryMonitorAdminController.groovy | 37 ---- .../hoist/admin/MonitorAdminController.groovy | 13 -- .../hoist/admin/ServiceAdminController.groovy | 46 ---- .../admin/WebSocketAdminController.groovy | 27 --- .../cluster/BaseClusterController.groovy | 34 +++ .../cluster/ClusterAdminController.groovy | 54 +++++ ...onnectionPoolMonitorAdminController.groovy | 52 +++++ .../admin/cluster/EnvAdminController.groovy | 31 +++ .../cluster/HzObjectAdminController.groovy | 53 +++++ .../cluster/LogViewerAdminController.groovy | 172 ++++++++++++++ .../MemoryMonitorAdminController.groovy | 62 ++++++ .../MonitorResultsAdminController.groovy | 33 +++ .../ServiceManagerAdminController.groovy | 50 +++++ .../cluster/WebSocketAdminController.groovy | 41 ++++ .../io/xh/hoist/impl/XhController.groovy | 3 + .../hoist/security/AccessInterceptor.groovy | 11 +- .../domain/io/xh/hoist/log/LogLevel.groovy | 42 +++- .../domain/io/xh/hoist/track/TrackLog.groovy | 4 + grails-app/init/io/xh/hoist/BootStrap.groovy | 10 +- .../init/io/xh/hoist/ClusterConfig.groovy | 209 ++++++++++++++++++ .../hoist/admin/ServiceManagerService.groovy | 66 ++++++ .../alertbanner/AlertBannerService.groovy | 39 ++-- .../ClientErrorEmailService.groovy | 4 + .../clienterror/ClientErrorService.groovy | 45 ++-- .../hoist/cluster/ClusterAdminService.groovy | 176 +++++++++++++++ .../io/xh/hoist/cluster/ClusterService.groovy | 192 ++++++++++++++++ .../io/xh/hoist/config/ConfigService.groovy | 24 +- .../io/xh/hoist/email/EmailService.groovy | 15 ++ .../environment/EnvironmentService.groovy | 3 +- .../hoist/export/GridExportImplService.groovy | 19 +- .../feedback/FeedbackEmailService.groovy | 10 +- .../xh/hoist/feedback/FeedbackService.groovy | 19 +- .../io/xh/hoist/log/LogArchiveService.groovy | 7 +- .../io/xh/hoist/log/LogLevelService.groovy | 11 + .../io/xh/hoist/log/LogReaderService.groovy | 4 + .../ConnectionPoolMonitoringService.groovy | 15 +- .../monitor/MemoryMonitoringService.groovy | 15 +- .../monitor/MonitoringEmailService.groovy | 10 +- .../xh/hoist/monitor/MonitoringService.groovy | 103 +++++---- .../io/xh/hoist/track/TrackService.groovy | 8 +- .../groovy/io/xh/hoist/BaseService.groovy | 162 ++++++++++++-- .../io/xh/hoist/HoistCoreGrailsPlugin.groovy | 7 +- .../groovy/io/xh/hoist/browser/Utils.groovy | 2 - .../groovy/io/xh/hoist/cache/Cache.groovy | 42 +++- .../groovy/io/xh/hoist/cache/Entry.groovy | 50 ++++- .../io/xh/hoist/cluster/ClusterRequest.groovy | 25 +++ .../xh/hoist/cluster/ClusterResponse.groovy | 8 + .../xh/hoist/cluster/ReplicatedValue.groovy | 29 +++ .../hoist/cluster/ReplicatedValueEntry.groovy | 52 +++++ .../configuration/ApplicationConfig.groovy | 26 ++- .../hoist/configuration/LogbackConfig.groovy | 21 +- ...enderer.groovy => ExceptionHandler.groovy} | 66 +++--- .../InstanceNotFoundException.groovy | 14 ++ .../io/xh/hoist/json/JSONSerializer.java | 3 +- .../serializer/ThrowableSerializer.groovy | 41 ++++ .../hoist/log/ClusterInstanceConverter.groovy | 19 ++ .../xh/hoist/log/LogSupportConverter.groovy | 5 +- .../io/xh/hoist/role/BaseRoleService.groovy | 1 + .../role/provided/DefaultRoleService.groovy | 25 ++- .../hoist/security/HoistSecurityFilter.groovy | 29 +-- src/main/groovy/io/xh/hoist/util/Timer.groovy | 70 ++++-- src/main/groovy/io/xh/hoist/util/Utils.groovy | 64 +++++- .../websocket/HoistWebSocketChannel.groovy | 1 - src/main/resources/hazelcast-hibernate.xml | 14 ++ 72 files changed, 2260 insertions(+), 625 deletions(-) delete mode 100644 grails-app/controllers/io/xh/hoist/admin/ConnectionPoolMonitorAdminController.groovy delete mode 100644 grails-app/controllers/io/xh/hoist/admin/EhCacheAdminController.groovy delete mode 100644 grails-app/controllers/io/xh/hoist/admin/EnvAdminController.groovy delete mode 100644 grails-app/controllers/io/xh/hoist/admin/LogViewerAdminController.groovy delete mode 100644 grails-app/controllers/io/xh/hoist/admin/MemoryMonitorAdminController.groovy delete mode 100644 grails-app/controllers/io/xh/hoist/admin/ServiceAdminController.groovy delete mode 100644 grails-app/controllers/io/xh/hoist/admin/WebSocketAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/BaseClusterController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/ClusterAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/ConnectionPoolMonitorAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/EnvAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/HzObjectAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/LogViewerAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/MemoryMonitorAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/MonitorResultsAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/ServiceManagerAdminController.groovy create mode 100644 grails-app/controllers/io/xh/hoist/admin/cluster/WebSocketAdminController.groovy create mode 100755 grails-app/init/io/xh/hoist/ClusterConfig.groovy create mode 100644 grails-app/services/io/xh/hoist/admin/ServiceManagerService.groovy create mode 100644 grails-app/services/io/xh/hoist/cluster/ClusterAdminService.groovy create mode 100644 grails-app/services/io/xh/hoist/cluster/ClusterService.groovy create mode 100644 src/main/groovy/io/xh/hoist/cluster/ClusterRequest.groovy create mode 100644 src/main/groovy/io/xh/hoist/cluster/ClusterResponse.groovy create mode 100644 src/main/groovy/io/xh/hoist/cluster/ReplicatedValue.groovy create mode 100644 src/main/groovy/io/xh/hoist/cluster/ReplicatedValueEntry.groovy rename src/main/groovy/io/xh/hoist/exception/{ExceptionRenderer.groovy => ExceptionHandler.groovy} (68%) create mode 100644 src/main/groovy/io/xh/hoist/exception/InstanceNotFoundException.groovy create mode 100644 src/main/groovy/io/xh/hoist/json/serializer/ThrowableSerializer.groovy create mode 100644 src/main/groovy/io/xh/hoist/log/ClusterInstanceConverter.groovy create mode 100644 src/main/resources/hazelcast-hibernate.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index d221f6c7..0f941734 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,73 @@ ## 20.0-SNAPSHOT - unreleased +### 🎁 New Features + +* Hoist Core v20 provides support for running multi-instance clusters of Hoist application servers. + Cluster management is provided by the use of Hazelcast (www.hazelcast.com), an open-source library + providing embedded java support for inter-server communication, co-ordination, and data sharing. + See the new `ClusterService.groovy` service, which provides the clustering implementation and main + API entry point for accessing the cluster. + ** Applications/client plugins upgrading to v18 will need to provide a cluster configuration class + with the name `ClusterConfig.groovy`. See toolbox for an example of this file. + ** Applications should fix their Hazelcast version with the following line in their gradle.properties: + `hazelcast.version=5.3.6` + ** Applications that intend to run with more than one server *must* enable sticky sessions when + routing clients to servers. This is critical for the correct operation of authentication + and web socket communications. + ** Many applications will *not* need to implement additional changes beyond the above to + run with multi-instances; Hoist will setup the cluster, elect a master instance, provide + cluster-aware hibernate caching and logging, and ensure cross-server consistency for its own + APIs. + ** However, complex applications -- especially applications with state, workflow, or business + logic -- should take care to ensure the app is safe to run in multi-instance mode. Distributed + data structures (e.g. Hazelcast Maps) should be used as needed, as well as limiting certain + actions to the "master" server. See toolbox, or Hoist for help. + ** `hoist-react >= 64.0` is required. +* New support for reporting of service statistics for trobuleshooting/monitoring. Implement + `BaseService.getAdminStats()` to provide diagnostic metadata about the state of your service that + will then be displayed in the admin client. +* All `Throwable`s are now serialized to JSON by default using Hoist's standard customization of + Jackson. + +### Breaking Changes +* The following server-side Hoist events are now implemented as cluster-wide Hazelcast messages + rather than single-server Grails events: + ** 'xhFeedbackReceived', 'xhClientErrorReceived', 'xhConfigChanged', and 'xhMonitorStatusReport' + Any applications that are listening to these events with `BaseService.subscribe` should instead use + the new cluster aware method `BaseService.subscribeToTopic`. +* The `exceptionRenderer` singleton has been simplified and renamed as `xhExceptionHandler`. This + change was needed to better support cross-cluster exception handling. This object is used by + Hoist internally for catching uncaught exceptions and this change is not expected to impact + most applications. + +* New support for Role Management. + * Hoist now supports an out-of-the-box, database-driven system for maintaining a hierarchical + set of roles and associating them with individual users. + * New system supports app and plug-in specific integrations to AD and other enterprise systems. + * Hoist-react `v64` is now required and will provide an administrative UI to visualize and + manage the new role system. + * See `DefaultRoleService` for more information. + +### ⚙️ Technical + +* Add `xh/echoHeaders` utility endpoint. Useful for verifying headers (e.g. `jespa_connection_id`) + that are installed by or must pass through multiple ingresses/load balancers. +* Remove HTML tag escaping when parsing alert banner create/update request JSON. + +### 💥 Breaking Changes + +* Applications will typically need to adjust their implementation of `BaseRoleService`. Most + applications are expected to adopt the new provided `DefaultRoleService`, and may be required to + migrate existing code/data to the new API. Applications that wish to continue to use a completely + custom `BaseRoleService` will need to implement one additional method: `getUsersForRole`. + +### 📚 Libraries +* grails `6.1.0` +* gorm `8.0.1` +* hazelcast `5.3.6` + + ## 19.0.0 - 2024-04-04 ### 💥 Breaking Changes (upgrade difficulty: 🟢 LOW - latest Hoist React + DB col additions) diff --git a/build.gradle b/build.gradle index 9a5432cc..33f31419 100644 --- a/build.gradle +++ b/build.gradle @@ -54,6 +54,9 @@ dependencies { //-------------------- // Hoist Additions //-------------------- + + api "com.hazelcast:hazelcast" + api "org.apache.tomcat:tomcat-jdbc" api "org.codehaus.groovy:groovy-dateutil:$groovyVersion" @@ -61,9 +64,7 @@ dependencies { api("org.quartz-scheduler:quartz:2.3.2") {exclude group: 'slf4j-api', module: 'c3p0'} api 'org.grails.plugins:quartz:2.0.13' - api "org.hibernate:hibernate-core:5.6.11.Final" - api "org.hibernate:hibernate-ehcache:5.6.11.Final" - api "net.sf.ehcache:ehcache:2.10.9.2" + api "org.hibernate:hibernate-jcache" api "org.apache.poi:poi:4.1.2" api "org.apache.poi:poi-ooxml:4.1.2" @@ -72,10 +73,14 @@ dependencies { api "org.apache.directory.api:api-all:2.1.3" api "org.springframework:spring-websocket" - api "org.apache.httpcomponents.client5:httpclient5:5.1.2" + api "org.apache.httpcomponents.client5:httpclient5:5.2.1" api 'org.jasypt:jasypt:1.9.3' api "commons-io:commons-io:2.8.0" api "org.owasp.encoder:encoder:1.2.3" + + api "com.esotericsoftware:kryo:5.5.0" + api "info.jerrinot:subzero-core:0.11" + } diff --git a/gradle.properties b/gradle.properties index 739c4207..caca7093 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,11 +1,12 @@ xhReleaseVersion=20.0-SNAPSHOT -grailsVersion=6.0.0 -grailsGradlePluginVersion=6.0.0 -grailsHibernatePluginVersion=8.0.0 +grailsVersion=6.1.2 +grailsGradlePluginVersion=6.1.2 +grailsHibernatePluginVersion=8.1.0 groovyVersion=3.0.11 -gormVersion=8.0.0 +gormVersion=8.1.0 logback.version=1.2.7 +hazelcast.version=5.3.6 org.gradle.daemon=true org.gradle.parallel=true diff --git a/grails-app/controllers/io/xh/hoist/BaseController.groovy b/grails-app/controllers/io/xh/hoist/BaseController.groovy index cd68a83b..04e583ea 100644 --- a/grails-app/controllers/io/xh/hoist/BaseController.groovy +++ b/grails-app/controllers/io/xh/hoist/BaseController.groovy @@ -11,7 +11,7 @@ package io.xh.hoist import grails.async.Promise import grails.async.web.WebPromises import groovy.transform.CompileStatic -import io.xh.hoist.exception.ExceptionRenderer +import io.xh.hoist.exception.ExceptionHandler import io.xh.hoist.json.JSONParser import io.xh.hoist.json.JSONSerializer import io.xh.hoist.log.LogSupport @@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory abstract class BaseController implements LogSupport, IdentitySupport { IdentityService identityService - ExceptionRenderer exceptionRenderer + ExceptionHandler xhExceptionHandler /** * Render an object to JSON. @@ -82,7 +82,7 @@ abstract class BaseController implements LogSupport, IdentitySupport { WebPromises.task { c.call() }.onError { Throwable t -> - exceptionRenderer.handleException(t, request, response, this) + handleUncaughtInternal(t) } } @@ -95,7 +95,16 @@ abstract class BaseController implements LogSupport, IdentitySupport { // Implementation //------------------- void handleException(Exception ex) { - exceptionRenderer.handleException(ex, request, response, this) + handleUncaughtInternal(ex) + } + + private void handleUncaughtInternal(Throwable t) { + xhExceptionHandler.handleException( + exception: t, + logTo: this, + logMessage: [_action: actionName], + renderTo: response + ) } // Provide cached logger to LogSupport for possible performance benefit diff --git a/grails-app/controllers/io/xh/hoist/admin/ConnectionPoolMonitorAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/ConnectionPoolMonitorAdminController.groovy deleted file mode 100644 index a530237f..00000000 --- a/grails-app/controllers/io/xh/hoist/admin/ConnectionPoolMonitorAdminController.groovy +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file belongs to Hoist, an application development toolkit - * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) - * - * Copyright © 2023 Extremely Heavy Industries Inc. - */ -package io.xh.hoist.admin - -import io.xh.hoist.BaseController -import io.xh.hoist.security.Access - -@Access(['HOIST_ADMIN_READER']) -class ConnectionPoolMonitorAdminController extends BaseController { - - def connectionPoolMonitoringService - - def index() { - def svc = connectionPoolMonitoringService - renderJSON( - enabled: svc.enabled, - snapshots: svc.snapshots, - poolConfiguration: svc.poolConfiguration - ) - } - - @Access(['HOIST_ADMIN']) - def takeSnapshot() { - renderJSON(connectionPoolMonitoringService.takeSnapshot()) - } - - @Access(['HOIST_ADMIN']) - def resetStats() { - renderJSON(connectionPoolMonitoringService.resetStats()) - } - -} diff --git a/grails-app/controllers/io/xh/hoist/admin/EhCacheAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/EhCacheAdminController.groovy deleted file mode 100644 index 9082710d..00000000 --- a/grails-app/controllers/io/xh/hoist/admin/EhCacheAdminController.groovy +++ /dev/null @@ -1,64 +0,0 @@ -/* - * This file belongs to Hoist, an application development toolkit - * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) - * - * Copyright © 2023 Extremely Heavy Industries Inc. - */ - -package io.xh.hoist.admin - -import io.xh.hoist.BaseController -import io.xh.hoist.security.Access -import net.sf.ehcache.CacheManager - -@Access(['HOIST_ADMIN_READER']) -class EhCacheAdminController extends BaseController { - - def listCaches() { - def manager = cacheManager, - caches = [] - - if (manager) { - caches = manager.cacheNames.collect { - def cache = manager.getCache(it) - return [ - name: cache.name, - entries: cache.size, - heapSize: (cache.calculateInMemorySize() / 1000000).toDouble().round(1) + 'MB', - evictionPolicy: cache.memoryStoreEvictionPolicy.name, - status: cache.status.toString() - ] - } - } - - renderJSON(caches) - } - - @Access(['HOIST_ADMIN']) - def clearAllCaches() { - def caches = cacheManager.cacheNames - caches.each {clearCache(it)} - renderJSON(success: true) - } - - @Access(['HOIST_ADMIN']) - def clearCaches() { - def caches = params.names instanceof String ? [params.names] : params.names - caches.each {clearCache(it)} - renderJSON(success: true) - } - - - //------------------------ - // Implementation - //------------------------ - private CacheManager getCacheManager() { - CacheManager.ALL_CACHE_MANAGERS.first() - } - - private void clearCache(String name) { - cacheManager.clearAllStartingWith(name) - logInfo('Cleared hibernate cache: ' + name) - } - -} diff --git a/grails-app/controllers/io/xh/hoist/admin/EnvAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/EnvAdminController.groovy deleted file mode 100644 index 764ba2da..00000000 --- a/grails-app/controllers/io/xh/hoist/admin/EnvAdminController.groovy +++ /dev/null @@ -1,26 +0,0 @@ -/* - * This file belongs to Hoist, an application development toolkit - * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) - * - * Copyright © 2023 Extremely Heavy Industries Inc. - */ -package io.xh.hoist.admin - -import io.xh.hoist.BaseController -import io.xh.hoist.security.Access - -import static io.xh.hoist.util.Utils.isSensitiveParamName - -@Access(['HOIST_ADMIN_READER']) -class EnvAdminController extends BaseController { - - def index() { - renderJSON([ - environment: System.getenv().collectEntries { - [it.key, isSensitiveParamName(it.key) ? '*****' : it.value] - }, - properties: System.getProperties() - ]) - } - -} diff --git a/grails-app/controllers/io/xh/hoist/admin/LogViewerAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/LogViewerAdminController.groovy deleted file mode 100644 index d3f2a9d9..00000000 --- a/grails-app/controllers/io/xh/hoist/admin/LogViewerAdminController.groovy +++ /dev/null @@ -1,108 +0,0 @@ -/* - * This file belongs to Hoist, an application development toolkit - * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) - * - * Copyright © 2023 Extremely Heavy Industries Inc. - */ - -package io.xh.hoist.admin - -import io.xh.hoist.BaseController -import io.xh.hoist.configuration.LogbackConfig -import groovy.io.FileType -import io.xh.hoist.security.Access - -@Access(['HOIST_ADMIN_READER']) -class LogViewerAdminController extends BaseController { - - def logArchiveService, - logReaderService - - def listFiles() { - def logRootPath = logReaderService.logDir.absolutePath, - files = availableFiles.collect { - [ - filename : it.key, - size : it.value.size(), - lastModified: it.value.lastModified() - ] - } - renderJSON(files: files, logRootPath: logRootPath) - } - - def getFile(String filename, Integer startLine, Integer maxLines, String pattern, Boolean caseSensitive) { - if (!availableFiles[filename]) throwUnavailable() - - // Catch any exceptions and render clean failure - the admin client auto-polls for log file - // updates, and we don't want to spam the logs with a repeated stacktrace. - try { - def content = logReaderService.readFile(filename, startLine, maxLines, pattern, caseSensitive) - renderJSON(success: true, filename: filename, content: content) - } catch (Exception e) { - renderJSON(success: false, filename: filename, content: [], exception: e.message) - } - } - - def download(String filename) { - if (!availableFiles[filename]) throwUnavailable() - def file = logReaderService.get(filename) - render( - file: file, - fileName: filename, - contentType: 'application/octet-stream' - ) - } - - /** - * Deletes one or more files from the log directory. - * @param filenames - (required) - */ - @Access(['HOIST_ADMIN']) - def deleteFiles() { - def filenames = params.list('filenames'), - available = availableFiles - - filenames.each {String filename -> - def toDelete = available[filename] - if (!toDelete) throwUnavailable() - - def deleted = toDelete.delete() - if (!deleted) logWarn("Failed to delete log: '$filename'.") - } - - renderJSON(success:true) - } - - /** - * Run log archiving process immediately. - * @param daysThreshold - (optional) min age in days of files to archive - null to use configured default. - */ - @Access(['HOIST_ADMIN']) - def archiveLogs(Integer daysThreshold) { - def ret = logArchiveService.archiveLogs(daysThreshold) - renderJSON([archived: ret]) - } - - - //---------------- - // Implementation - //---------------- - private Map getAvailableFiles() { - def baseDir = new File(LogbackConfig.logRootPath), - basePath = baseDir.toPath(), - files = [] - - baseDir.eachFileRecurse(FileType.FILES) { - def matches = it.name ==~ /.*\.log/ - if (matches) files << it - } - - files.collectEntries { File f -> - [basePath.relativize(f.toPath()).toString(), f] - } - } - - private static throwUnavailable() { - throw new RuntimeException('Filename not valid or available') - } -} diff --git a/grails-app/controllers/io/xh/hoist/admin/MemoryMonitorAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/MemoryMonitorAdminController.groovy deleted file mode 100644 index bc20a32b..00000000 --- a/grails-app/controllers/io/xh/hoist/admin/MemoryMonitorAdminController.groovy +++ /dev/null @@ -1,37 +0,0 @@ -/* - * This file belongs to Hoist, an application development toolkit - * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) - * - * Copyright © 2023 Extremely Heavy Industries Inc. - */ - -package io.xh.hoist.admin - -import io.xh.hoist.BaseController -import io.xh.hoist.security.Access - -@Access(['HOIST_ADMIN_READER']) -class MemoryMonitorAdminController extends BaseController { - - def memoryMonitoringService - - def snapshots() { - renderJSON(memoryMonitoringService.snapshots) - } - - @Access(['HOIST_ADMIN']) - def takeSnapshot() { - renderJSON(memoryMonitoringService.takeSnapshot()) - } - - @Access(['HOIST_ADMIN']) - def requestGc() { - renderJSON(memoryMonitoringService.requestGc()) - } - - @Access(['HOIST_ADMIN']) - def dumpHeap(String filename) { - memoryMonitoringService.dumpHeap(filename) - renderJSON(success: true) - } -} diff --git a/grails-app/controllers/io/xh/hoist/admin/MonitorAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/MonitorAdminController.groovy index 2cc2846e..ab357e4d 100644 --- a/grails-app/controllers/io/xh/hoist/admin/MonitorAdminController.groovy +++ b/grails-app/controllers/io/xh/hoist/admin/MonitorAdminController.groovy @@ -13,8 +13,6 @@ import io.xh.hoist.security.Access @Access(['HOIST_ADMIN_READER']) class MonitorAdminController extends AdminRestController { - def monitoringService - static restTarget = Monitor static trackChanges = true @@ -25,15 +23,4 @@ class MonitorAdminController extends AdminRestController { protected void preprocessSubmit(Map submit) { submit.lastUpdatedBy = authUsername } - - @Access(['HOIST_ADMIN']) - def forceRunAllMonitors() { - monitoringService.forceRun() - renderJSON(success:true) - } - - def results() { - renderJSON(monitoringService.getResults()) - } - } diff --git a/grails-app/controllers/io/xh/hoist/admin/ServiceAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/ServiceAdminController.groovy deleted file mode 100644 index eb714582..00000000 --- a/grails-app/controllers/io/xh/hoist/admin/ServiceAdminController.groovy +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file belongs to Hoist, an application development toolkit - * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) - * - * Copyright © 2023 Extremely Heavy Industries Inc. - */ - -package io.xh.hoist.admin - -import io.xh.hoist.BaseController -import io.xh.hoist.BaseService -import io.xh.hoist.security.Access - -@Access(['HOIST_ADMIN_READER']) -class ServiceAdminController extends BaseController { - - def listServices() { - def ret = getServices().collect{[name: it.key]} - renderJSON(ret) - } - - @Access(['HOIST_ADMIN']) - def clearCaches() { - def allServices = getServices(), - services = params.names instanceof String ? [params.names] : params.names - - services.each { - def svc = allServices[it] - if (svc) { - svc.clearCaches() - logInfo('Cleared service cache', it) - } - - } - renderJSON(success: true) - } - - - //------------------------ - // Implementation - //------------------------ - private Map getServices() { - return grailsApplication.mainContext.getBeansOfType(BaseService.class, false, false) - } - -} diff --git a/grails-app/controllers/io/xh/hoist/admin/WebSocketAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/WebSocketAdminController.groovy deleted file mode 100644 index cac9ef94..00000000 --- a/grails-app/controllers/io/xh/hoist/admin/WebSocketAdminController.groovy +++ /dev/null @@ -1,27 +0,0 @@ -/* - * This file belongs to Hoist, an application development toolkit - * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) - * - * Copyright © 2023 Extremely Heavy Industries Inc. - */ - -package io.xh.hoist.admin - -import io.xh.hoist.BaseController -import io.xh.hoist.security.Access - -@Access(['HOIST_ADMIN_READER']) -class WebSocketAdminController extends BaseController { - - def webSocketService - - def allChannels() { - renderJSON(webSocketService.allChannels) - } - - @Access(['HOIST_ADMIN']) - def pushToChannel(String channelKey, String topic, String message) { - webSocketService.pushToChannel(channelKey, topic, message) - renderJSON(success: true) - } -} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/BaseClusterController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/BaseClusterController.groovy new file mode 100644 index 00000000..3df0979b --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/BaseClusterController.groovy @@ -0,0 +1,34 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.admin.cluster + +import io.xh.hoist.BaseController +import io.xh.hoist.cluster.ClusterRequest + +abstract class BaseClusterController extends BaseController { + + def clusterService + + protected void runOnInstance(ClusterRequest task, String instance) { + def ret = clusterService.submitToInstance(task, instance) + if (ret.exception) { + // Just render exception, was already logged on target instance + xhExceptionHandler.handleException(exception: ret.exception, renderTo: response) + return + } + renderJSON(ret.value) + } + + protected void runOnMaster(ClusterRequest task) { + runOnInstance(task, clusterService.masterName) + } + + protected void runOnAllInstances(ClusterRequest task) { + renderJSON(clusterService.submitToAllInstances(task)) + } +} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/ClusterAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/ClusterAdminController.groovy new file mode 100644 index 00000000..0bbb1243 --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/ClusterAdminController.groovy @@ -0,0 +1,54 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.admin.cluster + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access +import io.xh.hoist.util.Utils + +import static io.xh.hoist.util.DateTimeUtils.SECONDS + +import static grails.async.Promises.task + +import static java.lang.Thread.sleep + +@Access(['HOIST_ADMIN_READER']) +class ClusterAdminController extends BaseClusterController { + + def clusterAdminService, + trackService + + def allInstances() { + renderJSON(clusterAdminService.allStats) + } + + @Access(['HOIST_ADMIN']) + def shutdownInstance(String instance) { + trackService.track( + category: 'Cluster Admin', + msg: 'Initiated Instance Shutdown', + severity: 'WARN', + logData: true, + data: [instance: instance] + ) + logWarn('Initiated Instance Shutdown', [instance: instance]) + runOnInstance(new ShutdownInstance(), instance) + // Wait enough to let async call below complete + sleep(5 * SECONDS) + } + static class ShutdownInstance extends ClusterRequest { + def doCall() { + // Run async to allow this call to successfully return. + task { + sleep(1 * SECONDS) + Utils.clusterService.shutdownInstance() + } + [success: true] + } + } +} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/ConnectionPoolMonitorAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/ConnectionPoolMonitorAdminController.groovy new file mode 100644 index 00000000..71137568 --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/ConnectionPoolMonitorAdminController.groovy @@ -0,0 +1,52 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ +package io.xh.hoist.admin.cluster + + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.appContext + +@Access(['HOIST_ADMIN_READER']) +class ConnectionPoolMonitorAdminController extends BaseClusterController { + + def snapshots(String instance) { + runOnInstance(new Snapshots(), instance) + } + static class Snapshots extends ClusterRequest { + def doCall() { + def svc = appContext.connectionPoolMonitoringService + return [ + enabled : svc.enabled, + snapshots : svc.snapshots, + poolConfiguration: svc.poolConfiguration + ] + } + } + + + @Access(['HOIST_ADMIN']) + def takeSnapshot(String instance) { + runOnInstance(new TakeSnapshot(), instance) + } + static class TakeSnapshot extends ClusterRequest { + def doCall() { + appContext.connectionPoolMonitoringService.takeSnapshot() + } + } + + @Access(['HOIST_ADMIN']) + def resetStats() { + runOnInstance(new ResetStats()) + } + static class ResetStats extends ClusterRequest { + def doCall() { + appContext.connectionPoolMonitoringService.resetStats() + } + } +} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/EnvAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/EnvAdminController.groovy new file mode 100644 index 00000000..54a0727f --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/EnvAdminController.groovy @@ -0,0 +1,31 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ +package io.xh.hoist.admin.cluster + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.isSensitiveParamName + + +@Access(['HOIST_ADMIN_READER']) +class EnvAdminController extends BaseClusterController { + + def index(String instance) { + runOnInstance(new Index(), instance) + } + static class Index extends ClusterRequest { + def doCall() { + [ + environment: System.getenv().collectEntries { + [it.key, isSensitiveParamName(it.key) ? '*****' : it.value] + }, + properties: System.properties + ] + } + } +} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/HzObjectAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/HzObjectAdminController.groovy new file mode 100644 index 00000000..1abea732 --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/HzObjectAdminController.groovy @@ -0,0 +1,53 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2022 Extremely Heavy Industries Inc. + */ +package io.xh.hoist.admin.cluster + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.appContext + + +@Access(['HOIST_ADMIN_READER']) +class HzObjectAdminController extends BaseClusterController { + + def listObjects(String instance) { + runOnInstance(new ListObjects(), instance) + } + + static class ListObjects extends ClusterRequest { + def doCall() { + appContext.clusterAdminService.listObjects() + } + } + + @Access(['HOIST_ADMIN']) + def clearObjects(String instance) { + runOnInstance(new ClearObjects(names: params.list('names')), instance) + } + + static class ClearObjects extends ClusterRequest { + List names + + def doCall() { + appContext.clusterAdminService.clearObjects(names) + return [success: true] + } + } + + @Access(['HOIST_ADMIN']) + def clearHibernateCaches(String instance) { + runOnInstance(new ClearHibernateCaches(), instance) + } + + static class ClearHibernateCaches extends ClusterRequest { + def doCall() { + appContext.clusterAdminService.clearHibernateCaches() + return [success: true] + } + } +} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/LogViewerAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/LogViewerAdminController.groovy new file mode 100644 index 00000000..3014fc08 --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/LogViewerAdminController.groovy @@ -0,0 +1,172 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.admin.cluster + +import groovy.io.FileType +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.configuration.LogbackConfig +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.getAppContext + +@Access(['HOIST_ADMIN_READER']) +class LogViewerAdminController extends BaseClusterController { + + def listFiles(String instance) { + runOnInstance(new ListFiles(), instance) + } + + static class ListFiles extends ClusterRequest { + def doCall() { + def logRootPath = appContext.logReaderService.logDir.absolutePath, + files = availableFiles.collect { + [ + filename : it.key, + size : it.value.size(), + lastModified: it.value.lastModified() + ] + } + return [files: files, logRootPath: logRootPath] + } + } + + + def getFile( + String filename, + Integer startLine, + Integer maxLines, + String pattern, + Boolean caseSensitive, + String instance + ) { + runOnInstance( + new GetFile( + filename: filename, + startLine: startLine, + maxLines: maxLines, + pattern: pattern, + caseSensitive: caseSensitive + ), + instance + ) + } + + static class GetFile extends ClusterRequest { + String filename + Integer startLine + Integer maxLines + String pattern + Boolean caseSensitive + + def doCall() { + if (!availableFiles[filename]) throwUnavailable() + + // Catch any exceptions and render clean failure - the admin client auto-polls for log file + // updates, and we don't want to spam the logs with a repeated stacktrace. + try { + def content = appContext.logReaderService.readFile(filename, startLine, maxLines, pattern, caseSensitive) + return [success: true, filename: filename, content: content] + } catch (Exception e) { + return [success: false, filename: filename, content: [], exception: e.message] + } + } + } + + def download(String filename, String instance) { + def task = new Download(filename: filename), + ret = clusterService.submitToInstance(task, instance) + + if (ret.exception) { + // Just render exception, was already logged on target instance + xhExceptionHandler.handleException(exception: ret.exception, renderTo: response) + return + } + + render( + file: ret.value, + fileName: filename, + contentType: 'application/octet-stream' + ) + } + + static class Download extends ClusterRequest { + String filename + + File doCall() { + if (!availableFiles[filename]) throwUnavailable() + return appContext.logReaderService.get(filename) + } + } + + + /** + * Deletes one or more files from the log directory. + * @param filenames - (required) + */ + @Access(['HOIST_ADMIN']) + def deleteFiles(String instance) { + runOnInstance(new DeleteFiles(filenames: params.list('filenames')), instance) + } + + static class DeleteFiles extends ClusterRequest { + List filenames + + def doCall() { + def available = availableFiles + + filenames.each { filename -> + def toDelete = available[filename] + if (!toDelete) throwUnavailable() + + def deleted = toDelete.delete() + if (!deleted) logWarn("Failed to delete log: '$filename'.") + } + [success: true] + } + } + + /** + * Run log archiving process immediately. + * @param daysThreshold - (optional) min age in days of files to archive - null to use configured default. + */ + @Access(['HOIST_ADMIN']) + def archiveLogs(Integer daysThreshold, String instance) { + runOnInstance(new ArchiveLogs(daysThreshold: daysThreshold), instance) + } + + static class ArchiveLogs extends ClusterRequest { + Integer daysThreshold + + def doCall() { + def ret = appContext.logArchiveService.archiveLogs(daysThreshold) + return [archived: ret] + } + } + + //---------------- + // Implementation + //---------------- + static Map getAvailableFiles() { + def baseDir = new File(LogbackConfig.logRootPath), + basePath = baseDir.toPath() + + List files = [] + baseDir.eachFileRecurse(FileType.FILES) { + def matches = it.name ==~ /.*\.log/ + if (matches) files << it + } + + files.collectEntries { File f -> + [basePath.relativize(f.toPath()).toString(), f] + } + } + + static void throwUnavailable() { + throw new RuntimeException('Filename not valid or available') + } +} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/MemoryMonitorAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/MemoryMonitorAdminController.groovy new file mode 100644 index 00000000..58d8fc57 --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/MemoryMonitorAdminController.groovy @@ -0,0 +1,62 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.admin.cluster + + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.appContext + +@Access(['HOIST_ADMIN_READER']) +class MemoryMonitorAdminController extends BaseClusterController { + + def snapshots(String instance) { + runOnInstance(new Snapshots(), instance) + } + static class Snapshots extends ClusterRequest { + def doCall() { + appContext.memoryMonitoringService.snapshots + } + } + + @Access(['HOIST_ADMIN']) + def takeSnapshot(String instance) { + runOnInstance(new TakeSnapshot(), instance) + } + static class TakeSnapshot extends ClusterRequest { + def doCall() { + appContext.memoryMonitoringService.takeSnapshot() + } + } + + + @Access(['HOIST_ADMIN']) + def requestGc(String instance) { + runOnInstance(new RequestGc(), instance) + } + static class RequestGc extends ClusterRequest { + def doCall() { + appContext.memoryMonitoringService.requestGc() + } + } + + + @Access(['HOIST_ADMIN']) + def dumpHeap(String filename, String instance) { + runOnInstance(new DumpHeap(filename: filename), instance) + } + static class DumpHeap extends ClusterRequest { + String filename + + def doCall() { + appContext.memoryMonitoringService.dumpHeap(filename) + return [success: true] + } + } +} \ No newline at end of file diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/MonitorResultsAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/MonitorResultsAdminController.groovy new file mode 100644 index 00000000..c2c6bb80 --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/MonitorResultsAdminController.groovy @@ -0,0 +1,33 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.admin.cluster + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.getAppContext + +@Access(['HOIST_ADMIN_READER']) +class MonitorResultsAdminController extends BaseClusterController { + + def monitoringService + + def results() { + renderJSON(monitoringService.getResults()) + } + + @Access(['HOIST_ADMIN']) + def forceRunAllMonitors() { + runOnMaster(new ForceRunAllMonitors()) + } + static class ForceRunAllMonitors extends ClusterRequest { + def doCall() { + appContext.monitoringService.forceRun() + } + } +} diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/ServiceManagerAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/ServiceManagerAdminController.groovy new file mode 100644 index 00000000..a641128d --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/ServiceManagerAdminController.groovy @@ -0,0 +1,50 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ +package io.xh.hoist.admin.cluster + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.appContext + +@Access(['HOIST_ADMIN_READER']) +class ServiceManagerAdminController extends BaseClusterController { + + def listServices(String instance) { + runOnInstance(new ListServices(), instance) + } + static class ListServices extends ClusterRequest { + def doCall() { + appContext.serviceManagerService.listServices() + } + } + + def getStats(String instance, String name) { + def task = new GetStats(name: name) + runOnInstance(task, instance) + } + static class GetStats extends ClusterRequest { + String name + def doCall() { + appContext.serviceManagerService.getStats(name) + } + } + + @Access(['HOIST_ADMIN']) + def clearCaches(String instance) { + def task = new ClearCaches(names: params.list('names')) + instance ? runOnInstance(task, instance) : runOnAllInstances(task) + } + static class ClearCaches extends ClusterRequest { + List names + + def doCall() { + appContext.serviceManagerService.clearCaches(names) + return [success: true] + } + } +} \ No newline at end of file diff --git a/grails-app/controllers/io/xh/hoist/admin/cluster/WebSocketAdminController.groovy b/grails-app/controllers/io/xh/hoist/admin/cluster/WebSocketAdminController.groovy new file mode 100644 index 00000000..fdf878e5 --- /dev/null +++ b/grails-app/controllers/io/xh/hoist/admin/cluster/WebSocketAdminController.groovy @@ -0,0 +1,41 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.admin.cluster + +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.security.Access + +import static io.xh.hoist.util.Utils.getAppContext + +@Access(['HOIST_ADMIN_READER']) +class WebSocketAdminController extends BaseClusterController { + + def allChannels(String instance) { + runOnInstance(new AllChannels(), instance) + } + static class AllChannels extends ClusterRequest { + def doCall() { + appContext.webSocketService.allChannels*.formatForJSON() + } + } + + @Access(['HOIST_ADMIN']) + def pushToChannel(String channelKey, String topic, String message, String instance) { + runOnInstance(new PushToChannel(channelKey: channelKey, topic: topic, message: message), instance) + } + static class PushToChannel extends ClusterRequest { + String channelKey + String topic + String message + + def doCall() { + appContext.webSocketService.pushToChannel(channelKey, topic, message) + return [success: true] + } + } +} diff --git a/grails-app/controllers/io/xh/hoist/impl/XhController.groovy b/grails-app/controllers/io/xh/hoist/impl/XhController.groovy index 4d9a8edf..de6a3d0c 100644 --- a/grails-app/controllers/io/xh/hoist/impl/XhController.groovy +++ b/grails-app/controllers/io/xh/hoist/impl/XhController.groovy @@ -10,6 +10,7 @@ package io.xh.hoist.impl import groovy.transform.CompileStatic import io.xh.hoist.BaseController import io.xh.hoist.alertbanner.AlertBannerService +import io.xh.hoist.cluster.ClusterService import io.xh.hoist.config.ConfigService import io.xh.hoist.clienterror.ClientErrorService import io.xh.hoist.exception.NotFoundException @@ -43,6 +44,7 @@ class XhController extends BaseController { TrackService trackService EnvironmentService environmentService BaseUserService userService + ClusterService clusterService //------------------------ // Identity / Auth @@ -222,6 +224,7 @@ class XhController extends BaseController { def options = configService.getMap('xhAppVersionCheck', [:]) renderJSON( *: options, + instanceName: clusterService.instanceName, appVersion: Utils.appVersion, appBuild: Utils.appBuild ) diff --git a/grails-app/controllers/io/xh/hoist/security/AccessInterceptor.groovy b/grails-app/controllers/io/xh/hoist/security/AccessInterceptor.groovy index 343cbe36..7a95ea46 100644 --- a/grails-app/controllers/io/xh/hoist/security/AccessInterceptor.groovy +++ b/grails-app/controllers/io/xh/hoist/security/AccessInterceptor.groovy @@ -8,7 +8,7 @@ package io.xh.hoist.security import groovy.transform.CompileStatic -import io.xh.hoist.exception.ExceptionRenderer +import io.xh.hoist.exception.ExceptionHandler import io.xh.hoist.exception.NotAuthorizedException import io.xh.hoist.user.HoistUser import io.xh.hoist.user.IdentityService @@ -18,7 +18,7 @@ import java.lang.reflect.Method class AccessInterceptor { IdentityService identityService - ExceptionRenderer exceptionRenderer + ExceptionHandler xhExceptionHandler AccessInterceptor() { matchAll() @@ -69,7 +69,12 @@ class AccessInterceptor { You do not have the application role(s) required. Currently logged in as: $username. """) - exceptionRenderer.handleException(ex, request, response, identityService) + xhExceptionHandler.handleException( + exception: ex, + logTo: identityService, + logMessage: [_action: actionName], + renderTo: response + ) return false } diff --git a/grails-app/domain/io/xh/hoist/log/LogLevel.groovy b/grails-app/domain/io/xh/hoist/log/LogLevel.groovy index 2ca5a13f..f8b10a1d 100644 --- a/grails-app/domain/io/xh/hoist/log/LogLevel.groovy +++ b/grails-app/domain/io/xh/hoist/log/LogLevel.groovy @@ -10,15 +10,19 @@ package io.xh.hoist.log import io.xh.hoist.json.JSONFormat import io.xh.hoist.util.Utils +import static grails.async.Promises.task + class LogLevel implements JSONFormat { String name String level - String getDefaultLevel() {logLevelService.getDefaultLevel(name)} - String getEffectiveLevel() {logLevelService.getEffectiveLevel(name)} Date lastUpdated String lastUpdatedBy + String getDefaultLevel() { logLevelService.getDefaultLevel(name) } + + String getEffectiveLevel() { logLevelService.getEffectiveLevel(name) } + public static List LEVELS = ['Trace', 'Debug', 'Info', 'Warn', 'Error', 'Inherit', 'Off'] static mapping = { @@ -33,18 +37,38 @@ class LogLevel implements JSONFormat { lastUpdatedBy(nullable: true, maxSize: 50) } + def afterUpdate() { + noteLogLevelChanged() + } + + def afterDelete() { + noteLogLevelChanged() + } + + def afterInsert() { + noteLogLevelChanged() + } + Map formatForJSON() { return [ - id: id, - name: name, - level: level, - defaultLevel: defaultLevel, - effectiveLevel: effectiveLevel, - lastUpdated: lastUpdated, - lastUpdatedBy: lastUpdatedBy + id : id, + name : name, + level : level, + defaultLevel : defaultLevel, + effectiveLevel: effectiveLevel, + lastUpdated : lastUpdated, + lastUpdatedBy : lastUpdatedBy ] } private getLogLevelService() { Utils.appContext.logLevelService } + private noteLogLevelChanged() { + // called in a new thread and with a delay to make sure the change has had the time to propagate + task { + Thread.sleep(500) + logLevelService.noteLogLevelChanged() + } + } + } diff --git a/grails-app/domain/io/xh/hoist/track/TrackLog.groovy b/grails-app/domain/io/xh/hoist/track/TrackLog.groovy index 0d59a6f4..9783b456 100644 --- a/grails-app/domain/io/xh/hoist/track/TrackLog.groovy +++ b/grails-app/domain/io/xh/hoist/track/TrackLog.groovy @@ -36,6 +36,10 @@ class TrackLog implements JSONFormat { dateCreated index: 'idx_xh_track_log_date_created' } + static cache = { + evictionConfig.size = 20000 + } + static constraints = { msg(maxSize: 255) username(maxSize: 50) diff --git a/grails-app/init/io/xh/hoist/BootStrap.groovy b/grails-app/init/io/xh/hoist/BootStrap.groovy index 62317d6d..c81c6ce9 100644 --- a/grails-app/init/io/xh/hoist/BootStrap.groovy +++ b/grails-app/init/io/xh/hoist/BootStrap.groovy @@ -7,17 +7,20 @@ package io.xh.hoist import grails.util.Holders +import io.xh.hoist.cluster.ClusterService import io.xh.hoist.util.Utils import java.time.ZoneId import static io.xh.hoist.util.DateTimeUtils.serverZoneId +import static io.xh.hoist.BaseService.parallelInit import static java.lang.Runtime.runtime class BootStrap { def logLevelService, configService, + clusterService, prefService def init = {servletContext -> @@ -28,8 +31,9 @@ class BootStrap { ensureExpectedServerTimeZone() def services = Utils.xhServices.findAll {it.class.canonicalName.startsWith('io.xh.hoist')} - BaseService.parallelInit([logLevelService]) - BaseService.parallelInit(services) + parallelInit([logLevelService]) + parallelInit([clusterService]) + parallelInit(services) } def destroy = {} @@ -50,6 +54,8 @@ class BootStrap { \n Hoist v${hoist.version} - ${Utils.appEnvironment} Extremely Heavy - https://xh.io + + Cluster ${ClusterService.clusterName} + + Instance ${ClusterService.instanceName} + ${runtime.availableProcessors()} available processors + ${String.format('%,d', (runtime.maxMemory() / 1000000).toLong())}mb available memory + JVM TimeZone is ${serverZoneId} diff --git a/grails-app/init/io/xh/hoist/ClusterConfig.groovy b/grails-app/init/io/xh/hoist/ClusterConfig.groovy new file mode 100755 index 00000000..39cea6df --- /dev/null +++ b/grails-app/init/io/xh/hoist/ClusterConfig.groovy @@ -0,0 +1,209 @@ +package io.xh.hoist + +import com.hazelcast.config.CacheSimpleConfig +import com.hazelcast.config.Config +import com.hazelcast.config.EvictionConfig +import com.hazelcast.config.EvictionPolicy +import com.hazelcast.config.InMemoryFormat +import com.hazelcast.config.MapConfig +import com.hazelcast.config.ReplicatedMapConfig +import com.hazelcast.config.SetConfig +import com.hazelcast.config.TopicConfig +import com.hazelcast.map.IMap +import com.hazelcast.replicatedmap.ReplicatedMap +import com.hazelcast.topic.ITopic +import com.hazelcast.collection.ISet +import com.hazelcast.config.MaxSizePolicy +import com.hazelcast.config.NearCacheConfig +import grails.core.GrailsClass +import info.jerrinot.subzero.SubZero +import io.xh.hoist.cache.Entry +import io.xh.hoist.cluster.ClusterResponse +import io.xh.hoist.cluster.ReplicatedValueEntry +import io.xh.hoist.util.Utils +import static io.xh.hoist.util.InstanceConfigUtils.getInstanceConfig + +import static grails.util.Holders.grailsApplication + +class ClusterConfig { + + /** + * Name of Hazelcast cluster. + * + * This value identifies the cluster to attach to, create and is unique to this + * application, version, and environment. + * + * To customize, override generateClusterName(). + */ + final String clusterName = generateClusterName() + + + /** + * Instance name of Hazelcast member. + * To customize, override generateInstanceName(). + */ + final String instanceName = generateInstanceName() + + + /** + * Are multi-instance clusters enabled? + * + * Defaults to true. Is set to false, Hoist will not create multi-instance clusters and may + * use simpler in-memory data-structures in place of their Hazelcast counterparts. Use this + * for applications that do not require multi-instance and do not wish to pay the serialization penalty of storing + * shared data in Hazelcast. + * + * Applications and plug-ins may set this value explicitly via the `multiInstanceEnabled` + * instance config, or override this method to implement additional logic. + */ + boolean getMultiInstanceEnabled() { + return getInstanceConfig('multiInstanceEnabled') !== 'false' + } + + /** + * Override this method to customize the cluster name of the Hazelcast cluster. + */ + protected String generateClusterName() { + def ret = Utils.appCode + '_' + Utils.appEnvironment + '_' + Utils.appVersion + if (!multiInstanceEnabled) ret += '_' + UUID.randomUUID().toString().take(8) + return ret + } + + /** + * Override this method to customize the instance name of the Hazelcast member. + */ + protected String generateInstanceName() { + UUID.randomUUID().toString().take(8) + } + + /** + * Produce configuration for the hazelcast cluster. + * + * Hoist uses simple Hazelcast's "multicast" cluster discovery by default. While often + * appropriate for local development, this may not be appropriate for your production + * application and can be replaced here with alternative cluster discovery mechanisms. + * + * This method should also be used to specify custom configurations of distributed + * hazelcast objects. + * + * Override this method to provide plugin or app specific configuration. + */ + Config createConfig() { + def ret = new Config() + + ret.instanceName = instanceName + ret.clusterName = clusterName + ret.memberAttributeConfig.setAttribute('instanceName', instanceName) + + ret.networkConfig.join.multicastConfig.enabled = true + + createDefaultConfigs(ret) + createHibernateConfigs(ret) + createServiceConfigs(ret) + + SubZero.useForClasses(ret, classesForKryo().toArray() as Class[]) + + return ret + } + + /** + * Override this method to specify additional classes that should be serialized with Kryo + * when being stored in Hazelcast structures. + * + * Kryo provides a more efficient serialization format and is recommended over default + * Java serialization for objects being stored in Hazelcast. + * + * Note that objects being stored in a Hoist `ReplicatedValue` or cluster-enabled `Cache` will not + * need their classes specified in this method. These objects are always serialized via Kryo. + */ + protected List classesForKryo() { + return [ReplicatedValueEntry, Entry, ClusterResponse] + } + + /** + * Override this to create additional default configs in the application. + */ + protected void createDefaultConfigs(Config config) { + config.getMapConfig('default').with { + statisticsEnabled = true + inMemoryFormat = InMemoryFormat.OBJECT + nearCacheConfig = new NearCacheConfig().setInMemoryFormat(InMemoryFormat.OBJECT) + } + config.getReplicatedMapConfig('default').with { + statisticsEnabled = true + inMemoryFormat = InMemoryFormat.OBJECT + } + config.getTopicConfig('default').with { + statisticsEnabled = true + } + config.getSetConfig('default').with { + statisticsEnabled = true + } + config.getCacheConfig('default').with { + statisticsEnabled = true + evictionConfig.maxSizePolicy = MaxSizePolicy.ENTRY_COUNT + evictionConfig.evictionPolicy = EvictionPolicy.LRU + evictionConfig.size = 5000 + } + + config.getCacheConfig('default-update-timestamps-region').with { + evictionConfig = new EvictionConfig(evictionConfig).setSize(1000) + } + + config.getCacheConfig('default-query-results-region').with { + evictionConfig = new EvictionConfig(evictionConfig).setSize(1000) + } + } + + //------------------------ + // Implementation + //------------------------ + private void createHibernateConfigs(Config config) { + grailsApplication.domainClasses.each { GrailsClass gc -> + Closure customizer = gc.getPropertyValue('cache') as Closure + if (customizer) { + // IMPORTANT -- We do an explicit clone, because wild card matching in hazelcast will + // actually just return the *shared* config (?!), and never want to let app edit that. + // Also need to explicitly clone the evictionConfig due to a probable Hz bug. + def baseConfig = config.findCacheConfig(gc.fullName), + cacheConfig = new CacheSimpleConfig(baseConfig) + cacheConfig.evictionConfig = new EvictionConfig(baseConfig.evictionConfig) + customizer.delegate = cacheConfig + customizer.resolveStrategy = Closure.DELEGATE_FIRST + customizer(cacheConfig) + } + } + } + + private void createServiceConfigs(Config config) { + grailsApplication.serviceClasses.each { GrailsClass gc -> + Map objs = gc.getPropertyValue('clusterConfigs') + if (!objs) return + objs.forEach {String key, List value -> + def customizer = value[1] as Closure, + objConfig + // IMPORTANT -- We do an explicit clone, because wild card matching in hazelcast will + // actually just return the *shared* config (?!), and never want to let app edit that. + switch (value[0]) { + case IMap: + objConfig = new MapConfig(config.findMapConfig(gc.fullName + '_' + key)) + break + case ReplicatedMap: + objConfig = new ReplicatedMapConfig(config.findReplicatedMapConfig(gc.fullName + '_' + key)) + break + case ISet: + objConfig = new SetConfig(config.findSetConfig(gc.fullName + '_' + key)) + break + case ITopic: + objConfig = new TopicConfig(config.findTopicConfig(key)) + break + default: + throw new RuntimeException('Unable to configure Cluster object') + } + customizer.delegate = objConfig + customizer.resolveStrategy = Closure.DELEGATE_FIRST + customizer(objConfig) + } + } + } +} \ No newline at end of file diff --git a/grails-app/services/io/xh/hoist/admin/ServiceManagerService.groovy b/grails-app/services/io/xh/hoist/admin/ServiceManagerService.groovy new file mode 100644 index 00000000..9e698ca9 --- /dev/null +++ b/grails-app/services/io/xh/hoist/admin/ServiceManagerService.groovy @@ -0,0 +1,66 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.admin + +import io.xh.hoist.BaseService + +class ServiceManagerService extends BaseService { + + def grailsApplication, + clusterAdminService + + Collection listServices() { + + + getServicesInternal().collect { name, svc -> + return [ + name: name, + initializedDate: svc.initializedDate, + lastCachesCleared: svc.lastCachesCleared + ] + } + } + + Map getStats(String name) { + def svc = grailsApplication.mainContext.getBean(name), + prefix = svc.class.name + '_', + timers = svc.timers*.adminStats, + distObjs = clusterService.distributedObjects + .findAll { it.getName().startsWith(prefix) } + .collect {clusterAdminService.getAdminStatsForObject(it)} + + Map ret = svc.adminStats + if (timers || distObjs) { + ret = ret.clone() + if (distObjs) ret.distributedObjects = distObjs + if (timers.size() == 1) { + ret.timer = timers[0] + } else if (timers.size() > 1) { + ret.timers = timers + } + } + + return ret + } + + void clearCaches(List names) { + def allServices = getServicesInternal() + + names.each { + def svc = allServices[it] + if (svc) { + svc.clearCaches() + logInfo('Cleared service cache', it) + } + } + } + + private Map getServicesInternal() { + return grailsApplication.mainContext.getBeansOfType(BaseService.class, false, false) + } +} diff --git a/grails-app/services/io/xh/hoist/alertbanner/AlertBannerService.groovy b/grails-app/services/io/xh/hoist/alertbanner/AlertBannerService.groovy index 669246f8..e7bbd414 100644 --- a/grails-app/services/io/xh/hoist/alertbanner/AlertBannerService.groovy +++ b/grails-app/services/io/xh/hoist/alertbanner/AlertBannerService.groovy @@ -7,6 +7,8 @@ package io.xh.hoist.alertbanner + +import io.xh.hoist.cache.Cache import io.xh.hoist.BaseService import io.xh.hoist.config.ConfigService import io.xh.hoist.json.JSONParser @@ -36,18 +38,24 @@ class AlertBannerService extends BaseService { private final static String presetsBlobName = 'xhAlertBannerPresets'; private final emptyAlert = [active: false] - private Map cachedBanner = emptyAlert + private Cache cache void init() { - createTimer( - runFn: this.&refreshCachedBanner, - interval: 2 * MINUTES, - runImmediatelyAndBlock: true - ) + cache = new Cache(name: 'cachedBanner', svc: this, expireTime: 2 * MINUTES, replicate: true) + super.init() } Map getAlertBanner() { - cachedBanner + return cache.getOrCreate('value') { + def conf = configService.getMap('xhAlertBannerConfig', [:]) + if (conf.enabled) { + def spec = getAlertSpec() + if (spec.active && (!spec.expires || spec.expires > currentTimeMillis())) { + return spec + } + } + return emptyAlert + } } //-------------------- @@ -68,7 +76,7 @@ class AlertBannerService extends BaseService { } else { svc.create([type: blobType, name: blobName, value: value], blobOwner) } - refreshCachedBanner() + cache.clear() } List getAlertPresets() { @@ -88,12 +96,11 @@ class AlertBannerService extends BaseService { } } - //---------------------------- // Implementation //----------------------------- private Map readFromSpec() { - def conf = configService.getMap('xhAlertBannerConfig', [:]) + def conf = configService.getMap('xhAlertBannerConfig') if (conf.enabled) { def spec = getAlertSpec() if (spec.active && (!spec.expires || spec.expires > currentTimeMillis())) { @@ -103,13 +110,13 @@ class AlertBannerService extends BaseService { return emptyAlert } - private void refreshCachedBanner() { - cachedBanner = readFromSpec() - logDebug("Refreshing Alert Banner state: " + cachedBanner.toMapString()) - } - void clearCaches() { super.clearCaches() - refreshCachedBanner() + cache.clear() } + + Map getAdminStats() {[ + config: configForAdminStats('xhAlertBannerConfig') + ]} + } diff --git a/grails-app/services/io/xh/hoist/clienterror/ClientErrorEmailService.groovy b/grails-app/services/io/xh/hoist/clienterror/ClientErrorEmailService.groovy index d0cdf3d2..90a241cf 100644 --- a/grails-app/services/io/xh/hoist/clienterror/ClientErrorEmailService.groovy +++ b/grails-app/services/io/xh/hoist/clienterror/ClientErrorEmailService.groovy @@ -78,4 +78,8 @@ class ClientErrorEmailService extends BaseService { return null } } + + Map getAdminStats() {[ + config: [toAddress: emailService.parseMailConfig('xhEmailSupport')] + ]} } diff --git a/grails-app/services/io/xh/hoist/clienterror/ClientErrorService.groovy b/grails-app/services/io/xh/hoist/clienterror/ClientErrorService.groovy index 0f1a4205..54998d1b 100644 --- a/grails-app/services/io/xh/hoist/clienterror/ClientErrorService.groovy +++ b/grails-app/services/io/xh/hoist/clienterror/ClientErrorService.groovy @@ -7,13 +7,11 @@ package io.xh.hoist.clienterror -import grails.events.* +import com.hazelcast.map.IMap import grails.gorm.transactions.Transactional import io.xh.hoist.BaseService import io.xh.hoist.util.Utils -import java.util.concurrent.ConcurrentHashMap - import static io.xh.hoist.browser.Utils.getBrowser import static io.xh.hoist.browser.Utils.getDevice import static io.xh.hoist.util.InstanceConfigUtils.getInstanceConfig @@ -31,20 +29,27 @@ import static java.lang.System.currentTimeMillis * this prevents us from ever overwhelming the server due to client issues, * and also allows us to produce a digest form of the email. */ -class ClientErrorService extends BaseService implements EventPublisher { +class ClientErrorService extends BaseService { def clientErrorEmailService, configService - private Map errors = new ConcurrentHashMap() + static clusterConfigs = [ + clientErrors: [IMap, { + evictionConfig.size = 100 + }] + ] + private IMap errors = getIMap('clientErrors') private int getMaxErrors() {configService.getMap('xhClientErrorConfig').maxErrors as int} private int getAlertInterval() {configService.getMap('xhClientErrorConfig').intervalMins * MINUTES} void init() { + super.init() createTimer( - interval: { alertInterval }, - delay: 15 * SECONDS + interval: { alertInterval }, + delay: 15 * SECONDS, + masterOnly: true ) } @@ -57,7 +62,6 @@ class ClientErrorService extends BaseService implements EventPublisher { */ void submit(String message, String error, String appVersion, String url, boolean userAlerted) { def request = currentRequest - if (!request) { throw new RuntimeException('Cannot submit a client error outside the context of an HttpRequest.') } @@ -65,7 +69,8 @@ class ClientErrorService extends BaseService implements EventPublisher { def userAgent = request.getHeader('User-Agent') if (errors.size() < maxErrors) { - errors[authUsername + currentTimeMillis()] = [ + def now = currentTimeMillis() + errors[authUsername + now] = [ msg : message, error : error, username : authUsername, @@ -76,7 +81,7 @@ class ClientErrorService extends BaseService implements EventPublisher { appEnvironment: Utils.appEnvironment, url : url?.take(500), userAlerted : userAlerted, - dateCreated : new Date(), + dateCreated : now, impersonating: identityService.impersonating ? username : null ] logDebug("Client Error received from $authUsername", "queued for processing") @@ -90,25 +95,29 @@ class ClientErrorService extends BaseService implements EventPublisher { //--------------------------------------------------------- @Transactional void onTimer() { - if (!errors) return + if (!errors) return - // swap out buffer def maxErrors = getMaxErrors(), errs = errors.values().take(maxErrors), count = errs.size() - errors = new ConcurrentHashMap() + errors.clear() if (getInstanceConfig('disableTrackLog') != 'true') { withDebug("Processing $count Client Errors") { - clientErrorEmailService.sendMail(errs, count == maxErrors) - - errs.each { + def errors = errs.collect { def ce = new ClientError(it) - ce.dateCreated = it.dateCreated + ce.dateCreated = new Date(it.dateCreated) ce.save(flush: true) - notify('xhClientErrorReceived', ce) } + getTopic('xhClientErrorReceived').publishAllAsync(errors) + clientErrorEmailService.sendMail(errs, count == maxErrors) } } } + + Map getAdminStats() {[ + config: configForAdminStats('xhClientErrorConfig'), + pendingErrorCount: errors.size() + ]} + } diff --git a/grails-app/services/io/xh/hoist/cluster/ClusterAdminService.groovy b/grails-app/services/io/xh/hoist/cluster/ClusterAdminService.groovy new file mode 100644 index 00000000..24d56c39 --- /dev/null +++ b/grails-app/services/io/xh/hoist/cluster/ClusterAdminService.groovy @@ -0,0 +1,176 @@ +package io.xh.hoist.cluster + +import com.hazelcast.cache.impl.CacheProxy +import com.hazelcast.collection.ISet +import com.hazelcast.core.DistributedObject +import com.hazelcast.executor.impl.ExecutorServiceProxy +import com.hazelcast.map.IMap +import com.hazelcast.nearcache.NearCacheStats +import com.hazelcast.replicatedmap.ReplicatedMap +import com.hazelcast.topic.ITopic +import io.xh.hoist.BaseService +import io.xh.hoist.util.Utils + +import static io.xh.hoist.util.Utils.appContext + +/** + * Reports on all instances within the current Hazelcast cluster, including a general list of all + * live instances with summary stats/metrics for each + more detailed information on distributed + * objects as seen by all instances, or any particular instance. + */ +class ClusterAdminService extends BaseService { + + Map getAdminStats() { + return [ + name : clusterService.instanceName, + address : clusterService.localMember.address.toString(), + isMaster : clusterService.isMaster, + isReady : clusterService.isReady, + memory : appContext.memoryMonitoringService.latestSnapshot, + connectionPool: appContext.connectionPoolMonitoringService.latestSnapshot, + wsConnections : appContext.webSocketService.allChannels.size(), + startupTime : Utils.startupTime + ] + } + + static class AdminStatsTask extends ClusterRequest { + def doCall() { + return appContext.clusterAdminService.adminStats + } + } + + Collection getAllStats() { + clusterService.submitToAllInstances(new AdminStatsTask()) + .collect { name, result -> + def ret = [ + name : name, + isLocal: name == clusterService.instanceName, + isReady: false + ] + if (result.value) { + ret << result.value + } + return ret + } + } + + Collection listObjects() { + clusterService + .hzInstance + .distributedObjects + .findAll { !(it instanceof ExecutorServiceProxy) } + .collect { getAdminStatsForObject(it) } + } + + void clearObjects(List names) { + def all = clusterService.distributedObjects + names.each { name -> + def obj = all.find { it.getName() == name } + if (obj instanceof ReplicatedMap || + obj instanceof IMap || + obj instanceof CacheProxy || + obj instanceof ISet + ) { + obj.clear() + logInfo("Cleared " + name) + } else { + logWarn('Cannot clear object - unsupported type', name) + } + } + } + + void clearHibernateCaches() { + appContext.beanDefinitionNames + .findAll { it.startsWith('sessionFactory') } + .each { appContext.getBean(it)?.cache.evictAllRegions() } + } + + Map getAdminStatsForObject(DistributedObject obj) { + switch (obj) { + case ReplicatedMap: + def stats = obj.getReplicatedMapStats() + return [ + name : obj.getName(), + type : 'Replicated Map', + size : obj.size(), + lastUpdateTime : stats.lastUpdateTime ?: null, + lastAccessTime : stats.lastAccessTime ?: null, + + hits : stats.hits, + gets : stats.getOperationCount, + puts : stats.putOperationCount + ] + case IMap: + def stats = obj.getLocalMapStats() + return [ + name : obj.getName(), + type : 'IMap', + size : obj.size(), + lastUpdateTime : stats.lastUpdateTime ?: null, + lastAccessTime : stats.lastAccessTime ?: null, + + ownedEntryCount: stats.ownedEntryCount, + hits : stats.hits, + gets : stats.getOperationCount, + sets : stats.setOperationCount, + puts : stats.putOperationCount, + nearCache : getNearCacheStats(stats.nearCacheStats), + ] + case ISet: + def stats = obj.getLocalSetStats() + return [ + name : obj.getName(), + type : 'Set', + size : obj.size(), + lastUpdateTime: stats.lastUpdateTime ?: null, + lastAccessTime: stats.lastAccessTime ?: null, + ] + case ITopic: + def stats = obj.getLocalTopicStats() + return [ + name : obj.getName(), + type : 'Topic', + publishOperationCount: stats.publishOperationCount, + receiveOperationCount: stats.receiveOperationCount + ] + case CacheProxy: + def evictionConfig = obj.cacheConfig.evictionConfig, + stats = obj.localCacheStatistics + return [ + name : obj.getName(), + type : 'Cache', + size : obj.size(), + lastUpdateTime : stats.lastUpdateTime ?: null, + lastAccessTime : stats.lastAccessTime ?: null, + + ownedEntryCount : stats.ownedEntryCount, + cacheHits : stats.cacheHits, + cacheHitPercentage: stats.cacheHitPercentage?.round(0), + config : [ + size : evictionConfig.size, + maxSizePolicy : evictionConfig.maxSizePolicy, + evictionPolicy: evictionConfig.evictionPolicy + ] + ] + default: + return [ + name : obj.getName(), + type: obj.class.toString() + ] + } + } + + //-------------------- + // Implementation + //-------------------- + private Map getNearCacheStats(NearCacheStats stats) { + if (!stats) return null + [ + ownedEntryCount : stats.ownedEntryCount, + lastPersistenceTime: stats.lastPersistenceTime, + hits : stats.hits, + misses : stats.misses, + ratio : stats.ratio.round(2) + ] + } +} diff --git a/grails-app/services/io/xh/hoist/cluster/ClusterService.groovy b/grails-app/services/io/xh/hoist/cluster/ClusterService.groovy new file mode 100644 index 00000000..d51720f9 --- /dev/null +++ b/grails-app/services/io/xh/hoist/cluster/ClusterService.groovy @@ -0,0 +1,192 @@ +package io.xh.hoist.cluster + +import com.hazelcast.cluster.Cluster +import com.hazelcast.cluster.Member +import com.hazelcast.cluster.MembershipEvent +import com.hazelcast.cluster.MembershipListener +import com.hazelcast.core.DistributedObject +import com.hazelcast.core.Hazelcast +import com.hazelcast.core.HazelcastInstance +import com.hazelcast.core.IExecutorService +import io.xh.hoist.BaseService +import io.xh.hoist.ClusterConfig +import io.xh.hoist.util.Utils +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.context.ApplicationListener + +import javax.management.InstanceNotFoundException + +class ClusterService extends BaseService implements ApplicationListener { + + /** + * Name of Hazelcast cluster. + * + * This value identifies the cluster to attach to, create and is unique to this + * application, version, and environment. + */ + static final String clusterName + + /** + * Name of this instance. + * + * A randomly chosen unique identifier. + */ + static final String instanceName + + /** + * The underlying embedded Hazelcast instance. + * Use for accessing the native Hazelcast APIs. + */ + static HazelcastInstance hzInstance + + private static ClusterConfig clusterConfig + + static { + // Create cluster/instance identifiers statically so logging can access early in lifecycle + if (Utils.appCode) { // ... do not create during build + clusterConfig = createConfig() + clusterName = clusterConfig.clusterName + instanceName = clusterConfig.instanceName + System.setProperty('io.xh.hoist.hzInstanceName', instanceName) + } + } + + /** Are multi-instance clusters enabled? */ + static boolean getMultiInstanceEnabled() { + clusterConfig.multiInstanceEnabled + } + + /** + * Called by Framework to initialize the Hazelcast instance. + * @internal + */ + static initializeInstance() { + hzInstance = Hazelcast.newHazelcastInstance(clusterConfig.createConfig()) + } + + void init() { + adjustMasterStatus() + cluster.addMembershipListener([ + memberAdded : { MembershipEvent e -> adjustMasterStatus(e.members) }, + memberRemoved: { MembershipEvent e -> adjustMasterStatus(e.members) } + ] as MembershipListener) + + super.init() + } + + private boolean _isMaster = false + + + /** + * Is this instance ready for requests? + */ + boolean isReady = false + + /** + * The Hazelcast member representing the 'master' instance. + * + * Apps typically use the master instance to execute any run-once logic on the cluster. + * + * We use a simple master definition documented by Hazelcast: The oldest member. + * See https://docs.hazelcast.org/docs/4.0/javadoc/com/hazelcast/cluster/Cluster.html#getMembers + */ + Member getMaster() { + cluster.members.iterator().next() + } + + /** The instance name of the master server.*/ + String getMasterName() { + master.getAttribute('instanceName') + } + + /** Is the local instance the master instance? */ + boolean getIsMaster() { + // Cache until we ensure our implementation lightweight enough -- also supports logging. + return _isMaster + } + + /** The Hazelcast member representing this instance. */ + Member getLocalMember() { + return cluster.localMember + } + + /** + * Shutdown this instance. + */ + void shutdownInstance() { + System.exit(0) + } + + /** + * The distributed objects available in the cluster + */ + Collection getDistributedObjects() { + hzInstance.distributedObjects + } + + //------------------------ + // Distributed execution + //------------------------ + IExecutorService getExecutorService() { + return hzInstance.getExecutorService('default') + } + + ClusterResponse submitToInstance(ClusterRequest c, String instance) { + executorService.submitToMember(c, getMember(instance)).get() + } + + Map> submitToAllInstances(ClusterRequest c) { + executorService + .submitToAllMembers(c) + .collectEntries { member, f -> [member.getAttribute('instanceName'), f.get()] } + } + + //------------------------------------ + // Implementation + //------------------------------------ + private Cluster getCluster() { + hzInstance.cluster + } + + private void adjustMasterStatus(Set members = cluster.members) { + // Accept members explicitly to avoid race conditions when responding to MembershipEvents + // (see https://docs.hazelcast.org/docs/4.0/javadoc/com/hazelcast/cluster/MembershipEvent.html) + // Not sure if we ever abdicate, could network failures cause master to leave and rejoin cluster? + def newIsMaster = members.iterator().next().localMember() + if (_isMaster != newIsMaster) { + _isMaster = newIsMaster + _isMaster ? + logInfo("I have assumed the role of Master. All hail me, '$instanceName'") : + logInfo('I have abdicated the role of Master.') + } + } + + private Member getMember(String instanceName) { + def ret = cluster.members.find { it.getAttribute('instanceName') == instanceName } + if (!ret) throw new InstanceNotFoundException("Unable to find cluster instance $instanceName") + return ret + } + + private static ClusterConfig createConfig() { + def clazz + try { + clazz = Class.forName(Utils.appPackage + '.ClusterConfig') + } catch (ClassNotFoundException e) { + clazz = Class.forName('io.xh.hoist.ClusterConfig') + } + return (clazz.getConstructor().newInstance() as ClusterConfig) + } + + void onApplicationEvent(ApplicationReadyEvent event) { + isReady = true + } + + Map getAdminStats() {[ + clusterId : cluster.clusterState.id, + instanceName: instanceName, + masterName : masterName, + isMaster : isMaster, + members : cluster.members.collect { it.getAttribute('instanceName') } + ]} + +} \ No newline at end of file diff --git a/grails-app/services/io/xh/hoist/config/ConfigService.groovy b/grails-app/services/io/xh/hoist/config/ConfigService.groovy index 643da9da..7b75bbf4 100644 --- a/grails-app/services/io/xh/hoist/config/ConfigService.groovy +++ b/grails-app/services/io/xh/hoist/config/ConfigService.groovy @@ -8,7 +8,6 @@ package io.xh.hoist.config import grails.compiler.GrailsCompileStatic -import grails.events.EventPublisher import grails.gorm.transactions.ReadOnly import grails.gorm.transactions.Transactional import groovy.transform.CompileDynamic @@ -32,7 +31,7 @@ import static io.xh.hoist.json.JSONSerializer.serializePretty * Fires an `xhConfigChanged` event when a config value is updated. */ @GrailsCompileStatic -class ConfigService extends BaseService implements EventPublisher { +class ConfigService extends BaseService { String getString(String name, String notFoundValue = null) { return (String) getInternalByName(name, 'string', notFoundValue) @@ -66,6 +65,11 @@ class ConfigService extends BaseService implements EventPublisher { return (String) getInternalByName(name, 'pwd', notFoundValue) } + + /** + * Return a map of all config values needed by client. + * All passwords will be obscured. + */ @ReadOnly boolean hasConfig(String name) { return AppConfig.findByName(name, [cache: true]) != null @@ -88,6 +92,19 @@ class ConfigService extends BaseService implements EventPublisher { return ret } + /** + * Return a map of specified config values, appropriate for display in admin client. + * Note this may include configs that are not typically sent to clients + * as specified by 'clientVisible'. All passwords will be obscured, however. + */ + @ReadOnly + Map getForAdminStats(String... names) { + return names.toList().collectEntries { + def config = AppConfig.findByName(it, [cache: true]) + [it, config?.externalValue(obscurePassword: true, jsonAsObject: true)] + } + } + /** * Parse a config which may contain a string or comma delimited list * into a List of split and trimmed strings. @@ -190,7 +207,8 @@ class ConfigService extends BaseService implements EventPublisher { } void fireConfigChanged(AppConfig obj) { - notify('xhConfigChanged', [key: obj.name, value: obj.externalValue()]) + def topic = clusterService.getTopic('xhConfigChanged') + topic.publishAsync([key: obj.name, value: obj.externalValue()]) } //------------------- diff --git a/grails-app/services/io/xh/hoist/email/EmailService.groovy b/grails-app/services/io/xh/hoist/email/EmailService.groovy index c63e9789..2ac67a73 100644 --- a/grails-app/services/io/xh/hoist/email/EmailService.groovy +++ b/grails-app/services/io/xh/hoist/email/EmailService.groovy @@ -23,6 +23,9 @@ class EmailService extends BaseService { def configService def groovyPageRenderer + private Date lastSentDate = null + private Long emailsSent = 0 + /** * Send email as specified by args param. * Application environment will be appended to subject unless environment is Production. @@ -125,6 +128,8 @@ class EmailService extends BaseService { attach f.fileName as String, f.contentType as String, f.contentSource } } + emailsSent++ + lastSentDate = new Date() if (doLog) { def recipCount = toRecipients.size() + (ccRecipients?.size() ?: 0) @@ -156,6 +161,16 @@ class EmailService extends BaseService { return s == 'none' ? null : formatAddresses(s) } + Map getAdminStats() {[ + config: configForAdminStats( + 'xhEmailOverride', + 'xhEmailFilter', + 'xhEmailDefaultSender', + 'xhEmailDefaultDomain' + ), + emailsSent: emailsSent, + lastSentDate: lastSentDate + ]} //------------------------ // Implementation diff --git a/grails-app/services/io/xh/hoist/environment/EnvironmentService.groovy b/grails-app/services/io/xh/hoist/environment/EnvironmentService.groovy index 27b841e6..5a5e2753 100644 --- a/grails-app/services/io/xh/hoist/environment/EnvironmentService.groovy +++ b/grails-app/services/io/xh/hoist/environment/EnvironmentService.groovy @@ -13,6 +13,7 @@ import grails.util.Holders import io.xh.hoist.BaseService import io.xh.hoist.util.Utils + /** * Service with metadata describing the runtime environment of Hoist and this application. * For the AppEnvironment (e.g. Development/Production), reference `Utils.appEnvironment`. @@ -58,7 +59,6 @@ class EnvironmentService extends BaseService { appVersion: Utils.appVersion, appBuild: Utils.appBuild, appEnvironment: Utils.appEnvironment.toString(), - startupTime: Utils.startupTime, grailsVersion: GrailsUtil.grailsVersion, javaVersion: System.getProperty('java.version'), serverTimeZone: serverTz.toZoneId().id, @@ -66,6 +66,7 @@ class EnvironmentService extends BaseService { appTimeZone: appTz.toZoneId().id, appTimeZoneOffset: appTz.getOffset(now), webSocketsEnabled: webSocketService.enabled, + instanceName: clusterService.instanceName ] hoistGrailsPlugins.each {it -> diff --git a/grails-app/services/io/xh/hoist/export/GridExportImplService.groovy b/grails-app/services/io/xh/hoist/export/GridExportImplService.groovy index 034a7bcd..c71f4009 100644 --- a/grails-app/services/io/xh/hoist/export/GridExportImplService.groovy +++ b/grails-app/services/io/xh/hoist/export/GridExportImplService.groovy @@ -41,6 +41,9 @@ class GridExportImplService extends BaseService { def configService + private Date lastExportDate = null + private Long exportCount = 0 + void init() { if (instanceLog.debugEnabled) { GraphicsEnvironment ge = GraphicsEnvironment.localGraphicsEnvironment @@ -61,11 +64,16 @@ class GridExportImplService extends BaseService { rows: params.rows.size(), cols: params.meta.size() ]) { - return [ + def ret = [ file : getFileData(params.type, params.rows, params.meta), contentType: getContentType(params.type), fileName : getFileName(params.filename, params.type) ] + + lastExportDate = new Date() + exportCount++; + return ret + } } @@ -359,4 +367,13 @@ class GridExportImplService extends BaseService { configService.getMap('xhExportConfig') } + + Map getAdminStats() { + return [ + config: configForAdminStats('xhExportConfig'), + exportCount: exportCount, + lastExportDate: lastExportDate + ] + } + } diff --git a/grails-app/services/io/xh/hoist/feedback/FeedbackEmailService.groovy b/grails-app/services/io/xh/hoist/feedback/FeedbackEmailService.groovy index a06bf31b..417b1ca2 100644 --- a/grails-app/services/io/xh/hoist/feedback/FeedbackEmailService.groovy +++ b/grails-app/services/io/xh/hoist/feedback/FeedbackEmailService.groovy @@ -15,7 +15,11 @@ class FeedbackEmailService extends BaseService { def emailService void init() { - subscribe('xhFeedbackReceived', this.&emailFeedback) + subscribeToTopic( + topic: 'xhFeedbackReceived', + onMessage: this.&emailFeedback, + masterOnly: true + ) } //------------------------ @@ -45,4 +49,8 @@ class FeedbackEmailService extends BaseService { return [msgText, metaText].findAll{it}.join('

') } + Map getAdminStats() {[ + config: [toAddress: emailService.parseMailConfig('xhEmailSupport')] + ]} + } diff --git a/grails-app/services/io/xh/hoist/feedback/FeedbackService.groovy b/grails-app/services/io/xh/hoist/feedback/FeedbackService.groovy index 81baf05c..9d3c12c9 100644 --- a/grails-app/services/io/xh/hoist/feedback/FeedbackService.groovy +++ b/grails-app/services/io/xh/hoist/feedback/FeedbackService.groovy @@ -10,12 +10,11 @@ package io.xh.hoist.feedback import grails.gorm.transactions.Transactional import io.xh.hoist.BaseService import io.xh.hoist.util.Utils -import grails.events.* import static io.xh.hoist.util.Utils.getCurrentRequest import static io.xh.hoist.browser.Utils.getBrowser import static io.xh.hoist.browser.Utils.getDevice -class FeedbackService extends BaseService implements EventPublisher { +class FeedbackService extends BaseService { /** * Create a feedback entry. Username, browser + environment info, and datetime will be set automatically. @@ -27,16 +26,16 @@ class FeedbackService extends BaseService implements EventPublisher { def request = currentRequest, userAgent = request?.getHeader('User-Agent'), values = [ - msg: message, - username: authUsername ?: 'ANON', - userAgent: userAgent, - browser: getBrowser(userAgent), - device: getDevice(userAgent), - appVersion: appVersion ?: Utils.appVersion, - appEnvironment: Utils.appEnvironment + msg : message, + username : authUsername ?: 'ANON', + userAgent : userAgent, + browser : getBrowser(userAgent), + device : getDevice(userAgent), + appVersion : appVersion ?: Utils.appVersion, + appEnvironment: Utils.appEnvironment ] def fb = new Feedback(values) fb.save(flush: true) - notify('xhFeedbackReceived', fb) + getTopic('xhFeedbackReceived').publishAsync(fb) } } diff --git a/grails-app/services/io/xh/hoist/log/LogArchiveService.groovy b/grails-app/services/io/xh/hoist/log/LogArchiveService.groovy index 1be56ba6..b675b4fd 100644 --- a/grails-app/services/io/xh/hoist/log/LogArchiveService.groovy +++ b/grails-app/services/io/xh/hoist/log/LogArchiveService.groovy @@ -70,7 +70,9 @@ class LogArchiveService extends BaseService { // Implementation //------------------------ private void onTimer() { - archiveLogs((Integer) config.archiveAfterDays) + if (isMaster) { + archiveLogs((Integer) config.archiveAfterDays) + } } private File getArchiveDir(String logPath, String category) { @@ -170,4 +172,7 @@ class LogArchiveService extends BaseService { return configService.getMap('xhLogArchiveConfig') } + Map getAdminStats() {[ + config: configForAdminStats('xhLogArchiveConfig') + ]} } diff --git a/grails-app/services/io/xh/hoist/log/LogLevelService.groovy b/grails-app/services/io/xh/hoist/log/LogLevelService.groovy index ee681076..1cca82a8 100644 --- a/grails-app/services/io/xh/hoist/log/LogLevelService.groovy +++ b/grails-app/services/io/xh/hoist/log/LogLevelService.groovy @@ -12,6 +12,8 @@ import io.xh.hoist.BaseService import ch.qos.logback.classic.Logger import ch.qos.logback.classic.LoggerContext import ch.qos.logback.classic.Level +import io.xh.hoist.cluster.ClusterRequest +import io.xh.hoist.util.Utils import org.slf4j.LoggerFactory import static io.xh.hoist.util.DateTimeUtils.MINUTES @@ -79,6 +81,15 @@ class LogLevelService extends BaseService { return logger.effectiveLevel.toString().toLowerCase().capitalize() } + void noteLogLevelChanged() { + clusterService.submitToAllInstances (new CalculateAdjustments()) + } + static class CalculateAdjustments extends ClusterRequest { + def doCall() { + Utils.appContext.logLevelService.calculateAdjustments() + } + } + //------------------------ // Implementation //------------------------ diff --git a/grails-app/services/io/xh/hoist/log/LogReaderService.groovy b/grails-app/services/io/xh/hoist/log/LogReaderService.groovy index de92d6ed..30455966 100644 --- a/grails-app/services/io/xh/hoist/log/LogReaderService.groovy +++ b/grails-app/services/io/xh/hoist/log/LogReaderService.groovy @@ -137,4 +137,8 @@ class LogReaderService extends BaseService { throw new RoutineRuntimeException('Query took too long. Log search aborted.') } } + + Map getAdminStats() {[ + config: configForAdminStats('xhEnableLogViewer', 'xhLogSearchTimeoutMs') + ]} } diff --git a/grails-app/services/io/xh/hoist/monitor/ConnectionPoolMonitoringService.groovy b/grails-app/services/io/xh/hoist/monitor/ConnectionPoolMonitoringService.groovy index 73653c98..df5c2c23 100644 --- a/grails-app/services/io/xh/hoist/monitor/ConnectionPoolMonitoringService.groovy +++ b/grails-app/services/io/xh/hoist/monitor/ConnectionPoolMonitoringService.groovy @@ -53,11 +53,15 @@ class ConnectionPoolMonitoringService extends BaseService { return _snapshots } + Map getLatestSnapshot() { + return _snapshots?.max { it.key }?.value + } + /** Take a snapshot of pool usage, add to in-memory history, and return. */ Map takeSnapshot() { ensureEnabled() - def newSnap = getStats() + def newSnap = getSnap() _snapshots[newSnap.timestamp] = newSnap // Don't allow snapshot history to grow endlessly - @@ -84,11 +88,10 @@ class ConnectionPoolMonitoringService extends BaseService { takeSnapshot() } - //------------------------ // Implementation //------------------------ - private Map getStats() { + private Map getSnap() { def ds = pooledDataSource return [ timestamp: currentTimeMillis(), @@ -128,5 +131,11 @@ class ConnectionPoolMonitoringService extends BaseService { void clearCaches() { this._snapshots.clear() + super.clearCaches() } + + Map getAdminStats() {[ + config: configForAdminStats('xhConnPoolMonitoringConfig'), + latestSnapshot: latestSnapshot + ]} } diff --git a/grails-app/services/io/xh/hoist/monitor/MemoryMonitoringService.groovy b/grails-app/services/io/xh/hoist/monitor/MemoryMonitoringService.groovy index 8e6ef80e..100c9bc2 100644 --- a/grails-app/services/io/xh/hoist/monitor/MemoryMonitoringService.groovy +++ b/grails-app/services/io/xh/hoist/monitor/MemoryMonitoringService.groovy @@ -46,6 +46,10 @@ class MemoryMonitoringService extends BaseService { return _snapshots } + Map getLatestSnapshot() { + return _snapshots?.max {it.key}?.value + } + /** * Dump the heap to a file for analysis. */ @@ -73,7 +77,7 @@ class MemoryMonitoringService extends BaseService { * Take a snapshot of JVM memory usage, store in this service's in-memory history, and return. */ Map takeSnapshot() { - def newSnap = getStats() + def newSnap = getSnap() _snapshots[newSnap.timestamp] = newSnap @@ -111,11 +115,10 @@ class MemoryMonitoringService extends BaseService { ] } - //------------------------ // Implementation //------------------------ - private Map getStats() { + private Map getSnap() { def mb = 1024 * 1024, timestamp = currentTimeMillis(), gcStats = getGCStats(timestamp), @@ -175,5 +178,11 @@ class MemoryMonitoringService extends BaseService { void clearCaches() { _snapshots.clear() + super.clearCaches() } + + Map getAdminStats() {[ + config: configForAdminStats('xhMemoryMonitoringConfig'), + latestSnapshot: latestSnapshot, + ]} } \ No newline at end of file diff --git a/grails-app/services/io/xh/hoist/monitor/MonitoringEmailService.groovy b/grails-app/services/io/xh/hoist/monitor/MonitoringEmailService.groovy index 1f3e34f9..2d1e5042 100644 --- a/grails-app/services/io/xh/hoist/monitor/MonitoringEmailService.groovy +++ b/grails-app/services/io/xh/hoist/monitor/MonitoringEmailService.groovy @@ -20,7 +20,11 @@ class MonitoringEmailService extends BaseService { def emailService void init() { - subscribe('xhMonitorStatusReport', this.&emailReport) + subscribeToTopic( + topic: 'xhMonitorStatusReport', + onMessage: this.&emailReport, + masterOnly: true + ) } //------------------------ @@ -51,4 +55,8 @@ class MonitoringEmailService extends BaseService { }.join('
') } + Map getAdminStats() {[ + config: [toAddress: emailService.parseMailConfig('xhMonitorEmailRecipients')] + ]} + } diff --git a/grails-app/services/io/xh/hoist/monitor/MonitoringService.groovy b/grails-app/services/io/xh/hoist/monitor/MonitoringService.groovy index 86fa7ebc..67379e12 100644 --- a/grails-app/services/io/xh/hoist/monitor/MonitoringService.groovy +++ b/grails-app/services/io/xh/hoist/monitor/MonitoringService.groovy @@ -8,13 +8,11 @@ package io.xh.hoist.monitor import grails.async.Promises -import grails.events.EventPublisher import grails.gorm.transactions.ReadOnly import io.xh.hoist.BaseService +import io.xh.hoist.cluster.ReplicatedValue import io.xh.hoist.util.Timer -import java.util.concurrent.ConcurrentHashMap - import static grails.async.Promises.task import static io.xh.hoist.monitor.MonitorStatus.* import static io.xh.hoist.util.DateTimeUtils.MINUTES @@ -32,40 +30,48 @@ import static java.lang.System.currentTimeMillis * * If enabled via config, this service will also write monitor run results to a dedicated log file. */ -class MonitoringService extends BaseService implements EventPublisher { +class MonitoringService extends BaseService { def configService, monitorResultService - private Map _results = new ConcurrentHashMap() - private Map _problems = new ConcurrentHashMap() - private Timer _monitorTimer - private Timer _notifyTimer - private boolean _alertMode = false - private Long _lastNotified + // Shared state for all servers to read + private ReplicatedValue> _results = getReplicatedValue('results') + + // Notification state for master to read only + private ReplicatedValue> problems = getReplicatedValue('problems') + private ReplicatedValue alertMode = getReplicatedValue('alertMode') + private ReplicatedValue lastNotified = getReplicatedValue('lastNotified') + + private Timer monitorTimer + private Timer notifyTimer void init() { - _monitorTimer = createTimer( + monitorTimer = createTimer( + masterOnly: true, + name: 'monitorTimer', + runFn: this.&onMonitorTimer, interval: {monitorInterval}, - delay: startupDelay, - runFn: this.&onMonitorTimer + delay: startupDelay ) - _notifyTimer = createTimer ( - interval: {notifyInterval}, - runFn: this.&onNotifyTimer + notifyTimer = createTimer ( + masterOnly: true, + name: 'notifyTimer', + runFn: this.&onNotifyTimer, + interval: {notifyInterval} ) } void forceRun() { - _monitorTimer.forceRun() + monitorTimer.forceRun() } @ReadOnly Map getResults() { + def results = _results.get() Monitor.list().collectEntries { - def result = _results[it.code] ?: monitorResultService.unknownMonitorResult(it) - [it.code, result] + [it.code, results?[it.code] ?: monitorResultService.unknownMonitorResult(it)] } } @@ -82,12 +88,12 @@ class MonitoringService extends BaseService implements EventPublisher { task { monitorResultService.runMonitor(m, timeout) } } - Map newResults = Promises + Map newResults = Promises .waitAll(tasks) - .collectEntries(new ConcurrentHashMap()) { [it.code, it] } + .collectEntries{ [it.code, it] } - markLastStatus(newResults, _results) - _results = newResults + markLastStatus(newResults, _results.get()) + _results.set(newResults) if (monitorConfig.writeToMonitorLog != false) logResults() evaluateProblems() } @@ -96,7 +102,7 @@ class MonitoringService extends BaseService implements EventPublisher { private void markLastStatus(Map newResults, Map oldResults) { def now = new Date() newResults.values().each {result -> - def oldResult = oldResults[result.code], + def oldResult = oldResults?[result.code], lastStatus = oldResult ? oldResult.status : UNKNOWN, statusChanged = lastStatus != result.status @@ -108,17 +114,16 @@ class MonitoringService extends BaseService implements EventPublisher { } private void evaluateProblems() { - Map flaggedResults = results.findAll {it.value.status >= WARN} + Map flaggedResults = results.findAll { it.value.status >= WARN } // 0) Remove all problems that are no longer problems - def removes = _problems.keySet().findAll {!flaggedResults[it]} - removes.each {_problems.remove(it)} + def probs = problems.get()?.findAll {flaggedResults[it.key]} ?: [:] // 1) (Re)Mark all existing problems - flaggedResults.each {code, result -> - def problem = _problems[code] + flaggedResults.each { code, result -> + def problem = probs[code] if (!problem) { - problem = _problems[code] = [result: result, cyclesAsFail: 0, cyclesAsWarn: 0] + problem = probs[code] = [result: result, cyclesAsFail: 0, cyclesAsWarn: 0] } if (result.status == FAIL) { @@ -128,29 +133,31 @@ class MonitoringService extends BaseService implements EventPublisher { problem.cyclesAsWarn++ } } + problems.set(probs) // 2) Handle alert mode transition -- notify immediately + // Note that we may get an extra transition if new master introduced in alerting def currAlertMode = calcAlertMode() - if (currAlertMode != _alertMode) { - _alertMode = currAlertMode + if (currAlertMode != alertMode.get()) { + alertMode.set(currAlertMode) notifyAlertModeChange() } } private void notifyAlertModeChange() { if (!isDevelopmentMode()) { - notify('xhMonitorStatusReport', generateStatusReport()) - _lastNotified = currentTimeMillis() + getTopic('xhMonitorStatusReport').publishAsync(generateStatusReport()) + lastNotified.set(currentTimeMillis()) } } private boolean calcAlertMode() { - if (_alertMode && _problems) return true + if (alertMode.get() && problems.get()) return true def failThreshold = monitorConfig.failNotifyThreshold, warnThreshold = monitorConfig.warnNotifyThreshold - return _problems.values().any { + return problems.get().values().any { it.cyclesAsFail >= failThreshold || it.cyclesAsWarn >= warnThreshold } } @@ -176,15 +183,15 @@ class MonitoringService extends BaseService implements EventPublisher { } private void onNotifyTimer() { - if (!_alertMode || !_lastNotified) return + if (!alertMode.get() || !lastNotified.get()) return def now = currentTimeMillis(), - timeThresholdMet = now > _lastNotified + monitorConfig.monitorRepeatNotifyMins * MINUTES + timeThresholdMet = now > lastNotified.get() + monitorConfig.monitorRepeatNotifyMins * MINUTES if (timeThresholdMet) { def report = generateStatusReport() logDebug("Emitting monitor status report: ${report.title}") - notify('xhMonitorStatusReport', report) - _lastNotified = now + getTopic('xhMonitorStatusReport').publishAsync(report) + lastNotified.set(currentTimeMillis()) } } @@ -215,10 +222,16 @@ class MonitoringService extends BaseService implements EventPublisher { void clearCaches() { super.clearCaches() - _results.clear() - _problems.clear() - if (monitorInterval > 0) { - _monitorTimer.forceRun() + if (isMaster) { + _results.set(null) + problems.set(null) + if (monitorInterval > 0) { + monitorTimer.forceRun() + } } } -} + + Map getAdminStats() {[ + config: configForAdminStats('xhMonitoringEnabled', 'xhMonitorConfig'), + ]} +} \ No newline at end of file diff --git a/grails-app/services/io/xh/hoist/track/TrackService.groovy b/grails-app/services/io/xh/hoist/track/TrackService.groovy index c9fa7be8..0c3dcae0 100644 --- a/grails-app/services/io/xh/hoist/track/TrackService.groovy +++ b/grails-app/services/io/xh/hoist/track/TrackService.groovy @@ -7,7 +7,6 @@ package io.xh.hoist.track -import grails.events.EventPublisher import groovy.transform.CompileStatic import io.xh.hoist.BaseService import io.xh.hoist.config.ConfigService @@ -35,7 +34,7 @@ import static io.xh.hoist.util.Utils.getCurrentRequest * active / accessible (intended for local development environments). */ @CompileStatic -class TrackService extends BaseService implements EventPublisher { +class TrackService extends BaseService { ConfigService configService @@ -82,7 +81,6 @@ class TrackService extends BaseService implements EventPublisher { return configService.getMap('xhActivityTrackingConfig') } - //------------------------- // Implementation //------------------------- @@ -165,4 +163,8 @@ class TrackService extends BaseService implements EventPublisher { } } } + + Map getAdminStats() {[ + config: configForAdminStats('xhActivityTrackingConfig') + ]} } diff --git a/src/main/groovy/io/xh/hoist/BaseService.groovy b/src/main/groovy/io/xh/hoist/BaseService.groovy index e9c269db..2bd254cf 100644 --- a/src/main/groovy/io/xh/hoist/BaseService.groovy +++ b/src/main/groovy/io/xh/hoist/BaseService.groovy @@ -7,11 +7,17 @@ package io.xh.hoist +import com.hazelcast.collection.ISet +import com.hazelcast.map.IMap +import com.hazelcast.replicatedmap.ReplicatedMap +import com.hazelcast.topic.ITopic +import com.hazelcast.topic.Message import grails.async.Promises -import io.xh.hoist.util.Utils import grails.util.GrailsClassUtils import groovy.transform.CompileDynamic -import io.xh.hoist.exception.ExceptionRenderer +import io.xh.hoist.cluster.ClusterService +import io.xh.hoist.cluster.ReplicatedValue +import io.xh.hoist.exception.ExceptionHandler import io.xh.hoist.log.LogSupport import io.xh.hoist.user.IdentitySupport import io.xh.hoist.util.Timer @@ -21,10 +27,13 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.DisposableBean +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit import static grails.async.Promises.task import static io.xh.hoist.util.DateTimeUtils.SECONDS +import static io.xh.hoist.util.Utils.appContext +import static io.xh.hoist.util.Utils.getConfigService /** * Standard superclass for all Hoist and Application-level services. @@ -35,11 +44,19 @@ import static io.xh.hoist.util.DateTimeUtils.SECONDS abstract class BaseService implements LogSupport, IdentitySupport, DisposableBean { IdentityService identityService - ExceptionRenderer exceptionRenderer - protected boolean _initialized = false + ClusterService clusterService + + ExceptionHandler xhExceptionHandler + + Date initializedDate = null + Date lastCachesCleared = null + + protected final List timers = [] + private boolean _destroyed = false + private Map _repValuesMap + private final Logger _log = LoggerFactory.getLogger(this.class) - private final List _timers = [] /** * Initialize a collection of BaseServices in parallel. @@ -68,20 +85,44 @@ abstract class BaseService implements LogSupport, IdentitySupport, DisposableBea * @param timeout - maximum time to wait for each service to init (in ms). */ final void initialize(Long timeout = 30 * SECONDS) { - if (_initialized) return + if (initializedDate) return try { withInfo("Initializing") { task { init() }.get(timeout, TimeUnit.MILLISECONDS) setupClearCachesConfigs() - _initialized = true } } catch (Throwable t) { - exceptionRenderer.handleException(t, this) + xhExceptionHandler.handleException(exception: t, logTo: this) + } finally { + initializedDate = new Date(); } } + //----------------------------------------------------------------- + // Distributed Resources + // Use static reference to ClusterService to allow access pre-init. + //------------------------------------------------------------------ + IMap getIMap(String id) { + ClusterService.hzInstance.getMap(hzName(id)) + } + + ISet getISet(String id) { + ClusterService.hzInstance.getSet(hzName(id)) + } + + ReplicatedMap getReplicatedMap(String id) { + ClusterService.hzInstance.getReplicatedMap(hzName(id)) + } + + ReplicatedValue getReplicatedValue(String key) { + new ReplicatedValue(key, repValuesMap) + } + + ITopic getTopic(String id) { + ClusterService.hzInstance.getTopic(id) + } /** * Create a new managed Timer bound to this service. @@ -94,13 +135,16 @@ abstract class BaseService implements LogSupport, IdentitySupport, DisposableBea args.runFn = this.&onTimer } def ret = new Timer(args) - _timers << ret + timers << ret return ret } /** * Managed Subscription to a Grails Event. * + * NOTE: Use this method to subscribe to local Grails events on the given server + * instance only. To subscribe to cluster-wide topics, use 'subscribeToTopic' instead. + * * This method will catch (and log) any exceptions thrown by its handler closure. * This is important because the core grails EventBus.subscribe() will silently swallow * exceptions, and stop processing subsequent handlers. @@ -109,7 +153,7 @@ abstract class BaseService implements LogSupport, IdentitySupport, DisposableBea * hot-reloading scenario where multiple instances of singleton services may be created. */ protected void subscribe(String eventName, Closure c) { - Utils.appContext.eventBus.subscribe(eventName) {Object... args -> + appContext.eventBus.subscribe(eventName) {Object... args -> if (destroyed) return try { logDebug("Receiving event '$eventName'") @@ -120,6 +164,49 @@ abstract class BaseService implements LogSupport, IdentitySupport, DisposableBea } } + + /** + * + * Managed Subscription to a cluster topic. + * + * NOTE: Use this method to subscribe to cluster-wide topics. To subscribe to local + * Grails events on this instance only, use subscribe instead. + * + * This method will catch (and log) any exceptions thrown by its handler closure. + * + * This subscription also avoids firing handlers on destroyed services. This is important in a + * hot-reloading scenario where multiple instances of singleton services may be created. + */ + protected void subscribeToTopic(Map config) { + def topic = config.topic as String, + onMessage = config.onMessage as Closure, + masterOnly = config.masterOnly as Boolean + + + getTopic(topic).addMessageListener { Message m -> + if (destroyed || (masterOnly && !isMaster)) return + try { + logDebug("Receiving message on topic '$topic'") + if (onMessage.maximumNumberOfParameters == 1) { + onMessage.call(m.messageObject) + } else { + onMessage.call(m.messageObject, m) + } + } catch (Exception e) { + logError("Exception handling message on topic '$topic'", e) + } + } + } + + + //------------------ + // Cluster Support + //------------------ + /** Is this instance the master instance */ + protected boolean getIsMaster() { + clusterService.isMaster + } + //----------------------------------- // Core template methods for override //----------------------------------- @@ -133,23 +220,42 @@ abstract class BaseService implements LogSupport, IdentitySupport, DisposableBea * resetting other stateful service objects such as HttpClients. The Hoist admin client provides a UI to call * this method on all BaseServices within a running application as an operational / troubleshooting tool. */ - void clearCaches() {} + void clearCaches() { + lastCachesCleared = new Date() + } /** * Cleanup or release any service resources - e.g. cancel any timers. * Called by Spring on a clean shutdown of the application. */ void destroy() { - _timers.each { + timers.each { it.cancel() } _destroyed = true } + /** + * Return meta data about this service for troubleshooting and monitoring. + * This data will be exposed via the Hoist admin client. + * + * Note that information about service timers and distributed objects does not need to be + * included here and will be automatically included by the framework. + */ + Map getAdminStats(){[:]} + + /** + * Return a map of specified config values, appropriate for including in + * implementations of getAdminStats(). + */ + protected Map configForAdminStats(String... names) { + getConfigService().getForAdminStats(names) + } + //-------------------- // Implemented methods //-------------------- - boolean isInitialized() {_initialized} + boolean isInitialized() {!!initializedDate} boolean isDestroyed() {_destroyed} HoistUser getUser() {identityService.user} @@ -165,17 +271,35 @@ abstract class BaseService implements LogSupport, IdentitySupport, DisposableBea } if (deps) { - subscribe('xhConfigChanged') {Map ev -> - if (deps.contains(ev.key)) { - logInfo("Clearing caches due to config change", ev.key) - clearCaches() + subscribeToTopic( + topic: 'xhConfigChanged', + onMessage: { Map msg -> + def key = msg.key + if (deps.contains(key)) { + logInfo("Clearing caches due to config change", key) + clearCaches() + } } - } + ) } } // Provide cached logger to LogSupport for possible performance benefit - private final Logger _log = LoggerFactory.getLogger(this.class) Logger getInstanceLog() { _log } + + //------------------------ + // Internal implementation + //------------------------ + protected String hzName(String key) { + this.class.name + '_' + key + } + + protected Map getRepValuesMap() { + _repValuesMap ?= ( + ClusterService.multiInstanceEnabled ? + getReplicatedMap('replicatedValues') : + new ConcurrentHashMap() + ) + } } diff --git a/src/main/groovy/io/xh/hoist/HoistCoreGrailsPlugin.groovy b/src/main/groovy/io/xh/hoist/HoistCoreGrailsPlugin.groovy index b56bd049..e46d55ec 100644 --- a/src/main/groovy/io/xh/hoist/HoistCoreGrailsPlugin.groovy +++ b/src/main/groovy/io/xh/hoist/HoistCoreGrailsPlugin.groovy @@ -8,7 +8,8 @@ package io.xh.hoist import grails.plugins.Plugin -import io.xh.hoist.exception.ExceptionRenderer +import io.xh.hoist.cluster.ClusterService +import io.xh.hoist.exception.ExceptionHandler import io.xh.hoist.security.HoistSecurityFilter import io.xh.hoist.websocket.HoistWebSocketConfigurer import org.springframework.boot.web.servlet.FilterRegistrationBean @@ -34,6 +35,8 @@ class HoistCoreGrailsPlugin extends Plugin { Closure doWithSpring() { {-> + ClusterService.initializeInstance() + hoistIdentityFilter(FilterRegistrationBean) { filter = bean(HoistSecurityFilter) order = Ordered.HIGHEST_PRECEDENCE + 40 @@ -43,7 +46,7 @@ class HoistCoreGrailsPlugin extends Plugin { hoistWebSocketConfigurer(HoistWebSocketConfigurer) } - exceptionRenderer(ExceptionRenderer) + xhExceptionHandler(ExceptionHandler) } } diff --git a/src/main/groovy/io/xh/hoist/browser/Utils.groovy b/src/main/groovy/io/xh/hoist/browser/Utils.groovy index bf3a1afd..4101fcaa 100644 --- a/src/main/groovy/io/xh/hoist/browser/Utils.groovy +++ b/src/main/groovy/io/xh/hoist/browser/Utils.groovy @@ -7,13 +7,11 @@ package io.xh.hoist.browser -import groovy.util.logging.Slf4j import static io.xh.hoist.browser.Browser.* import static io.xh.hoist.browser.Device.* import javax.servlet.http.HttpServletRequest -@Slf4j class Utils { private static Map BROWSERS_MATCHERS = [ diff --git a/src/main/groovy/io/xh/hoist/cache/Cache.groovy b/src/main/groovy/io/xh/hoist/cache/Cache.groovy index cc9a0219..74eb812c 100644 --- a/src/main/groovy/io/xh/hoist/cache/Cache.groovy +++ b/src/main/groovy/io/xh/hoist/cache/Cache.groovy @@ -9,6 +9,7 @@ package io.xh.hoist.cache import groovy.transform.CompileStatic import io.xh.hoist.BaseService +import io.xh.hoist.cluster.ClusterService import java.util.concurrent.ConcurrentHashMap @@ -18,24 +19,44 @@ import static java.lang.System.currentTimeMillis @CompileStatic class Cache { - - private final ConcurrentHashMap> _map = new ConcurrentHashMap() + private final Map> _map private Date _lastCull - public final String name // name for status logging disambiguation [default anon] - public final BaseService svc // service using this cache (for logging purposes) + /** Optional name for status logging disambiguation. */ + public final String name + + /** Service using this cache (for logging purposes). */ + public final BaseService svc + + /** Closure to determine if an entry should be expired. */ public final Closure expireFn + + /** Time after which an entry should be expired. */ public final Long expireTime + + /** Closure to determine the timestamp of an entry. */ public final Closure timestampFn + /** Whether this cache should be replicated across a cluster. */ + public final boolean replicate + Cache(Map options) { - name = (String) options.name ?: 'anon' + name = (String) options.name svc = (BaseService) options.svc expireTime = (Long) options.expireTime expireFn = (Closure) options.expireFn timestampFn = (Closure) options.timestampFn + replicate = (boolean) options.replicate if (!svc) throw new RuntimeException("Missing required argument 'svc' for Cache") + if (replicate && ClusterService.multiInstanceEnabled) { + if (!name) { + throw new RuntimeException("Cannot create a replicated cache without a unique name") + } + _map = svc.getReplicatedMap(name) + } else { + _map = new ConcurrentHashMap() + } } V get(K key) { @@ -46,6 +67,7 @@ class Cache { cullEntries() def ret = _map[key] if (ret && shouldExpire(ret)) { + _map[key]?.isRemoving = true _map.remove(key) return null } @@ -54,7 +76,8 @@ class Cache { void put(K key, V obj) { cullEntries() - _map.put(key, new Entry(obj)) + _map[key]?.isRemoving = true + _map.put(key, new Entry(key.toString(), obj)) } V getOrCreate(K key, Closure c) { @@ -108,10 +131,13 @@ class Cache { } if (cullKeys.size()) { - svc.logDebug("Cache '$name' culling ${cullKeys.size()} out of ${_map.size()} entries") + svc.logDebug("Cache '${name?: "anon"}' culling ${cullKeys.size()} out of ${_map.size()} entries") } - cullKeys.each {_map.remove(it)} + cullKeys.each { + _map[it]?.isRemoving = true + _map.remove(it) + } } } diff --git a/src/main/groovy/io/xh/hoist/cache/Entry.groovy b/src/main/groovy/io/xh/hoist/cache/Entry.groovy index 749bee70..caf3eb7f 100644 --- a/src/main/groovy/io/xh/hoist/cache/Entry.groovy +++ b/src/main/groovy/io/xh/hoist/cache/Entry.groovy @@ -7,16 +7,56 @@ package io.xh.hoist.cache +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.KryoSerializable +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output import groovy.transform.CompileStatic +import io.xh.hoist.log.LogSupport + +import static java.lang.System.currentTimeMillis @CompileStatic -class Entry { +class Entry implements KryoSerializable, LogSupport { + String key + T value + boolean isRemoving + Long dateEntered + + Entry(String key, T value) { + this.key = key + this.value = value + this.isRemoving = false + this.dateEntered = System.currentTimeMillis() + } - final V value - final Long dateEntered = System.currentTimeMillis() + Entry() {} - Entry(Object value) { - this.value = (V) value + void write(Kryo kryo, Output output) { + output.writeBoolean(isRemoving) + output.writeString(key) + if (!isRemoving) { + withSingleTrace('Serializing') { + output.writeLong(dateEntered) + kryo.writeClassAndObject(output, value) + } + } } + void read(Kryo kryo, Input input) { + isRemoving = input.readBoolean() + key = input.readString() + if (!isRemoving) { + withSingleTrace('Deserializing') { + dateEntered = input.readLong() + value = kryo.readClassAndObject(input) as T + } + } + } + + private void withSingleTrace(String msg, Closure c) { + Long start = currentTimeMillis() + c() + logTrace(msg, key, [_elapsedMs: currentTimeMillis() - start]) + } } diff --git a/src/main/groovy/io/xh/hoist/cluster/ClusterRequest.groovy b/src/main/groovy/io/xh/hoist/cluster/ClusterRequest.groovy new file mode 100644 index 00000000..36a99b56 --- /dev/null +++ b/src/main/groovy/io/xh/hoist/cluster/ClusterRequest.groovy @@ -0,0 +1,25 @@ +package io.xh.hoist.cluster + +import io.xh.hoist.log.LogSupport +import java.util.concurrent.Callable +import static io.xh.hoist.util.Utils.getExceptionHandler + +abstract class ClusterRequest implements Callable>, Serializable, LogSupport { + + ClusterResponse call() { + try { + return new ClusterResponse(value: doCall()) + } catch (Throwable t) { + exceptionHandler.handleException( + exception: t, + logTo: this, + logMessage: [_action: this.class.simpleName] + ) + return new ClusterResponse(exception: t) + } + } + + abstract T doCall() +} + + diff --git a/src/main/groovy/io/xh/hoist/cluster/ClusterResponse.groovy b/src/main/groovy/io/xh/hoist/cluster/ClusterResponse.groovy new file mode 100644 index 00000000..1964fb09 --- /dev/null +++ b/src/main/groovy/io/xh/hoist/cluster/ClusterResponse.groovy @@ -0,0 +1,8 @@ +package io.xh.hoist.cluster + +class ClusterResponse { + T value + Throwable exception +} + + diff --git a/src/main/groovy/io/xh/hoist/cluster/ReplicatedValue.groovy b/src/main/groovy/io/xh/hoist/cluster/ReplicatedValue.groovy new file mode 100644 index 00000000..fe7d4074 --- /dev/null +++ b/src/main/groovy/io/xh/hoist/cluster/ReplicatedValue.groovy @@ -0,0 +1,29 @@ +package io.xh.hoist.cluster +/** + * A cluster available value that can be read and written by any node in the cluster. + * Designed for small unrelated values that are needed across the cluster. + * + * This value will be stored in ths replicated map provided to this object. + */ +class ReplicatedValue { + + final String key + final Map> mp + + ReplicatedValue(String key, Map mp) { + this.key = key + this.mp = mp + } + + T get() { + mp[key]?.value as T + } + + void set(T value) { + mp[key]?.isRemoving = true + + value == null ? + mp.remove(key) : + mp.put(key, new ReplicatedValueEntry(key, value)) + } +} diff --git a/src/main/groovy/io/xh/hoist/cluster/ReplicatedValueEntry.groovy b/src/main/groovy/io/xh/hoist/cluster/ReplicatedValueEntry.groovy new file mode 100644 index 00000000..ebb0a254 --- /dev/null +++ b/src/main/groovy/io/xh/hoist/cluster/ReplicatedValueEntry.groovy @@ -0,0 +1,52 @@ +package io.xh.hoist.cluster + +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.KryoSerializable +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output +import groovy.transform.CompileStatic +import io.xh.hoist.log.LogSupport + +import static java.lang.System.currentTimeMillis + +@CompileStatic +class ReplicatedValueEntry implements KryoSerializable, LogSupport { + String key + T value + boolean isRemoving + + ReplicatedValueEntry() {} + + ReplicatedValueEntry(String key, T value) { + this.key = key + this.value = value + this.isRemoving = false + } + + void write(Kryo kryo, Output output) { + output.writeBoolean(isRemoving) + output.writeString(key) + if (!isRemoving) { + withSingleTrace('Serializing') { + kryo.writeClassAndObject(output, value) + } + } + } + + void read(Kryo kryo, Input input) { + isRemoving = input.readBoolean() + key = input.readString() + if (!isRemoving) { + withSingleTrace('Deserializing') { + value = kryo.readClassAndObject(input) as T + } + } + } + + private void withSingleTrace(String msg, Closure c) { + Long start = currentTimeMillis() + c() + logTrace(msg, key, [_elapsedMs: currentTimeMillis() - start]) + } +} + diff --git a/src/main/groovy/io/xh/hoist/configuration/ApplicationConfig.groovy b/src/main/groovy/io/xh/hoist/configuration/ApplicationConfig.groovy index 14677847..11ee9be5 100644 --- a/src/main/groovy/io/xh/hoist/configuration/ApplicationConfig.groovy +++ b/src/main/groovy/io/xh/hoist/configuration/ApplicationConfig.groovy @@ -89,12 +89,34 @@ class ApplicationConfig { } } - hibernate { + hazelcast { + jcache { + provider { + type = 'member' + } + } + client.statistics.enabled = true + } + hibernate { + javax { + cache { + provider = 'com.hazelcast.cache.impl.HazelcastServerCachingProvider' + uri = 'hazelcast-hibernate.xml' + } + persistence { + sharedCache { + mode = 'ENABLE_SELECTIVE' + } + } + } cache { use_second_level_cache = true queries = true use_query_cache = true - region.factory_class = 'org.hibernate.cache.ehcache.SingletonEhCacheRegionFactory' + generate_statistics = true + region { + factory_class = 'org.hibernate.cache.jcache.JCacheRegionFactory' + } } show_sql = false } diff --git a/src/main/groovy/io/xh/hoist/configuration/LogbackConfig.groovy b/src/main/groovy/io/xh/hoist/configuration/LogbackConfig.groovy index 19777d47..3a840b45 100644 --- a/src/main/groovy/io/xh/hoist/configuration/LogbackConfig.groovy +++ b/src/main/groovy/io/xh/hoist/configuration/LogbackConfig.groovy @@ -15,6 +15,8 @@ import ch.qos.logback.core.encoder.Encoder import ch.qos.logback.core.encoder.LayoutWrappingEncoder import ch.qos.logback.core.rolling.RollingFileAppender import ch.qos.logback.core.rolling.TimeBasedRollingPolicy +import io.xh.hoist.cluster.ClusterService +import io.xh.hoist.log.ClusterInstanceConverter import io.xh.hoist.util.Utils import java.nio.file.Paths @@ -41,32 +43,32 @@ class LogbackConfig { * Layout used for for logging to stdout * String or a Closure that produces a Layout */ - static Object stdoutLayout = '%d{yyyy-MM-dd HH:mm:ss.SSS} | %c{0} [%p] | %m%n' + static Object stdoutLayout = '%d{yyyy-MM-dd HH:mm:ss.SSS} | %instance | %c{0} [%p] | %m%n' /** * Layout for logs created by dailyLog() function * String or a Closure that produces a Layout * This layout will be used by the built-in rolling daily log provided by hoist. */ - static Object dailyLayout = '%d{HH:mm:ss.SSS} | %c{0} [%p] | %m%n' + static Object dailyLayout = '%d{HH:mm:ss.SSS} | %instance | %c{0} [%p] | %m%n' /** * Layout for logs created by monthlyLog() function * String or a Closure that produces a Layout */ - static Object monthlyLayout = '%d{MM-dd HH:mm:ss.SSS} | %c{0} [%p] | %m%n' + static Object monthlyLayout = '%d{MM-dd HH:mm:ss.SSS} | %instance | %c{0} [%p] | %m%n' /** * Layout used for logging monitor results to a dedicated log. * String or a Closure that produces a Layout */ - static Object monitorLayout = '%d{HH:mm:ss.SSS} | %m%n' + static Object monitorLayout = '%d{HH:mm:ss.SSS} | %instance | %m%n' /** * Layout used for logging client-side tracking results to a dedicated log. * String or a Closure that produces a Layout */ - static Object trackLayout = '%d{HH:mm:ss.SSS} | %m%n' + static Object trackLayout = '%d{HH:mm:ss.SSS} | %instance | %m%n' /** @@ -87,7 +89,7 @@ class LogbackConfig { static void defaultConfig(Script script) { withDelegate(script) { - def appLogName = Utils.appCode, + def appLogName = "${Utils.appCode}-${ClusterService.instanceName}", trackLogName = "$appLogName-track", monitorLogName = "$appLogName-monitor" @@ -97,6 +99,7 @@ class LogbackConfig { conversionRule("m", LogSupportConverter) conversionRule("msg", LogSupportConverter) conversionRule("message", LogSupportConverter) + conversionRule("instance", ClusterInstanceConverter) //---------------------------------- // Appenders @@ -124,11 +127,13 @@ class LogbackConfig { logger('io.xh.hoist.track.TrackService', INFO, [trackLogName, 'stdout'], false) // Quiet noisy loggers - logger('org.hibernate', ERROR) logger('org.springframework', ERROR) - logger('net.sf.ehcache', ERROR) + logger('org.hibernate', ERROR) logger('org.apache.directory.ldap.client.api.LdapNetworkConnection', ERROR) + // Stifle warning about disabled strong consistency library -- requires 3 node min. + logger('com.hazelcast.cp.CPSubsystem', ERROR) + // Turn off built-in global grails stacktrace logger. It can easily swamp logs! // If needed, it can be (carefully) re-enabled by in admin console. // Applications should *not* typically enable -- instead Hoist stacktraces can be diff --git a/src/main/groovy/io/xh/hoist/exception/ExceptionRenderer.groovy b/src/main/groovy/io/xh/hoist/exception/ExceptionHandler.groovy similarity index 68% rename from src/main/groovy/io/xh/hoist/exception/ExceptionRenderer.groovy rename to src/main/groovy/io/xh/hoist/exception/ExceptionHandler.groovy index 84047f38..95dd25b7 100644 --- a/src/main/groovy/io/xh/hoist/exception/ExceptionRenderer.groovy +++ b/src/main/groovy/io/xh/hoist/exception/ExceptionHandler.groovy @@ -9,11 +9,9 @@ package io.xh.hoist.exception import grails.util.GrailsUtil import groovy.transform.CompileStatic -import io.xh.hoist.json.JSONFormat import io.xh.hoist.json.JSONSerializer import io.xh.hoist.log.LogSupport -import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse import java.util.concurrent.ExecutionException @@ -31,27 +29,35 @@ import static org.apache.hc.core5.http.HttpStatus.SC_INTERNAL_SERVER_ERROR * These two contexts capture the overwhelming majority of code execution in a Hoist server. */ @CompileStatic -class ExceptionRenderer { +class ExceptionHandler { /** - * Main entry point for request based code (e.g. controllers) + * Sanitizes, pre-processes, and logs exception. + * + * Used by BaseController, ClusterRequest, Timer, and AccessInterceptor to handle + * otherwise unhandled exception. */ - void handleException(Throwable t, HttpServletRequest request, HttpServletResponse response, LogSupport logSupport) { - t = preprocess(t) - logException(t, logSupport) - - response.setStatus(getHttpStatus(t)) - response.setContentType('application/json') - response.writer.write(toJSON(t)) - response.flushBuffer() - } + void handleException(Map options) { + Throwable t = options.exception as Throwable + HttpServletResponse renderTo = options.renderTo as HttpServletResponse + LogSupport logTo = options.logTo as LogSupport + Object logMessage = options.logMessage - /** - * Main entry point for non-request based code (e.g. Timers) - */ - void handleException(Throwable t, LogSupport logSupport) { t = preprocess(t) - logException(t, logSupport) + if (logTo) { + if (logMessage) { + shouldLogDebug(t) ? logTo.logDebug(logMessage, t) : logTo.logError(logMessage, t) + } else { + shouldLogDebug(t) ? logTo.logDebug(t) : logTo.logError(t) + } + } + + if (renderTo) { + renderTo.setStatus(getHttpStatus(t)) + renderTo.setContentType('application/json') + renderTo.writer.write(JSONSerializer.serialize(t)) + renderTo.flushBuffer() + } } /** @@ -77,20 +83,11 @@ class ExceptionRenderer { return t } - protected void logException(Throwable t, LogSupport logSupport) { - if (shouldlogDebug(t)) { - logSupport.logDebug(null, t) - } else { - logSupport.logError(null, t) - } - } - - protected boolean shouldlogDebug(Throwable t) { + protected boolean shouldLogDebug(Throwable t) { return t instanceof RoutineException } protected int getHttpStatus(Throwable t) { - if (t instanceof HttpException && !(t instanceof ExternalHttpException)) { return ((HttpException) t).statusCode } @@ -100,19 +97,6 @@ class ExceptionRenderer { SC_INTERNAL_SERVER_ERROR } - protected String toJSON(Throwable t) { - def ret = t instanceof JSONFormat ? - t : - [ - name : t.class.simpleName, - message: t.message, - cause : t.cause?.message, - isRoutine: t instanceof RoutineException - ].findAll {it.value} - return JSONSerializer.serialize(ret); - } - - //--------------------------- // Implementation //--------------------------- diff --git a/src/main/groovy/io/xh/hoist/exception/InstanceNotFoundException.groovy b/src/main/groovy/io/xh/hoist/exception/InstanceNotFoundException.groovy new file mode 100644 index 00000000..7b98afed --- /dev/null +++ b/src/main/groovy/io/xh/hoist/exception/InstanceNotFoundException.groovy @@ -0,0 +1,14 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.exception + +class InstanceNotFoundException extends RoutineRuntimeException { + InstanceNotFoundException(String s) { + super(s) + } +} \ No newline at end of file diff --git a/src/main/groovy/io/xh/hoist/json/JSONSerializer.java b/src/main/groovy/io/xh/hoist/json/JSONSerializer.java index 28cdce10..5e56f256 100644 --- a/src/main/groovy/io/xh/hoist/json/JSONSerializer.java +++ b/src/main/groovy/io/xh/hoist/json/JSONSerializer.java @@ -50,7 +50,8 @@ public class JSONSerializer { .addSerializer(JSONFormatCached.class, new JSONFormatCachedSerializer()) .addSerializer(JSONFormat.class, new JSONFormatSerializer()) .addSerializer(Double.class, new DoubleSerializer()) - .addSerializer(Float.class, new FloatSerializer()); + .addSerializer(Float.class, new FloatSerializer()) + .addSerializer(Throwable.class, new ThrowableSerializer()); // ... plus one overwrite of JSR 310 standard hoistModule.addSerializer(LocalDate.class, new LocalDateSerializer()); diff --git a/src/main/groovy/io/xh/hoist/json/serializer/ThrowableSerializer.groovy b/src/main/groovy/io/xh/hoist/json/serializer/ThrowableSerializer.groovy new file mode 100644 index 00000000..a0358a84 --- /dev/null +++ b/src/main/groovy/io/xh/hoist/json/serializer/ThrowableSerializer.groovy @@ -0,0 +1,41 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.json.serializer; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import groovy.transform.CompileStatic +import io.xh.hoist.exception.RoutineException +import io.xh.hoist.json.JSONFormat + +@CompileStatic +class ThrowableSerializer extends StdSerializer { + + ThrowableSerializer() { + this(null) + } + + ThrowableSerializer(Class t) { + super(t) + } + + @Override + void serialize(Throwable t, JsonGenerator jgen, SerializerProvider provider) throws IOException { + def ret = t instanceof JSONFormat ? + t.formatForJSON() : + [ + name : t.class.simpleName, + message : t.message, + cause : t.cause?.message, + isRoutine: t instanceof RoutineException + ].findAll { it.value } + + jgen.writeObject(ret) + } +} diff --git a/src/main/groovy/io/xh/hoist/log/ClusterInstanceConverter.groovy b/src/main/groovy/io/xh/hoist/log/ClusterInstanceConverter.groovy new file mode 100644 index 00000000..3d34f6b6 --- /dev/null +++ b/src/main/groovy/io/xh/hoist/log/ClusterInstanceConverter.groovy @@ -0,0 +1,19 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2023 Extremely Heavy Industries Inc. + */ + +package io.xh.hoist.log + +import ch.qos.logback.classic.pattern.ClassicConverter +import ch.qos.logback.classic.spi.ILoggingEvent +import io.xh.hoist.cluster.ClusterService + +class ClusterInstanceConverter extends ClassicConverter { + String convert(ILoggingEvent event) { + return ClusterService.instanceName + } +} + diff --git a/src/main/groovy/io/xh/hoist/log/LogSupportConverter.groovy b/src/main/groovy/io/xh/hoist/log/LogSupportConverter.groovy index 22c4e6d9..a50d00a5 100644 --- a/src/main/groovy/io/xh/hoist/log/LogSupportConverter.groovy +++ b/src/main/groovy/io/xh/hoist/log/LogSupportConverter.groovy @@ -10,8 +10,7 @@ package io.xh.hoist.log import ch.qos.logback.classic.pattern.ClassicConverter import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.classic.spi.ThrowableProxy - -import static io.xh.hoist.util.Utils.exceptionRenderer +import static io.xh.hoist.util.Utils.getExceptionHandler /** * Layout Converter to output log messages in a human readable layout. @@ -93,7 +92,7 @@ class LogSupportConverter extends ClassicConverter { protected String formatThrowable(Throwable t) { try { - return exceptionRenderer.summaryTextForThrowable(t) + return exceptionHandler.summaryTextForThrowable(t) } catch (Exception ignored) { return t.message } diff --git a/src/main/groovy/io/xh/hoist/role/BaseRoleService.groovy b/src/main/groovy/io/xh/hoist/role/BaseRoleService.groovy index f42dcc6c..9f4d4def 100644 --- a/src/main/groovy/io/xh/hoist/role/BaseRoleService.groovy +++ b/src/main/groovy/io/xh/hoist/role/BaseRoleService.groovy @@ -7,6 +7,7 @@ package io.xh.hoist.role +import grails.gorm.transactions.ReadOnly import groovy.transform.CompileStatic import io.xh.hoist.BaseService diff --git a/src/main/groovy/io/xh/hoist/role/provided/DefaultRoleService.groovy b/src/main/groovy/io/xh/hoist/role/provided/DefaultRoleService.groovy index 23a71809..c8e6a06d 100644 --- a/src/main/groovy/io/xh/hoist/role/provided/DefaultRoleService.groovy +++ b/src/main/groovy/io/xh/hoist/role/provided/DefaultRoleService.groovy @@ -8,9 +8,9 @@ package io.xh.hoist.role.provided import grails.gorm.transactions.ReadOnly +import io.xh.hoist.cluster.ReplicatedValue import io.xh.hoist.role.BaseRoleService import io.xh.hoist.user.HoistUser -import io.xh.hoist.util.DateTimeUtils import io.xh.hoist.util.Timer import java.util.concurrent.ConcurrentHashMap @@ -20,9 +20,9 @@ import static io.xh.hoist.util.Utils.isLocalDevelopment import static io.xh.hoist.util.Utils.isProduction import static java.util.Collections.emptySet import static java.util.Collections.emptyMap -import static java.util.Collections.unmodifiableMap import static java.util.Collections.unmodifiableSet import static io.xh.hoist.util.InstanceConfigUtils.getInstanceConfig +import static io.xh.hoist.util.DateTimeUtils.SECONDS /** * Optional concrete implementation of BaseRoleService for applications that wish to leverage @@ -81,7 +81,7 @@ class DefaultRoleService extends BaseRoleService { defaultRoleUpdateService private Timer timer - protected Map> _allRoleAssignments = emptyMap() + protected ReplicatedValue>> _allRoleAssignments = getReplicatedValue('roleAssignments') protected ConcurrentMap> _roleAssignmentsByUser = new ConcurrentHashMap<>() protected Map _usersForDirectoryGroups = emptyMap() @@ -91,9 +91,10 @@ class DefaultRoleService extends BaseRoleService { ensureRequiredConfigAndRolesCreated() timer = createTimer( - interval: { config.refreshIntervalSecs as int * DateTimeUtils.SECONDS }, + interval: { config.refreshIntervalSecs as int * SECONDS }, runFn: this.&refreshRoleAssignments, - runImmediatelyAndBlock: true + runImmediatelyAndBlock: true, + masterOnly: true ) } @@ -102,7 +103,7 @@ class DefaultRoleService extends BaseRoleService { //------------------------------------ @Override Map> getAllRoleAssignments() { - _allRoleAssignments + _allRoleAssignments.get() } @Override @@ -283,7 +284,7 @@ class DefaultRoleService extends BaseRoleService { void refreshRoleAssignments() { withDebug('Refreshing role caches') { - _allRoleAssignments = unmodifiableMap(generateRoleAssignments()) + _allRoleAssignments.set(generateRoleAssignments()) _roleAssignmentsByUser = new ConcurrentHashMap() } } @@ -359,10 +360,18 @@ class DefaultRoleService extends BaseRoleService { } void clearCaches() { - _allRoleAssignments = emptyMap() + _allRoleAssignments.set(emptyMap()) _roleAssignmentsByUser = new ConcurrentHashMap() _usersForDirectoryGroups = emptyMap() timer.forceRun() super.clearCaches() } + + + Map getAdminStats() {[ + roleAssignments: allRoleAssignments?.size(), + roleAssignmentsByUser: _roleAssignmentsByUser?.size(), + usersForDirectoryGroups: _usersForDirectoryGroups?.size() + ]} + } diff --git a/src/main/groovy/io/xh/hoist/security/HoistSecurityFilter.groovy b/src/main/groovy/io/xh/hoist/security/HoistSecurityFilter.groovy index 303f2936..b61d794c 100644 --- a/src/main/groovy/io/xh/hoist/security/HoistSecurityFilter.groovy +++ b/src/main/groovy/io/xh/hoist/security/HoistSecurityFilter.groovy @@ -8,21 +8,19 @@ package io.xh.hoist.security import groovy.transform.CompileStatic -import io.xh.hoist.exception.ExceptionRenderer +import io.xh.hoist.cluster.ClusterService +import io.xh.hoist.exception.ExceptionHandler import io.xh.hoist.exception.RoutineRuntimeException import io.xh.hoist.log.LogSupport import io.xh.hoist.util.Utils -import org.springframework.boot.context.event.ApplicationReadyEvent -import org.springframework.context.ApplicationListener +import static io.xh.hoist.util.Utils.appContext import javax.servlet.* import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse @CompileStatic -class HoistSecurityFilter implements Filter, LogSupport, ApplicationListener { - private boolean isReady = false - +class HoistSecurityFilter implements Filter, LogSupport { void init(FilterConfig filterConfig) {} void destroy() {} @@ -30,13 +28,14 @@ class HoistSecurityFilter implements Filter, LogSupport, ApplicationListener 0 && intervalElapsed(intervalMs, lastRun)) || forceRun) { + if ((intervalMs > 0 && intervalElapsed(intervalMs, lastRunCompleted)) || forceRun) { boolean wasForced = forceRun doRun() if (wasForced) forceRun = false diff --git a/src/main/groovy/io/xh/hoist/util/Utils.groovy b/src/main/groovy/io/xh/hoist/util/Utils.groovy index bc92beb2..eebd21a3 100644 --- a/src/main/groovy/io/xh/hoist/util/Utils.groovy +++ b/src/main/groovy/io/xh/hoist/util/Utils.groovy @@ -7,6 +7,12 @@ package io.xh.hoist.util +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output +import com.esotericsoftware.kryo.serializers.JavaSerializer +import io.xh.hoist.cluster.ClusterService +import io.xh.hoist.exception.ExceptionHandler import io.xh.hoist.json.JSONParser import grails.util.Environment import grails.util.Holders @@ -15,7 +21,7 @@ import io.xh.hoist.AppEnvironment import io.xh.hoist.BaseService import io.xh.hoist.config.ConfigService import io.xh.hoist.environment.EnvironmentService -import io.xh.hoist.exception.ExceptionRenderer +import io.xh.hoist.log.LogSupport import io.xh.hoist.pref.PrefService import io.xh.hoist.role.BaseRoleService import io.xh.hoist.security.BaseAuthenticationService @@ -26,6 +32,10 @@ import org.grails.web.servlet.mvc.GrailsWebRequest import org.springframework.context.ApplicationContext import org.springframework.web.context.request.RequestContextHolder import javax.servlet.http.HttpServletRequest +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterInputStream + +import static java.lang.System.currentTimeMillis class Utils { @@ -89,6 +99,10 @@ class Utils { return (ConfigService) appContext.configService } + static ClusterService getClusterService() { + return (ClusterService) appContext.clusterService + } + static PrefService getPrefService() { return (PrefService) appContext.prefService } @@ -109,10 +123,6 @@ class Utils { return (BaseRoleService) appContext.roleService } - static ExceptionRenderer getExceptionRenderer() { - return (ExceptionRenderer) appContext.exceptionRenderer - } - static BaseAuthenticationService getAuthenticationService() { return (BaseAuthenticationService) appContext.authenticationService } @@ -125,6 +135,10 @@ class Utils { return Holders.applicationContext } + static ExceptionHandler getExceptionHandler() { + return (ExceptionHandler) appContext.xhExceptionHandler + } + /** * Get the current request. * @@ -188,5 +202,45 @@ class Utils { static void withDelegate(Object o, Closure c) { c.delegate = o c.call() + } + + static testSerialization(Object obj, + Class clazz, + LogSupport logSupport, + Map opts + ) { + Kryo kryo = new Kryo() + kryo.registrationRequired = false + if (opts.java) { + kryo.register(clazz, new JavaSerializer()) + } + kryo.reset() + kryo.setReferences(opts.refs) + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(32 * 1024) + + Long startTime = currentTimeMillis() + OutputStream outputStream = byteStream + if (opts.compress) outputStream = new DeflaterOutputStream(outputStream) + Output output = new Output(outputStream) + kryo.writeObject(output, obj); + output.close() + Long serializeTime = currentTimeMillis() + + InputStream inputStream = new ByteArrayInputStream(byteStream.toByteArray()) + if (opts.compress) inputStream = new InflaterInputStream(inputStream) + Input input = new Input(inputStream) + Object object2 = kryo.readObject(input, clazz) + Long endTime = currentTimeMillis() + + logSupport.logInfo( + opts, + "(${serializeTime - startTime}/${endTime-serializeTime})ms", + "${(byteStream.size() / 1000000).round(2)}MB", + "${endTime-startTime}ms" + ) + + + + } } diff --git a/src/main/groovy/io/xh/hoist/websocket/HoistWebSocketChannel.groovy b/src/main/groovy/io/xh/hoist/websocket/HoistWebSocketChannel.groovy index 013b14fe..d73f5b4a 100644 --- a/src/main/groovy/io/xh/hoist/websocket/HoistWebSocketChannel.groovy +++ b/src/main/groovy/io/xh/hoist/websocket/HoistWebSocketChannel.groovy @@ -8,7 +8,6 @@ package io.xh.hoist.websocket import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j import io.xh.hoist.json.JSONFormat import io.xh.hoist.log.LogSupport import io.xh.hoist.user.HoistUser diff --git a/src/main/resources/hazelcast-hibernate.xml b/src/main/resources/hazelcast-hibernate.xml new file mode 100644 index 00000000..4a0067e7 --- /dev/null +++ b/src/main/resources/hazelcast-hibernate.xml @@ -0,0 +1,14 @@ + + + + + ${io.xh.hoist.hzInstanceName} +