[feat]: create a warning if an instance has too many objects (#3070)

* create a warning if an instance has too many objects

* fix limit

* ensure hostname ist set in adapter tests too

---------

Co-authored-by: Bluefox <[email protected]>
Co-authored-by: Ingo Fischer <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -5,6 +5,7 @@
 -->
 
 ## __WORK IN PROGRESS__
+* (@foxriver76) Added objects warn limit per instance
 * (@Apollon77) Allows only numbers for `ts` and `lc` fields in state when provided for setState
 * (@GermanBluefox) Added typing for `visIconSets` in `io-package.json`(for vis-2 SVG icon sets)
 * (@GermanBluefox) Added typing for `smartName` in the enum objects

README.md

@@ -91,6 +91,7 @@ The main configuration is stored in `iobroker-data/iobroker.json`. Normally, the
 - [Error Reporting via ioBroker Sentry](#error-reporting-via-iobroker-sentry)
 - [Notification System](#notification-system)
 - [Disk space warnings](#disk-space-warnings)
+- [Objects warn limit](#objects-warn-limit)
 - [Controlling and monitoring of adapter processes](#controlling-and-monitoring-of-adapter-processes)
 - [Multihost](#multihost)
 - [TIERS: Start instances in an ordered manner](#tiers-start-instances-in-an-ordered-manner)
@@ -552,6 +553,12 @@ All three are optional and can be a string or null/undefined if omitted.
 The js-controller will generate a notification of in the scope `system` and the category `diskSpaceIssues` on warning level, if your free disk space falls under a specified threshold. 
 By default, this threshold is 5 % of disk space. Via the state `system.host.<hostname>.diskWarning` you can override this level to any level between `0` and `100`. 
 
+### Objects warn limit
+**Feature status:** New in 7.1.0
+
+The js-controller will generate a notification of in the scope `system` and the category `numberObjectsLimitExceeded` on warning level, if your number of objects for an adapter instance exceed a specified threshold.
+By default, this is set to `5000` objects. Via the state `system.host.adapter.<adapter>.<instance>.objectsWarnLimit` you can override this threshold to any positive number.
+
 ### Logging
 #### Log levels
 **Feature status:** stable

packages/adapter/src/lib/adapter/adapter.ts

@@ -138,7 +138,7 @@ import type {
     UserInterfaceClientRemoveMessage,
 } from '@/lib/_Types.js';
 import { UserInterfaceMessagingController } from '@/lib/adapter/userInterfaceMessagingController.js';
-import { SYSTEM_ADAPTER_PREFIX } from '@iobroker/js-controller-common-db/constants';
+import { SYSTEM_ADAPTER_PREFIX, DEFAULT_OBJECTS_WARN_LIMIT } from '@iobroker/js-controller-common-db/constants';
 import { isLogLevel } from '@iobroker/js-controller-common-db/tools';
 
 const controllerVersion = packJson.version;
@@ -11506,30 +11506,15 @@ export class AdapterClass extends EventEmitter {
 
                 if (!this._config.isInstall && (!process.argv || !this._config.forceIfDisabled)) {
                     const id = `system.adapter.${this.namespace}`;
-                    this.outputCount += 2;
-                    this.#states.setState(`${id}.alive`, { val: true, ack: true, expire: 30, from: id });
-                    let done = false;
-                    this.#states.setState(
-                        `${id}.connected`,
-                        {
-                            val: true,
-                            ack: true,
-                            expire: 30,
-                            from: id,
-                        },
-                        () => {
-                            if (!done) {
-                                done = true;
-                                this.terminate(EXIT_CODES.NO_ADAPTER_CONFIG_FOUND);
-                            }
-                        },
-                    );
-                    setTimeout(() => {
-                        if (!done) {
-                            done = true;
-                            this.terminate(EXIT_CODES.NO_ADAPTER_CONFIG_FOUND);
-                        }
-                    }, 1_000);
+                    await this.#setStateWithOutputCount(`${id}.alive`, { val: true, ack: true, expire: 30, from: id });
+                    await this.#setStateWithOutputCount(`${id}.connected`, {
+                        val: true,
+                        ack: true,
+                        expire: 30,
+                        from: id,
+                    });
+
+                    this.terminate(EXIT_CODES.NO_ADAPTER_CONFIG_FOUND);
                     return;
                 }
             }
@@ -11590,7 +11575,7 @@ export class AdapterClass extends EventEmitter {
             // @ts-expect-error
             this.config = adapterConfig.native;
             // @ts-expect-error
-            this.host = adapterConfig.common.host;
+            this.host = adapterConfig.common.host || tools.getHostName();
             // @ts-expect-error
             this.common = adapterConfig.common;
 
@@ -11635,7 +11620,7 @@ export class AdapterClass extends EventEmitter {
             this.config = adapterConfig.native || {};
             // @ts-expect-error
             this.common = adapterConfig.common || {};
-            this.host = this.common?.host || tools.getHostName() || os.hostname();
+            this.host = this.common?.host || tools.getHostName();
         }
 
         this.adapterConfig = adapterConfig;
@@ -11706,13 +11691,19 @@ export class AdapterClass extends EventEmitter {
             from: `system.adapter.${this.namespace}`,
         });
 
+        try {
+            await this.#checkObjectsWarnLimit();
+        } catch (e) {
+            this._logger.error(`${this.namespaceLog} Could not check objects warn limit: ${e.message}`);
+        }
+
         if (this._options.instance === undefined) {
             this.version = this.pack?.version
                 ? this.pack.version
                 : this.ioPack?.common
                   ? this.ioPack.common.version
                   : 'unknown';
-            // display if it's a non-official version - only if installedFrom is explicitly given and differs it's not npm
+
             // display if it's a non-official version - only if installedFrom is explicitly given and differs it's not npm
             const isNpmVersion = isInstalledFromNpm({
                 adapterName: this.name,
@@ -11982,6 +11973,67 @@ export class AdapterClass extends EventEmitter {
         });
     }
 
+    /**
+     * Check if the number of objects exceeds the warning limit
+     */
+    async #checkObjectsWarnLimit(): Promise<void> {
+        if (!this.#objects || !this.#states) {
+            throw new Error(tools.ERRORS.ERROR_DB_CLOSED);
+        }
+
+        const warnLimitId = `${SYSTEM_ADAPTER_PREFIX + this.namespace}.objectsWarnLimit`;
+
+        const warnLimitState = await this.#getStateWithInputCount(warnLimitId);
+
+        const objectsWarnLimit =
+            typeof warnLimitState?.val === 'number' ? warnLimitState.val : DEFAULT_OBJECTS_WARN_LIMIT;
+
+        if (warnLimitState?.ack === false) {
+            await this.#setStateWithOutputCount(warnLimitId, { val: objectsWarnLimit, ack: true });
+        }
+
+        const keys = await this.#objects.getKeysAsync(`${this.namespace}*`);
+        const objectsCount = keys?.length ?? 0;
+
+        if (objectsCount > objectsWarnLimit) {
+            const message = `This instance has ${objectsCount} objects, the limit for this instance is set to ${objectsWarnLimit}.`;
+            this._logger.warn(`${this.namespaceLog} ${message}`);
+            await this.registerNotification('system', 'numberObjectsLimitExceeded', message);
+        }
+    }
+
+    /**
+     * Get a state and automatically increase the input count
+     *
+     * @param id id of the state
+     */
+    #getStateWithInputCount(id: string): ioBroker.GetStatePromise {
+        if (!this.#states) {
+            throw new Error(tools.ERRORS.ERROR_DB_CLOSED);
+        }
+
+        this.inputCount++;
+        return this.#states.getState(id);
+    }
+
+    /**
+     * Set a state and automatically increase the output count
+     *
+     * @param id if of the state
+     * @param state state to set
+     */
+    #setStateWithOutputCount(
+        id: string,
+        state: ioBroker.SettableState | ioBroker.StateValue,
+    ): ioBroker.SetStatePromise {
+        if (!this.#states) {
+            throw new Error(tools.ERRORS.ERROR_DB_CLOSED);
+        }
+
+        this.outputCount++;
+        return this.#states.setState(id, state);
+    }
+
     private async _extendObjects(tasks: Record<string, any>, callback: () => void): Promise<void> {
         if (!tasks || !tasks.length) {
             return tools.maybeCallback(callback);

packages/common-db/src/lib/common/constants.ts

@@ -8,3 +8,5 @@ export const SYSTEM_REPOSITORIES_ID = 'system.repositories';
 export const SYSTEM_CONFIG_ID = 'system.config';
 /** Unicode symbol to be appended on endkey of getObjectView */
 export const HIGHEST_UNICODE_SYMBOL = '\u9999';
+/** Default limit for generating a warning if exceeding the number of objects per instance */
+export const DEFAULT_OBJECTS_WARN_LIMIT = 5_000;

packages/common-db/src/lib/common/tools.ts

@@ -32,6 +32,7 @@ import type * as DiskUsage from 'diskusage';
 import * as url from 'node:url';
 import { createRequire } from 'node:module';
 import type { WithRequired } from '@iobroker/types-dev';
+import { DEFAULT_OBJECTS_WARN_LIMIT } from '@/lib/common/constants.js';
 
 // eslint-disable-next-line unicorn/prefer-module
 const thisDir = url.fileURLToPath(new URL('.', import.meta.url || `file://${__filename}`));
@@ -3536,6 +3537,20 @@ export function getInstanceIndicatorObjects(namespace: string): ioBroker.StateOb
             },
             native: {},
         },
+        {
+            _id: `${id}.objectsWarnLimit`,
+            type: 'state',
+            common: {
+                name: `${namespace} objects warn limit`,
+                type: 'number',
+                read: true,
+                write: true,
+                desc: 'If the number of objects of this adapter instance exceeds this limit, the user will receive a warning',
+                role: 'state',
+                def: DEFAULT_OBJECTS_WARN_LIMIT,
+            },
+            native: {},
+        },
     ];
 }
 

packages/controller/io-package.json

@@ -1310,6 +1310,38 @@
           },
           "regex": [],
           "limit": 1
+        },
+        {
+          "category": "numberObjectsLimitExceeded",
+          "name": {
+            "en": "The configured limit of objects for this adapter instance has been exceeded.",
+            "de": "Das konfigurierte Limit an Objekten für diese Adapterinstanz wurde überschritten.",
+            "ru": "Превышен настроенный лимит объектов для данного экземпляра адаптера.",
+            "pt": "O limite configurado de objectos para esta instância de adaptador foi ultrapassado.",
+            "nl": "De geconfigureerde limiet van objecten voor deze instantie van de adapter is overschreden.",
+            "fr": "La limite d'objets configurée pour cette instance d'adaptateur a été dépassée.",
+            "it": "È stato superato il limite di oggetti configurato per questa istanza dell'adattatore.",
+            "es": "Se ha superado el límite configurado de objetos para esta instancia de adaptador.",
+            "pl": "Skonfigurowany limit obiektów dla tej instancji adaptera został przekroczony.",
+            "uk": "Перевищено ліміт об'єктів для цього екземпляра адаптера.",
+            "zh-cn": "The configured limit of objects for this adapter instance has been exceeded."
+          },
+          "severity": "alert",
+          "description": {
+            "en": "This adapter instance has exceeded the configured limit of objects. Too many objects are often caused by misconfigurations or bugs. This can lead to massive performance issues. If this is expected, you can increase the limit for this adapter instance.",
+            "de": "Diese Adapterinstanz hat das konfigurierte Limit an Objekten überschritten. Zu viele Objekte werden oft durch Fehlkonfigurationen oder Bugs verursacht. Dies kann zu massiven Performance-Problemen führen. Wenn dies zu erwarten ist, können Sie das Limit für diese Adapterinstanz erhöhen.",
+            "ru": "Этот экземпляр адаптера превысил установленный лимит объектов. Слишком большое количество объектов часто вызвано неправильной конфигурацией или ошибками. Это может привести к серьезным проблемам с производительностью. Если это ожидаемо, вы можете увеличить лимит для этого экземпляра адаптера.",
+            "pt": "Esta instância do adaptador excedeu o limite configurado de objectos. A existência de demasiados objectos é frequentemente causada por configurações incorrectas ou erros. Isto pode levar a grandes problemas de desempenho. Se isto for expetável, pode aumentar o limite para esta instância do adaptador.",
+            "nl": "Deze instantie van de adapter heeft de geconfigureerde limiet van objecten overschreden. Te veel objecten worden vaak veroorzaakt door verkeerde configuraties of bugs. Dit kan leiden tot enorme prestatieproblemen. Als dit wordt verwacht, kun je de limiet voor deze adapterinstantie verhogen.",
+            "fr": "Cette instance d'adaptateur a dépassé la limite d'objets configurée. Un trop grand nombre d'objets est souvent dû à une mauvaise configuration ou à des bogues. Cela peut entraîner d'importants problèmes de performance. Si cela est prévisible, vous pouvez augmenter la limite de cette instance d'adaptateur.",
+            "it": "L'istanza di questo adattatore ha superato il limite di oggetti configurato. Un numero eccessivo di oggetti è spesso causato da configurazioni errate o da bug. Ciò può causare gravi problemi di prestazioni. Se questo è previsto, è possibile aumentare il limite per questa istanza dell'adattatore.",
+            "es": "Esta instancia de adaptador ha superado el límite configurado de objetos. El exceso de objetos suele deberse a errores o errores de configuración. Esto puede conducir a problemas de rendimiento masivos. Si esto es lo esperado, puedes aumentar el límite para esta instancia de adaptador.",
+            "pl": "Ta instancja adaptera przekroczyła skonfigurowany limit obiektów. Zbyt duża liczba obiektów jest często spowodowana błędną konfiguracją lub błędami. Może to prowadzić do poważnych problemów z wydajnością. Jeśli jest to oczekiwane, można zwiększyć limit dla tej instancji adaptera.",
+            "uk": "Цей екземпляр адаптера перевищив встановлений ліміт об'єктів. Надмірна кількість об'єктів часто спричинена неправильними конфігураціями або помилками. Це може призвести до значних проблем з продуктивністю. Якщо це очікується, ви можете збільшити ліміт для цього екземпляра адаптера.",
+            "zh-cn": "This adapter instance has exceeded the configured limit of objects. Too many objects are often caused by misconfigurations or bugs. This can lead to massive performance issues. If this is expected, you can increase the limit for this adapter instance."
+          },
+          "regex": [],
+          "limit": 1
         }
       ]
     }

packages/types-dev/index.d.ts

@@ -376,7 +376,16 @@ declare global {
                 | 'nonExistingFileErrors'
                 | 'remoteHostErrors'
                 | 'restartLoop'
-                | 'fileToJsonl';
+                | 'fileToJsonl'
+                | 'numberObjectsLimitExceeded'
+                | 'systemRebootRequired'
+                | 'dockerUpdate'
+                | 'packageUpdates'
+                | 'securityIssues'
+                | 'databaseErrors'
+                | 'blockedVersions'
+                | 'automaticAdapterUpgradeSuccessful'
+                | 'automaticAdapterUpgradeFailed';
             [other: string]: string;
         }