diff --git a/package.json b/package.json index de3131b9b3d..92e94ef38fa 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,7 @@ "node": ">=18" }, "dependencies": { - "@datadog/libdatadog": "^0.4.0", + "@datadog/libdatadog": "^0.5.0", "@datadog/native-appsec": "8.4.0", "@datadog/native-iast-rewriter": "2.8.0", "@datadog/native-iast-taint-tracking": "3.3.0", diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 05d11f25afb..e362206465d 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -11,7 +11,7 @@ const tagger = require('./tagger') const get = require('../../datadog-core/src/utils/src/get') const has = require('../../datadog-core/src/utils/src/has') const set = require('../../datadog-core/src/utils/src/set') -const { isTrue, isFalse } = require('./util') +const { isTrue, isFalse, normalizeProfilingEnabledValue } = require('./util') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('./plugins/util/tags') const { getGitMetadataFromGitProperties, removeUserSensitiveInfo } = require('./git_properties') const { updateConfig } = require('./telemetry') @@ -236,6 +236,12 @@ function reformatSpanSamplingRules (rules) { class Config { constructor (options = {}) { + if (!isInServerlessEnvironment()) { + // Bail out early if we're in a serverless environment, stable config isn't supported + const StableConfig = require('./config_stable') + this.stableConfig = new StableConfig() + } + options = { ...options, appsec: options.appsec != null ? options.appsec : options.experimental?.appsec, @@ -244,13 +250,24 @@ class Config { // Configure the logger first so it can be used to warn about other configs const logConfig = log.getConfig() - this.debug = logConfig.enabled + this.debug = log.isEnabled( + this.stableConfig?.fleetEntries?.DD_TRACE_DEBUG, + this.stableConfig?.localEntries?.DD_TRACE_DEBUG + ) this.logger = coalesce(options.logger, logConfig.logger) - this.logLevel = coalesce(options.logLevel, logConfig.logLevel) - + this.logLevel = log.getLogLevel( + options.logLevel, + this.stableConfig?.fleetEntries?.DD_TRACE_LOG_LEVEL, + this.stableConfig?.localEntries?.DD_TRACE_LOG_LEVEL + ) log.use(this.logger) log.toggle(this.debug, this.logLevel) + // Process stable config warnings, if any + for (const warning of this.stableConfig?.warnings ?? []) { + log.warn(warning) + } + checkIfBothOtelAndDdEnvVarSet() const DD_API_KEY = coalesce( @@ -337,7 +354,9 @@ class Config { } this._applyDefaults() + this._applyLocalStableConfig() this._applyEnvironment() + this._applyFleetStableConfig() this._applyOptions(options) this._applyCalculated() this._applyRemote({}) @@ -576,6 +595,45 @@ class Config { this._setValue(defaults, 'trace.dynamoDb.tablePrimaryKeys', undefined) } + _applyLocalStableConfig () { + const obj = setHiddenProperty(this, '_localStableConfig', {}) + this._applyStableConfig(this.stableConfig?.localEntries ?? {}, obj) + } + + _applyFleetStableConfig () { + const obj = setHiddenProperty(this, '_fleetStableConfig', {}) + this._applyStableConfig(this.stableConfig?.fleetEntries ?? {}, obj) + } + + _applyStableConfig (config, obj) { + const { + DD_APPSEC_ENABLED, + DD_APPSEC_SCA_ENABLED, + DD_DATA_STREAMS_ENABLED, + DD_DYNAMIC_INSTRUMENTATION_ENABLED, + DD_ENV, + DD_IAST_ENABLED, + DD_LOGS_INJECTION, + DD_PROFILING_ENABLED, + DD_RUNTIME_METRICS_ENABLED, + DD_SERVICE, + DD_VERSION + } = config + + this._setBoolean(obj, 'appsec.enabled', DD_APPSEC_ENABLED) + this._setBoolean(obj, 'appsec.sca.enabled', DD_APPSEC_SCA_ENABLED) + this._setBoolean(obj, 'dsmEnabled', DD_DATA_STREAMS_ENABLED) + this._setBoolean(obj, 'dynamicInstrumentation.enabled', DD_DYNAMIC_INSTRUMENTATION_ENABLED) + this._setString(obj, 'env', DD_ENV) + this._setBoolean(obj, 'iast.enabled', DD_IAST_ENABLED) + this._setBoolean(obj, 'logInjection', DD_LOGS_INJECTION) + const profilingEnabled = normalizeProfilingEnabledValue(DD_PROFILING_ENABLED) + this._setString(obj, 'profiling.enabled', profilingEnabled) + this._setBoolean(obj, 'runtimeMetrics', DD_RUNTIME_METRICS_ENABLED) + this._setString(obj, 'service', DD_SERVICE) + this._setString(obj, 'version', DD_VERSION) + } + _applyEnvironment () { const { AWS_LAMBDA_FUNCTION_NAME, @@ -831,16 +889,13 @@ class Config { this._envUnprocessed.peerServiceMapping = DD_TRACE_PEER_SERVICE_MAPPING } this._setString(env, 'port', DD_TRACE_AGENT_PORT) - const profilingEnabledEnv = coalesce( - DD_EXPERIMENTAL_PROFILING_ENABLED, - DD_PROFILING_ENABLED, - this._isInServerlessEnvironment() ? 'false' : undefined + const profilingEnabled = normalizeProfilingEnabledValue( + coalesce( + DD_EXPERIMENTAL_PROFILING_ENABLED, + DD_PROFILING_ENABLED, + this._isInServerlessEnvironment() ? 'false' : undefined + ) ) - const profilingEnabled = isTrue(profilingEnabledEnv) - ? 'true' - : isFalse(profilingEnabledEnv) - ? 'false' - : profilingEnabledEnv === 'auto' ? 'auto' : undefined this._setString(env, 'profiling.enabled', profilingEnabled) this._setString(env, 'profiling.exporters', DD_PROFILING_EXPORTERS) this._setBoolean(env, 'profiling.sourceMap', DD_PROFILING_SOURCE_MAP && !isFalse(DD_PROFILING_SOURCE_MAP)) @@ -1347,9 +1402,33 @@ class Config { // eslint-disable-next-line @stylistic/js/max-len // https://github.com/DataDog/dd-go/blob/prod/trace/apps/tracer-telemetry-intake/telemetry-payload/static/config_norm_rules.json _merge () { - const containers = [this._remote, this._options, this._env, this._calculated, this._defaults] - const origins = ['remote_config', 'code', 'env_var', 'calculated', 'default'] - const unprocessedValues = [this._remoteUnprocessed, this._optsUnprocessed, this._envUnprocessed, {}, {}] + const containers = [ + this._remote, + this._options, + this._fleetStableConfig, + this._env, + this._localStableConfig, + this._calculated, + this._defaults + ] + const origins = [ + 'remote_config', + 'code', + 'fleet_stable_config', + 'env_var', + 'local_stable_config', + 'calculated', + 'default' + ] + const unprocessedValues = [ + this._remoteUnprocessed, + this._optsUnprocessed, + {}, + this._envUnprocessed, + {}, + {}, + {} + ] const changes = [] for (const name in this._defaults) { diff --git a/packages/dd-trace/src/config_stable.js b/packages/dd-trace/src/config_stable.js new file mode 100644 index 00000000000..49c50865429 --- /dev/null +++ b/packages/dd-trace/src/config_stable.js @@ -0,0 +1,100 @@ +const os = require('os') +const fs = require('fs') + +class StableConfig { + constructor () { + this.warnings = [] // Logger hasn't been initialized yet, so we can't use log.warn + this.localEntries = {} + this.fleetEntries = {} + this.wasm_loaded = false + + const { localConfigPath, fleetConfigPath } = this._getStableConfigPaths() + if (!fs.existsSync(localConfigPath) && !fs.existsSync(fleetConfigPath)) { + // Bail out early if files don't exist to avoid unnecessary library loading + return + } + + const localConfig = this._readConfigFromPath(localConfigPath) + const fleetConfig = this._readConfigFromPath(fleetConfigPath) + if (!localConfig && !fleetConfig) { + // Bail out early if files are empty or we can't read them to avoid unnecessary library loading + return + } + + // Note: we don't enforce loading because there may be cases where the library is not available and we + // want to avoid breaking the application. In those cases, we will not have the file-based configuration. + let libdatadog + try { + libdatadog = require('@datadog/libdatadog') + this.wasm_loaded = true + } catch (e) { + this.warnings.push('Can\'t load libdatadog library') + return + } + + const libconfig = libdatadog.maybeLoad('library_config') + if (libconfig === undefined) { + this.warnings.push('Can\'t load library_config library') + return + } + + try { + const configurator = new libconfig.JsConfigurator() + configurator.set_envp(Object.entries(process.env).map(([key, value]) => `${key}=${value}`)) + configurator.set_args(process.argv) + configurator.get_configuration(localConfig.toString(), fleetConfig.toString()).forEach((entry) => { + if (entry.source === 'local_stable_config') { + this.localEntries[entry.name] = entry.value + } else if (entry.source === 'fleet_stable_config') { + this.fleetEntries[entry.name] = entry.value + } + }) + } catch (e) { + this.warnings.push(`Error parsing configuration from file: ${e.message}`) + } + } + + _readConfigFromPath (path) { + try { + return fs.readFileSync(path, 'utf8') + } catch (err) { + if (err.code !== 'ENOENT') { + this.warnings.push(`Error reading config file at ${path}. ${err.code}: ${err.message}`) + } + return '' // Always return a string to avoid undefined.toString() errors + } + } + + _getStableConfigPaths () { + let localConfigPath = '' + let fleetConfigPath = '' + switch (os.type().toLowerCase()) { + case 'linux': + localConfigPath = '/etc/datadog-agent/application_monitoring.yaml' + fleetConfigPath = '/etc/datadog-agent/managed/datadog-agent/stable/application_monitoring.yaml' + break + case 'darwin': + localConfigPath = '/opt/datadog-agent/etc/application_monitoring.yaml' + fleetConfigPath = '/opt/datadog-agent/etc/managed/datadog-agent/stable/application_monitoring.yaml' + break + case 'win32': + localConfigPath = 'C:\\ProgramData\\Datadog\\application_monitoring.yaml' + fleetConfigPath = 'C:\\ProgramData\\Datadog\\managed\\datadog-agent\\stable\\application_monitoring.yaml' + break + default: + break + } + + // Allow overriding the paths for testing + if (process.env.DD_TEST_LOCAL_CONFIG_PATH !== undefined) { + localConfigPath = process.env.DD_TEST_LOCAL_CONFIG_PATH + } + if (process.env.DD_TEST_FLEET_CONFIG_PATH !== undefined) { + fleetConfigPath = process.env.DD_TEST_FLEET_CONFIG_PATH + } + + return { localConfigPath, fleetConfigPath } + } +} + +module.exports = StableConfig diff --git a/packages/dd-trace/src/log/index.js b/packages/dd-trace/src/log/index.js index db3a475e120..fe2d5d4a607 100644 --- a/packages/dd-trace/src/log/index.js +++ b/packages/dd-trace/src/log/index.js @@ -105,23 +105,36 @@ const log = { deprecate (code, message) { return this._deprecate(code, message) + }, + + isEnabled (fleetStableConfigValue = undefined, localStableConfigValue = undefined) { + return isTrue(coalesce( + fleetStableConfigValue, + process.env?.DD_TRACE_DEBUG, + process.env?.OTEL_LOG_LEVEL === 'debug' || undefined, + localStableConfigValue, + config.enabled + )) + }, + + getLogLevel ( + optionsValue = undefined, + fleetStableConfigValue = undefined, + localStableConfigValue = undefined + ) { + return coalesce( + optionsValue, + fleetStableConfigValue, + process.env?.DD_TRACE_LOG_LEVEL, + process.env?.OTEL_LOG_LEVEL, + localStableConfigValue, + config.logLevel + ) } } log.reset() -const enabled = isTrue(coalesce( - process.env.DD_TRACE_DEBUG, - process.env.OTEL_LOG_LEVEL === 'debug', - config.enabled -)) - -const logLevel = coalesce( - process.env.DD_TRACE_LOG_LEVEL, - process.env.OTEL_LOG_LEVEL, - config.logLevel -) - -log.toggle(enabled, logLevel) +log.toggle(log.isEnabled(), log.getLogLevel()) module.exports = log diff --git a/packages/dd-trace/src/util.js b/packages/dd-trace/src/util.js index ca1ce1323cb..23e51be97ec 100644 --- a/packages/dd-trace/src/util.js +++ b/packages/dd-trace/src/util.js @@ -79,11 +79,20 @@ function hasOwn (object, prop) { return Object.prototype.hasOwnProperty.call(object, prop) } +function normalizeProfilingEnabledValue (configValue) { + return isTrue(configValue) + ? 'true' + : isFalse(configValue) + ? 'false' + : configValue === 'auto' ? 'auto' : undefined +} + module.exports = { isTrue, isFalse, isError, globMatch, calculateDDBasePath, - hasOwn + hasOwn, + normalizeProfilingEnabledValue } diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 39446fc17f6..ce74a187d94 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -2432,4 +2432,168 @@ describe('Config', () => { expect(config).to.have.nested.property('stats.enabled', false) }) }) + + context('library config', () => { + const StableConfig = require('../src/config_stable') + const path = require('path') + // os.tmpdir returns undefined on Windows somehow + const baseTempDir = os.platform() !== 'win32' ? os.tmpdir() : 'C:\\Windows\\Temp' + let env + let tempDir + beforeEach(() => { + env = process.env + tempDir = fs.mkdtempSync(path.join(baseTempDir, 'config-test-')) + process.env.DD_TEST_LOCAL_CONFIG_PATH = path.join(tempDir, 'local.yaml') + process.env.DD_TEST_FLEET_CONFIG_PATH = path.join(tempDir, 'fleet.yaml') + }) + + afterEach(() => { + process.env = env + fs.rmdirSync(tempDir, { recursive: true }) + }) + + it('should apply host wide config', () => { + fs.writeFileSync( + process.env.DD_TEST_LOCAL_CONFIG_PATH, + ` +apm_configuration_default: + DD_RUNTIME_METRICS_ENABLED: true +`) + const config = new Config() + expect(config).to.have.property('runtimeMetrics', true) + }) + + it('should apply service specific config', () => { + fs.writeFileSync( + process.env.DD_TEST_LOCAL_CONFIG_PATH, + ` +rules: + - selectors: + - origin: language + matches: + - nodejs + operator: equals + configuration: + DD_SERVICE: my-service +`) + const config = new Config() + expect(config).to.have.property('service', 'my-service') + }) + + it('should respect the priority sources', () => { + // 1. Default + const config1 = new Config() + expect(config1).to.have.property('service', 'node') + + // 2. Local stable > Default + fs.writeFileSync( + process.env.DD_TEST_LOCAL_CONFIG_PATH, + ` +rules: + - selectors: + - origin: language + matches: + - nodejs + operator: equals + configuration: + DD_SERVICE: service_local_stable +`) + const config2 = new Config() + expect(config2).to.have.property( + 'service', + 'service_local_stable', + 'default < local stable config' + ) + + // 3. Env > Local stable > Default + process.env.DD_SERVICE = 'service_env' + const config3 = new Config() + expect(config3).to.have.property( + 'service', + 'service_env', + 'default < local stable config < env var' + ) + + // 4. Fleet Stable > Env > Local stable > Default + fs.writeFileSync( + process.env.DD_TEST_FLEET_CONFIG_PATH, + ` +rules: + - selectors: + - origin: language + matches: + - nodejs + operator: equals + configuration: + DD_SERVICE: service_fleet_stable +`) + const config4 = new Config() + expect(config4).to.have.property( + 'service', + 'service_fleet_stable', + 'default < local stable config < env var < fleet stable config' + ) + + // 5. Code > Fleet Stable > Env > Local stable > Default + const config5 = new Config({ service: 'service_code' }) + expect(config5).to.have.property( + 'service', + 'service_code', + 'default < local stable config < env var < fleet config < code' + ) + }) + + it('should ignore unknown keys', () => { + fs.writeFileSync( + process.env.DD_TEST_LOCAL_CONFIG_PATH, + ` +apm_configuration_default: + DD_RUNTIME_METRICS_ENABLED: true + DD_FOOBAR_ENABLED: baz +`) + const stableConfig = new StableConfig() + expect(stableConfig.warnings).to.have.lengthOf(0) + + const config = new Config() + expect(config).to.have.property('runtimeMetrics', true) + }) + + it('should log a warning if the YAML files are malformed', () => { + fs.writeFileSync( + process.env.DD_TEST_LOCAL_CONFIG_PATH, + ` + apm_configuration_default: +DD_RUNTIME_METRICS_ENABLED true +`) + const stableConfig = new StableConfig() + expect(stableConfig.warnings).to.have.lengthOf(1) + }) + + it('should only load the WASM module if the stable config files exist', () => { + const stableConfig1 = new StableConfig() + expect(stableConfig1).to.have.property('wasm_loaded', false) + + fs.writeFileSync( + process.env.DD_TEST_LOCAL_CONFIG_PATH, + ` +apm_configuration_default: + DD_RUNTIME_METRICS_ENABLED: true +`) + const stableConfig2 = new StableConfig() + expect(stableConfig2).to.have.property('wasm_loaded', true) + }) + + it('should not load the WASM module in a serverless environment', () => { + fs.writeFileSync( + process.env.DD_TEST_LOCAL_CONFIG_PATH, + ` +apm_configuration_default: + DD_RUNTIME_METRICS_ENABLED: true +`) + + process.env.AWS_LAMBDA_FUNCTION_NAME = 'my-great-lambda-function' + const stableConfig = new Config() + expect(stableConfig).to.not.have.property('stableConfig') + }) + }) }) diff --git a/yarn.lock b/yarn.lock index 218b3a7c05b..d9f4b4653c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -401,10 +401,10 @@ resolved "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz" integrity "sha1-u1BFecHK6SPmV2pPXaQ9Jfl729k= sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" -"@datadog/libdatadog@^0.4.0": - version "0.4.0" - resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.4.0.tgz#aeeea02973f663b555ad9ac30c4015a31d561598" - integrity sha512-kGZfFVmQInzt6J4FFGrqMbrDvOxqwk3WqhAreS6n9b/De+iMVy/NMu3V7uKsY5zAvz+uQw0liDJm3ZDVH/MVVw== +"@datadog/libdatadog@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.5.0.tgz#0ef2a2a76bb9505a0e7e5bc9be1415b467dbf368" + integrity sha512-YvLUVOhYVjJssm0f22/RnDQMc7ZZt/w1bA0nty1vvjyaDz5EWaHfWaaV4GYpCt5MRvnGjCBxIwwbRivmGseKeQ== "@datadog/native-appsec@8.4.0": version "8.4.0"