diff --git a/docs/data-shippers/extensions-logging.asciidoc b/docs/data-shippers/extensions-logging.asciidoc deleted file mode 100644 index 556829ca..00000000 --- a/docs/data-shippers/extensions-logging.asciidoc +++ /dev/null @@ -1,444 +0,0 @@ -[[extensions-logging-data-shipper]] -=== Elastic.Extensions.Logging - -Elastic logger provider for Microsoft.Extensions.Logging. - -Writes direct to Elasticsearch using the https://www.elastic.co/guide/en/ecs/current/ecs-reference.html[Elastic Common Schema (ECS)], -with semantic logging of structured data from message and scope values. The results can be viewed and queried in the Kibana console. - -==== Installation - -Add a reference to the `Elastic.Extensions.Logging` package: - -[source,xml] -[subs="attributes"] ----- - ----- - -==== Usage - -Then, add the provider to the loggingBuilder during host construction, using the provided extension method. - -[source,c#] ----- -using Elastic.Extensions.Logging; - -// ... - - .ConfigureLogging((hostContext, loggingBuilder) => - { - loggingBuilder.AddElasticsearch(); - }) - ----- - -The default configuration will write to a local Elasticsearch running at http://localhost:9200/. - -Once you have sent some log event, open Kibana (e.g. http://localhost:5601/) and define an -index pattern for "dotnet-*" with the time filter "@timestamp". - -You can then discover the log events for the index. Some useful columns to add -are `log.level`, `log.logger`, `event.code`, `message`, `tags`, and `process.thread.id`. - -If you are running multiple applications or on multiple servers, you might want to -include `service.type`, `service.version`, and `host.hostname`. - -Additional fields are defined below, and all individual message and scope values are logged -as `labels.*` custom key/value pairs, e.g. `labels.CustomerId`. - -==== Basic configuration - -For deployment you will usually want to override the configuration with your actual server location(s). -The other useful value to configure is a tag for the environment, e.g. Development/Staging/Production. - -[source,json] ----- -{ - "Logging": { - "Elasticsearch": { - "NodeUris": [ "https://elastic-staging.example.com:9200" ], - "Tags": [ "Staging" ] - } - } -} ----- - -*NOTE:* You don't need any configuration to just use a local Elasticsearch instance, as it defaults to http://localhost:9200/. - -==== Configuration settings - -The logger provider will be automatically configured with any logging settings under the alias `Elasticsearch`. - -The following default settings are used. - -[source,json] ----- -{ - "Logging": { - "Elasticsearch": { - "IncludeHost": true, - "IncludeProcess": true, - "IncludeScopes": true, - "IncludeUser": true, - "Index": "dotnet-{0:yyyy.MM.dd}", - "IndexOffset": null, - "IsEnabled": true, - "ListSeparator": ", ", - "MapCorrelationValues": true, - "Tags": [], - "ShipTo": { - "NodePoolType": "SingleNode", - "NodeUris": [ "http://localhost:9200" ] - } - } - } -} ----- - -|=== -|Setting |Type |Description - -|IncludeHost |boolean |Default `true`; set to `false` to disable logging host values. -|IncludeProcess |boolean |Default `true`; set to `false` to disable logging process values. -|IncludeScopes |boolean |Default `true`; set to `false` to disable logging scope values. -|IncludeUser |boolean |Default `true`; set to `false` to disable logging user details. -|Index |format |Format string used to generate the Elasticsearch `index`, using the current timestamp. Default is `dotnet-{0:yyyy.MM.dd}`. -|IndexOffset |timespan |Override to set the offset used to generate the `index`. Default value is `null`, which uses the system local offset; use `"00:00&"` for UTC. -|IsEnabled |boolean |Default `true`; set to `false` to disable the logger. -|ListSeparator |string |Separator to use for `IEnumerable` in `labels.*` values. Default is `", "`. -|Tags |array |Additional tags to include in the message. Useful to specify the environment or other details, e.g. `[ "Staging", "Priority"]` -|=== - -The ShipTo settings can have the following properties, depending on the type of connection pool. - -|=== -|Setting |Type |Description - -|ApiKey |string |API Key, where connection pool type is Cloud, and authenticating via API Key. -|CloudId |string |Cloud ID, where connection pool type is Cloud. -|NodePoolType |enum |Default is `Singlenode`, or `Sniffing` for multiple nodes, or `Cloud` if `CloudId` is provided. Other supported values are `Static` or `Sticky`. -|NodeUris |array |URI(s) of the Elasticsearch nodes to connect to. Default is a single node `[ "http://localhost:9200" ]` -|Password |string |Password, where connection pool type is Cloud, and authenticating via username/password. -|Username |string |Username, where connection pool type is Cloud, and authenticating via username/password. -|=== - -If you want to configure from a different section, it can be configured manually: - -[source,c#] ----- - .ConfigureLogging((hostContext, loggingBuilder) => - { - loggingBuilder.AddElasticsearch(options => - hostContext.Configuration.Bind("Logging:CustomElasticsearch", options)); - }) ----- - -Configuration can, of course, also be done in code, e.g. to add the environment as a tag. - -==== Elastic Cloud configuration - -If `CloudId` is provided, the `ConnectionPoolType` defaults to `Cloud`: - -[source,json] ----- -{ - "Logging": { - "Elasticsearch": { - "ShipTo": { - "CloudId": "12345", - "ApiKey": "abcdef" - } - } - } -} ----- - -==== Output - Elastic Common Schema (ECS) - -Log messages sent to Elasticsearch follow the https://www.elastic.co/guide/en/ecs/current/ecs-reference.html[Elastic Common Schema (ECS)]. - -==== Example document - -The `_source` field is the message sent from the LoggerProvider, along with the `_index` and `_id` (a GUID). - -[source,json] ----- -{ - "_index": "dotnet-2020.04.12", - "_type": "_doc", - "_id": "563503a8-9d10-46ff-a09f-c6ccbf124db9", - "_version": 1, - "_score": null, - "_source": { - "MessageTemplate": "Unexpected error processing customer {CustomerId}.", - "Scopes": [ - "IP address 2001:db8:85a3::8a2e:370:7334", - "PlainScope" - ], - "agent": { - "version": "1.0.0+bd3ad6", - "type": "Elastic.Extensions.Logging.LoggerProvider" - }, - "ecs": { - "version": "1.5.0" - }, - "error": { - "message": "Calculation error", - "type": "System.Exception", - "stack_trace": "System.Exception: Calculation error\n ---> System.DivideByZeroException: Attempted to divide by zero.\n at HelloElasticsearch.Worker.ExecuteAsync(CancellationToken stoppingToken) in /home/sly/Code/essential-logging/examples/HelloElasticsearch/Worker.cs:line 80\n --- End of inner exception stack trace ---\n at HelloElasticsearch.Worker.ExecuteAsync(CancellationToken stoppingToken) in /home/sly/Code/essential-logging/examples/HelloElasticsearch/Worker.cs:line 84" - }, - "event": { - "code": "5000", - "action": "ErrorProcessingCustomer", - "severity": 3 - }, - "host": { - "os": { - "platform": "Unix", - "full": "Linux 4.15.0-91-generic #92-Ubuntu SMP Fri Feb 28 11:09:48 UTC 2020", - "version": "4.15.0.91" - }, - "hostname": "VUB1804", - "architecture": "X64" - }, - "log": { - "level": "Error", - "logger": "HelloElasticsearch.Worker" - }, - "process": { - "thread": { - "id": 10 - }, - "pid": 25982, - "name": "HelloElasticsearch" - }, - "service": { - "type": "HelloElasticsearch", - "version": "1.0.0" - }, - "user": { - "id": "sgryphon+es@live.com", - "name": "sly", - "domain": "VUB1804" - }, - "@timestamp": "2020-04-13T21:25:22.3352989+10:00", - "tags": [ - "Development" - ], - "labels": { - "ip": "2001:db8:85a3::8a2e:370:7334", - "CustomerId": "12345" - }, - "message": "Unexpected error processing customer 12345.", - "trace": { - "id": "c20bde1071f7cf4e9a6f368c824e05f7" - }, - "transaction": { - "id": "92ba5ee64d963746" - } - }, - "fields": { - "@timestamp": [ - "2020-04-13T11:25:22.335Z" - ] - }, - "sort": [ - 1586777122335 - ] -} ----- - -==== Standard Fields - -|=== -|Field |Type |Description - -|@timestamp |date |`DateTimeOffset` when the message was logged, including local offset. -|message |string |The formatted log message and arguments. -|tags |array |Custom tags from configuration, e.g. `[ "Staging", "Priority" ]`. Can have multiple values. -|event.action |string |The name of the logged EventId, e.g. `ErrorProcessingCustomer`. -|event.code |string |The numeric value (as a string) of the EventId, e.g. `5000`. -|event.severity |long |The syslog severity corresponding to the log level, 2 = critical, 3 = error, 4 = warning, 6 = information, 7 = debug and trace. (Also used in the Systemd format of ConsoleLoggerProvider) -|log.level |string |The log level: `Critical`, `Error`, `Warning`, `Information`, `Debug`, or `Trace`. -|log.logger |string |The category name (namespace and class) of the logger, e.g. `HelloElasticsearch.Worker`. -|=== - -The `event.severity` field is numeric and can be used to order events by level, e.g. Kibana query `event.severity <= 4` will get all messages that have log level `Warning` or worse. - -==== Error fields - -If the log message includes an exception, the details are reported in the error fields. - -|=== -|Field |Type |Description - -|error.message |string |The `Message` property of any exception. -|error.stack_trace |string |Full details of the exception, `Exception.ToString()`, including the stack trace and the stack trace of any inner exceptions. -|error.type |string |The type of the error message, e.g. `System.DivideByZeroException` -|=== - -==== Custom fields - -Following the ECS conventions, these use alternative Title casing, to identify them as non-standard fields. - -|=== -|Field |Type |Description - -|MessageTemplate |string |The original message template, e.g. "Unexpected error processing customer {CustomerId}." -|Scopes |array |Array of string formatted scope values, in the order added. -|=== - -==== Label values - -|=== -|Field |Type |Description - -|labels.* |string |Custom key/value pairs of all named parameter values and named scope values. All values are strings (no nested objects). -|=== - -Label values can be accessed by their key, for example if the message, or scope, includes a parameter CustomerId, then the value will be logged as `labels.CustomerId` can be searched in Kibana using "labels.CustomerId: 12345". - -*Example:* - -The following will generate two labels, `labels.EndTime` from the message and `labels.CustomerId` from the scope: - -[source,c#] ----- -using (_logger.BeginScope("{CustomerId}", customerId)) -{ - _logger.LogWarning("End of processing reached at {EndTime}.", end); -} ----- - -Labels are taken from the message (state) and any scope values (may be disabled via the configuration options). In Microsoft.Extensions.Logging an -internal FormattedLogValues is used in the ILogger overloads for log levels and scopes; it implements the `IEnumerable<KeyValuePair<string,object>>` -interface that is used to extract the individual parameter values. - -The `labels` property in ECS should not contain nested objects, so values are converted to keyword strings. For most objects this is just calling ToString(), -with specific formats for some types, e.g. calling string on a list is usually not very useful, so the contents of the list is logged instead. - -*Labels value formatting* - -|=== -|Type |Formatting - -|byte |Hex, e.g. "9A" -|byte[] |Prefixed hex, e.g. "0x12789AF0" -|DateTimeOffset |ISO format, e.g. "2020-01-02T03:04:05.000000+06:00" -|DateTime |In most cases `DateTimeOffset` should be used instead (1). Where `DateTime` is used for date only (with no time component), it is formatted as a date, e.g. "2020-01-02". If it has a time component, the roundtrip ("o") format is used. -|IEnumerable |Values separated by ", " (configurable) -|IDictionary<string, object> |A string containing key value pairs, e.g. `token="0x12789AF0" count="5"` -|_other values_ |The result of `ToString()`, including scalar values, e.g. the number `5.3` is logged as the string "5.3" -|=== - -*(1) See https://docs.microsoft.com/en-us/dotnet/standard/datetime/choosing-between-datetime* - -==== Agent fields - -These identify the version of the logger provider being used. - -|=== -|Field |Type |Description - -|agent.type |string |Name of the logger provider assembly, `Elastic.Extensions.Logging.LoggerProvider`. -|agent.version |string |Informational version number of the logger assembly, e.g. `1.1.1+bd3ad63`. -|ecs.version |string |Version of ECS standard used, currently `1.5`. -|=== - -==== Service fields - -This identifies the application/service that is running and generating the logs. - -The values are pulled from the entry assemb, `Assembly.GetEntryAssembly()`, using the `Name` -and `AssemblyInformationalVersionAttribute` values (if informational version is not set -it falls back to assembly `Version`). - -|=== -|Field |Type |Description - -|service.type |string |Name of the entry assembly, `HelloElasticsearch`. -|service.version |string |Informational version number of the entry assembly, e.g. `1.2.0-beta.1+79d095a`. -|=== - -*Note:* You should be using a build process that sets the assembly informational version correctly. -e.g. If you have a dotnet project using git you can install the local tool `GitVersion.Tool`, -and use it to automatically generate semantic version numbers from the git branch information. - -To install the tool: - -[source,powershell] ----- -dotnet new tool-manifest -dotnet tool install GitVersion.Tool ----- - -Then use the tool to create a semantic version number that can be used in your build process: - -[source,powershell] ----- -dotnet tool restore -dotnet gitversion ----- - -You are welcome to use the link:../../build.ps1[`build.ps1`] script in this repository as an example. - -==== Tracing fields - -|=== -|Field |Type |Description - -|trace.id |string |Cross-service trace correlation identifier. From `Activity.Current.RootId` from `System.Diagnostics`, with a fallback to `CorrelationManager.ActivityId`. Can be overridden by a message or scope value `trace.id`. -|transaction.id |string |Transaction for this service, e.g. individual request identifier. If in W3C format, parse out the SpanId from `Activity.Current.Id` from `System.Diagnostics`, otherwise just use the full `Activity.Current.Id` (e.g. if hierarchical). Can be overridden by message or scope value `transaction.id`. -|=== - -ASP.NET will automatically pass correlation identifiers between tiers; from 3.0 it also supports the W3C Trace Context standard (https://www.w3.org/TR/trace-context/). - -The value of `Activity.Current.RootId` is used as the cross-service identifier (in W3C format this is the Trace ID), -if in W3C format the Span ID portion of `Activity.Current.Id` is used for the transaction, otherwise the full value is used (this is consistent with the way ASP.NET works). - -It is recommended to turn on W3C format, for compatibility with other systems: - -[source,c#] ----- -Activity.DefaultIdFormat = ActivityIdFormat.W3C; ----- - -==== Host fields - -NOTE: Can be disabled via configuration. - -|=== -|Field |Type |Description - -|host.architecture |string |The processor architecture, e.g. X64. Value of `RuntimeInformation.OSArchitecture`. -|host.hostname |string |The computer name. Value of `Environment.MachineName`. -|host.os.full |string |Full description of the operation system. Value of `RuntimeInformation.OSDescription`. -|host.os.platform |string |Operating system platform. Value of `Environment.OSVersion.Platform`. -|host.os.version |string |Operating system version. Value of `Environment.OSVersion.Version`. -|=== - -==== Process fields - -NOTE: Can be disabled via configuration. - -|=== -|Field |Type |Description - -|process.name |string |The current process name. From `Process.GetCurrentProcess()`. -|process.pid |long |The current process ID. From `Process.GetCurrentProcess()`. -|process.thread.id |long |Current thread ID. Value of `Thread.CurrentThread.ManagedThreadId`. -|process.thread.name |string |Name of the thread. From `Thread.CurrentThread.Name`. -|=== - -==== User fields - -NOTE: Can be disabled via configuration. - -|=== -|Field |Type |Description - -|user.domain |string |The current domain, either the machine name or a Windows domain. Value of `Environment.UserDomainName`. -|user.id |string |Current user principal name, if set. Value of `Thread.CurrentPrincipal.Identity.Name`. -|user.name |string |The current user. Value of `Environment.UserName`. -|=== \ No newline at end of file diff --git a/docs/data-shippers/index.asciidoc b/docs/data-shippers/index.asciidoc deleted file mode 100644 index ca041b61..00000000 --- a/docs/data-shippers/index.asciidoc +++ /dev/null @@ -1,55 +0,0 @@ -ifdef::env-github[] -NOTE: For the best reading experience, -please view this documentation at https://www.elastic.co/guide/en/ecs-logging/dotnet[elastic.co] -endif::[] - -[[data-shippers]] -== Data Shippers - -Our datashippers integrate with logging frameworks to facilitate -sending events (such as logs) to various outputs. - -Currently these shippers support Elastic Cloud & Elasticsearch but -other outputs are in the works. - -=== Elasticsearch Security - -If Elasticsearch's security is enabled you will need to ensure you configure a user or API key with enough privileges - -==== Bootstrap - -In order for the datashippers to have enough privileges to bootstrap the -target datastreams with all the ECS mappings, templates and settings the authenticated -security principal needs the following minimum privileges: - -[options="header"] -|==== -|Type | Privileges - -|Cluster -|`monitor`, `manage_ilm`, `manage_index_templates`, `manage_pipeline` - -|Index -|`manage`, `create_doc` -|==== - -==== No bootstrap - -If the datashippers are configured to skip bootstrapping the target destinations all together, -the security principal requires the following minimum privileges to push data. - -[options="header"] -|==== -|Type | Privileges - -|Cluster -|`monitor` - -|Index -|`auto_configure` `create_doc` -|==== - -include::./ingest-commonschema.asciidoc[Elastic.Ingest.Elasticsearch.CommonSchema] -include::./serilog.asciidoc[Serilog] -include::./extensions-logging.asciidoc[Microsoft.Extensions.Logging] -include::./benchmark-dotnet.asciidoc[BenchmarkDotNet] diff --git a/docs/data-shippers/serilog.asciidoc b/docs/data-shippers/serilog.asciidoc deleted file mode 100644 index 9f0c69af..00000000 --- a/docs/data-shippers/serilog.asciidoc +++ /dev/null @@ -1,157 +0,0 @@ -[[serilog-data-shipper]] -=== Elastic.Serilog.Sinks - -A https://serilog.net/[Serilog] sink that writes logs directly to https://www.elastic.co/elasticsearch/[Elasticsearch] or https://www.elastic.co/cloud[Elastic Cloud] - -==== Installation - -Add a reference to the `Elastic.Serilog.Sinks` package: - -[source,xml] -[subs="attributes"] ----- - ----- - -==== Usage - -There's a few ways that you can extend a `Serilog` `LoggerConfiguration`: - -[source,csharp] ----- -Log.Logger = new LoggerConfiguration() - .MinimumLevel.Debug() - .Enrich.FromLogContext() - ----- - -*NOTE:* Don't forget we also publish an https://github.com/elastic/ecs-dotnet/blob/main/src/Elastic.Apm.SerilogEnricher/readme.md[`Elastic.Apm.SerilogEnricher`] for the Elastic APM Agent! - -Writing to `Elasticsearch` - -[source,csharp] ----- -.WriteTo.Elasticsearch(new [] { new Uri("http://localhost:9200" )}, opts => -{ - opts.DataStream = new DataStreamName("logs", "console-example", "demo"); - opts.BootstrapMethod = BootstrapMethod.Failure; - opts.ConfigureChannel = channelOpts => - { - channelOpts.BufferOptions = new BufferOptions - { - ConcurrentConsumers = 10 - }; - }; -}, transport => -{ - // transport.Authentication(new BasicAuthentication(username, password)); // Basic Auth - // transport.Authentication(new ApiKey(base64EncodedApiKey)); // ApiKey -}) - ----- - -Writing to `Elastic Cloud`: - -[source,csharp] ----- -.WriteTo.ElasticCloud("cloudId", "cloudUser", "cloudPass", opts => ----- - -`opts` is an instance of `ElasticsearchSinkOptions` with the following options - - -==== Configuration - -|=== -|Option |Description - -|`Transport` |An instance of `Elastic.Transport` that dictates where and how we are communicating to. Defaults to `http://localhost:9200` -|`DataStream` |Where to write data, defaults to the `logs-dotnet-default` datastream. -|`BootstrapMethod` |Wheter the sink should attempt to install component and index templates to ensure the datastream has ECS mappings. Can be be either `None` (the default), `Silent` (attempt but fail silently), `Failure` (attempt and fail with exceptions if bootstrapping fails). -|`TextFormatting` |Allows explicit control of over the `EcsTextFormatterConfiguration` used to emit ECS json documents. See https://github.com/elastic/ecs-dotnet/tree/main/src/Elastic.CommonSchema.Serilog[`Elastic.CommonSchema.Serilog`] for available options. -|`ConfigureChannel` |A callback receiving the `DatastreamChannelOptions` which allows you to control sizing, backpressure etc. See https://github.com/elastic/elastic-ingest-dotnet/blob/main/src/Elastic.Ingest.Elasticsearch/README.md#elasticingestelasticsearch[`Elastic.Ingest.Elasticsearch`] for more information. -|=== - -Note that you can also pass `ElasticsearchSinkOptions` directly - -[source,csharp] ----- -.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(client.Transport)) ----- - -This allows you to reuse the `Transport` used by the Elasticsearch Client for instance. - -==== Authentication - -When {es} security features are enabled, requests without a valid authentication header will be rejected. -You can enable authentication via one of the methods below: - -**Basic Auth** - -[source,csharp] ----- -.WriteTo.Elasticsearch(new [] { new Uri("http://localhost:9200" )}, opts => -{ - ... -}, transport => -{ - transport.Authentication(new BasicAuthentication(username, password)); <1> -}) - ----- -<1> Basic authentication - -**API Key** - -[source,csharp] ----- -.WriteTo.Elasticsearch(new [] { new Uri("http://localhost:9200" )}, opts => -{ - ... -}, transport => -{ - transport.Authentication(new ApiKey(base64EncodedApiKey)); <1> -}) - ----- -<1> API Key - -To learn more about authentication with the {stack}, see {ref}/setting-up-authentication.html[User Authentication]. - -==== ECS Aware Message Templates - -This sink by proxy of its formatter allows you to set ECS fields directly from the message template using properties that adhere to the -https://messagetemplates.org/ format. - -The available ECS message template properties are listed under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId` - -[source,chsarp] ----- -Log.Information("The time is {TraceId}", "my-trace-id"); ----- - -Will override `trace.id` on the resulting ECS json document. - -==== Troubleshooting - -In case of issues, you can enable the [Serilog Self-Log feature](https://github.com/serilog/serilog/wiki/Debugging-and-Diagnostics#selflog) to expose any error you might have encountered. - -==== Comparison with https://github.com/serilog-contrib/serilog-sinks-elasticsearch[`Serilog.Sinks.Elasticsearch`] - -* `Serilog.Sinks.Elasticsearch` is an amazing community led sink that has a ton of options and works against older Elasticsearch versions `< 8.0`. -* `Serilog.Sinks.Elasticsearch` is unofficially supported by Elastic with some of the .NET team helping to maintain it. -* `Elastic.Serilog.Sinks` is *officially* supported by Elastic and was purposely build to adhere to newer best practices around logging, datastreams and ILM. -* `Elastic.Serilog.Sinks` is purposely build to have fewer configuration options and be more prescriptive than `Serilog.Sinks.Elasticsearch`. -* That is not to say there aren't plenty of configuration hooks in `Elastic.Serilog.Sinks` - -===== Notable absent features: - -* `Elastic.Serilog.Sinks` only works with `Elasticsearch 8.x` and up. -* This is because the bootrapping (`BootstrapMethod`) attempts to load templates build for Elasticsearch 8.0 and up. -* `Elastic.Serilog.Sinks` has only one way it emits data to Elasticsearch confirming to the https://github.com/elastic/ecs-logging[ecs-logging specification] -* That doesn't mean you can not introduce your own additional properties though. -* `Elastic.Serilog.Sinks` has no durable mode. -* If you need higher guarantees on log delivery use https://github.com/serilog/serilog-sinks-file[`Serilog.Sinks.File`] with our https://www.nuget.org/packages/Elastic.CommonSchema.Serilog/[ECS log formatter] for Serilog and use https://www.elastic.co/beats/filebeat[filebeat] to ship these logs. -* Check out {fleet-guide}/fleet-overview.html[Elastic Agent and Fleet] to simplify collecting logs and metrics on the edge. - -If you miss a particular feature from `Serilog.Sinks.Elasticsearch` in `Elastic.Serilog.Sinks` please open a https://github.com/elastic/ecs-dotnet/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=%5BFEATURE%5D[feature request]! We'd love to grow this sink organically moving forward. diff --git a/docs/docset.yml b/docs/docset.yml new file mode 100644 index 00000000..9b2f2d6e --- /dev/null +++ b/docs/docset.yml @@ -0,0 +1,489 @@ +project: 'ECS Logging .NET' +cross_links: + - apm-agent-dotnet + - beats + - docs-content + - ecs + - ecs-logging +toc: + - toc: reference +subs: + ref: "https://www.elastic.co/guide/en/elasticsearch/reference/current" + ref-bare: "https://www.elastic.co/guide/en/elasticsearch/reference" + ref-8x: "https://www.elastic.co/guide/en/elasticsearch/reference/8.1" + ref-80: "https://www.elastic.co/guide/en/elasticsearch/reference/8.0" + ref-7x: "https://www.elastic.co/guide/en/elasticsearch/reference/7.17" + ref-70: "https://www.elastic.co/guide/en/elasticsearch/reference/7.0" + ref-60: "https://www.elastic.co/guide/en/elasticsearch/reference/6.0" + ref-64: "https://www.elastic.co/guide/en/elasticsearch/reference/6.4" + xpack-ref: "https://www.elastic.co/guide/en/x-pack/6.2" + logstash-ref: "https://www.elastic.co/guide/en/logstash/current" + kibana-ref: "https://www.elastic.co/guide/en/kibana/current" + kibana-ref-all: "https://www.elastic.co/guide/en/kibana" + beats-ref-root: "https://www.elastic.co/guide/en/beats" + beats-ref: "https://www.elastic.co/guide/en/beats/libbeat/current" + beats-ref-60: "https://www.elastic.co/guide/en/beats/libbeat/6.0" + beats-ref-63: "https://www.elastic.co/guide/en/beats/libbeat/6.3" + beats-devguide: "https://www.elastic.co/guide/en/beats/devguide/current" + auditbeat-ref: "https://www.elastic.co/guide/en/beats/auditbeat/current" + packetbeat-ref: "https://www.elastic.co/guide/en/beats/packetbeat/current" + metricbeat-ref: "https://www.elastic.co/guide/en/beats/metricbeat/current" + filebeat-ref: "https://www.elastic.co/guide/en/beats/filebeat/current" + functionbeat-ref: "https://www.elastic.co/guide/en/beats/functionbeat/current" + winlogbeat-ref: "https://www.elastic.co/guide/en/beats/winlogbeat/current" + heartbeat-ref: "https://www.elastic.co/guide/en/beats/heartbeat/current" + journalbeat-ref: "https://www.elastic.co/guide/en/beats/journalbeat/current" + ingest-guide: "https://www.elastic.co/guide/en/ingest/current" + fleet-guide: "https://www.elastic.co/guide/en/fleet/current" + apm-guide-ref: "https://www.elastic.co/guide/en/apm/guide/current" + apm-guide-7x: "https://www.elastic.co/guide/en/apm/guide/7.17" + apm-app-ref: "https://www.elastic.co/guide/en/kibana/current" + apm-agents-ref: "https://www.elastic.co/guide/en/apm/agent" + apm-android-ref: "https://www.elastic.co/guide/en/apm/agent/android/current" + apm-py-ref: "https://www.elastic.co/guide/en/apm/agent/python/current" + apm-py-ref-3x: "https://www.elastic.co/guide/en/apm/agent/python/3.x" + apm-node-ref-index: "https://www.elastic.co/guide/en/apm/agent/nodejs" + apm-node-ref: "https://www.elastic.co/guide/en/apm/agent/nodejs/current" + apm-node-ref-1x: "https://www.elastic.co/guide/en/apm/agent/nodejs/1.x" + apm-rum-ref: "https://www.elastic.co/guide/en/apm/agent/rum-js/current" + apm-ruby-ref: "https://www.elastic.co/guide/en/apm/agent/ruby/current" + apm-java-ref: "https://www.elastic.co/guide/en/apm/agent/java/current" + apm-go-ref: "https://www.elastic.co/guide/en/apm/agent/go/current" + apm-dotnet-ref: "https://www.elastic.co/guide/en/apm/agent/dotnet/current" + apm-php-ref: "https://www.elastic.co/guide/en/apm/agent/php/current" + apm-ios-ref: "https://www.elastic.co/guide/en/apm/agent/swift/current" + apm-lambda-ref: "https://www.elastic.co/guide/en/apm/lambda/current" + apm-attacher-ref: "https://www.elastic.co/guide/en/apm/attacher/current" + docker-logging-ref: "https://www.elastic.co/guide/en/beats/loggingplugin/current" + esf-ref: "https://www.elastic.co/guide/en/esf/current" + kinesis-firehose-ref: "https://www.elastic.co/guide/en/kinesis/{{kinesis_version}}" + estc-welcome-current: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current" + estc-welcome: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current" + estc-welcome-all: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions" + hadoop-ref: "https://www.elastic.co/guide/en/elasticsearch/hadoop/current" + stack-ref: "https://www.elastic.co/guide/en/elastic-stack/current" + stack-ref-67: "https://www.elastic.co/guide/en/elastic-stack/6.7" + stack-ref-68: "https://www.elastic.co/guide/en/elastic-stack/6.8" + stack-ref-70: "https://www.elastic.co/guide/en/elastic-stack/7.0" + stack-ref-80: "https://www.elastic.co/guide/en/elastic-stack/8.0" + stack-ov: "https://www.elastic.co/guide/en/elastic-stack-overview/current" + stack-gs: "https://www.elastic.co/guide/en/elastic-stack-get-started/current" + stack-gs-current: "https://www.elastic.co/guide/en/elastic-stack-get-started/current" + javaclient: "https://www.elastic.co/guide/en/elasticsearch/client/java-api/current" + java-api-client: "https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current" + java-rest: "https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current" + jsclient: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current" + jsclient-current: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current" + es-ruby-client: "https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current" + es-dotnet-client: "https://www.elastic.co/guide/en/elasticsearch/client/net-api/current" + es-php-client: "https://www.elastic.co/guide/en/elasticsearch/client/php-api/current" + es-python-client: "https://www.elastic.co/guide/en/elasticsearch/client/python-api/current" + defguide: "https://www.elastic.co/guide/en/elasticsearch/guide/2.x" + painless: "https://www.elastic.co/guide/en/elasticsearch/painless/current" + plugins: "https://www.elastic.co/guide/en/elasticsearch/plugins/current" + plugins-8x: "https://www.elastic.co/guide/en/elasticsearch/plugins/8.1" + plugins-7x: "https://www.elastic.co/guide/en/elasticsearch/plugins/7.17" + plugins-6x: "https://www.elastic.co/guide/en/elasticsearch/plugins/6.8" + glossary: "https://www.elastic.co/guide/en/elastic-stack-glossary/current" + upgrade_guide: "https://www.elastic.co/products/upgrade_guide" + blog-ref: "https://www.elastic.co/blog/" + curator-ref: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current" + curator-ref-current: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current" + metrics-ref: "https://www.elastic.co/guide/en/metrics/current" + metrics-guide: "https://www.elastic.co/guide/en/metrics/guide/current" + logs-ref: "https://www.elastic.co/guide/en/logs/current" + logs-guide: "https://www.elastic.co/guide/en/logs/guide/current" + uptime-guide: "https://www.elastic.co/guide/en/uptime/current" + observability-guide: "https://www.elastic.co/guide/en/observability/current" + observability-guide-all: "https://www.elastic.co/guide/en/observability" + siem-guide: "https://www.elastic.co/guide/en/siem/guide/current" + security-guide: "https://www.elastic.co/guide/en/security/current" + security-guide-all: "https://www.elastic.co/guide/en/security" + endpoint-guide: "https://www.elastic.co/guide/en/endpoint/current" + sql-odbc: "https://www.elastic.co/guide/en/elasticsearch/sql-odbc/current" + ecs-ref: "https://www.elastic.co/guide/en/ecs/current" + ecs-logging-ref: "https://www.elastic.co/guide/en/ecs-logging/overview/current" + ecs-logging-go-logrus-ref: "https://www.elastic.co/guide/en/ecs-logging/go-logrus/current" + ecs-logging-go-zap-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current" + ecs-logging-go-zerolog-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current" + ecs-logging-java-ref: "https://www.elastic.co/guide/en/ecs-logging/java/current" + ecs-logging-dotnet-ref: "https://www.elastic.co/guide/en/ecs-logging/dotnet/current" + ecs-logging-nodejs-ref: "https://www.elastic.co/guide/en/ecs-logging/nodejs/current" + ecs-logging-php-ref: "https://www.elastic.co/guide/en/ecs-logging/php/current" + ecs-logging-python-ref: "https://www.elastic.co/guide/en/ecs-logging/python/current" + ecs-logging-ruby-ref: "https://www.elastic.co/guide/en/ecs-logging/ruby/current" + ml-docs: "https://www.elastic.co/guide/en/machine-learning/current" + eland-docs: "https://www.elastic.co/guide/en/elasticsearch/client/eland/current" + eql-ref: "https://eql.readthedocs.io/en/latest/query-guide" + extendtrial: "https://www.elastic.co/trialextension" + wikipedia: "https://en.wikipedia.org/wiki" + forum: "https://discuss.elastic.co/" + xpack-forum: "https://discuss.elastic.co/c/50-x-pack" + security-forum: "https://discuss.elastic.co/c/x-pack/shield" + watcher-forum: "https://discuss.elastic.co/c/x-pack/watcher" + monitoring-forum: "https://discuss.elastic.co/c/x-pack/marvel" + graph-forum: "https://discuss.elastic.co/c/x-pack/graph" + apm-forum: "https://discuss.elastic.co/c/apm" + enterprise-search-ref: "https://www.elastic.co/guide/en/enterprise-search/current" + app-search-ref: "https://www.elastic.co/guide/en/app-search/current" + workplace-search-ref: "https://www.elastic.co/guide/en/workplace-search/current" + enterprise-search-node-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/enterprise-search-node/current" + enterprise-search-php-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/php/current" + enterprise-search-python-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/python/current" + enterprise-search-ruby-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/ruby/current" + elastic-maps-service: "https://maps.elastic.co" + integrations-docs: "https://docs.elastic.co/en/integrations" + integrations-devguide: "https://www.elastic.co/guide/en/integrations-developer/current" + time-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#time-units" + byte-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#byte-units" + apm-py-ref-v: "https://www.elastic.co/guide/en/apm/agent/python/current" + apm-node-ref-v: "https://www.elastic.co/guide/en/apm/agent/nodejs/current" + apm-rum-ref-v: "https://www.elastic.co/guide/en/apm/agent/rum-js/current" + apm-ruby-ref-v: "https://www.elastic.co/guide/en/apm/agent/ruby/current" + apm-java-ref-v: "https://www.elastic.co/guide/en/apm/agent/java/current" + apm-go-ref-v: "https://www.elastic.co/guide/en/apm/agent/go/current" + apm-ios-ref-v: "https://www.elastic.co/guide/en/apm/agent/swift/current" + apm-dotnet-ref-v: "https://www.elastic.co/guide/en/apm/agent/dotnet/current" + apm-php-ref-v: "https://www.elastic.co/guide/en/apm/agent/php/current" + ecloud: "Elastic Cloud" + esf: "Elastic Serverless Forwarder" + ess: "Elasticsearch Service" + ece: "Elastic Cloud Enterprise" + eck: "Elastic Cloud on Kubernetes" + serverless-full: "Elastic Cloud Serverless" + serverless-short: "Serverless" + es-serverless: "Elasticsearch Serverless" + es3: "Elasticsearch Serverless" + obs-serverless: "Elastic Observability Serverless" + sec-serverless: "Elastic Security Serverless" + serverless-docs: "https://docs.elastic.co/serverless" + cloud: "https://www.elastic.co/guide/en/cloud/current" + ess-utm-params: "?page=docs&placement=docs-body" + ess-baymax: "?page=docs&placement=docs-body" + ess-trial: "https://cloud.elastic.co/registration?page=docs&placement=docs-body" + ess-product: "https://www.elastic.co/cloud/elasticsearch-service?page=docs&placement=docs-body" + ess-console: "https://cloud.elastic.co?page=docs&placement=docs-body" + ess-console-name: "Elasticsearch Service Console" + ess-deployments: "https://cloud.elastic.co/deployments?page=docs&placement=docs-body" + ece-ref: "https://www.elastic.co/guide/en/cloud-enterprise/current" + eck-ref: "https://www.elastic.co/guide/en/cloud-on-k8s/current" + ess-leadin: "You can run Elasticsearch on your own hardware or use our hosted Elasticsearch Service that is available on AWS, GCP, and Azure. https://cloud.elastic.co/registration{ess-utm-params}[Try the Elasticsearch Service for free]." + ess-leadin-short: "Our hosted Elasticsearch Service is available on AWS, GCP, and Azure, and you can https://cloud.elastic.co/registration{ess-utm-params}[try it for free]." + ess-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elasticsearch Service\"]" + ece-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud_ece.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elastic Cloud Enterprise\"]" + cloud-only: "This feature is designed for indirect use by https://cloud.elastic.co/registration{ess-utm-params}[Elasticsearch Service], https://www.elastic.co/guide/en/cloud-enterprise/{ece-version-link}[Elastic Cloud Enterprise], and https://www.elastic.co/guide/en/cloud-on-k8s/current[Elastic Cloud on Kubernetes]. Direct use is not supported." + ess-setting-change: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"{ess-trial}\", title=\"Supported on {ess}\"] indicates a change to a supported https://www.elastic.co/guide/en/cloud/current/ec-add-user-settings.html[user setting] for Elasticsearch Service." + ess-skip-section: "If you use Elasticsearch Service, skip this section. Elasticsearch Service handles these changes for you." + api-cloud: "https://www.elastic.co/docs/api/doc/cloud" + api-ece: "https://www.elastic.co/docs/api/doc/cloud-enterprise" + api-kibana-serverless: "https://www.elastic.co/docs/api/doc/serverless" + es-feature-flag: "This feature is in development and not yet available for use. This documentation is provided for informational purposes only." + es-ref-dir: "'{{elasticsearch-root}}/docs/reference'" + apm-app: "APM app" + uptime-app: "Uptime app" + synthetics-app: "Synthetics app" + logs-app: "Logs app" + metrics-app: "Metrics app" + infrastructure-app: "Infrastructure app" + siem-app: "SIEM app" + security-app: "Elastic Security app" + ml-app: "Machine Learning" + dev-tools-app: "Dev Tools" + ingest-manager-app: "Ingest Manager" + stack-manage-app: "Stack Management" + stack-monitor-app: "Stack Monitoring" + alerts-ui: "Alerts and Actions" + rules-ui: "Rules" + rac-ui: "Rules and Connectors" + connectors-ui: "Connectors" + connectors-feature: "Actions and Connectors" + stack-rules-feature: "Stack Rules" + user-experience: "User Experience" + ems: "Elastic Maps Service" + ems-init: "EMS" + hosted-ems: "Elastic Maps Server" + ipm-app: "Index Pattern Management" + ingest-pipelines: "ingest pipelines" + ingest-pipelines-app: "Ingest Pipelines" + ingest-pipelines-cap: "Ingest pipelines" + ls-pipelines: "Logstash pipelines" + ls-pipelines-app: "Logstash Pipelines" + maint-windows: "maintenance windows" + maint-windows-app: "Maintenance Windows" + maint-windows-cap: "Maintenance windows" + custom-roles-app: "Custom Roles" + data-source: "data view" + data-sources: "data views" + data-source-caps: "Data View" + data-sources-caps: "Data Views" + data-source-cap: "Data view" + data-sources-cap: "Data views" + project-settings: "Project settings" + manage-app: "Management" + index-manage-app: "Index Management" + data-views-app: "Data Views" + rules-app: "Rules" + saved-objects-app: "Saved Objects" + tags-app: "Tags" + api-keys-app: "API keys" + transforms-app: "Transforms" + connectors-app: "Connectors" + files-app: "Files" + reports-app: "Reports" + maps-app: "Maps" + alerts-app: "Alerts" + crawler: "Enterprise Search web crawler" + ents: "Enterprise Search" + app-search-crawler: "App Search web crawler" + agent: "Elastic Agent" + agents: "Elastic Agents" + fleet: "Fleet" + fleet-server: "Fleet Server" + integrations-server: "Integrations Server" + ingest-manager: "Ingest Manager" + ingest-management: "ingest management" + package-manager: "Elastic Package Manager" + integrations: "Integrations" + package-registry: "Elastic Package Registry" + artifact-registry: "Elastic Artifact Registry" + aws: "AWS" + stack: "Elastic Stack" + xpack: "X-Pack" + es: "Elasticsearch" + kib: "Kibana" + esms: "Elastic Stack Monitoring Service" + esms-init: "ESMS" + ls: "Logstash" + beats: "Beats" + auditbeat: "Auditbeat" + filebeat: "Filebeat" + heartbeat: "Heartbeat" + metricbeat: "Metricbeat" + packetbeat: "Packetbeat" + winlogbeat: "Winlogbeat" + functionbeat: "Functionbeat" + journalbeat: "Journalbeat" + es-sql: "Elasticsearch SQL" + esql: "ES|QL" + elastic-agent: "Elastic Agent" + k8s: "Kubernetes" + log-driver-long: "Elastic Logging Plugin for Docker" + security: "X-Pack security" + security-features: "security features" + operator-feature: "operator privileges feature" + es-security-features: "Elasticsearch security features" + stack-security-features: "Elastic Stack security features" + endpoint-sec: "Endpoint Security" + endpoint-cloud-sec: "Endpoint and Cloud Security" + elastic-defend: "Elastic Defend" + elastic-sec: "Elastic Security" + elastic-endpoint: "Elastic Endpoint" + swimlane: "Swimlane" + sn: "ServiceNow" + sn-itsm: "ServiceNow ITSM" + sn-itom: "ServiceNow ITOM" + sn-sir: "ServiceNow SecOps" + jira: "Jira" + ibm-r: "IBM Resilient" + webhook: "Webhook" + webhook-cm: "Webhook - Case Management" + opsgenie: "Opsgenie" + bedrock: "Amazon Bedrock" + gemini: "Google Gemini" + hive: "TheHive" + monitoring: "X-Pack monitoring" + monitor-features: "monitoring features" + stack-monitor-features: "Elastic Stack monitoring features" + watcher: "Watcher" + alert-features: "alerting features" + reporting: "X-Pack reporting" + report-features: "reporting features" + graph: "X-Pack graph" + graph-features: "graph analytics features" + searchprofiler: "Search Profiler" + xpackml: "X-Pack machine learning" + ml: "machine learning" + ml-cap: "Machine learning" + ml-init: "ML" + ml-features: "machine learning features" + stack-ml-features: "Elastic Stack machine learning features" + ccr: "cross-cluster replication" + ccr-cap: "Cross-cluster replication" + ccr-init: "CCR" + ccs: "cross-cluster search" + ccs-cap: "Cross-cluster search" + ccs-init: "CCS" + ilm: "index lifecycle management" + ilm-cap: "Index lifecycle management" + ilm-init: "ILM" + dlm: "data lifecycle management" + dlm-cap: "Data lifecycle management" + dlm-init: "DLM" + search-snap: "searchable snapshot" + search-snaps: "searchable snapshots" + search-snaps-cap: "Searchable snapshots" + slm: "snapshot lifecycle management" + slm-cap: "Snapshot lifecycle management" + slm-init: "SLM" + rollup-features: "data rollup features" + ipm: "index pattern management" + ipm-cap: "Index pattern" + rollup: "rollup" + rollup-cap: "Rollup" + rollups: "rollups" + rollups-cap: "Rollups" + rollup-job: "rollup job" + rollup-jobs: "rollup jobs" + rollup-jobs-cap: "Rollup jobs" + dfeed: "datafeed" + dfeeds: "datafeeds" + dfeed-cap: "Datafeed" + dfeeds-cap: "Datafeeds" + ml-jobs: "machine learning jobs" + ml-jobs-cap: "Machine learning jobs" + anomaly-detect: "anomaly detection" + anomaly-detect-cap: "Anomaly detection" + anomaly-job: "anomaly detection job" + anomaly-jobs: "anomaly detection jobs" + anomaly-jobs-cap: "Anomaly detection jobs" + dataframe: "data frame" + dataframes: "data frames" + dataframe-cap: "Data frame" + dataframes-cap: "Data frames" + watcher-transform: "payload transform" + watcher-transforms: "payload transforms" + watcher-transform-cap: "Payload transform" + watcher-transforms-cap: "Payload transforms" + transform: "transform" + transforms: "transforms" + transform-cap: "Transform" + transforms-cap: "Transforms" + dataframe-transform: "transform" + dataframe-transform-cap: "Transform" + dataframe-transforms: "transforms" + dataframe-transforms-cap: "Transforms" + dfanalytics-cap: "Data frame analytics" + dfanalytics: "data frame analytics" + dataframe-analytics-config: "'{dataframe} analytics config'" + dfanalytics-job: "'{dataframe} analytics job'" + dfanalytics-jobs: "'{dataframe} analytics jobs'" + dfanalytics-jobs-cap: "'{dataframe-cap} analytics jobs'" + cdataframe: "continuous data frame" + cdataframes: "continuous data frames" + cdataframe-cap: "Continuous data frame" + cdataframes-cap: "Continuous data frames" + cdataframe-transform: "continuous transform" + cdataframe-transforms: "continuous transforms" + cdataframe-transforms-cap: "Continuous transforms" + ctransform: "continuous transform" + ctransform-cap: "Continuous transform" + ctransforms: "continuous transforms" + ctransforms-cap: "Continuous transforms" + oldetection: "outlier detection" + oldetection-cap: "Outlier detection" + olscore: "outlier score" + olscores: "outlier scores" + fiscore: "feature influence score" + evaluatedf-api: "evaluate {dataframe} analytics API" + evaluatedf-api-cap: "Evaluate {dataframe} analytics API" + binarysc: "binary soft classification" + binarysc-cap: "Binary soft classification" + regression: "regression" + regression-cap: "Regression" + reganalysis: "regression analysis" + reganalysis-cap: "Regression analysis" + depvar: "dependent variable" + feature-var: "feature variable" + feature-vars: "feature variables" + feature-vars-cap: "Feature variables" + classification: "classification" + classification-cap: "Classification" + classanalysis: "classification analysis" + classanalysis-cap: "Classification analysis" + infer-cap: "Inference" + infer: "inference" + lang-ident-cap: "Language identification" + lang-ident: "language identification" + data-viz: "Data Visualizer" + file-data-viz: "File Data Visualizer" + feat-imp: "feature importance" + feat-imp-cap: "Feature importance" + nlp: "natural language processing" + nlp-cap: "Natural language processing" + apm-agent: "APM agent" + apm-go-agent: "Elastic APM Go agent" + apm-go-agents: "Elastic APM Go agents" + apm-ios-agent: "Elastic APM iOS agent" + apm-ios-agents: "Elastic APM iOS agents" + apm-java-agent: "Elastic APM Java agent" + apm-java-agents: "Elastic APM Java agents" + apm-dotnet-agent: "Elastic APM .NET agent" + apm-dotnet-agents: "Elastic APM .NET agents" + apm-node-agent: "Elastic APM Node.js agent" + apm-node-agents: "Elastic APM Node.js agents" + apm-php-agent: "Elastic APM PHP agent" + apm-php-agents: "Elastic APM PHP agents" + apm-py-agent: "Elastic APM Python agent" + apm-py-agents: "Elastic APM Python agents" + apm-ruby-agent: "Elastic APM Ruby agent" + apm-ruby-agents: "Elastic APM Ruby agents" + apm-rum-agent: "Elastic APM Real User Monitoring (RUM) JavaScript agent" + apm-rum-agents: "Elastic APM RUM JavaScript agents" + apm-lambda-ext: "Elastic APM AWS Lambda extension" + project-monitors: "project monitors" + project-monitors-cap: "Project monitors" + private-location: "Private Location" + private-locations: "Private Locations" + pwd: "YOUR_PASSWORD" + esh: "ES-Hadoop" + default-dist: "default distribution" + oss-dist: "OSS-only distribution" + observability: "Observability" + api-request-title: "Request" + api-prereq-title: "Prerequisites" + api-description-title: "Description" + api-path-parms-title: "Path parameters" + api-query-parms-title: "Query parameters" + api-request-body-title: "Request body" + api-response-codes-title: "Response codes" + api-response-body-title: "Response body" + api-example-title: "Example" + api-examples-title: "Examples" + api-definitions-title: "Properties" + multi-arg: "†footnoteref:[multi-arg,This parameter accepts multiple arguments.]" + multi-arg-ref: "†footnoteref:[multi-arg]" + yes-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png[Yes,20,15]" + no-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png[No,20,15]" + es-repo: "https://github.com/elastic/elasticsearch/" + es-issue: "https://github.com/elastic/elasticsearch/issues/" + es-pull: "https://github.com/elastic/elasticsearch/pull/" + es-commit: "https://github.com/elastic/elasticsearch/commit/" + kib-repo: "https://github.com/elastic/kibana/" + kib-issue: "https://github.com/elastic/kibana/issues/" + kibana-issue: "'{kib-repo}issues/'" + kib-pull: "https://github.com/elastic/kibana/pull/" + kibana-pull: "'{kib-repo}pull/'" + kib-commit: "https://github.com/elastic/kibana/commit/" + ml-repo: "https://github.com/elastic/ml-cpp/" + ml-issue: "https://github.com/elastic/ml-cpp/issues/" + ml-pull: "https://github.com/elastic/ml-cpp/pull/" + ml-commit: "https://github.com/elastic/ml-cpp/commit/" + apm-repo: "https://github.com/elastic/apm-server/" + apm-issue: "https://github.com/elastic/apm-server/issues/" + apm-pull: "https://github.com/elastic/apm-server/pull/" + kibana-blob: "https://github.com/elastic/kibana/blob/current/" + apm-get-started-ref: "https://www.elastic.co/guide/en/apm/get-started/current" + apm-server-ref: "https://www.elastic.co/guide/en/apm/server/current" + apm-server-ref-v: "https://www.elastic.co/guide/en/apm/server/current" + apm-server-ref-m: "https://www.elastic.co/guide/en/apm/server/master" + apm-server-ref-62: "https://www.elastic.co/guide/en/apm/server/6.2" + apm-server-ref-64: "https://www.elastic.co/guide/en/apm/server/6.4" + apm-server-ref-70: "https://www.elastic.co/guide/en/apm/server/7.0" + apm-overview-ref-v: "https://www.elastic.co/guide/en/apm/get-started/current" + apm-overview-ref-70: "https://www.elastic.co/guide/en/apm/get-started/7.0" + apm-overview-ref-m: "https://www.elastic.co/guide/en/apm/get-started/master" + infra-guide: "https://www.elastic.co/guide/en/infrastructure/guide/current" + a-data-source: "a data view" + icon-bug: "pass:[]" + icon-checkInCircleFilled: "pass:[]" + icon-warningFilled: "pass:[]" diff --git a/docs/enrichers/apm-serilog.asciidoc b/docs/enrichers/apm-serilog.asciidoc deleted file mode 100644 index a6a38238..00000000 --- a/docs/enrichers/apm-serilog.asciidoc +++ /dev/null @@ -1,44 +0,0 @@ -[[apm-serilog-enricher]] -=== APM Serilog Enricher - -This enricher adds the transaction id and trace id to every Serilog log message that is created during a transaction. - -==== Installation - -Add a reference to the http://nuget.org/packages/Elastic.Apm.SerilogEnricher[Elastic.Apm.SerilogEnricher] package: - -[source,xml] -[subs="attributes"] ----- - ----- - -==== Usage - -[source,csharp] ----- -var logger = new LoggerConfiguration() - .Enrich.WithElasticApmCorrelationInfo() - .WriteTo.Console(outputTemplate: "[{ElasticApmTraceId} {ElasticApmTransactionId} {ElasticApmSpanId} {Message:lj} {NewLine}{Exception}") - .CreateLogger(); ----- - -==== Properties - -In the code snippet above `Enrich.WithElasticApmCorrelationInfo()` enables the enricher from this project, -which will set 3 properties for log lines that are created during a transaction: - -* `ElasticApmTraceId` -* `ElasticApmTransactionId` -* `ElasticApmSpanId` - -These two properties are printed to the Console using the `outputTemplate` parameter, of course they can -be used with any sink, you could consider using a filesystem sink and -https://www.elastic.co/downloads/beats/filebeat[Elastic Filebeat] for durable and reliable ingestion. -This enricher is also compatible with the -https://www.nuget.org/packages/Elastic.CommonSchema.Serilog[Elastic.CommonSchema.Serilog] package. - -==== Prerequisite - -The prerequisite for this to work is a configured https://github.com/elastic/apm-agent-dotnet[Elastic APM Agent]. -If the agent is not configured the enricher won't add anything to the logs. \ No newline at end of file diff --git a/docs/enrichers/index.asciidoc b/docs/enrichers/index.asciidoc deleted file mode 100644 index daf0a1fd..00000000 --- a/docs/enrichers/index.asciidoc +++ /dev/null @@ -1,13 +0,0 @@ -ifdef::env-github[] -NOTE: For the best reading experience, -please view this documentation at https://www.elastic.co/guide/en/ecs-logging/dotnet[elastic.co] -endif::[] - -== Enrichers - -Enrichers can be installed next to `Log Formatters` and `Data Shippers` to automatically -enrich the ECS json that gets produced. - -include::./apm-serilog.asciidoc[Serilog] -include::./apm-nlog.asciidoc[NLog] - diff --git a/docs/formatters/index.asciidoc b/docs/formatters/index.asciidoc deleted file mode 100644 index 73f8598e..00000000 --- a/docs/formatters/index.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -ifdef::env-github[] -NOTE: For the best reading experience, -please view this documentation at https://www.elastic.co/guide/en/ecs-logging/dotnet[elastic.co] -endif::[] - -== Formatters - -Our log formatters allow you to change the way various logging frameworks -log to IO (file/console) to use ECS json. - -[float] -=== ECS Aware Message Templates - -Our log formatters allow you to set ECS fields directly from the message template using properties that adhere to the -https://messagetemplates.org/ format. - -[source,chsarp] ----- -Log.Information("The time is {TraceId}", "my-trace-id"); ----- - -Will directly override `trace.id` on the resulting ECS json document. - -All supported ECS message template properties are available as constants under the `LogTemplateProperties` static class. -For example `LogTemplateProperties.TraceId` will return `"TraceId"`. - - - -[float] -=== Ingest ECS log files - -If you are using one of our formatter libraries to log to file or stdout/stderr you can use the following options -to get these logs into Elasticsearch or Elastic Cloud: - -NOTE: We also support writing logs directly to Elasticsearch or Elastic Cloud. See our <> - -include::{ecs-repo-dir}/setup.asciidoc[tag=configure-filebeat] - -include::./serilog.asciidoc[Serilog] -include::./nlog.asciidoc[NLog] -include::./log4net.asciidoc[log4net] - diff --git a/docs/formatters/nlog.asciidoc b/docs/formatters/nlog.asciidoc deleted file mode 100644 index aefbdf61..00000000 --- a/docs/formatters/nlog.asciidoc +++ /dev/null @@ -1,174 +0,0 @@ -[[nlog-formatter]] -=== NLog Layout - -This Layout implementation formats an NLog event into a JSON representation that adheres to the Elastic Common Schema specification. - -==== Installation - -Add a reference to the Elastic.CommonSchema.NLog package: - -[source,xml] -[subs="attributes"] ----- - ----- - -==== Usage - -===== Setup Programatically - -[source,csharp] ----- -Layout.Register("EcsLayout"); // Register the ECS layout. -var config = new LoggingConfiguration(); -var consoleTarget = new ConsoleTarget("console") { Layout = new EcsLayout() }; // Use the ECS layout. -config.AddRule(LogLevel.Debug, LogLevel.Fatal, consoleTarget); -LogManager.Configuration = config; -var logger = LogManager.GetCurrentClassLogger(); - ----- - -In the code snippet above `Layout.Register("EcsLayout")` registers the `EcsLayout` with NLog. -The `Layout = new EcsLayout()` line then instructs NLog to use the registered layout. -The sample above uses the console target, but you are free to use any target of your choice; perhaps consider using a -filesystem target and https://www.elastic.co/downloads/beats/filebeat[Elastic Filebeat] for durable and reliable ingestion. - -===== Setup using NLog.config - -[source,xml] ----- - - - - - - - - - - - - - - - - ----- - -==== EcsLayout Parameter Options - -* *Metadata Options* -* _IncludeEventProperties_ - Include LogEvent properties as metadata. Default: `true` -* _IncludeScopeProperties_ - Include NLog Scope Context Properties as metadata. Default: `false` -* _ExcludeProperties_ - Comma separated string with names which properties to exclude. -* *Event Options* - -* _EventAction_ - -* _EventCategory_ - -* _EventId_ - -* _EventKind_ - -* _EventSeverity_ - -* *Agent Options* - -* _AgentId_ - -* _AgentName_ - -* _AgentType_ - -* _AgentVersion_ - -* *Process Options* - -* _ProcessExecutable_ - Default: `${processname:FullName=true}` -* _ProcessId_ - Default: `${processid}` -* _ProcessName_ - Default: `${processname:FullName=false}` -* _ProcessThreadId_ - Default: `${threadid}` -* _ProcessTitle_ - Default: `${processinfo:MainWindowTitle}` -* *Server Options* - -* _ServerAddress_ - -* _ServerIp_ - -* _ServerUser_ - Default: `${environment-user}` -* *Host Options* - -* _HostId_ - -* _HostIp_ - Default: `${local-ip:cachedSeconds=60}` -* _HostName_ - Default: `${machinename}` -* *Log Origin Options* - -* _LogOriginCallSiteMethod_ - Default: `${exception:format=method}` -* _LogOriginCallSiteFile_ - Default: `${exception:format=source}` -* _LogOriginCallSiteLine_ - -* *Http Options* - -* _HttpRequestId_ - Default: `${aspnet-trace-identifier}` -* _HttpRequestMethod_ - Default: `${aspnet-request-method}` -* _HttpRequestBytes_ - Default: `${aspnet-request-contentlength}` -* _HttpRequestReferrer_ - Default: `${aspnet-request-referrer}` -* _HttpResponseStatusCode_ - Default: `${aspnet-response-statuscode}` -* *Url Options* - -* _UrlScheme_ - Default: `${aspnet-request-url:IncludeScheme=true:IncludeHost=false:IncludePath=false}` -* _UrlDomain_ - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=true:IncludePath=false}` -* _UrlPath_ - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=false:IncludePath=true}` -* _UrlPort_ - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=false:IncludePath=false:IncludePort=true}` -* _UrlQuery_ - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=false:IncludePath=false:IncludeQueryString=true}` -* _UrlUserName_ - Default: `${aspnet-user-identity}` -* *Trace Options* - -* _ApmTraceId_ - Default: `${ElasticApmTraceId}` -* *Transaction Options* - -* _ApmTransactionId_ - Default: `${ElasticApmTransactionId}` -* - -==== ECS Aware Message Templates - -Additionally any valid ECS log template properties that is available under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId` -is supported and will directly set the appropriate ECS fields. - -[source,chsarp] ----- -logger.Info("The time is {TraceId}", "my-trace-id"); ----- - -Will override `trace.id` on the resulting ECS json document. - -==== Example output from EcsLayout - -An example of the output is given below: - -[source,json] ----- -{ - "@timestamp":"2020-02-20T16:07:06.7109766+11:00", - "log.level":"Info", - "message":"Info \"X\" 2.2", - "metadata":{ - "value_x":"X", - "some_y":2.2 - }, - "ecs":{ - "version":"1.4.0" - }, - "event":{ - "severity":6, - "timezone":"AUS Eastern Standard Time", - "created":"2020-02-20T16:07:06.7109766+11:00" - }, - "host":{ - "name":"LAPTOP" - }, - "log":{ - "logger":"Elastic.CommonSchema.NLog", - "original":"Info {ValueX} {SomeY}" - }, - "process":{ - "thread":{ - "id":17592 - }, - "pid":17592, - "name":"dotnet", - "executable":"C:\\Program Files\\dotnet\\dotnet.exe" - } -} ----- diff --git a/docs/images/dotnet-overview.png b/docs/images/dotnet-overview.png deleted file mode 100644 index d2d1c189..00000000 Binary files a/docs/images/dotnet-overview.png and /dev/null differ diff --git a/docs/index.asciidoc b/docs/index.asciidoc deleted file mode 100644 index b6b96f5d..00000000 --- a/docs/index.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -:ecs-repo-dir: {ecs-logging-root}/docs/ -:ecs-logging-dotnet-version: 8.6.0 - -include::{asciidoc-dir}/../../shared/versions/stack/current.asciidoc[] -include::{asciidoc-dir}/../../shared/attributes.asciidoc[] - -ifdef::env-github[] -NOTE: For the best reading experience, -please view this documentation at https://www.elastic.co/guide/en/ecs-logging/dotnet[elastic.co] -endif::[] - -= ECS Logging .NET Reference - -ifndef::env-github[] -include::./intro.asciidoc[Introduction] -include::./setup.asciidoc[Set up] -include::./types/types.asciidoc[Types] -include::./formatters/index.asciidoc[Formatters] -include::./data-shippers/index.asciidoc[Data Shippers] -include::./enrichers/index.asciidoc[Enrichers] -endif::[] \ No newline at end of file diff --git a/docs/intro.asciidoc b/docs/intro.asciidoc deleted file mode 100644 index 65baaf34..00000000 --- a/docs/intro.asciidoc +++ /dev/null @@ -1,79 +0,0 @@ -[[intro]] -== Introduction - -ECS logging are integrations for your favorite .NET logging libraries. -They make it easy to format your logs into ECS-compatible JSON. - -TIP: Want to learn more about ECS, ECS logging, and other available language plugins? -See the {ecs-logging-ref}/intro.html[ECS logging overview]. - -The .NET ECS libraries fall into several categories: - -[float] -=== Model - -At its core all .NET ECS libraries are powered by `Elastic.CommonSchema` -which strongly types the ECS specification to .NET classes. - -This library can be used to map your events to ECS in a typesafe fashion. - -[source,csharp] ----- -var doc = EcsDocument.CreateNewWithDefaults(); -doc = new EcsDocument(); ----- - -<>. - -[float] -=== Log Formatters - -Our log formatters allow you to change the way various logging frameworks -log to IO (file/console) to use ECS json. - -* <> -* <> -* <> - -[float] -=== Data Shippers - -Our datashippers integrate with logging frameworks to facilitate -sending events (logs) to various off Elastic receivers. - -Currently these shippers support Elastic Cloud & Elasticsearch but -other outputs are in the works. - -* <> -* <> -* <> - -All the data shippers utilize <> to send events to Elasticsearch. - -[float] -=== Enrichers - -Enrichers can be installed next to `Log Formatters` and `Data Shippers` to automatically -enrich the ECS json that gets produced. - -* <> -* <> - -[float] -=== Architecture - -The libraries reuse the same components that power other .NET libraries from Elastic--ensuring a common way to both configure and monitor Elastic's .NET libraries. - -image:images/ecs-dotnet-overview.png["ECS.NET Artichtecture overview"] - -At its core all data shippers depend on https://github.com/elastic/elastic-transport-net[Elastic.Transport] to -coordinate HttpRequests. This library is also the heart of Elastic's clients and ensures -we share best practices and configuration options. - -The data shipper libraries all depend upon the push based ingestion -abstractions from https://github.com/elastic/elastic-ingest-dotnet[Elastic.Ingest.*] that -presents an easy to use `System.Threading.Channels` backed method to push events as batches at variable rates to external datasources. - -<>. - - diff --git a/docs/reference/_elasticsearch_security.md b/docs/reference/_elasticsearch_security.md new file mode 100644 index 00000000..698b0c92 --- /dev/null +++ b/docs/reference/_elasticsearch_security.md @@ -0,0 +1,29 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/_elasticsearch_security.html +--- + +# Elasticsearch security [_elasticsearch_security] + +If Elasticsearch’s security is enabled you will need to ensure you configure a user or API key with enough privileges + +## Bootstrap [_bootstrap] + +In order for the datashippers to have enough privileges to bootstrap the target datastreams with all the ECS mappings, templates and settings the authenticated security principal needs the following minimum privileges: + +| Type | Privileges | +| --- | --- | +| Cluster | `monitor`, `manage_ilm`, `manage_index_templates`, `manage_pipeline` | +| Index | `manage`, `create_doc` | + + +## No bootstrap [_no_bootstrap] + +If the datashippers are configured to skip bootstrapping the target destinations all together, the security principal requires the following minimum privileges to push data. + +| Type | Privileges | +| --- | --- | +| Cluster | `monitor` | +| Index | `auto_configure` `create_doc` | + + diff --git a/docs/reference/_enrichers_2.md b/docs/reference/_enrichers_2.md new file mode 100644 index 00000000..f1dd8f1c --- /dev/null +++ b/docs/reference/_enrichers_2.md @@ -0,0 +1,11 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/_enrichers_2.html +--- + +# Enrichers [_enrichers_2] + +Enrichers can be installed next to `Log Formatters` and `Data Shippers` to automatically enrich the ECS json that gets produced. + + + diff --git a/docs/reference/_extending_ecsdocument.md b/docs/reference/_extending_ecsdocument.md new file mode 100644 index 00000000..8be5a38e --- /dev/null +++ b/docs/reference/_extending_ecsdocument.md @@ -0,0 +1,44 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/_extending_ecsdocument.html +--- + +# Extending EcsDocument [_extending_ecsdocument] + +In instances where using the `IDictionary Metadata` property is not sufficient, or there is a clearer definition of the structure of the ECS-compatible document you would like to index, it is possible to subclass the `EcsDocument` object and provide your own property definitions. + +Through `TryRead`/`ReceiveProperty`/`WriteAdditionalProperties` you can hook into the `EcsDocumentJsonConverter` and read/write additional properties. + +```csharp +/// +/// An extended ECS document with an additional property +/// +[JsonConverter(typeof(EcsDocumentJsonConverterFactory))] +public class MyEcsDocument : EcsDocument +{ + [JsonPropertyName("my_root_property"), DataMember(Name = "my_root_property")] + public MyCustomType MyRootProperty { get; set; } + + protected override bool TryRead(string propertyName, out Type type) + { + type = propertyName switch + { + "my_root_property" => typeof(MyCustomType), + _ => null + }; + return type != null; + } + + protected override bool ReceiveProperty(string propertyName, object value) => + propertyName switch + { + "my_root_property" => null != (MyRootProperty = value as MyCustomType), + _ => false + }; + + protected override void WriteAdditionalProperties(Action write) => write("my_root_property", MyCustomType); +} +``` + +The Elastic.CommonSchema.BenchmarkDotNetExporter project takes this approach in the [Domain source directory](https://github.com/elastic/ecs-dotnet/tree/main/src/Elastic.CommonSchema.BenchmarkDotNetExporter), where the BenchmarkDocument subclasses EcsDocument. + diff --git a/docs/reference/_formatters.md b/docs/reference/_formatters.md new file mode 100644 index 00000000..cf806666 --- /dev/null +++ b/docs/reference/_formatters.md @@ -0,0 +1,125 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/_formatters.html +--- + +# Formatters [_formatters] + +Our log formatters allow you to change the way various logging frameworks log to IO (file/console) to use ECS json. + + +## ECS aware message templates [_ecs_aware_message_templates] + +Our log formatters allow you to set ECS fields directly from the message template using properties that adhere to the [https://messagetemplates.org/](https://messagetemplates.org/) format. + +```csharp +Log.Information("The time is {TraceId}", "my-trace-id"); +``` + +Will directly override `trace.id` on the resulting ECS json document. + +All supported ECS message template properties are available as constants under the `LogTemplateProperties` static class. For example `LogTemplateProperties.TraceId` will return `"TraceId"`. + + +## Ingest ECS log files [_ingest_ecs_log_files] + +If you are using one of our formatter libraries to log to file or stdout/stderr you can use the following options to get these logs into Elasticsearch or Elastic Cloud: + +::::{note} +We also support writing logs directly to Elasticsearch or Elastic Cloud. See our [various data shipper integrations](/reference/data-shippers.md) +:::: + + +:::::::{tab-set} + +::::::{tab-item} Log file +1. Follow the [Filebeat quick start](beats://docs/reference/filebeat/filebeat-installation-configuration.md) +2. Add the following configuration to your `filebeat.yaml` file. + +For Filebeat 7.16+ + +```yaml +filebeat.inputs: +- type: filestream <1> + paths: /path/to/logs.json + parsers: + - ndjson: + overwrite_keys: true <2> + add_error_key: true <3> + expand_keys: true <4> + +processors: <5> + - add_host_metadata: ~ + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ +``` + +1. Use the filestream input to read lines from active log files. +2. Values from the decoded JSON object overwrite the fields that {{filebeat}} normally adds (type, source, offset, etc.) in case of conflicts. +3. {{filebeat}} adds an "error.message" and "error.type: json" key in case of JSON unmarshalling errors. +4. {{filebeat}} will recursively de-dot keys in the decoded JSON, and expand them into a hierarchical object structure. +5. Processors enhance your data. See [processors](beats://docs/reference/filebeat/filtering-enhancing-data.md) to learn more. + + +For Filebeat < 7.16 + +```yaml +filebeat.inputs: +- type: log + paths: /path/to/logs.json + json.keys_under_root: true + json.overwrite_keys: true + json.add_error_key: true + json.expand_keys: true + +processors: +- add_host_metadata: ~ +- add_cloud_metadata: ~ +- add_docker_metadata: ~ +- add_kubernetes_metadata: ~ +``` +:::::: + +::::::{tab-item} Kubernetes +1. Make sure your application logs to stdout/stderr. +2. Follow the [Run Filebeat on Kubernetes](beats://docs/reference/filebeat/running-on-kubernetes.md) guide. +3. Enable [hints-based autodiscover](beats://docs/reference/filebeat/configuration-autodiscover-hints.md) (uncomment the corresponding section in `filebeat-kubernetes.yaml`). +4. Add these annotations to your pods that log using ECS loggers. This will make sure the logs are parsed appropriately. + +```yaml +annotations: + co.elastic.logs/json.overwrite_keys: true <1> + co.elastic.logs/json.add_error_key: true <2> + co.elastic.logs/json.expand_keys: true <3> +``` + +1. Values from the decoded JSON object overwrite the fields that {{filebeat}} normally adds (type, source, offset, etc.) in case of conflicts. +2. {{filebeat}} adds an "error.message" and "error.type: json" key in case of JSON unmarshalling errors. +3. {{filebeat}} will recursively de-dot keys in the decoded JSON, and expand them into a hierarchical object structure. +:::::: + +::::::{tab-item} Docker +1. Make sure your application logs to stdout/stderr. +2. Follow the [Run Filebeat on Docker](beats://docs/reference/filebeat/running-on-docker.md) guide. +3. Enable [hints-based autodiscover](beats://docs/reference/filebeat/configuration-autodiscover-hints.md). +4. Add these labels to your containers that log using ECS loggers. This will make sure the logs are parsed appropriately. + +```yaml +labels: + co.elastic.logs/json.overwrite_keys: true <1> + co.elastic.logs/json.add_error_key: true <2> + co.elastic.logs/json.expand_keys: true <3> +``` + +1. Values from the decoded JSON object overwrite the fields that {{filebeat}} normally adds (type, source, offset, etc.) in case of conflicts. +2. {{filebeat}} adds an "error.message" and "error.type: json" key in case of JSON unmarshalling errors. +3. {{filebeat}} will recursively de-dot keys in the decoded JSON, and expand them into a hierarchical object structure. +:::::: + +::::::: +For more information, see the [Filebeat reference](beats://docs/reference/filebeat/configuring-howto-filebeat.md). + + + + diff --git a/docs/reference/_usage.md b/docs/reference/_usage.md new file mode 100644 index 00000000..1080ee9f --- /dev/null +++ b/docs/reference/_usage.md @@ -0,0 +1,95 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/_usage.html +--- + +# Usage [_usage] + +## Creating an ECS event [_creating_an_ecs_event] + +The recommended way to create instances of `EcsDocument` is through: + +```csharp +var doc = EcsDocument.CreateNewWithDefaults(); +``` + +This will automatically assign most common ECS fields that can be inferred from the running process. + +However there is no requirement to do so, simply creating a new `EcsDocument` instance directly is completely valid and supported. + +```csharp +var ecsDocument = new EcsDocument +{ + Timestamp = DateTimeOffset.Parse("2019-10-23T19:44:38.485Z"), + Dns = new Dns + { + Id = "23666", + OpCode = "QUERY", + Type = "answer", + QuestionName = "www.example.com", + QuestionType = "A", + QuestionClass = "IN", + QuestionRegisteredDomain = "example.com", + HeaderFlags = new[] { "RD", "RA" }, + ResponseCode = "NOERROR", + ResolvedIp = new[] { "10.0.190.47", "10.0.190.117" }, + Answers = new[] + { + new DnsAnswers + { + Data = "10.0.190.47", + Name = "www.example.com", + Type = "A", + Class = "IN", + Ttl = 59 + }, + new DnsAnswers + { + Data = "10.0.190.117", + Name = "www.example.com", + Type = "A", + Class = "IN", + Ttl = 59 + } + } + }, + Network = new Network + { + Type = "ipv4", + Transport = "udp", + Protocol = "dns", + Direction = "outbound", + CommunityId = "1:19beef+RWVW9+BEEF/Q45VFU+2Y=", + Bytes = 126 + }, + Source = new Source { Ip = "192.168.86.26", Port = 5785, Bytes = 31 }, + Destination = new Destination { Ip = "8.8.4.4", Port = 53, Bytes = 95 }, + Client = new Client { Ip = "192.168.86.26", Port = 5785, Bytes = 31 }, + Server = new Server { Ip = "8.8.4.4", Port = 53, Bytes = 95 }, + Event = new Event + { + Duration = 122433000, + Start = DateTimeOffset.Parse("2019-10-23T19:44:38.485Z"), + End = DateTimeOffset.Parse("2019-10-23T19:44:38.607Z"), + Kind = "event", + Category = new[] { "network_traffic" } + }, + Ecs = new Ecs { Version = "1.2.0" }, + Metadata = new Dictionary { { "client", "ecs-dotnet" } } +}; +``` + + +## Dynamically assign ECS fields [_dynamically_assign_ecs_fields] + +Additionally, ECS fields can be dynamically assigned through + +```csharp +ecsDocument.AssignProperty("orchestrator.cluster.id", "id"); +``` + +This will assign `ecsDocument.Orchestrator.ClusterId` to `"id"` and automatically create a new `Orchestrator` instance if needed. + +Any `string` or `boolean` value that is not a known `ecs` field will be assigned to `labels.*` and everything else to `metatadata.*` + + diff --git a/docs/enrichers/apm-nlog.asciidoc b/docs/reference/apm-nlog-enricher.md similarity index 57% rename from docs/enrichers/apm-nlog.asciidoc rename to docs/reference/apm-nlog-enricher.md index 07203af7..358eeae7 100644 --- a/docs/enrichers/apm-nlog.asciidoc +++ b/docs/reference/apm-nlog-enricher.md @@ -1,5 +1,9 @@ -[[apm-nlog-enricher]] -=== APM NLog Layout +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/apm-nlog-enricher.html +--- + +# APM NLog layout [apm-nlog-enricher] Allows you to add the following place holders in your NLog templates: @@ -12,52 +16,53 @@ Allows you to add the following place holders in your NLog templates: Which will be replaced with the appropriate Elastic APM variables if available -==== Installation +## Installation [_installation_10] + +Add a reference to the [Elastic.Apm.NLog](http://nuget.org/packages/Elastic.Apm.NLog) package: -Add a reference to the http://nuget.org/packages/Elastic.Apm.NLog[Elastic.Apm.NLog] package: +```xml + +``` -[source,xml] -[subs="attributes"] ----- - ----- -==== Usage +## Usage [_usage_10] -===== How to use from API +### How to use from API [_how_to_use_from_api] -[source,csharp] ----- +```csharp // Logged message will be in format of `trace-id|transation-id|span-id|InTransaction` // or `|||InTransaction` if the place holders are not available var consoleTarget = new ConsoleTarget("console"); -consoleTarget.Layout = +consoleTarget.Layout = "${ElasticApmServiceName}|${ElasticApmTraceId}|${ElasticApmTransactionId}|${ElasticApmSpanId}|${message}"; config.AddRule(LogLevel.Debug, LogLevel.Fatal, consoleTarget); LogManager.Configuration = config; var logger = LogManager.GetCurrentClassLogger(); +``` ----- -===== How to use from NLog.config +### How to use from NLog.config [_how_to_use_from_nlog_config] -[source,xml] ----- +```xml - ----- +``` + + + +## Prerequisite [_prerequisite_2] + +The prerequisite for this to work is a configured [Elastic APM Agent](https://github.com/elastic/apm-agent-dotnet). If the agent is not configured the APM place holders will be empty. -==== Prerequisite -The prerequisite for this to work is a configured https://github.com/elastic/apm-agent-dotnet[Elastic APM Agent]. If the agent is not configured the APM place holders will be empty. \ No newline at end of file diff --git a/docs/reference/apm-serilog-enricher.md b/docs/reference/apm-serilog-enricher.md new file mode 100644 index 00000000..dcbf11b4 --- /dev/null +++ b/docs/reference/apm-serilog-enricher.md @@ -0,0 +1,44 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/apm-serilog-enricher.html +--- + +# APM serilog enricher [apm-serilog-enricher] + +This enricher adds the transaction id and trace id to every Serilog log message that is created during a transaction. + +## Installation [_installation_9] + +Add a reference to the [Elastic.Apm.SerilogEnricher](http://nuget.org/packages/Elastic.Apm.SerilogEnricher) package: + +```xml + +``` + + +## Usage [_usage_9] + +```csharp +var logger = new LoggerConfiguration() + .Enrich.WithElasticApmCorrelationInfo() + .WriteTo.Console(outputTemplate: "[{ElasticApmTraceId} {ElasticApmTransactionId} {ElasticApmSpanId} {Message:lj} {NewLine}{Exception}") + .CreateLogger(); +``` + + +## Properties [_properties] + +In the code snippet above `Enrich.WithElasticApmCorrelationInfo()` enables the enricher from this project, which will set 3 properties for log lines that are created during a transaction: + +* `ElasticApmTraceId` +* `ElasticApmTransactionId` +* `ElasticApmSpanId` + +These two properties are printed to the Console using the `outputTemplate` parameter, of course they can be used with any sink, you could consider using a filesystem sink and [Elastic Filebeat](https://www.elastic.co/downloads/beats/filebeat) for durable and reliable ingestion. This enricher is also compatible with the [Elastic.CommonSchema.Serilog](https://www.nuget.org/packages/Elastic.CommonSchema.Serilog) package. + + +## Prerequisite [_prerequisite] + +The prerequisite for this to work is a configured [Elastic APM Agent](https://github.com/elastic/apm-agent-dotnet). If the agent is not configured the enricher won’t add anything to the logs. + + diff --git a/docs/data-shippers/benchmark-dotnet.asciidoc b/docs/reference/benchmark-dotnet-data-shipper.md similarity index 88% rename from docs/data-shippers/benchmark-dotnet.asciidoc rename to docs/reference/benchmark-dotnet-data-shipper.md index db131603..a2896b25 100644 --- a/docs/data-shippers/benchmark-dotnet.asciidoc +++ b/docs/reference/benchmark-dotnet-data-shipper.md @@ -1,22 +1,24 @@ -[[benchmark-dotnet-data-shipper]] -=== BenchmarkDotnet Exporter +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/benchmark-dotnet-data-shipper.html +--- -An exporter for https://github.com/dotnet/BenchmarkDotNet[BenchmarkDotnet] that will index benchmarking results directly into Elasticsearch. +# BenchmarkDotnet exporter [benchmark-dotnet-data-shipper] -==== Installation +An exporter for [BenchmarkDotnet](https://github.com/dotnet/BenchmarkDotNet) that will index benchmarking results directly into Elasticsearch. -Add a reference to the http://nuget.org/packages/Elastic.CommonSchema.BenchmarkDotNetExporter[Elastic.CommonSchema.BenchmarkDotNetExporter] package: +## Installation [_installation_8] -[source,xml] -[subs="attributes"] ----- - ----- +Add a reference to the [Elastic.CommonSchema.BenchmarkDotNetExporter](http://nuget.org/packages/Elastic.CommonSchema.BenchmarkDotNetExporter) package: -==== Usage +```xml + +``` -[source,csharp] ----- + +## Usage [_usage_8] + +```csharp var options = new ElasticsearchBenchmarkExporterOptions(url) { GitBranch = "externally-provided-branch", @@ -27,24 +29,20 @@ var exporter = new ElasticsearchBenchmarkExporter(options); var config = CreateDefaultConfig().With(exporter); BenchmarkRunner.Run(typeof(Md5VsSha256), config); +``` ----- - -The code snippet above configures the `ElasticsearchBenchmarkExporter` with the supplied `ElasticsearchBenchmarkExporterOptions`. It is possible to configure the exporter to use https://www.elastic.co/cloud/[Elastic Cloud] as follows: +The code snippet above configures the `ElasticsearchBenchmarkExporter` with the supplied `ElasticsearchBenchmarkExporterOptions`. It is possible to configure the exporter to use [Elastic Cloud](https://www.elastic.co/cloud/) as follows: -[source,csharp] ----- +```csharp var options = new ElasticsearchBenchmarkExporterOptions(url) { CloudId = "CLOUD_ID_HERE" }; - ----- +``` Example _source from a search in Elasticsearch after a benchmark run: -[source,json] ----- +```json { "_index":"benchmark-dotnet-2020-01-01", "_type":"_doc", @@ -218,4 +216,6 @@ Example _source from a search in Elasticsearch after a benchmark run: } } } ----- \ No newline at end of file +``` + + diff --git a/docs/reference/data-shippers.md b/docs/reference/data-shippers.md new file mode 100644 index 00000000..cc60b66c --- /dev/null +++ b/docs/reference/data-shippers.md @@ -0,0 +1,16 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/data-shippers.html +--- + +# Data shippers [data-shippers] + +Our datashippers integrate with logging frameworks to facilitate sending events (such as logs) to various outputs. + +Currently these shippers support Elastic Cloud & Elasticsearch but other outputs are in the works. + + + + + + diff --git a/docs/reference/ecs-dotnet.md b/docs/reference/ecs-dotnet.md new file mode 100644 index 00000000..8abbf4da --- /dev/null +++ b/docs/reference/ecs-dotnet.md @@ -0,0 +1,38 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/ecs-dotnet.html +--- + +# .NET model of ECS [ecs-dotnet] + +The `Elastic.CommonSchema` project contains a full C# representation of the [Elastic Common Schema (ECS)](https://github.com/elastic/ecs). The intention of this library is to form a reliable and correct basis for integrating into Elasticsearch, using both Microsoft .NET and ECS. + +These types can be used either as-is or in conjunction with the [Official .NET clients for Elasticsearch](https://github.com/elastic/elasticsearch-net). The types are annotated with the corresponding `DataMember` attributes, enabling out-of-the-box serialization support with the Elasticsearch.net clients. + + +## Installation [_installation] + +Add a reference to the Elastic.CommonSchema package: + +```xml + +``` + +::::{tip} +Use [Elastic.Ingest.Elasticsearch.CommonSchema](https://github.com/elastic/ecs-dotnet/tree/main/src/Elastic.Ingest.Elasticsearch.CommonSchema) to easily persist ECS documents to Elasticsearch or Elastic Cloud. +:::: + + + +### Versioning [_versioning] + +The version of the Elastic.CommonSchema package matches the published ECS version, with the same corresponding branch names: + +* Nested Schema (The C# types are generated from this YAML file): [https://github.com/elastic/ecs/blob/v1.4.0/generated/ecs/ecs_nested.yml](https://github.com/elastic/ecs/blob/v1.4.0/generated/ecs/ecs_nested.yml) +* .NET types: [https://github.com/elastic/ecs-dotnet/tree/v1.4.0](https://github.com/elastic/ecs-dotnet/tree/v1.4.0) + +The version numbers of the NuGet package must match the exact version of ECS used within Elasticsearch. Attempting to use mismatched versions, for example a NuGet package with version 1.2.0 against an Elasticsearch index configured to use an ECS template with version 1.1.0, will result in indexing and data problems. + + + + diff --git a/docs/data-shippers/ingest-commonschema.asciidoc b/docs/reference/ecs-ingest-channels.md similarity index 54% rename from docs/data-shippers/ingest-commonschema.asciidoc rename to docs/reference/ecs-ingest-channels.md index 58e5b9de..dcd4fb53 100644 --- a/docs/data-shippers/ingest-commonschema.asciidoc +++ b/docs/reference/ecs-ingest-channels.md @@ -1,28 +1,30 @@ -[[ecs-ingest-channels]] -=== ECS Ingest Channels +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/ecs-ingest-channels.html +--- -A specialization of https://www.nuget.org/packages/Elastic.Ingest.Elasticsearch#readme-body-tab[`Elastic.Ingest.Elasticsearch`] that offers two channel implementations that make it easy to write ECS formatted data and bootstrap the target datastreams/indices with ECS mappings and settings. +# ECS ingest channels [ecs-ingest-channels] -==== Installation +A specialization of [`Elastic.Ingest.Elasticsearch`](https://www.nuget.org/packages/Elastic.Ingest.Elasticsearch#readme-body-tab) that offers two channel implementations that make it easy to write ECS formatted data and bootstrap the target datastreams/indices with ECS mappings and settings. + +## Installation [_installation_5] Add a reference to the `Elastic.Ingest.Elasticsearch.CommonSchema` package: -[source,xml] -[subs="attributes"] ----- - ----- +```xml + +``` + -==== Usage +## Usage [_usage_5] -===== EcsDataStreamChannel +### EcsDataStreamChannel [_ecsdatastreamchanneltevent] -A channel that specializes to writing data with a timestamp to Elasticsearch data streams. +A channel that specializes to writing data with a timestamp to Elasticsearch data streams. A channel can be created to push data to the `logs-dotnet-default` data stream. -[source,csharp] ----- +```csharp var dataStream = new DataStreamName("logs", "dotnet"); var bufferOptions = new BufferOptions { } var options = new DataStreamChannelOptions(transport) @@ -31,31 +33,32 @@ var options = new DataStreamChannelOptions(transport) BufferOptions = bufferOptions }; var channel = new EcsDataStreamChannel(options); +``` ----- +::::{tip} +Learn more about Elastic’s data stream naming convention in [this blog post](https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme). +:::: -TIP: Learn more about Elastic's data stream naming convention in https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[this blog post]. We can now push data to Elasticsearch using the `EcsDataStreamChannel` -[source,csharp] ----- -var doc = new EcsDocument -{ - Timestamp = DateTimeOffset.Now, - Message = "Hello World!", +```csharp +var doc = new EcsDocument +{ + Timestamp = DateTimeOffset.Now, + Message = "Hello World!", } channel.TryWrite(doc); ----- +``` -===== EcsIndexChannel + +### EcsIndexChannel [_ecsindexchanneltevent] A channel that specializes in writing catalog data to Elastic indices. We can create an `EcsIndexChannel<>` to push `EcsDocument` (or subclassed) instances. -[source,csharp] ----- +```csharp var options = new IndexChannelOptions(transport) { IndexFormat = "catalog-data-{0:yyyy.MM.dd}", @@ -63,20 +66,19 @@ var options = new IndexChannelOptions(transport) TimestampLookup = c => c.Timestamp, }; var channel = new EcsIndexChannel(options); ----- +``` Now we can push data using: -[source,csharp] ----- -var doc = new CatalogDocument -{ - Created = date, - Title = "Hello World!", - Id = "hello-world" +```csharp +var doc = new CatalogDocument +{ + Created = date, + Title = "Hello World!", + Id = "hello-world" } channel.TryWrite(doc); ----- +``` This will push data to `catalog-data-2023.01.1` because `TimestampLookup` yields `Timestamp` to `IndexFormat`. @@ -84,24 +86,26 @@ This will push data to `catalog-data-2023.01.1` because `TimestampLookup` yields `BulkOperationIdLookup` determines if the document should be pushed to Elasticsearch using a `create` or `index` operation. -[float] -=== Bootstrapping target Datastream or Index + +## Bootstrapping target datastream or index [_bootstrapping_target_datastream_or_index] Optionally the target data stream or index can be bootstrapped using the following. -[source,csharp] ----- -await channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, "7-days-default"); ----- +```csharp +await channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, "7-days-default"); +``` -This will bootstrap: +This will bootstrap: * Set up component templates for all ECS fieldsets -* reference: {ecs-ref}/ecs-field-reference.html -* templates: https://github.com/elastic/ecs/tree/main/generated/elasticsearch/composable/component +* reference: {{ecs-ref}}/ecs-field-reference.html +* templates: [https://github.com/elastic/ecs/tree/main/generated/elasticsearch/composable/component](https://github.com/elastic/ecs/tree/main/generated/elasticsearch/composable/component) * Create a special `*-settings` component template for the datastream/indices that sets up ILM. -* Set up an {ref}/index-templates.html[index template] for the target data streams or indices. +* Set up an [index template](docs-content://manage-data/data-store/templates.md) for the target data streams or indices. If the index template already exists no further bootstrapping will occur. -Just like `Elastic.Ingest.Elasticsearch` the channel is aware that `logs` and `metrics` have default component templates and ensures the new index tempate references them. \ No newline at end of file +Just like `Elastic.Ingest.Elasticsearch` the channel is aware that `logs` and `metrics` have default component templates and ensures the new index tempate references them. + + + diff --git a/docs/reference/extensions-logging-data-shipper.md b/docs/reference/extensions-logging-data-shipper.md new file mode 100644 index 00000000..a34a9953 --- /dev/null +++ b/docs/reference/extensions-logging-data-shipper.md @@ -0,0 +1,422 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/extensions-logging-data-shipper.html +--- + +# Elastic.Extensions.Logging [extensions-logging-data-shipper] + +Elastic logger provider for Microsoft.Extensions.Logging. + +Writes direct to Elasticsearch using the [Elastic Common Schema (ECS)][Elastic Common Schema (ECS)](ecs://docs/reference/index.md)), with semantic logging of structured data from message and scope values. The results can be viewed and queried in the Kibana console. + +## Installation [_installation_7] + +Add a reference to the `Elastic.Extensions.Logging` package: + +```xml + +``` + + +## Usage [_usage_7] + +Then, add the provider to the loggingBuilder during host construction, using the provided extension method. + +```csharp +using Elastic.Extensions.Logging; + +// ... + + .ConfigureLogging((hostContext, loggingBuilder) => + { + loggingBuilder.AddElasticsearch(); + }) +``` + +The default configuration will write to a local Elasticsearch running at [http://localhost:9200/](http://localhost:9200/). + +Once you have sent some log event, open Kibana (e.g. [http://localhost:5601/](http://localhost:5601/)) and define an index pattern for "dotnet-*" with the time filter "@timestamp". + +You can then discover the log events for the index. Some useful columns to add are `log.level`, `log.logger`, `event.code`, `message`, `tags`, and `process.thread.id`. + +If you are running multiple applications or on multiple servers, you might want to include `service.type`, `service.version`, and `host.hostname`. + +Additional fields are defined below, and all individual message and scope values are logged as `labels.*` custom key/value pairs, e.g. `labels.CustomerId`. + + +## Basic configuration [_basic_configuration] + +For deployment you will usually want to override the configuration with your actual server location(s). The other useful value to configure is a tag for the environment, e.g. Development/Staging/Production. + +```json +{ + "Logging": { + "Elasticsearch": { + "NodeUris": [ "https://elastic-staging.example.com:9200" ], + "Tags": [ "Staging" ] + } + } +} +``` + +**NOTE:** You don’t need any configuration to just use a local Elasticsearch instance, as it defaults to [http://localhost:9200/](http://localhost:9200/). + + +## Configuration settings [_configuration_settings] + +The logger provider will be automatically configured with any logging settings under the alias `Elasticsearch`. + +The following default settings are used. + +```json +{ + "Logging": { + "Elasticsearch": { + "IncludeHost": true, + "IncludeProcess": true, + "IncludeScopes": true, + "IncludeUser": true, + "Index": "dotnet-{0:yyyy.MM.dd}", + "IndexOffset": null, + "IsEnabled": true, + "ListSeparator": ", ", + "MapCorrelationValues": true, + "Tags": [], + "ShipTo": { + "NodePoolType": "SingleNode", + "NodeUris": [ "http://localhost:9200" ] + } + } + } +} +``` + +| Setting | Type | Description | +| --- | --- | --- | +| IncludeHost | boolean | Default `true`; set to `false` to disable logging host values. | +| IncludeProcess | boolean | Default `true`; set to `false` to disable logging process values. | +| IncludeScopes | boolean | Default `true`; set to `false` to disable logging scope values. | +| IncludeUser | boolean | Default `true`; set to `false` to disable logging user details. | +| Index | format | Format string used to generate the Elasticsearch `index`, using the current timestamp. Default is `dotnet-{0:yyyy.MM.dd}`. | +| IndexOffset | timespan | Override to set the offset used to generate the `index`. Default value is `null`, which uses the system local offset; use `"00:00&"` for UTC. | +| IsEnabled | boolean | Default `true`; set to `false` to disable the logger. | +| ListSeparator | string | Separator to use for `IEnumerable` in `labels.*` values. Default is `", "`. | +| Tags | array | Additional tags to include in the message. Useful to specify the environment or other details, e.g. `[ "Staging", "Priority"]` | + +The ShipTo settings can have the following properties, depending on the type of connection pool. + +| Setting | Type | Description | +| --- | --- | --- | +| ApiKey | string | API Key, where connection pool type is Cloud, and authenticating via API Key. | +| CloudId | string | Cloud ID, where connection pool type is Cloud. | +| NodePoolType | enum | Default is `Singlenode`, or `Sniffing` for multiple nodes, or `Cloud` if `CloudId` is provided. Other supported values are `Static` or `Sticky`. | +| NodeUris | array | URI(s) of the Elasticsearch nodes to connect to. Default is a single node `[ "http://localhost:9200" ]` | +| Password | string | Password, where connection pool type is Cloud, and authenticating via username/password. | +| Username | string | Username, where connection pool type is Cloud, and authenticating via username/password. | + +If you want to configure from a different section, it can be configured manually: + +```csharp + .ConfigureLogging((hostContext, loggingBuilder) => + { + loggingBuilder.AddElasticsearch(options => + hostContext.Configuration.Bind("Logging:CustomElasticsearch", options)); + }) +``` + +Configuration can, of course, also be done in code, e.g. to add the environment as a tag. + + +## Elastic Cloud configuration [_elastic_cloud_configuration] + +If `CloudId` is provided, the `ConnectionPoolType` defaults to `Cloud`: + +```json +{ + "Logging": { + "Elasticsearch": { + "ShipTo": { + "CloudId": "12345", + "ApiKey": "abcdef" + } + } + } +} +``` + + +## Output - Elastic Common Schema (ECS) [_output_elastic_common_schema_ecs] + +Log messages sent to Elasticsearch follow the [Elastic Common Schema (ECS)][Elastic Common Schema (ECS)](ecs://docs/reference/index.md)). + + +## Example document [_example_document] + +The `_source` field is the message sent from the LoggerProvider, along with the `_index` and `_id` (a GUID). + +```json +{ + "_index": "dotnet-2020.04.12", + "_type": "_doc", + "_id": "563503a8-9d10-46ff-a09f-c6ccbf124db9", + "_version": 1, + "_score": null, + "_source": { + "MessageTemplate": "Unexpected error processing customer {CustomerId}.", + "Scopes": [ + "IP address 2001:db8:85a3::8a2e:370:7334", + "PlainScope" + ], + "agent": { + "version": "1.0.0+bd3ad6", + "type": "Elastic.Extensions.Logging.LoggerProvider" + }, + "ecs": { + "version": "1.5.0" + }, + "error": { + "message": "Calculation error", + "type": "System.Exception", + "stack_trace": "System.Exception: Calculation error\n ---> System.DivideByZeroException: Attempted to divide by zero.\n at HelloElasticsearch.Worker.ExecuteAsync(CancellationToken stoppingToken) in /home/sly/Code/essential-logging/examples/HelloElasticsearch/Worker.cs:line 80\n --- End of inner exception stack trace ---\n at HelloElasticsearch.Worker.ExecuteAsync(CancellationToken stoppingToken) in /home/sly/Code/essential-logging/examples/HelloElasticsearch/Worker.cs:line 84" + }, + "event": { + "code": "5000", + "action": "ErrorProcessingCustomer", + "severity": 3 + }, + "host": { + "os": { + "platform": "Unix", + "full": "Linux 4.15.0-91-generic #92-Ubuntu SMP Fri Feb 28 11:09:48 UTC 2020", + "version": "4.15.0.91" + }, + "hostname": "VUB1804", + "architecture": "X64" + }, + "log": { + "level": "Error", + "logger": "HelloElasticsearch.Worker" + }, + "process": { + "thread": { + "id": 10 + }, + "pid": 25982, + "name": "HelloElasticsearch" + }, + "service": { + "type": "HelloElasticsearch", + "version": "1.0.0" + }, + "user": { + "id": "sgryphon+es@live.com", + "name": "sly", + "domain": "VUB1804" + }, + "@timestamp": "2020-04-13T21:25:22.3352989+10:00", + "tags": [ + "Development" + ], + "labels": { + "ip": "2001:db8:85a3::8a2e:370:7334", + "CustomerId": "12345" + }, + "message": "Unexpected error processing customer 12345.", + "trace": { + "id": "c20bde1071f7cf4e9a6f368c824e05f7" + }, + "transaction": { + "id": "92ba5ee64d963746" + } + }, + "fields": { + "@timestamp": [ + "2020-04-13T11:25:22.335Z" + ] + }, + "sort": [ + 1586777122335 + ] +} +``` + + +## Standard fields [_standard_fields] + +| Field | Type | Description | +| --- | --- | --- | +| @timestamp | date | `DateTimeOffset` when the message was logged, including local offset. | +| message | string | The formatted log message and arguments. | +| tags | array | Custom tags from configuration, e.g. `[ "Staging", "Priority" ]`. Can have multiple values. | +| event.action | string | The name of the logged EventId, e.g. `ErrorProcessingCustomer`. | +| event.code | string | The numeric value (as a string) of the EventId, e.g. `5000`. | +| event.severity | long | The syslog severity corresponding to the log level, 2 = critical, 3 = error, 4 = warning, 6 = information, 7 = debug and trace. (Also used in the Systemd format of ConsoleLoggerProvider) | +| log.level | string | The log level: `Critical`, `Error`, `Warning`, `Information`, `Debug`, or `Trace`. | +| log.logger | string | The category name (namespace and class) of the logger, e.g. `HelloElasticsearch.Worker`. | + +The `event.severity` field is numeric and can be used to order events by level, e.g. Kibana query `event.severity <= 4` will get all messages that have log level `Warning` or worse. + + +## Error fields [_error_fields] + +If the log message includes an exception, the details are reported in the error fields. + +| Field | Type | Description | +| --- | --- | --- | +| error.message | string | The `Message` property of any exception. | +| error.stack_trace | string | Full details of the exception, `Exception.ToString()`, including the stack trace and the stack trace of any inner exceptions. | +| error.type | string | The type of the error message, e.g. `System.DivideByZeroException` | + + +## Custom fields [_custom_fields] + +Following the ECS conventions, these use alternative Title casing, to identify them as non-standard fields. + +| Field | Type | Description | +| --- | --- | --- | +| MessageTemplate | string | The original message template, e.g. "Unexpected error processing customer `{{CustomerId}}`." | +| Scopes | array | Array of string formatted scope values, in the order added. | + + +## Label values [_label_values] + +| Field | Type | Description | +| --- | --- | --- | +| labels.* | string | Custom key/value pairs of all named parameter values and named scope values. All values are strings (no nested objects). | + +Label values can be accessed by their key, for example if the message, or scope, includes a parameter CustomerId, then the value will be logged as `labels.CustomerId` can be searched in Kibana using "labels.CustomerId: 12345". + +**Example:** + +The following will generate two labels, `labels.EndTime` from the message and `labels.CustomerId` from the scope: + +```csharp +using (_logger.BeginScope("{CustomerId}", customerId)) +{ + _logger.LogWarning("End of processing reached at {EndTime}.", end); +} +``` + +Labels are taken from the message (state) and any scope values (may be disabled via the configuration options). In Microsoft.Extensions.Logging an internal FormattedLogValues is used in the ILogger overloads for log levels and scopes; it implements the `IEnumerable<KeyValuePair<string,object>>` interface that is used to extract the individual parameter values. + +The `labels` property in ECS should not contain nested objects, so values are converted to keyword strings. For most objects this is just calling ToString(), with specific formats for some types, e.g. calling string on a list is usually not very useful, so the contents of the list is logged instead. + +**Labels value formatting** + +| Type | Formatting | +| --- | --- | +| byte | Hex, e.g. "9A" | +| byte[] | Prefixed hex, e.g. "0x12789AF0" | +| DateTimeOffset | ISO format, e.g. "2020-01-02T03:04:05.000000+06:00" | +| DateTime | In most cases `DateTimeOffset` should be used instead (1). Where `DateTime` is used for date only (with no time component), it is formatted as a date, e.g. "2020-01-02". If it has a time component, the roundtrip ("o") format is used. | +| IEnumerable | Values separated by ", " (configurable) | +| IDictionary | A string containing key value pairs, e.g. `token="0x12789AF0" count="5"` | +| *other values* | The result of `ToString()`, including scalar values, e.g. the number `5.3` is logged as the string "5.3" | + +**(1) See [https://docs.microsoft.com/en-us/dotnet/standard/datetime/choosing-between-datetime**](https://docs.microsoft.com/en-us/dotnet/standard/datetime/choosing-between-datetime**) + + +## Agent fields [_agent_fields] + +These identify the version of the logger provider being used. + +| Field | Type | Description | +| --- | --- | --- | +| agent.type | string | Name of the logger provider assembly, `Elastic.Extensions.Logging.LoggerProvider`. | +| agent.version | string | Informational version number of the logger assembly, e.g. `1.1.1+bd3ad63`. | +| ecs.version | string | Version of ECS standard used, currently `1.5`. | + + +## Service fields [_service_fields] + +This identifies the application/service that is running and generating the logs. + +The values are pulled from the entry assemb, `Assembly.GetEntryAssembly()`, using the `Name` and `AssemblyInformationalVersionAttribute` values (if informational version is not set it falls back to assembly `Version`). + +| Field | Type | Description | +| --- | --- | --- | +| service.type | string | Name of the entry assembly, `HelloElasticsearch`. | +| service.version | string | Informational version number of the entry assembly, e.g. `1.2.0-beta.1+79d095a`. | + +**Note:** You should be using a build process that sets the assembly informational version correctly. e.g. If you have a dotnet project using git you can install the local tool `GitVersion.Tool`, and use it to automatically generate semantic version numbers from the git branch information. + +To install the tool: + +```powershell +dotnet new tool-manifest +dotnet tool install GitVersion.Tool +``` + +Then use the tool to create a semantic version number that can be used in your build process: + +```powershell +dotnet tool restore +dotnet gitversion +``` + +% You are welcome to use the [`build.ps1`](../../build.ps1) script in this repository as an example. + + +## Tracing fields [_tracing_fields] + +| Field | Type | Description | +| --- | --- | --- | +| trace.id | string | Cross-service trace correlation identifier. From `Activity.Current.RootId` from `System.Diagnostics`, with a fallback to `CorrelationManager.ActivityId`. Can be overridden by a message or scope value `trace.id`. | +| transaction.id | string | Transaction for this service, e.g. individual request identifier. If in W3C format, parse out the SpanId from `Activity.Current.Id` from `System.Diagnostics`, otherwise just use the full `Activity.Current.Id` (e.g. if hierarchical). Can be overridden by message or scope value `transaction.id`. | + +ASP.NET will automatically pass correlation identifiers between tiers; from 3.0 it also supports the W3C Trace Context standard ([https://www.w3.org/TR/trace-context/](https://www.w3.org/TR/trace-context/)). + +The value of `Activity.Current.RootId` is used as the cross-service identifier (in W3C format this is the Trace ID), if in W3C format the Span ID portion of `Activity.Current.Id` is used for the transaction, otherwise the full value is used (this is consistent with the way ASP.NET works). + +It is recommended to turn on W3C format, for compatibility with other systems: + +```csharp +Activity.DefaultIdFormat = ActivityIdFormat.W3C; +``` + + +## Host fields [_host_fields] + +::::{note} +Can be disabled via configuration. +:::: + + +| Field | Type | Description | +| --- | --- | --- | +| host.architecture | string | The processor architecture, e.g. X64. Value of `RuntimeInformation.OSArchitecture`. | +| host.hostname | string | The computer name. Value of `Environment.MachineName`. | +| host.os.full | string | Full description of the operation system. Value of `RuntimeInformation.OSDescription`. | +| host.os.platform | string | Operating system platform. Value of `Environment.OSVersion.Platform`. | +| host.os.version | string | Operating system version. Value of `Environment.OSVersion.Version`. | + + +## Process fields [_process_fields] + +::::{note} +Can be disabled via configuration. +:::: + + +| Field | Type | Description | +| --- | --- | --- | +| process.name | string | The current process name. From `Process.GetCurrentProcess()`. | +| process.pid | long | The current process ID. From `Process.GetCurrentProcess()`. | +| process.thread.id | long | Current thread ID. Value of `Thread.CurrentThread.ManagedThreadId`. | +| process.thread.name | string | Name of the thread. From `Thread.CurrentThread.Name`. | + + +## User fields [_user_fields] + +::::{note} +Can be disabled via configuration. +:::: + + +| Field | Type | Description | +| --- | --- | --- | +| user.domain | string | The current domain, either the machine name or a Windows domain. Value of `Environment.UserDomainName`. | +| user.id | string | Current user principal name, if set. Value of `Thread.CurrentPrincipal.Identity.Name`. | +| user.name | string | The current user. Value of `Environment.UserName`. | + + diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..6a3c1c0e --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,74 @@ +--- +mapped_pages: + # TO DO: Do we want this in addition to an intro page? + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/index.html + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/intro.html +--- + +# ECS Logging .NET [intro] + +ECS logging are integrations for your favorite .NET logging libraries. They make it easy to format your logs into ECS-compatible JSON. + +::::{tip} +Want to learn more about ECS, ECS logging, and other available language plugins? See the [ECS logging overview](ecs-logging://docs/reference/intro.md). +:::: + + +The .NET ECS libraries fall into several categories: + + +## Model [_model] + +At its core all .NET ECS libraries are powered by `Elastic.CommonSchema` which strongly types the ECS specification to .NET classes. + +This library can be used to map your events to ECS in a typesafe fashion. + +```csharp +var doc = EcsDocument.CreateNewWithDefaults(); +doc = new EcsDocument(); +``` + +[Lean more about Elastic.CommonSchema](/reference/ecs-dotnet.md). + + +## Log formatters [_log_formatters] + +Our log formatters allow you to change the way various logging frameworks log to IO (file/console) to use ECS json. + +* [`Elastic.CommonSchema.Serilog`](/reference/serilog-formatter.md) +* [`Elastic.CommonSchema.NLog`](/reference/nlog-formatter.md) +* [`Elastic.CommonSchema.Log4Net`](/reference/log4net-formatter.md) + + +## Data shippers [_data_shippers] + +Our datashippers integrate with logging frameworks to facilitate sending events (logs) to various off Elastic receivers. + +Currently these shippers support Elastic Cloud & Elasticsearch but other outputs are in the works. + +* [`Elastic.Serilog.Sinks`](/reference/serilog-data-shipper.md) +* [`Elastic.Extensions.Logging`](/reference/extensions-logging-data-shipper.md) +* [`Elastic.CommonSchema.BenchmarkDotNetExporter`](/reference/benchmark-dotnet-data-shipper.md) + +All the data shippers utilize [`Elastic.Ingest.Elasticsearch.CommonSchema`](/reference/ecs-ingest-channels.md) to send events to Elasticsearch. + + +## Enrichers [_enrichers] + +Enrichers can be installed next to `Log Formatters` and `Data Shippers` to automatically enrich the ECS json that gets produced. + +* [`Elastic.Apm.SerilogEnricher`](/reference/apm-serilog-enricher.md) +* [`Elastic.Apm.NLog`](/reference/apm-nlog-enricher.md) + + +## Architecture [_architecture] + +The libraries reuse the same components that power other .NET libraries from Elastic—​ensuring a common way to both configure and monitor Elastic’s .NET libraries. + +![ECS.NET Artichtecture overview](../images/ecs-dotnet-overview.png "") + +At its core all data shippers depend on [Elastic.Transport](https://github.com/elastic/elastic-transport-net) to coordinate HttpRequests. This library is also the heart of Elastic’s clients and ensures we share best practices and configuration options. + +The data shipper libraries all depend upon the push based ingestion abstractions from [Elastic.Ingest.*](https://github.com/elastic/elastic-ingest-dotnet) that presents an easy to use `System.Threading.Channels` backed method to push events as batches at variable rates to external datasources. + +[Lean more about Elastic.Ingest.Elasticsearch.CommonSchema](/reference/ecs-ingest-channels.md). diff --git a/docs/reference/intro_to_xyz.md b/docs/reference/intro_to_xyz.md new file mode 100644 index 00000000..e52874aa --- /dev/null +++ b/docs/reference/intro_to_xyz.md @@ -0,0 +1,19 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/intro_to_xyz.html +--- + +# A note on the Metadata property [intro_to_xyz] + +The C# `EcsDocument` type includes a property called `Metadata` with the signature: + +```csharp +/// +/// Container for additional metadata against this event. +/// +[JsonPropertyName("metadata"), DataMember(Name = "metadata")] +public IDictionary Metadata { get; set; } +``` + +This property is not part of the ECS specification, but is included as a means to index supplementary information. + diff --git a/docs/formatters/log4net.asciidoc b/docs/reference/log4net-formatter.md similarity index 69% rename from docs/formatters/log4net.asciidoc rename to docs/reference/log4net-formatter.md index 28b6a01c..c3b43cbc 100644 --- a/docs/formatters/log4net.asciidoc +++ b/docs/reference/log4net-formatter.md @@ -1,26 +1,28 @@ -[[log4net-formatter]] -=== log4net +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/log4net-formatter.html +--- + +# log4net [log4net-formatter] This Layout implementation formats a log4net event into a JSON representation that adheres to the Elastic Common Schema specification. -==== Installation +## Installation [_installation_4] + +Add a reference to the [Elastic.CommonSchema.Log4net](http://nuget.org/packages/Elastic.CommonSchema.Log4net) package: -Add a reference to the http://nuget.org/packages/Elastic.CommonSchema.Log4net[Elastic.CommonSchema.Log4net] package: +```xml + +``` -[source,xml] -[subs="attributes"] ----- - ----- -==== Usage +## Usage [_usage_4] -===== Setup using configuration +### Setup using configuration [_setup_using_configuration] -Specify layout type in appender's configuration: +Specify layout type in appender’s configuration: -[source,xml] ----- +```xml @@ -30,33 +32,31 @@ Specify layout type in appender's configuration: +``` ----- -===== Setup programatically +### Setup programatically [_setup_programatically_2] -[source,csharp] ----- +```csharp var hierarchy = (Hierarchy)LogManager.CreateRepository(Guid.NewGuid().ToString()); var appender = new ConsoleAppender { Layout = new EcsLayout() }; // Use the ECS layout. hierarchy.Root.AddAppender(appender); hierarchy.Root.Level = Level.All; hierarchy.Configured = true; +``` + +The `Layout = new EcsLayout()` line then instructs log4net to use ECS layout. The sample above uses the console appender, but you are free to use any appender of your choice, perhaps consider using a filesystem target and [Elastic Filebeat](https://www.elastic.co/downloads/beats/filebeat) for durable and reliable ingestion. ----- -The `Layout = new EcsLayout()` line then instructs log4net to use ECS layout. -The sample above uses the console appender, but you are free to use any appender of your choice, perhaps consider using a -filesystem target and https://www.elastic.co/downloads/beats/filebeat[Elastic Filebeat] for durable and reliable ingestion. -==== ECS Aware Properties +## ECS Aware Properties [_ecs_aware_properties] -Any valid ECS log template properties that is available under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId` -is supported and will directly set the appropriate ECS field. +Any valid ECS log template properties that is available under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId` is supported and will directly set the appropriate ECS field. -==== Output -Apart from {ecs-ref}/ecs-guidelines.html#_general_guidelines[mandatory fields], the output contains additional data: +## Output [_output] + +Apart from [mandatory fields](ecs://docs/reference/ecs-guidelines.md#_general_guidelines), the output contains additional data: * `log.origin.file.name` is taken from `LocationInformation` * `log.origin.file.line` is taken from `LocationInformation` @@ -65,7 +65,7 @@ Apart from {ecs-ref}/ecs-guidelines.html#_general_guidelines[mandatory fields], * `event.timezone` is equal to local timezone * `host.hostname` is taken from `HostName` property * `process.thread.id` is taken from `ThreadName` if it has numeric value -* `process.thread.name` is taken from `ThreadName` if it doesn't have numeric value +* `process.thread.name` is taken from `ThreadName` if it doesn’t have numeric value * `service.name` is taken from entry or calling assembly * `service.version` is taken from entry or calling assembly * `error.message` is taken from `ExceptionObject` @@ -75,8 +75,7 @@ Apart from {ecs-ref}/ecs-guidelines.html#_general_guidelines[mandatory fields], Sample log event output (formatted for readability): -[source,json] ----- +```json { "@timestamp": "2022-08-28T14:06:28.5121651+02:00", "log.level": "INFO", @@ -117,4 +116,6 @@ Sample log event output (formatted for readability): "version": "1.0.0.0" } } ----- \ No newline at end of file +``` + + diff --git a/docs/reference/nlog-formatter.md b/docs/reference/nlog-formatter.md new file mode 100644 index 00000000..492ff1cc --- /dev/null +++ b/docs/reference/nlog-formatter.md @@ -0,0 +1,164 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/nlog-formatter.html +--- + +# NLog layout [nlog-formatter] + +This Layout implementation formats an NLog event into a JSON representation that adheres to the Elastic Common Schema specification. + +## Installation [_installation_3] + +Add a reference to the Elastic.CommonSchema.NLog package: + +```xml + +``` + + +## Usage [_usage_3] + +### Setup programatically [_setup_programatically] + +```csharp +Layout.Register("EcsLayout"); // Register the ECS layout. +var config = new LoggingConfiguration(); +var consoleTarget = new ConsoleTarget("console") { Layout = new EcsLayout() }; // Use the ECS layout. +config.AddRule(LogLevel.Debug, LogLevel.Fatal, consoleTarget); +LogManager.Configuration = config; +var logger = LogManager.GetCurrentClassLogger(); +``` + +In the code snippet above `Layout.Register("EcsLayout")` registers the `EcsLayout` with NLog. The `Layout = new EcsLayout()` line then instructs NLog to use the registered layout. The sample above uses the console target, but you are free to use any target of your choice; perhaps consider using a filesystem target and [Elastic Filebeat](https://www.elastic.co/downloads/beats/filebeat) for durable and reliable ingestion. + + +### Setup using NLog.config [_setup_using_nlog_config] + +```xml + + + + + + + + + + + + + + + + +``` + + + +## EcsLayout Parameter Options [_ecslayout_parameter_options] + +* **Metadata Options** +* *IncludeEventProperties* - Include LogEvent properties as metadata. Default: `true` +* *IncludeScopeProperties* - Include NLog Scope Context Properties as metadata. Default: `false` +* *ExcludeProperties* - Comma separated string with names which properties to exclude. +* **Event Options** +* *EventAction* - +* *EventCategory* - +* *EventId* - +* *EventKind* - +* *EventSeverity* - +* **Agent Options** +* *AgentId* - +* *AgentName* - +* *AgentType* - +* *AgentVersion* - +* **Process Options** +* *ProcessExecutable* - Default: `${processname:FullName=true}` +* *ProcessId* - Default: `${processid}` +* *ProcessName* - Default: `${processname:FullName=false}` +* *ProcessThreadId* - Default: `${threadid}` +* *ProcessTitle* - Default: `${processinfo:MainWindowTitle}` +* **Server Options** +* *ServerAddress* - +* *ServerIp* - +* *ServerUser* - Default: `${environment-user}` +* **Host Options** +* *HostId* - +* *HostIp* - Default: `${local-ip:cachedSeconds=60}` +* *HostName* - Default: `${machinename}` +* **Log Origin Options** +* *LogOriginCallSiteMethod* - Default: `${exception:format=method}` +* *LogOriginCallSiteFile* - Default: `${exception:format=source}` +* *LogOriginCallSiteLine* - +* **Http Options** +* *HttpRequestId* - Default: `${aspnet-trace-identifier}` +* *HttpRequestMethod* - Default: `${aspnet-request-method}` +* *HttpRequestBytes* - Default: `${aspnet-request-contentlength}` +* *HttpRequestReferrer* - Default: `${aspnet-request-referrer}` +* *HttpResponseStatusCode* - Default: `${aspnet-response-statuscode}` +* **Url Options** +* *UrlScheme* - Default: `${aspnet-request-url:IncludeScheme=true:IncludeHost=false:IncludePath=false}` +* *UrlDomain* - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=true:IncludePath=false}` +* *UrlPath* - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=false:IncludePath=true}` +* *UrlPort* - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=false:IncludePath=false:IncludePort=true}` +* *UrlQuery* - Default: `${aspnet-request-url:IncludeScheme=false:IncludeHost=false:IncludePath=false:IncludeQueryString=true}` +* *UrlUserName* - Default: `${aspnet-user-identity}` +* **Trace Options** +* *ApmTraceId* - Default: `${ElasticApmTraceId}` +* **Transaction Options** +* *ApmTransactionId* - Default: `${ElasticApmTransactionId}` * + + +## ECS Aware Message Templates [_ecs_aware_message_templates_3] + +Additionally any valid ECS log template properties that is available under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId` is supported and will directly set the appropriate ECS fields. + +```csharp +logger.Info("The time is {TraceId}", "my-trace-id"); +``` + +Will override `trace.id` on the resulting ECS json document. + + +## Example output from EcsLayout [_example_output_from_ecslayout] + +An example of the output is given below: + +```json +{ + "@timestamp":"2020-02-20T16:07:06.7109766+11:00", + "log.level":"Info", + "message":"Info \"X\" 2.2", + "metadata":{ + "value_x":"X", + "some_y":2.2 + }, + "ecs":{ + "version":"1.4.0" + }, + "event":{ + "severity":6, + "timezone":"AUS Eastern Standard Time", + "created":"2020-02-20T16:07:06.7109766+11:00" + }, + "host":{ + "name":"LAPTOP" + }, + "log":{ + "logger":"Elastic.CommonSchema.NLog", + "original":"Info {ValueX} {SomeY}" + }, + "process":{ + "thread":{ + "id":17592 + }, + "pid":17592, + "name":"dotnet", + "executable":"C:\\Program Files\\dotnet\\dotnet.exe" + } +} +``` + + diff --git a/docs/reference/serilog-data-shipper.md b/docs/reference/serilog-data-shipper.md new file mode 100644 index 00000000..e67fc6d5 --- /dev/null +++ b/docs/reference/serilog-data-shipper.md @@ -0,0 +1,153 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/serilog-data-shipper.html +--- + +# Elastic.Serilog.Sinks [serilog-data-shipper] + +A [Serilog](https://serilog.net/) sink that writes logs directly to [Elasticsearch](https://www.elastic.co/elasticsearch/) or [Elastic Cloud](https://www.elastic.co/cloud) + +## Installation [_installation_6] + +Add a reference to the `Elastic.Serilog.Sinks` package: + +```xml + +``` + + +## Usage [_usage_6] + +There’s a few ways that you can extend a `Serilog` `LoggerConfiguration`: + +```csharp +Log.Logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.FromLogContext() +``` + +**NOTE:** Don’t forget we also publish an [`Elastic.Apm.SerilogEnricher`](https://github.com/elastic/ecs-dotnet/blob/main/src/Elastic.Apm.SerilogEnricher/readme.md) for the Elastic APM Agent! + +Writing to `Elasticsearch` + +```csharp +.WriteTo.Elasticsearch(new [] { new Uri("http://localhost:9200" )}, opts => +{ + opts.DataStream = new DataStreamName("logs", "console-example", "demo"); + opts.BootstrapMethod = BootstrapMethod.Failure; + opts.ConfigureChannel = channelOpts => + { + channelOpts.BufferOptions = new BufferOptions + { + ConcurrentConsumers = 10 + }; + }; +}, transport => +{ + // transport.Authentication(new BasicAuthentication(username, password)); // Basic Auth + // transport.Authentication(new ApiKey(base64EncodedApiKey)); // ApiKey +}) +``` + +Writing to `Elastic Cloud`: + +```csharp +.WriteTo.ElasticCloud("cloudId", "cloudUser", "cloudPass", opts => +``` + +`opts` is an instance of `ElasticsearchSinkOptions` with the following options + + +## Configuration [_configuration_2] + +| Option | Description | +| --- | --- | +| `Transport` | An instance of `Elastic.Transport` that dictates where and how we are communicating to. Defaults to `http://localhost:9200` | +| `DataStream` | Where to write data, defaults to the `logs-dotnet-default` datastream. | +| `BootstrapMethod` | Wheter the sink should attempt to install component and index templates to ensure the datastream has ECS mappings. Can be be either `None` (the default), `Silent` (attempt but fail silently), `Failure` (attempt and fail with exceptions if bootstrapping fails). | +| `TextFormatting` | Allows explicit control of over the `EcsTextFormatterConfiguration` used to emit ECS json documents. See [`Elastic.CommonSchema.Serilog`](https://github.com/elastic/ecs-dotnet/tree/main/src/Elastic.CommonSchema.Serilog) for available options. | +| `ConfigureChannel` | A callback receiving the `DatastreamChannelOptions` which allows you to control sizing, backpressure etc. See [`Elastic.Ingest.Elasticsearch`](https://github.com/elastic/elastic-ingest-dotnet/blob/main/src/Elastic.Ingest.Elasticsearch/README.md#elasticingestelasticsearch) for more information. | + +Note that you can also pass `ElasticsearchSinkOptions` directly + +```csharp +.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(client.Transport)) +``` + +This allows you to reuse the `Transport` used by the Elasticsearch Client for instance. + + +## Authentication [_authentication] + +When {{es}} security features are enabled, requests without a valid authentication header will be rejected. You can enable authentication via one of the methods below: + +**Basic Auth** + +```csharp +.WriteTo.Elasticsearch(new [] { new Uri("http://localhost:9200" )}, opts => +{ + ... +}, transport => +{ + transport.Authentication(new BasicAuthentication(username, password)); <1> +}) +``` + +1. Basic authentication + + +**API Key** + +```csharp +.WriteTo.Elasticsearch(new [] { new Uri("http://localhost:9200" )}, opts => +{ + ... +}, transport => +{ + transport.Authentication(new ApiKey(base64EncodedApiKey)); <1> +}) +``` + +1. API Key + + +To learn more about authentication with the {{stack}}, see [User Authentication](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/user-authentication.md). + + +## ECS aware message templates [_ecs_aware_message_templates_4] + +This sink by proxy of its formatter allows you to set ECS fields directly from the message template using properties that adhere to the [https://messagetemplates.org/](https://messagetemplates.org/) format. + +The available ECS message template properties are listed under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId` + +```csharp +Log.Information("The time is {TraceId}", "my-trace-id"); +``` + +Will override `trace.id` on the resulting ECS json document. + + +## Troubleshooting [_troubleshooting] + +In case of issues, you can enable the [Serilog Self-Log feature](https://github.com/serilog/serilog/wiki/Debugging-and-Diagnostics#selflog) to expose any error you might have encountered. + + +## Comparison with [`Serilog.Sinks.Elasticsearch`](https://github.com/serilog-contrib/serilog-sinks-elasticsearch) [_comparison_with_serilog_sinks_elasticsearchhttpsgithub_comserilog_contribserilog_sinks_elasticsearch] + +* `Serilog.Sinks.Elasticsearch` is an amazing community led sink that has a ton of options and works against older Elasticsearch versions `< 8.0`. +* `Serilog.Sinks.Elasticsearch` is unofficially supported by Elastic with some of the .NET team helping to maintain it. +* `Elastic.Serilog.Sinks` is **officially** supported by Elastic and was purposely build to adhere to newer best practices around logging, datastreams and ILM. +* `Elastic.Serilog.Sinks` is purposely build to have fewer configuration options and be more prescriptive than `Serilog.Sinks.Elasticsearch`. +* That is not to say there aren’t plenty of configuration hooks in `Elastic.Serilog.Sinks` + +### Notable absent features: [_notable_absent_features] + +* `Elastic.Serilog.Sinks` only works with `Elasticsearch 8.x` and up. +* This is because the bootrapping (`BootstrapMethod`) attempts to load templates build for Elasticsearch 8.0 and up. +* `Elastic.Serilog.Sinks` has only one way it emits data to Elasticsearch confirming to the [ecs-logging specification](https://github.com/elastic/ecs-logging) +* That doesn’t mean you can not introduce your own additional properties though. +* `Elastic.Serilog.Sinks` has no durable mode. +* If you need higher guarantees on log delivery use [`Serilog.Sinks.File`](https://github.com/serilog/serilog-sinks-file) with our [ECS log formatter](https://www.nuget.org/packages/Elastic.CommonSchema.Serilog/) for Serilog and use [filebeat](https://www.elastic.co/beats/filebeat) to ship these logs. +* Check out [Elastic Agent and Fleet](docs-content://reference/ingestion-tools/fleet/index.md) to simplify collecting logs and metrics on the edge. + +If you miss a particular feature from `Serilog.Sinks.Elasticsearch` in `Elastic.Serilog.Sinks` please open a [feature request](https://github.com/elastic/ecs-dotnet/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=%5BFEATURE%5D)! We’d love to grow this sink organically moving forward. diff --git a/docs/formatters/serilog.asciidoc b/docs/reference/serilog-formatter.md similarity index 58% rename from docs/formatters/serilog.asciidoc rename to docs/reference/serilog-formatter.md index c7888fe4..fc6c1058 100644 --- a/docs/formatters/serilog.asciidoc +++ b/docs/reference/serilog-formatter.md @@ -1,34 +1,34 @@ -[[serilog-formatter]] -=== Serilog formatter +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/serilog-formatter.html +--- + +# Serilog formatter [serilog-formatter] This `ITextFormatter` implementation formats a Serilog event into a JSON representation that adheres to the Elastic Common Schema specification. -==== Installation +## Installation [_installation_2] + +Add a reference to the [Elastic.CommonSchema.Serilog](http://nuget.org/packages/Elastic.CommonSchema.Serilog) package: -Add a reference to the http://nuget.org/packages/Elastic.CommonSchema.Serilog[Elastic.CommonSchema.Serilog] package: +```xml + +``` -[source,xml] -[subs="attributes"] ----- - ----- -==== Usage +## Usage [_usage_2] -[source,csharp] ----- +```csharp var logger = new LoggerConfiguration() .WriteTo.Console(new EcsTextFormatter()) .CreateLogger(); +``` ----- - -In the code snippet above `new EcsTextFormatter()` enables the text formatter and instructs Serilog to format the event as JSON. The sample above uses the Console sink, but you are free to use any sink of your choice, perhaps consider using a filesystem sink and https://www.elastic.co/downloads/beats/filebeat[Elastic Filebeat] for durable and reliable ingestion. +In the code snippet above `new EcsTextFormatter()` enables the text formatter and instructs Serilog to format the event as JSON. The sample above uses the Console sink, but you are free to use any sink of your choice, perhaps consider using a filesystem sink and [Elastic Filebeat](https://www.elastic.co/downloads/beats/filebeat) for durable and reliable ingestion. In ASP.NET (core) applications -[source,csharp] ----- +```csharp .UseSerilog((ctx, config) => { // Ensure HttpContextAccessor is accessible @@ -39,15 +39,13 @@ In ASP.NET (core) applications .Enrich.WithEcsHttpContext(httpAccessor) .WriteTo.Async(a => a.Console(new EcsTextFormatter())); }) - ----- +``` The `WithEcsHttpContext` ensures logs will be enriched with `HttpContext` data. An example of the output is given below: -[source,json] ----- +```json { "@timestamp": "2019-11-22T14:59:02.5903135+11:00", "log.level": "Information", @@ -70,30 +68,29 @@ An example of the output is given below: "executable": "System.Threading.ExecutionContext" } } +``` ----- -==== Configuration +## Configuration [_configuration] -|=== -|Option |Description +| Option | Description | +| --- | --- | +| `MapCurrentThead` | `true` map `ecs.process` by looking up the `Process` from the current thread | +| `MapHttpAdapter` | `null` a way to map `HttpContextAccessor` to ECS fields. | +| `LogEventPropertiesToFilter` | A `Set` of properties that should not be emitted as `labels.*` or `metadata.*` | +| `MapCustom` | A Func that allows you to mutate the EcsDocument before its fully converted. | -|`MapCurrentThead` | `true` map `ecs.process` by looking up the `Process` from the current thread -|`MapHttpAdapter` | `null` a way to map `HttpContextAccessor` to ECS fields. -|`LogEventPropertiesToFilter` | A `Set` of properties that should not be emitted as `labels.*` or `metadata.*` -|`MapCustom` | A Func that allows you to mutate the EcsDocument before its fully converted. -|=== -==== ECS Aware Message Templates +## ECS aware message templates [_ecs_aware_message_templates_2] -This formatter also allows you to set ECS fields directly from the message template using properties that adhere to the -https://messagetemplates.org/ format. +This formatter also allows you to set ECS fields directly from the message template using properties that adhere to the [https://messagetemplates.org/](https://messagetemplates.org/) format. The available ECS message template properties are listed under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId` -[source,chsarp] ----- +```csharp Log.Information("The time is {TraceId}", "my-trace-id"); ----- +``` + +Will override `trace.id` on the resulting ECS json document. + -Will override `trace.id` on the resulting ECS json document. diff --git a/docs/reference/setup.md b/docs/reference/setup.md new file mode 100644 index 00000000..373e9059 --- /dev/null +++ b/docs/reference/setup.md @@ -0,0 +1,128 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/ecs-logging/dotnet/current/setup.html +navigation_title: 'Get started' +--- + +# Get started with ECS Logging .NET [setup] + + +## Step 1: Configure application logging [setup-step-1] + +If you want to integrate with an existing logger emitting ECS json to a file or stdout/stderr. + +Choose one of our formatters: + +* [`Elastic.CommonSchema.Serilog`](/reference/serilog-formatter.md) +* [`Elastic.CommonSchema.NLog`](/reference/nlog-formatter.md) +* [`Elastic.CommonSchema.Log4Net`](/reference/log4net-formatter.md) + +If you want to write the logs directly to one of Elastic’s endpoints (e.g Elastic Cloud / Elasticsearch) + +Choose one of our data shipping loggers: + +* [`Elastic.Serilog.Sinks`](/reference/serilog-data-shipper.md) +* [`Elastic.Extensions.Logging`](/reference/extensions-logging-data-shipper.md) + + +## Step 2: Enable APM log correlation (optional) [setup-step-2] + +If you are using the Elastic APM .NET agent, [log correlation can be configured](apm-agent-dotnet://docs/reference/logs.md) to inject trace, transaction and span id fields into log events. + +By default the ECS logging integrations will read tracing information from [System.Diagnostics.Activity](https://learn.microsoft.com/en-us/dotnet/api/system.diagnostics.activity?view=net-7.0) if the APM logging corrolation libraries are not installed. + + +## Step 3: Configure Filebeat (optional) [setup-step-3] + +If you are using one of our log formatters you can use the following methods to ship these logs to Elastic. + +:::::::{tab-set} + +::::::{tab-item} Log file +1. Follow the [Filebeat quick start](beats://docs/reference/filebeat/filebeat-installation-configuration.md) +2. Add the following configuration to your `filebeat.yaml` file. + +For Filebeat 7.16+ + +```yaml +filebeat.inputs: +- type: filestream <1> + paths: /path/to/logs.json + parsers: + - ndjson: + overwrite_keys: true <2> + add_error_key: true <3> + expand_keys: true <4> + +processors: <5> + - add_host_metadata: ~ + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ +``` + +1. Use the filestream input to read lines from active log files. +2. Values from the decoded JSON object overwrite the fields that {{filebeat}} normally adds (type, source, offset, etc.) in case of conflicts. +3. {{filebeat}} adds an "error.message" and "error.type: json" key in case of JSON unmarshalling errors. +4. {{filebeat}} will recursively de-dot keys in the decoded JSON, and expand them into a hierarchical object structure. +5. Processors enhance your data. See [processors](beats://docs/reference/filebeat/filtering-enhancing-data.md) to learn more. + + +For Filebeat < 7.16 + +```yaml +filebeat.inputs: +- type: log + paths: /path/to/logs.json + json.keys_under_root: true + json.overwrite_keys: true + json.add_error_key: true + json.expand_keys: true + +processors: +- add_host_metadata: ~ +- add_cloud_metadata: ~ +- add_docker_metadata: ~ +- add_kubernetes_metadata: ~ +``` +:::::: + +::::::{tab-item} Kubernetes +1. Make sure your application logs to stdout/stderr. +2. Follow the [Run Filebeat on Kubernetes](beats://docs/reference/filebeat/running-on-kubernetes.md) guide. +3. Enable [hints-based autodiscover](beats://docs/reference/filebeat/configuration-autodiscover-hints.md) (uncomment the corresponding section in `filebeat-kubernetes.yaml`). +4. Add these annotations to your pods that log using ECS loggers. This will make sure the logs are parsed appropriately. + +```yaml +annotations: + co.elastic.logs/json.overwrite_keys: true <1> + co.elastic.logs/json.add_error_key: true <2> + co.elastic.logs/json.expand_keys: true <3> +``` + +1. Values from the decoded JSON object overwrite the fields that {{filebeat}} normally adds (type, source, offset, etc.) in case of conflicts. +2. {{filebeat}} adds an "error.message" and "error.type: json" key in case of JSON unmarshalling errors. +3. {{filebeat}} will recursively de-dot keys in the decoded JSON, and expand them into a hierarchical object structure. +:::::: + +::::::{tab-item} Docker +1. Make sure your application logs to stdout/stderr. +2. Follow the [Run Filebeat on Docker](beats://docs/reference/filebeat/running-on-docker.md) guide. +3. Enable [hints-based autodiscover](beats://docs/reference/filebeat/configuration-autodiscover-hints.md). +4. Add these labels to your containers that log using ECS loggers. This will make sure the logs are parsed appropriately. + +```yaml +labels: + co.elastic.logs/json.overwrite_keys: true <1> + co.elastic.logs/json.add_error_key: true <2> + co.elastic.logs/json.expand_keys: true <3> +``` + +1. Values from the decoded JSON object overwrite the fields that {{filebeat}} normally adds (type, source, offset, etc.) in case of conflicts. +2. {{filebeat}} adds an "error.message" and "error.type: json" key in case of JSON unmarshalling errors. +3. {{filebeat}} will recursively de-dot keys in the decoded JSON, and expand them into a hierarchical object structure. +:::::: + +::::::: +For more information, see the [Filebeat reference](beats://docs/reference/filebeat/configuring-howto-filebeat.md). + diff --git a/docs/reference/toc.yml b/docs/reference/toc.yml new file mode 100644 index 00000000..d6fa68fe --- /dev/null +++ b/docs/reference/toc.yml @@ -0,0 +1,24 @@ +toc: + - file: index.md + - file: setup.md + - file: ecs-dotnet.md + children: + - file: _usage.md + - file: intro_to_xyz.md + - file: _extending_ecsdocument.md + - file: _formatters.md + children: + - file: serilog-formatter.md + - file: nlog-formatter.md + - file: log4net-formatter.md + - file: data-shippers.md + children: + - file: _elasticsearch_security.md + - file: ecs-ingest-channels.md + - file: serilog-data-shipper.md + - file: extensions-logging-data-shipper.md + - file: benchmark-dotnet-data-shipper.md + - file: _enrichers_2.md + children: + - file: apm-serilog-enricher.md + - file: apm-nlog-enricher.md \ No newline at end of file diff --git a/docs/setup.asciidoc b/docs/setup.asciidoc deleted file mode 100644 index 2d0d058f..00000000 --- a/docs/setup.asciidoc +++ /dev/null @@ -1,41 +0,0 @@ -[[setup]] -== Get Started - -[float] -[[setup-step-1]] -=== Step 1: Configure application logging - -If you want to integrate with an existing logger emitting ECS json to a file or stdout/stderr. - -Choose one of our formatters: - -* <> -* <> -* <> - -If you want to write the logs directly to one of Elastic's endpoints (e.g Elastic Cloud / Elasticsearch) - -Choose one of our data shipping loggers: - -* <> -* <> - -[float] -[[setup-step-2]] -=== Step 2: Enable APM log correlation (optional) -If you are using the Elastic APM .NET agent, -{apm-dotnet-ref}/log-correlation.html[log correlation can be configured] to -inject trace, transaction and span id fields into log events. - -By default the ECS logging integrations will read tracing information from -https://learn.microsoft.com/en-us/dotnet/api/system.diagnostics.activity?view=net-7.0[System.Diagnostics.Activity] -if the APM logging corrolation libraries are not installed. - -[float] -[[setup-step-3]] -=== Step 3: Configure Filebeat (optional) - -If you are using one of our log formatters you can use the following methods -to ship these logs to Elastic. - -include::{ecs-repo-dir}/setup.asciidoc[tag=configure-filebeat] diff --git a/docs/types/types.asciidoc b/docs/types/types.asciidoc deleted file mode 100644 index ae5e75fc..00000000 --- a/docs/types/types.asciidoc +++ /dev/null @@ -1,183 +0,0 @@ -[[ecs-dotnet]] -== .NET Model of ECS - -The `Elastic.CommonSchema` project contains a full C# representation of the https://github.com/elastic/ecs[Elastic Common Schema (ECS)]. -The intention of this library is to form a reliable and correct basis for integrating into Elasticsearch, using both -Microsoft .NET and ECS. - -These types can be used either as-is or in conjunction with the https://github.com/elastic/elasticsearch-net[Official .NET clients for Elasticsearch]. The types are annotated with the corresponding `DataMember` attributes, enabling out-of-the-box serialization support with the Elasticsearch.net clients. - -[float] -=== Installation - -Add a reference to the Elastic.CommonSchema package: - -[source,xml] -[subs="attributes"] ----- - ----- - -TIP: Use https://github.com/elastic/ecs-dotnet/tree/main/src/Elastic.Ingest.Elasticsearch.CommonSchema[Elastic.Ingest.Elasticsearch.CommonSchema] to easily persist ECS documents to Elasticsearch or Elastic Cloud. - - -[float] -==== Versioning - -The version of the Elastic.CommonSchema package matches the published ECS version, with the same corresponding branch names: - -* Nested Schema (The C# types are generated from this YAML file): https://github.com/elastic/ecs/blob/v1.4.0/generated/ecs/ecs_nested.yml -* .NET types: https://github.com/elastic/ecs-dotnet/tree/v1.4.0 - -The version numbers of the NuGet package must match the exact version of ECS used within Elasticsearch. Attempting to use mismatched versions, for example a NuGet package with version 1.2.0 against an Elasticsearch index configured to use an ECS template with version 1.1.0, will result in indexing and data problems. - -=== Usage - -==== Creating an ECS event - -The recommended way to create instances of `EcsDocument` is through: - -[source,csharp] ----- -var doc = EcsDocument.CreateNewWithDefaults(); ----- - -This will automatically assign most common ECS fields that can be inferred from the running process. - -However there is no requirement to do so, simply creating a new `EcsDocument` instance directly -is completely valid and supported. - -[source,csharp] ----- -var ecsDocument = new EcsDocument -{ - Timestamp = DateTimeOffset.Parse("2019-10-23T19:44:38.485Z"), - Dns = new Dns - { - Id = "23666", - OpCode = "QUERY", - Type = "answer", - QuestionName = "www.example.com", - QuestionType = "A", - QuestionClass = "IN", - QuestionRegisteredDomain = "example.com", - HeaderFlags = new[] { "RD", "RA" }, - ResponseCode = "NOERROR", - ResolvedIp = new[] { "10.0.190.47", "10.0.190.117" }, - Answers = new[] - { - new DnsAnswers - { - Data = "10.0.190.47", - Name = "www.example.com", - Type = "A", - Class = "IN", - Ttl = 59 - }, - new DnsAnswers - { - Data = "10.0.190.117", - Name = "www.example.com", - Type = "A", - Class = "IN", - Ttl = 59 - } - } - }, - Network = new Network - { - Type = "ipv4", - Transport = "udp", - Protocol = "dns", - Direction = "outbound", - CommunityId = "1:19beef+RWVW9+BEEF/Q45VFU+2Y=", - Bytes = 126 - }, - Source = new Source { Ip = "192.168.86.26", Port = 5785, Bytes = 31 }, - Destination = new Destination { Ip = "8.8.4.4", Port = 53, Bytes = 95 }, - Client = new Client { Ip = "192.168.86.26", Port = 5785, Bytes = 31 }, - Server = new Server { Ip = "8.8.4.4", Port = 53, Bytes = 95 }, - Event = new Event - { - Duration = 122433000, - Start = DateTimeOffset.Parse("2019-10-23T19:44:38.485Z"), - End = DateTimeOffset.Parse("2019-10-23T19:44:38.607Z"), - Kind = "event", - Category = new[] { "network_traffic" } - }, - Ecs = new Ecs { Version = "1.2.0" }, - Metadata = new Dictionary { { "client", "ecs-dotnet" } } -}; - ----- - -[float] -=== Dynamically assign ECS fields - -Additionally, ECS fields can be dynamically assigned through - -[source,csharp] ----- -ecsDocument.AssignProperty("orchestrator.cluster.id", "id"); - ----- - -This will assign `ecsDocument.Orchestrator.ClusterId` to `"id"` and automatically create a new `Orchestrator` instance if needed. - -Any `string` or `boolean` value that is not a known `ecs` field will be assigned to `labels.*` and everything else to `metatadata.*` - -[id=intro_to_xyz,titleabbrev=" XYZ Intro"] -=== A note on the `Metadata` property - -The C# `EcsDocument` type includes a property called `Metadata` with the signature: - -[source,csharp] ----- -/// -/// Container for additional metadata against this event. -/// -[JsonPropertyName("metadata"), DataMember(Name = "metadata")] -public IDictionary Metadata { get; set; } ----- - -This property is not part of the ECS specification, but is included as a means to index supplementary information. - -=== Extending EcsDocument - -In instances where using the `IDictionary Metadata` property is not sufficient, or there is a clearer definition of the structure of the ECS-compatible document you would like to index, it is possible to subclass the `EcsDocument` object and provide your own property definitions. - -Through `TryRead`/`ReceiveProperty`/`WriteAdditionalProperties` you can hook into the `EcsDocumentJsonConverter` and read/write additional properties. - -[source,csharp] ----- -/// -/// An extended ECS document with an additional property -/// -[JsonConverter(typeof(EcsDocumentJsonConverterFactory))] -public class MyEcsDocument : EcsDocument -{ - [JsonPropertyName("my_root_property"), DataMember(Name = "my_root_property")] - public MyCustomType MyRootProperty { get; set; } - - protected override bool TryRead(string propertyName, out Type type) - { - type = propertyName switch - { - "my_root_property" => typeof(MyCustomType), - _ => null - }; - return type != null; - } - - protected override bool ReceiveProperty(string propertyName, object value) => - propertyName switch - { - "my_root_property" => null != (MyRootProperty = value as MyCustomType), - _ => false - }; - - protected override void WriteAdditionalProperties(Action write) => write("my_root_property", MyCustomType); -} ----- - -The Elastic.CommonSchema.BenchmarkDotNetExporter project takes this approach in the https://github.com/elastic/ecs-dotnet/tree/main/src/Elastic.CommonSchema.BenchmarkDotNetExporter[Domain source directory], where the BenchmarkDocument subclasses EcsDocument.