MLH-36 | Integrating OTEL with logback

Dockerfile

@@ -56,8 +56,7 @@ COPY atlas-hub/repair_index.py /opt/apache-atlas/bin/
 RUN chmod +x /opt/apache-atlas/bin/repair_index.py
 
 COPY atlas-hub/atlas_start.py.patch atlas-hub/atlas_config.py.patch /opt/apache-atlas/bin/
-COPY atlas-hub/pre-conf/atlas-log4j.xml /opt/apache-atlas/conf/
-COPY atlas-hub/pre-conf/atlas-log4j2.xml /opt/apache-atlas/conf/
+COPY atlas-hub/pre-conf/atlas-logback.xml /opt/apache-atlas/conf/
 COPY atlas-hub/pre-conf/atlas-auth/ /opt/apache-atlas/conf/
 
 RUN curl https://repo1.maven.org/maven2/org/jolokia/jolokia-jvm/1.6.2/jolokia-jvm-1.6.2-agent.jar -o /opt/apache-atlas/libext/jolokia-jvm-agent.jar

tools/classification-updater/src/main/resources/atlas-log4j.xml → addons/couchbase-bridge/src/test/resources/logback.xml

@@ -17,26 +17,22 @@
   ~ limitations under the License.
   -->
 
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<configuration>
+  <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+    <param name="Target" value="System.out"/>
+    <encoder>
+      <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern>
+    </encoder>
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
+  </appender>
 
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-    <appender name="FILE" class="org.apache.log4j.RollingFileAppender">
-        <param name="File" value="/var/log/atlas/classification-updater.log"/>
-        <param name="Append" value="true"/>
-        <param name="maxFileSize" value="100MB" />
-        <param name="maxBackupIndex" value="20" />
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
-        </layout>
-    </appender>
+  <logger name="org.apache.atlas" additivity="false" level="info">
+    <appender-ref ref="console"/>
+  </logger>
 
-    <logger name="org.apache.atlas.tools.BulkFetchAndUpdate" additivity="false">
-        <level value="info"/>
-        <appender-ref ref="FILE"/>
-    </logger>
-
-    <root>
-        <priority value="warn"/>
-        <appender-ref ref="FILE"/>
-    </root>
-</log4j:configuration>
+  <root level="DEBUG">
+    <appender-ref ref="console"/>
+  </root>
+</configuration>

addons/falcon-bridge/pom.xml

@@ -267,8 +267,8 @@
                             <value>application.log</value>
                         </systemProperty>
                         <systemProperty>
-                            <name>log4j.configuration</name>
-                            <value>file:///${project.build.directory}/test-classes/atlas-log4j.xml</value>
+                            <name>logback.configurationFile</name>
+                            <value>file:///${project.build.directory}/test-classes/atlas-logback.xml</value>
                         </systemProperty>
                         <systemProperty>
                             <name>atlas.graphdb.backend</name>

addons/falcon-bridge/src/test/resources/atlas-logback.xml

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<configuration>
+  <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+    <param name="Target" value="System.out"/>
+    <encoder>
+      <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern>
+    </encoder>
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
+  </appender>
+
+  <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${atlas.log.dir}/${atlas.log.file}</file>
+    <append>true</append>
+    <encoder>
+      <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern>
+    </encoder>
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${atlas.log.dir}/${atlas.log.file}-%d</fileNamePattern>
+      <maxHistory>20</maxHistory>
+      <cleanHistoryOnStart>true</cleanHistoryOnStart>
+    </rollingPolicy>
+  </appender>
+
+  <appender name="AUDIT" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${atlas.log.dir}/audit.log</file>
+    <append>true</append>
+    <encoder>
+      <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern>
+    </encoder>
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${atlas.log.dir}/audit-%d.log</fileNamePattern>
+      <maxHistory>20</maxHistory>
+      <cleanHistoryOnStart>false</cleanHistoryOnStart>
+    </rollingPolicy>
+  </appender>
+
+  <appender name="METRICS" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${atlas.log.dir}/metrics.log</file>
+    <append>true</append>
+    <encoder>
+      <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern>
+    </encoder>
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${atlas.log.dir}/metrics-%d.log</fileNamePattern>
+      <maxHistory>20</maxHistory>
+      <cleanHistoryOnStart>false</cleanHistoryOnStart>
+    </rollingPolicy>
+  </appender>
+
+  <appender name="FAILED" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${atlas.log.dir}/failed.log</file>
+    <append>true</append>
+    <encoder>
+      <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern>
+    </encoder>
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${atlas.log.dir}/failed-%d.log</fileNamePattern>
+      <maxHistory>20</maxHistory>
+      <cleanHistoryOnStart>false</cleanHistoryOnStart>
+    </rollingPolicy>
+  </appender>
+
+  <logger name="org.apache.atlas" additivity="false" level="info">
+    <appender-ref ref="FILE"/>
+  </logger>
+
+  <logger name="org.apache.atlas.impala.ImpalaLineageTool" additivity="false" level="debug">
+    <appender-ref ref="FILE"/>
+  </logger>
+
+  <logger name="org.apache.atlas.impala.hook.ImpalaLineageHook" additivity="false" level="debug">
+    <appender-ref ref="FILE"/>
+  </logger>
+
+  <logger name="org.janusgraph" additivity="false" level="warn">
+    <appender-ref ref="FILE"/>
+  </logger>
+
+  <logger name="org.springframework" additivity="false" level="warn">
+    <appender-ref ref="console"/>
+  </logger>
+
+  <logger name="org.eclipse" additivity="false" level="warn">
+    <appender-ref ref="console"/>
+  </logger>
+
+  <logger name="com.sun.jersey" additivity="false" level="warn">
+    <appender-ref ref="console"/>
+  </logger>
+
+
+  <!-- to avoid logs - The configuration log.flush.interval.messages = 1 was supplied but isn't a known config -->
+  <logger name="org.apache.kafka.common.config.AbstractConfig" additivity="false" level="error">
+    <appender-ref ref="FILE"/>
+  </logger>
+
+  <logger name="METRICS" additivity="false" level="debug">
+    <appender-ref ref="METRICS"/>
+  </logger>
+
+  <logger name="FAILED" additivity="false" level="info">
+    <appender-ref ref="FAILED"/>
+  </logger>
+
+  <logger name="AUDIT" additivity="false" level="info">
+    <appender-ref ref="AUDIT"/>
+  </logger>
+
+   <root level="warn">
+    <appender-ref ref="FILE"/>
+  </root>
+</configuration>

addons/hbase-bridge/pom.xml

@@ -339,6 +339,11 @@
                                             <artifactId>jsr311-api</artifactId>
                                             <version>${jsr.version}</version>
                                         </artifactItem>
+                                        <artifactItem>
+                                            <groupId>ch.qos.logback</groupId>
+                                            <artifactId>logback-classic</artifactId>
+                                            <version>${logback.version}</version>
+                                        </artifactItem>
                                     </artifactItems>
                                 </configuration>
                             </execution>
@@ -417,8 +422,8 @@
                             <value>application.log</value>
                         </systemProperty>
                         <systemProperty>
-                            <name>log4j.configuration</name>
-                            <value>file:///${project.build.directory}/test-classes/atlas-log4j.xml</value>
+                            <name>logback.configurationFile</name>
+                            <value>file:///${project.build.directory}/test-classes/atlas-logback.xml</value>
                         </systemProperty>
                         <systemProperty>
                             <name>atlas.graphdb.backend</name>