BIGTOP-4139: Support OpenEuler 22.03 for Bigtop Manager

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/enums/OSType.java

@@ -35,5 +35,7 @@ public enum OSType {
     DEBIAN11,
 
     FEDORA36,
+
+    OPENEULER22,
     ;
 }

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/utils/os/OSDetection.java

@@ -55,7 +55,7 @@ public class OSDetection {
 
     public static String getOS() {
         if (SystemUtils.IS_OS_LINUX) {
-            String os = getOSType().toLowerCase() + getOSVersion().toLowerCase();
+            String os = getOSType() + getOSVersion().toLowerCase();
             ifSupportedOS(os);
             return os;
         } else {
@@ -102,7 +102,7 @@ public static long totalDisk() {
     private static String getOSType() {
         String output = getOSRelease();
 
-        String osType = regexOS(ID_PATTERN, output);
+        String osType = regexOS(ID_PATTERN, output).toLowerCase();
 
         log.debug("osType: {}", osType);
         return osType;

bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/JobDao.java

@@ -23,13 +23,16 @@
 
 import org.apache.ibatis.annotations.Param;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 
 public interface JobDao extends BaseDao<JobPO> {
 
     List<JobPO> findAllByClusterId(@Param("clusterId") Long clusterId);
 
+    List<JobPO> findAllByIdsJoin(@Param("ids") Collection<Long> ids);
+
     Optional<JobPO> findByIdJoin(@Param("id") Long id);
 
     List<JobPO> findAllByClusterIsNull();

bigtop-manager-dao/src/main/resources/mapper/mysql/HostComponentMapper.xml

@@ -115,7 +115,7 @@
         <include refid="baseColumnsV2">
             <property name="alias" value="hc"/>
         </include>
-            , c.component_name
+            , c.component_name, h.hostname
         from
         host_component hc
         inner join (select * from host
@@ -134,7 +134,7 @@
         on hc.host_id = h.id
         inner join (select * from component
         <where>
-            <if test="componentName != 0">
+            <if test="componentName != null">
                 component_name = #{componentName}
             </if>
         </where>

bigtop-manager-dao/src/main/resources/mapper/mysql/JobMapper.xml

@@ -73,7 +73,7 @@
         </collection>
     </resultMap>
 
-    <select id="findAllByClusterId" parameterType="java.lang.Long" resultMap="jobMap">
+    <select id="findAllByIdsJoin" parameterType="java.lang.Long" resultMap="jobMap">
         select
         <include refid="baseColumnsV2">
             <property name="alias" value="h"/>
@@ -83,15 +83,19 @@
         from
         (select * from job
         <where>
-            <if test="clusterId != 0">
-                cluster_id = #{clusterId}
+            <if test="ids != null">
+                id in
+                <foreach collection="ids" item="id" index="index" open="(" close=")" separator=",">
+                    #{id}
+                </foreach>
             </if>
         </where>
         ) h
         inner join stage st
         on h.id = st.job_id
         inner join task tk
         on tk.job_id = h.id and st.id = tk.stage_id
+        order by h.id desc
     </select>
 
     <select id="findByIdJoin" parameterType="java.lang.Long" resultMap="jobMap">
@@ -115,16 +119,24 @@
         on tk.job_id = h.id and st.id = tk.stage_id
     </select>
 
+    <select id="findAllByClusterId" parameterType="java.lang.Long" resultType="org.apache.bigtop.manager.dao.po.JobPO">
+        select
+        <include refid="baseColumns"/>
+        from
+        job
+        <where>
+            <if test="clusterId != 0">
+                cluster_id = #{clusterId}
+            </if>
+        </where>
+    </select>
+
     <select id="findAllByClusterIsNull"
             resultType="org.apache.bigtop.manager.dao.po.JobPO">
         select
-        <include refid="baseColumnsV2">
-            <property name="alias" value="h"/>
-        </include>
+        <include refid="baseColumns"/>
         from
-        (select * from job where cluster_id is null) h
-        inner join stage st
-        on h.id = st.job_id
+        job where cluster_id is null
     </select>
 
 </mapper>

bigtop-manager-dao/src/main/resources/mapper/mysql/RepoMapper.xml

@@ -24,7 +24,7 @@
 <mapper namespace="org.apache.bigtop.manager.dao.repository.RepoDao">
 
     <sql id="baseColumns">
-        id, base_url, os, arch, repo_id, repo_name, cluster_id
+        id, base_url, os, arch, repo_id, repo_name, repo_type, cluster_id
     </sql>
 
     <select id="findByRepoName" resultType="org.apache.bigtop.manager.dao.po.RepoPO">
@@ -41,10 +41,10 @@
     </select>
 
     <insert id="saveAll" useGeneratedKeys="true" keyProperty="id" keyColumn="id">
-        insert into repo (base_url, os, arch, repo_id, repo_name, cluster_id, create_by, update_by, create_time, update_time)
+        insert into repo (base_url, os, arch, repo_id, repo_name, repo_type, cluster_id, create_by, update_by, create_time, update_time)
         values
         <foreach collection='clusters' item='cluster' separator=','>
-            (#{cluster.baseUrl},#{cluster.os},#{cluster.arch},#{cluster.repoId},#{cluster.repoName},#{cluster.clusterId},#{cluster.createBy},#{cluster.updateBy},#{cluster.createTime},#{cluster.updateTime})
+            (#{cluster.baseUrl},#{cluster.os},#{cluster.arch},#{cluster.repoId},#{cluster.repoName},#{cluster.repoType},#{cluster.clusterId},#{cluster.createBy},#{cluster.updateBy},#{cluster.createTime},#{cluster.updateTime})
         </foreach>
     </insert>
 

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java

@@ -79,6 +79,9 @@ public PageVO<JobVO> list(Long clusterId) {
             jobPOList = jobDao.findAllByClusterId(clusterId);
         }
         PageInfo<JobPO> pageInfo = new PageInfo<>(jobPOList);
+        List<JobPO> allByIdsJoin =
+                jobDao.findAllByIdsJoin(jobPOList.stream().map(JobPO::getId).toList());
+        pageInfo.setList(allByIdsJoin);
 
         return PageVO.of(pageInfo);
     }

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml

@@ -48,6 +48,14 @@ gpgcheck=0
                 <arch>x86_64</arch>
                 <base-url>http://123.56.2.244/bigtop/3.3.0/rocky8/x86_64/</base-url>
             </repo>
+            <repo>
+                <repo-id>Bigtop-3.3.0</repo-id>
+                <repo-name>Bigtop-3.3.0 for openEuler-22.03 x86_64</repo-name>
+                <repo-type>binary</repo-type>
+                <os>openeuler22</os>
+                <arch>x86_64</arch>
+                <base-url>http://123.56.2.244/bigtop/3.3.0/openeuler22/x86_64/</base-url>
+            </repo>
         </repos>
     </stack>
 </metainfo>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml

@@ -63,6 +63,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml

@@ -103,6 +103,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml

@@ -59,6 +59,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml

@@ -63,6 +63,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml

@@ -53,6 +53,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml

@@ -45,6 +45,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml

@@ -92,6 +92,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml

@@ -59,6 +59,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml

@@ -49,6 +49,7 @@
                 <operating-systems>
                     <os>centos7</os>
                     <os>rocky8</os>
+                    <os>openeuler22</os>
                 </operating-systems>
                 <architectures>
                     <arch>x86_64</arch>

bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java

@@ -85,7 +85,8 @@ public Map<String, Object> coreSite() {
         Map<String, Object> coreSite = LocalSettings.configurations(serviceName(), "core-site");
         List<String> namenodeList = LocalSettings.hosts("namenode");
         if (!namenodeList.isEmpty()) {
-            coreSite.put("fs.defaultFS", MessageFormat.format("hdfs://{0}:8020", namenodeList.get(0)));
+            coreSite.put(
+                    "fs.defaultFS", ((String) coreSite.get("fs.defaultFS")).replace("localhost", namenodeList.get(0)));
         }
         return coreSite;
     }
@@ -98,6 +99,18 @@ public Map<String, Object> hadoopPolicy() {
     @GlobalParams
     public Map<String, Object> hdfsSite() {
         Map<String, Object> hdfsSite = LocalSettings.configurations(serviceName(), "hdfs-site");
+        List<String> namenodeList = LocalSettings.hosts("namenode");
+        if (!namenodeList.isEmpty()) {
+            hdfsSite.put(
+                    "dfs.namenode.rpc-address",
+                    ((String) hdfsSite.get("dfs.namenode.rpc-address")).replace("0.0.0.0", namenodeList.get(0)));
+            hdfsSite.put(
+                    "dfs.datanode.https.address",
+                    ((String) hdfsSite.get("dfs.datanode.https.address")).replace("0.0.0.0", namenodeList.get(0)));
+            hdfsSite.put(
+                    "dfs.namenode.https-address",
+                    ((String) hdfsSite.get("dfs.namenode.https-address")).replace("0.0.0.0", namenodeList.get(0)));
+        }
 
         dfsDataDir = (String) hdfsSite.get("dfs.datanode.data.dir");
         dfsNameNodeDir = (String) hdfsSite.get("dfs.namenode.name.dir");

bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/repo/PackageManagerType.java

@@ -28,6 +28,7 @@
 import static org.apache.bigtop.manager.common.enums.OSType.DEBIAN10;
 import static org.apache.bigtop.manager.common.enums.OSType.DEBIAN11;
 import static org.apache.bigtop.manager.common.enums.OSType.FEDORA36;
+import static org.apache.bigtop.manager.common.enums.OSType.OPENEULER22;
 import static org.apache.bigtop.manager.common.enums.OSType.ROCKY8;
 import static org.apache.bigtop.manager.common.enums.OSType.UBUNTU20;
 import static org.apache.bigtop.manager.common.enums.OSType.UBUNTU22;
@@ -36,7 +37,7 @@
 public enum PackageManagerType {
     YUM(List.of(CENTOS7)),
 
-    DNF(List.of(ROCKY8, FEDORA36)),
+    DNF(List.of(ROCKY8, FEDORA36, OPENEULER22)),
 
     APT(List.of(UBUNTU20, UBUNTU22, DEBIAN10, DEBIAN11)),
     ;

dev-support/docker/openeuler22/Dockerfile

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM openeuler/openeuler:22.03
+
+RUN dnf -y install sudo wget openssh-clients openssh-server vim mariadb mariadb-server net-tools chrony krb5-server krb5-libs krb5-workstation git rpm-build python3 procps-ng
+RUN wget https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz -O /tmp/jdk.tar.gz --no-check-certificate \
+  && mkdir -p /usr/local/java && tar -xzf /tmp/jdk.tar.gz -C /usr/local/java --strip-components=1 \
+  && rm -f /tmp/jdk.tar.gz \
+  && ln -s /usr/local/java/bin/java /usr/bin/java
+RUN wget https://archive.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz -O /tmp/apache-maven.tar.gz --no-check-certificate \
+  && mkdir -p /usr/share/maven && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \
+  && rm -f /tmp/apache-maven.tar.gz \
+  && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+RUN wget https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement/master/files/docker/systemctl3.py -O /usr/bin/systemctl
+
+RUN /bin/sed -i 's,#   StrictHostKeyChecking ask,StrictHostKeyChecking no,g' /etc/ssh/ssh_config
+
+RUN ssh-keygen -f "/root/.ssh/id_rsa" -N ""
+
+EXPOSE 1-65535