Merging the latest changes into optimzier branch

.github/workflows/docker-image.yml

@@ -1,20 +1,24 @@
-name: Docker Image CI
+name: Docker Image Publish CI ( Dockerhub and ECR )
 
 on:
-  push:
-    branches: [ "master" ]
-  pull_request:
-    branches: [ "master" ]
+  release:
+    types: [created]
 
 jobs:
-
   build:
-
     runs-on: ubuntu-latest
-
+    outputs:
+      id: git_tag_output
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Determine tag type
+        id: tag_type
+        run: echo "::set-output name=is_release_tag::$(echo ${GITHUB_REF#refs/tags/} | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' || echo 'false')"
+      - name: Set version
+        id: version
+        run: |
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -25,11 +29,42 @@ jobs:
           DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
         run:
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-      - name: Build and push
-        uses: docker/build-push-action@v5
+      - name: Build and push dockerhub
+        uses: docker/build-push-action@v5 
         with:
           context: .
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: onedatashare/transfer_service:latest
+          tags: onedatashare/transfer_service:${{ env.VERSION }} 
+
+  push-to-ecr:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Determine tag type
+        id: tag_type
+        run: echo "::set-output name=is_release_tag::$(echo ${GITHUB_REF#refs/tags/} | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' || echo 'false')"
+      - name: Set version
+        id: version
+        run: |
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+      - name: Setup AWS ECR Details
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
 
+      - name: Login to Amazon ECR
+        id: login-pf-aws-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and push the tagged docker image to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-pf-aws-ecr.outputs.registry }}
+          ECR_REPOSITORY: onedatashare/transfer_service
+          IMAGE_TAG: ${{ env.VERSION }}
+        run: |
+          docker pull onedatashare/transfer_service:$IMAGE_TAG
+          docker tag onedatashare/transfer_service:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

Dockerfile

@@ -1,59 +1,32 @@
-FROM maven:3.9.5-amazoncorretto-21 AS build
+FROM maven:3.9.7-amazoncorretto-21 AS build
 
 COPY src /home/app/src
 COPY pom.xml /home/app
 RUN mvn -f /home/app/pom.xml clean package -DskipTests
 
 # Final Image
-FROM amazoncorretto:21-alpine-jdk
+FROM amazoncorretto:21-alpine3.18-jdk
 
+RUN apk update
 RUN apk --no-cache add python3-dev py3-pip build-base gcc linux-headers
-RUN pip3 install pmeter-ods==1.0.8
+
+RUN pip install pmeter-ods
 
 COPY --from=build /home/app/target/ods-transfer-service-0.0.1-SNAPSHOT.jar /usr/local/lib/ods-transfer-service-0.0.1-SNAPSHOT.jar
 
 ENV PIP_ROOT_USER_ACTION=ignore
-ENV NODE_NAME="${NODE_NAME}"
-ENV USER_NAME="${USER_NAME}"
-ENV APP_NAME="${USER_NAME}"-"${NODE_NAME}"
-
-ENV CONNECTOR_QUEUE="${APP_NAME}"
-ENV ODS_GDRIVE_CLIENT_ID="${ODS_GDRIVE_CLIENT_ID}"
-ENV ODS_GDRIVE_CLIENT_SECRET="${ODS_GDRIVE_CLIENT_SECRET}"
-ENV ODS_GDRIVE_PROJECT_ID="onedatashare-dev"
-ENV EUREKA_URI="${EUREKA_URI}"
-ENV EUREKA_PASS="${EUREKA_PASS}"
-ENV EUREKA_USER="${EUREKA_USER}"
-ENV FOLDER_WITH_CERTS="${FOLDER_WITH_CERTS}"
-COPY ${FOLDER_WITH_CERTS} /certs/
-ENV COCKROACH_URI="${COCKROACH_URI}"
-ENV COCKROACH_USER="${COCKROACH_USER}"
-ENV COCKROACH_PASS="${COCKROACH_PASS}"
-ENV RMQ_ADDRESS="amqps://b-0e720b16-3ea7-4227-ad65-6cce3704121c.mq.us-east-2.amazonaws.com:5671"
-
-#use ODS user for your private queue.
-#create creds through aws console
-ENV AMPQ_USER="${AMPQ_USER}"
-ENV AMPQ_PWD="${AMPQ_PWD}"
 
 #change to monitor the active NIC
 ENV PMETER_CLI_OPTIONS="-NS"
 ENV PMETER_NIC_INTERFACE="${PMETER_NIC_INTERFACE:-eth0}"
-ENV INFLUX_ORG="${INFLUX_ORG}"
-ENV INFLUX_BUCKET="${USER_NAME}"
-ENV INFLUX_TOKEN="${INFLUX_TOKEN}"
-ENV INFLUX_URI="https://influxdb.onedatashare.org"
 ENV ENABLE_PMETER="true"
 ENV PMETER_CRON_EXP="*/15 * * * * *"
-
-ENV OPTIMIZER_URL="${OPTIMIZER_URL}"
-ENV OPTIMIZER_ENABLE="${OPTIMIZER_ENABLE}"
-ENV SPRING_PROFILE="${SPRING_PROFILE:-hsql}"
+ENV SPRING_PROFILES_ACTIVE=aws,virtual,cockroach
 
 ENV PATH "/home/ods/.local/bin:${PATH}"
 
 RUN mkdir -p $HOME/.pmeter/
 RUN touch $HOME/.pmeter/transfer_service_pmeter_measure.txt
 
 EXPOSE 8092
-ENTRYPOINT ["java", "-Dspring.profiles.active=hsql","-jar", "/usr/local/lib/ods-transfer-service-0.0.1-SNAPSHOT.jar"]
+ENTRYPOINT ["java","-jar", "/usr/local/lib/ods-transfer-service-0.0.1-SNAPSHOT.jar"]

pom.xml

@@ -16,7 +16,7 @@
 
     <properties>
         <java.version>21</java.version>
-        <spring-cloud.version>2023.0.0-RC1</spring-cloud.version>
+        <spring-cloud.version>2023.0.2</spring-cloud.version>
     </properties>
 
     <repositories>
@@ -36,12 +36,43 @@
                 <enabled>false</enabled>
             </snapshots>
         </repository>
+        <repository>
+            <id>private-repository</id>
+            <name>Hazelcast Private Repository</name>
+            <url>https://repository.hazelcast.com/release/</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
     </repositories>
     <dependencies>
         <dependency>
             <groupId>com.fasterxml.jackson.datatype</groupId>
             <artifactId>jackson-datatype-jsr310</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.hazelcast</groupId>
+            <artifactId>hazelcast-enterprise</artifactId>
+            <version>5.5.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-vault-config</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.vault</groupId>
+            <artifactId>spring-vault-core</artifactId>
+            <version>3.1.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-bootstrap</artifactId>
+            <version>4.1.3</version>
+        </dependency>
         <dependency>
             <groupId>com.influxdb</groupId>
             <artifactId>influxdb-client-java</artifactId>
@@ -52,10 +83,6 @@
             <artifactId>commons-pool2</artifactId>
             <version>2.11.1</version>
         </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.box</groupId>
             <artifactId>box-java-sdk</artifactId>
@@ -130,20 +157,25 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-actuator</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.17.2</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
-<!--            <exclusions>-->
-<!--                <exclusion>-->
-<!--                    <groupId>org.springframework.boot</groupId>-->
-<!--                    <artifactId>spring-boot-starter-tomcat</artifactId>-->
-<!--                </exclusion>-->
-<!--            </exclusions>-->
-        </dependency>
-<!--        <dependency>-->
-<!--            <groupId>org.springframework.boot</groupId>-->
-<!--            <artifactId>spring-boot-starter-jetty</artifactId>-->
-<!--        </dependency>-->
+            <!--            <exclusions>-->
+            <!--                <exclusion>-->
+            <!--                    <groupId>org.springframework.boot</groupId>-->
+            <!--                    <artifactId>spring-boot-starter-tomcat</artifactId>-->
+            <!--                </exclusion>-->
+            <!--            </exclusions>-->
+        </dependency>
+        <!--        <dependency>-->
+        <!--            <groupId>org.springframework.boot</groupId>-->
+        <!--            <artifactId>spring-boot-starter-jetty</artifactId>-->
+        <!--        </dependency>-->
         <dependency>
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
@@ -157,6 +189,16 @@
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-s3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>auth</artifactId>
+            <version>2.25.67</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>sts</artifactId>
+            <version>2.25.67</version>
+        </dependency>
         <dependency>
             <groupId>jakarta.servlet</groupId>
             <artifactId>jakarta.servlet-api</artifactId>

src/main/java/org/onedatashare/transferservice/odstransferservice/Enum/MessageType.java

@@ -0,0 +1,5 @@
+package org.onedatashare.transferservice.odstransferservice.Enum;
+
+public enum MessageType {
+    CARBON_AVG_REQUEST, TRANSFER_JOB_REQUEST, APPLICATION_PARAM_CHANGE, STOP_JOB_REQUEST, CARBON_IP_REQUEST
+}

src/main/java/org/onedatashare/transferservice/odstransferservice/OdsTransferService.java

@@ -1,6 +1,5 @@
 package org.onedatashare.transferservice.odstransferservice;
 
-import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.client.discovery.EnableDiscoveryClient;

src/main/java/org/onedatashare/transferservice/odstransferservice/config/BatchConfig.java

@@ -7,38 +7,40 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.task.SimpleAsyncTaskExecutor;
 import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.retry.backoff.BackOffPolicy;
+import org.springframework.retry.backoff.ExponentialBackOffPolicy;
 import org.springframework.transaction.PlatformTransactionManager;
 
 import javax.sql.DataSource;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 @Configuration
 public class BatchConfig {
 
-//    @Bean
-//    public JobLauncher jobLauncher(JobRepository jobRepository) {
-//        TaskExecutorJobLauncher taskExecutorJobLauncher = new TaskExecutorJobLauncher();
-//        taskExecutorJobLauncher.setJobRepository(jobRepository);
-//        return taskExecutorJobLauncher;
-//    }
-
-    @Bean
-    public Set<Long> jobIds() {
-        return new HashSet<>();
-    }
-
     @Bean
     public PlatformTransactionManager transactionManager(DataSource dataSource) {
         return new DataSourceTransactionManager(dataSource);
     }
 
     @Bean
-    public JobLauncher asyncJobLauncher(JobRepository jobRepository) {
+    public JobLauncher jobLauncher(JobRepository jobRepository) {
         TaskExecutorJobLauncher jobLauncher = new TaskExecutorJobLauncher();
         jobLauncher.setJobRepository(jobRepository);
-        jobLauncher.setTaskExecutor(new SimpleAsyncTaskExecutor());
+        SimpleAsyncTaskExecutor taskExecutor = new SimpleAsyncTaskExecutor();
+        taskExecutor.setConcurrencyLimit(4);
+        jobLauncher.setTaskExecutor(taskExecutor);
         return jobLauncher;
     }
+
+
+    @Bean
+    public BackOffPolicy backOffPolicy() {
+        ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
+        backOffPolicy.setInitialInterval(TimeUnit.SECONDS.toMillis(5));
+        backOffPolicy.setMultiplier(2.0);
+        backOffPolicy.setMaxInterval(TimeUnit.DAYS.toMillis(1));
+        return backOffPolicy;
+    }
+
 }