Support Apps Export and Import

[sgokaram-saagie]

Supporting Exporting and Importing Apps.
https://github.com/saagie/gradle-saagie-dataops-plugin/wiki/projectsExport

[ZouhairBear]

• for app exportation , the related issue is : Support Apps Export from v2 #354

• for app importation , the related issue is : Support Apps Import from v2 #355

[ZouhairBear]

• test for script of all tasks

How to test :

Version to test : 2.1.19
Create a build file:
Example of build.newTask.gradle:

import io.saagie.plugin.dataops.tasks.projects.artifact.ProjectsExportJobV1Task
import io.saagie.plugin.dataops.tasks.projects.artifact.ProjectsImportJobTask
import groovy.json.JsonSlurper;
plugins {
    id 'groovy'
    id 'base'
    id 'io.saagie.gradle-saagie-dataops-plugin' version '2.1.19'
}
dependencies {
    implementation group: 'gradle.plugin.io.saagie', name: 'gradle-saagie-dataops-plugin', version: '2.1.12'
}
def exportLocation = ' ' // <=== edit this with the export path
def tmpLocation = ' ' // <=== edit this with the tmp file path
task projectExportSqoop(type: ProjectsExportJobV1Task) { // <====== Create the first job
    configuration = saagie {}
    doFirst {
        projectExportSqoop.configuration = saagie {
            server {
                url = saagieurl
                login = saagieuserid
                password = saagiepassword
                environment = saagieplatformid
                jwt = true
                acceptSelfSigned = true
            }
            job {
                ids = ['21025']
            }
            exportArtifacts {
                export_file = exportLocation + '/job/sqoop.zip'
                overwrite=true
                temporary_directory= tmpLocation
            }
        }
    }
    taskName = 'projectExportSqoop'
    group = 'Saagie'
    description= 'export v1 sqoop job for saagie'
}
task projectExportJava(type: ProjectsExportJobV1Task) { // <====== Create the second job
    configuration = saagie {}
    doFirst {
        projectExportJava.configuration = saagie {
            server {
                url = saagieurl
                login = saagieuserid
                password = saagiepassword
                environment = saagieplatformid
                jwt = true
                acceptSelfSigned = true
            }
            job {
                ids = ['21754']
            }
            exportArtifacts {
                export_file = exportLocation + '/job/java.zip'
                overwrite=true
                temporary_directory= tmpLocation
            }
        }
    }
    taskName = 'projectExportJava'
    group = 'Saagie'
    description= 'export v1 Java job for saagie'
}
task projectExportR(type: ProjectsExportJobV1Task) { // <====== Create the second job
    configuration = saagie {}
    doFirst {
        projectExportR.configuration = saagie {
            server {
                url = saagieurl
                login = saagieuserid
                password = saagiepassword
                environment = saagieplatformid
                jwt = true
                acceptSelfSigned = true
            }
            job {
                ids = ['20983']
            }
            exportArtifacts {
                export_file = exportLocation + '/job/r.zip'
                overwrite=true
                temporary_directory= tmpLocation
            }
        }
    }
    taskName = 'projectExportR'
    group = 'Saagie'
    description= 'export v1 R job for saagie'
}
task projectExportTalend(type: ProjectsExportJobV1Task) { // <====== Create the second job
    configuration = saagie {}
    doFirst {
        projectExportTalend.configuration = saagie {
            server {
                url = saagieurl
                login = saagieuserid
                password = saagiepassword
                environment = saagieplatformid
                jwt = true
                acceptSelfSigned = true
            }
            job {
                ids = ['21439']
            }
            exportArtifacts {
                export_file = exportLocation + '/job/talend.zip'
                overwrite=true
                temporary_directory= tmpLocation
            }
        }
    }
    taskName = 'projectExportTalend'
    group = 'Saagie'
    description= 'export v1 talend job for saagie'
}
task projectExportDocker(type: ProjectsExportJobV1Task) { // <====== Create the second job
    configuration = saagie {}
    doFirst {
        projectExportDocker.configuration = saagie {
            server {
                url = saagieurl
                login = saagieuserid
                password = saagiepassword
                environment = saagieplatformid
                jwt = true
                acceptSelfSigned = true
            }
            job {
                ids = ['13592']
            }
            exportArtifacts {
                export_file = exportLocation + '/job/docker.zip'
                overwrite=true
                temporary_directory= tmpLocation
            }
        }
    }
    taskName = 'projectExportDocker'
    group = 'Saagie'
    description= 'export v1 Docker job for saagie'
}
task projectExportPipeline(type: ProjectsExportJobV1Task) { // <====== Create the second job
    configuration = saagie {}
    doFirst {
        projectExportPipeline.configuration = saagie {
            server {
                url = saagieurl
                login = saagieuserid
                password = saagiepassword
                environment = saagieplatformid
                jwt = true
                acceptSelfSigned = true
            }
            pipeline {
                ids = ['526']
                include_job=true
                include_all_versions=true
            }
            exportArtifacts {
                export_file = exportLocation + '/pipeline/pipeline.zip'
                overwrite=true
                temporary_directory= tmpLocation
            }
        }
    }
    taskName = 'projectExportPipeline'
    group = 'Saagie'
    description= 'export Pipleline with out dependency'
}
task projectImportSqoop( // <======== Import artifacts, see that we use the gradle dependOn
        type: ProjectsImportJobTask) {
    dependsOn(['projectExportSqoop'])
    configuration = saagie {
        server {
            url = saagieurl
            login = saagieuserid
            password = saagiepassword
            environment = saagieplatformid
            jwt = true
        }
    }
    doFirst {
        projectImportSqoop.configuration = saagie {
            project {
                id = saagieprojectid
            }
            importArtifacts {
                import_file = exportLocation + '/job/sqoop.zip'
                temporary_directory= tmpLocation
            }
        }
    }
    description= 'import sqoop job for saagie'
    taskName = 'projectImportSqoop'
}
task projectImportJava( // <======== Import artifacts, see that we use the gradle dependOn
        type: ProjectsImportJobTask) {
    dependsOn(['projectExportJava'])
    configuration = saagie {
        server {
            url = saagieurl
            login = saagieuserid
            password = saagiepassword
            environment = saagieplatformid
            jwt = true
        }
    }
    doFirst {
        projectImportJava.configuration = saagie {
            project {
                id = saagieprojectid
            }
            importArtifacts {
                import_file = exportLocation + '/job/java.zip'
                temporary_directory= tmpLocation
            }
        }
    }
    description= 'import java job for saagie'
    taskName = 'projectImportJava'
}
task projectImportR( // <======== Import artifacts, see that we use the gradle dependOn
        type: ProjectsImportJobTask) {
    dependsOn(['projectExportR'])
    configuration = saagie {
        server {
            url = saagieurl
            login = saagieuserid
            password = saagiepassword
            environment = saagieplatformid
            jwt = true
        }
    }
    doFirst {
        projectImportR.configuration = saagie {
            project {
                id = saagieprojectid
            }
            importArtifacts {
                import_file = exportLocation + '/job/r.zip'
                temporary_directory= tmpLocation
            }
        }
    }
    description= 'import R job for saagie'
    taskName = 'projectImportR'
}
task projectImportTalend( // <======== Import artifacts, see that we use the gradle dependOn
        type: ProjectsImportJobTask) {
    dependsOn(['projectExportTalend'])
    configuration = saagie {
        server {
            url = saagieurl
            login = saagieuserid
            password = saagiepassword
            environment = saagieplatformid
            jwt = true
        }
    }
    doFirst {
        projectImportTalend.configuration = saagie {
            project {
                id = saagieprojectid
            }
            importArtifacts {
                import_file = exportLocation + '/job/talend.zip'
                temporary_directory= tmpLocation
            }
        }
    }
    description= 'import Talend job for saagie'
    taskName = 'projectImportTalend'
}

task projectImportDocker( // <======== Import artifacts, see that we use the gradle dependOn
        type: ProjectsImportJobTask) {
    dependsOn(['projectExportDocker'])
    configuration = saagie {
        server {
            url = saagieurl
            login = saagieuserid
            password = saagiepassword
            environment = saagieplatformid
            jwt = true
        }
    }
    doFirst {
        projectImportDocker.configuration = saagie {
            project {
                id = saagieprojectid
            }
            importArtifacts {
                import_file = exportLocation + '/job/docker.zip'
                temporary_directory= tmpLocation
            }
        }
    }
    description= 'import Docker job for saagie'
    taskName = 'projectImportDocker'
}
task projectImportPipeline( // <======== Import artifacts, see that we use the gradle dependOn
        type: ProjectsImportJobTask) {
    dependsOn(['projectExportPipeline'])
    configuration = saagie {
        server {
            url = saagieurl
            login = saagieuserid
            password = saagiepassword
            environment = saagieplatformid
            jwt = true
        }
    }
    doFirst {
        projectImportPipeline.configuration = saagie {
            project {
                id = saagieprojectid
            }
            importArtifacts {
                import_file = exportLocation + '/pipeline/pipeline.zip'
                temporary_directory= tmpLocation
            }
        }
    }
    description= 'Import Pipeline with Dependency Job and all versions'
    taskName = 'projectImportPipeline'
}
task projectTestAll( // <======== Import artifacts, see that we use the gradle dependOn
        type:Exec) {
    dependsOn(['projectImportSqoop','projectImportJava','projectImportR','projectImportTalend','projectImportDocker', 'projectImportPipeline' ])
    commandLine 'echo','Testing'
}

Executing the script using this commande :
The command line : gradle -b build.newTask.gradle projectTestAll

[ZouhairBear]

• I have tested the script for all tasks : it works as expected

[ZouhairBear]

• test for export all artifacts

How to test :

Version to test : 2.1.19

The Export all artifacts phase:

Create a build file:
Example of build.exportAllArtifcats.gradle:

plugins {
  id "io.saagie.gradle-saagie-dataops-plugin" version "2.1.19"
}
saagie {
    server {
        url = "REPLACE_SAAGIEURL"
        login = "REPLACE_SAAGIELOGIN"
	password = "REPLACE_SAAGIEPASSWORD"
        environment = "REPLACE_SAAGIE"
	jwt = true
    }
    project {
        id = "REPLACE_SAAGIE_PROJECT_ID"
   include_all_artifacts = true
    }

    env {
        scope = 'project'
    }
        }
    exportArtifacts {
          export_file = "./export_all_artifacts.zip" // <=== url  should exist  
          overwrite=true  // <=== temporary_directory ./tmp'  should exist        
    }
}

Export all project artifacts using this commande :
The command line : gradle -b build.exportAllArtifcats.gradle projectsExport

[ZouhairBear]

• I have tested the export project with all artifacts : it works as expected

[ZouhairBear]

• test for import all artifacts

How to test :

Version to test : 2.1.19

The Export all artifacts phase:

Create a build file:
Example of build.importAllArtifcats.gradle:

plugins {
  id "io.saagie.gradle-saagie-dataops-plugin" version "2.1.19"
}
saagie {
    server {
        url = "REPLACE_SAAGIEURL"
        login = "REPLACE_SAAGIELOGIN"
	password = "REPLACE_SAAGIEPASSWORD"
        environment = "REPLACE_SAAGIE"
	jwt = true
    }
    project {
        id = "REPLACE_SAAGIE_PROJECT_ID"
   
    }

   
        }
    exportArtifacts {
          export_file = "./export_all_artifacts.zip" // <=== url  should exist  
          overwrite=true  // <=== temporary_directory ./tmp'  should exist        
    }
}

Export all project artifacts using this commande :
The command line : gradle -b build.importAllArtifcats.gradle projectsImport