Merge pull request #141 from filecoin-project/feat/auto-clone-repo

Testing clone, modified environment variables and CI/CD

.github/workflows/deploy_to_prod.yml

@@ -8,6 +8,7 @@ on:
 jobs:
   deploy_to_prod:
     runs-on: ubuntu-latest
+    environment: production
 
     steps:
     - name: Checkout code
@@ -49,13 +50,12 @@ jobs:
             },
             "environment": { 
               "GH_PRIVATE_KEY": "${{secrets.GH_PRIVATE_KEY}}",
-              "GITHUB_OWNER": "${{secrets.GH_OWNER}}",
-              "GITHUB_REPO": "${{secrets.GH_REPO}}",
-              "GITHUB_APP_ID": "${{secrets.GH_APP_ID}}",
-              "GITHUB_INSTALLATION_ID": "${{secrets.GH_INSTALLATION_ID}}",
+              "GITHUB_APP_ID": "${{vars.GH_APP_ID}}",
               "FILPLUS_ENV": "prod",
               "RUST_LOG": "debug",
-              "BOT_USER": "${{secrets.BOT_USER}}"
+              "BOT_USER": "${{vars.BOT_USER}}",
+              "BACKEND_URL": "${{vars.BACKEND_URL}}",
+              "DB_URL": "${{secrets.DB_URL}}"
             }
           }
         }' > containers.json

.github/workflows/deploy_to_staging.yml

@@ -24,6 +24,7 @@ jobs:
     name: Deploy to staging
     needs: end_to_end_tests
     runs-on: ubuntu-latest
+    environment: staging
 
     steps:
     - name: Checkout code
@@ -64,8 +65,14 @@ jobs:
               "8080": "HTTP"
             },
             "environment": { 
-                "GH_PRIVATE_KEY": "${{secrets.GH_STAGING_PRIVATE_KEY}}",
-                "DB_URL": "${{secrets.STAGING_DB_URL}}"
+                "GH_PRIVATE_KEY": "${{secrets.GH_PRIVATE_KEY}}",
+                "DB_URL": "${{secrets.DB_URL}}"
+                "GITHUB_APP_ID": "${{vars.GH_APP_ID}}",
+                "FILPLUS_ENV": "prod",
+                "RUST_LOG": "debug",
+                "BOT_USER": "${{vars.BOT_USER}}",
+                "BACKEND_URL": "${{vars.BACKEND_URL}}",
+                "DB_URL": "${{secrets.DB_URL}}"
             }
           }
         }' > containers.json

fplus-http-server/src/router/allocator.rs

@@ -1,6 +1,6 @@
 use actix_web::{get, post, put, delete, web, HttpResponse, Responder};
 use fplus_database::database::allocators as allocators_db;
-use fplus_lib::core::{allocator::process_allocator_file, AllocatorUpdateInfo, ChangedAllocator};
+use fplus_lib::core::{allocator::{process_allocator_file, is_allocator_repo_created, create_allocator_repo}, AllocatorUpdateInfo, ChangedAllocator};
 
 /**
  * Get all allocators
@@ -44,14 +44,29 @@ pub async fn create_from_json(file: web::Json<ChangedAllocator>) -> actix_web::R
                 Some(model.application.verifiers_gh_handles.join(", ")) // Join verifiers in a string if exists
             };
 
-            match allocators_db::create_or_update_allocator(
-                model.owner,
-                model.repo,
+            let allocator_model = match allocators_db::create_or_update_allocator(
+                model.owner.clone(),
+                model.repo.clone(),
                 Some(model.installation_id as i64), 
                 Some(model.multisig_address),      
                 verifiers_gh_handles,
             ).await {
-                Ok(allocator_model) => Ok(HttpResponse::Ok().json(allocator_model)),
+                Ok(allocator_model) => allocator_model,
+                Err(e) => return Ok(HttpResponse::BadRequest().body(e.to_string())),
+            };
+
+            match is_allocator_repo_created(&model.owner, &model.repo).await {
+                Ok(true) => Ok(HttpResponse::Ok().json(allocator_model)),
+                Ok(false) => {
+                    //Create allocator repo. If it fails, return http error
+                    match create_allocator_repo(&model.owner, &model.repo).await {
+                        Ok(files_list) => {
+                            log::info!("Allocator repo created successfully: {:?}", files_list);
+                            Ok(HttpResponse::Ok().json(allocator_model))
+                        }
+                        Err(e) => Ok(HttpResponse::BadRequest().body(e.to_string())),
+                    }
+                },
                 Err(e) => Ok(HttpResponse::BadRequest().body(e.to_string())),
             }
         },

fplus-lib/src/config.rs

@@ -9,7 +9,7 @@ pub fn default_env_vars() -> &'static HashMap<&'static str, &'static str> {
         m.insert("GITHUB_OWNER", "filecoin-project");
         m.insert("GITHUB_REPO", "filecoin-plus-falcon");
         m.insert("GITHUB_APP_ID", "826129");
-        m.insert("GITHUB_INSTALLATION_ID", "47207972");
+        m.insert("GITHUB_INSTALLATION_ID", "48206379");
         m.insert("RUST_LOG", "info");
         m.insert("RUST_BACKTRACE", "1");
         m.insert("DB_URL", "");

fplus-lib/src/core/allocator/mod.rs

@@ -1,7 +1,7 @@
 use octocrab::models::repos::ContentItems;
 
 use crate::config::get_env_var_or_default;
-use crate::external_services::github::github_async_new;
+use crate::external_services::github::{github_async_new, GithubWrapper};
 use crate::{base64::decode_allocator_model, error::LDNError};
 
 use self::file::AllocatorModel;
@@ -12,17 +12,17 @@ pub async fn process_allocator_file(file_name: &str) -> Result<AllocatorModel, L
 
     let owner = get_env_var_or_default("ALLOCATOR_GOVERNANCE_OWNER");
     let repo = get_env_var_or_default("ALLOCATOR_GOVERNANCE_REPO");
+    let installation_id = get_env_var_or_default("GITHUB_INSTALLATION_ID");
     let branch = "main";
     let path = file_name.to_string();
 
-    let gh = github_async_new(owner.to_string(), repo.to_string()).await;
+    let gh = GithubWrapper::new(owner.clone(), repo.clone(), installation_id);
     let content_items = gh.get_file(&path, branch).await.map_err(|e| LDNError::Load(e.to_string()))?;
     let model = content_items_to_allocator_model(content_items).map_err(|e| LDNError::Load(e.to_string()))?;
 
     Ok(model)
 }
 
-
 fn content_items_to_allocator_model(file: ContentItems) -> Result<AllocatorModel, LDNError> {
     let encoded_content = match file.items.get(0).and_then(|f| f.content.clone()) {
         Some(content) => {
@@ -48,4 +48,89 @@ fn content_items_to_allocator_model(file: ContentItems) -> Result<AllocatorModel
             Err(LDNError::Load("Failed to parse allocator model".to_string()))
         }
     }
+}
+
+pub async fn is_allocator_repo_created(owner: &str, repo: &str) -> Result<bool, LDNError> {
+    let repo_flag_file = "invalisd.md";
+    let applications_directory = "applications";
+    let gh = github_async_new(owner.to_string(), repo.to_string()).await;
+    let all_files_result = gh.get_files(applications_directory).await.map_err(|e| {
+        LDNError::Load(format!("Failed to retrieve all files from GitHub. Reason: {}", e))
+    });
+
+    match all_files_result {
+        Ok(content_items) => {
+            let mut is_repo_created = false;
+            for file in content_items.items.iter() {
+                if file.name == repo_flag_file {
+                    is_repo_created = true;
+                    break;
+                }
+            }
+            Ok(is_repo_created)
+        },
+        Err(e) => {
+            if e.to_string().contains("GitHub: This repository is empty") || e.to_string().contains("GitHub: Not Found"){
+                Ok(false)
+            } else {
+                Err(e)
+            }
+        },
+    }
+}
+
+pub async fn create_allocator_repo(owner: &str, repo: &str) -> Result<(), LDNError> {
+    let gh = github_async_new(owner.to_string(), repo.to_string()).await;
+    let mut dirs = Vec::new();
+    dirs.push("".to_string());
+
+    while dirs.len() > 0 {
+        let dir = dirs.pop().unwrap();
+        let files_list = gh.get_files_from_public_repo("clriesco", "filplus-allocator-template", Some(&dir)).await.map_err(|e| {
+            LDNError::Load(format!("Failed to retrieve all files from GitHub. Reason: {}", e))
+        })?;
+
+        for file in files_list.items.iter() {
+            let file_path = file.path.clone();
+            if file.r#type == "dir" {
+                dirs.push(file_path);
+                continue;
+            }
+            let file = reqwest::Client::new()
+            .get(&file.download_url.clone().unwrap())
+            .send()
+            .await
+            .map_err(|e| LDNError::Load(format!("here {}", e)))?;
+            let file = file
+                .text()
+                .await
+                .map_err(|e| LDNError::Load(format!("here1 {}", e)))?;
+
+            //Get file from target repo. If file does not exist or fails to retrieve, create it
+            let target_file = gh.get_file(&file_path, "main").await.map_err(|e| {
+                LDNError::Load(format!("Failed to retrieve file from GitHub. Reason: {} in file {}", e, file_path))
+            });
+
+            match target_file {
+                Ok(target_file) => {
+                    if target_file.items.is_empty() {
+                        log::info!("Creating file in target repo: {}", file_path);
+                        gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
+                            LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path))
+                        })?;
+                    } else {
+                        log::info!("File already exists in target repo: {}", file_path);
+                    }
+                },
+                Err(_) => {
+                    log::info!("Creating file in target repo: {}", file_path);
+                    gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
+                        LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path))
+                    })?;
+                },
+            }
+        }
+    }
+
+    Ok(())
 }

fplus-lib/src/external_services/github.rs

@@ -711,4 +711,23 @@ impl GithubWrapper {
             .await?;
         Ok(pull_request.head.ref_field.clone())
     }
+
+    pub async fn get_files_from_public_repo(
+        &self,
+        owner: &str, 
+        repo: &str, 
+        path: Option<&str>
+    ) -> Result<ContentItems, OctocrabError> {
+        let branch = "main";
+        let octocrab = Octocrab::builder().build()?;
+        let gh = octocrab.repos(owner, repo);
+
+        //if path is not provided, take all files from root
+        let contents_items = match path {
+            Some(p) => gh.get_content().r#ref(branch).path(p).send().await?,
+            None => gh.get_content().r#ref(branch).send().await?,
+        };
+
+        Ok(contents_items)
+    }
 }