Merge pull request #187 from benjamin-747/main

add some bazel config in env file

.env

@@ -15,4 +15,10 @@ GIT_INTERNAL_DECODE_CACHE_SIZE = 1000 # Maximum number of git objects in LRU cac
 GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE = 10000 # The maximum number of git object in a "INSERT" SQL database operation
 GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE = 10 # The maximum number of parallel insertion threads in the database operation queue
 GIT_INTERNAL_DECODE_CACHE_TYEP = "lru" #{lru,redis}
-REDIS_CONFIG = "redis://127.0.0.1:6379"
+REDIS_CONFIG = "redis://127.0.0.1:6379"
+
+
+# Bazel build config, you can use service like buildfarm to enable RBE(remote build execution)
+BAZEL_BUILDP_PATH = "/tmp/.mega/bazel_build_projects" # Specify a temporary directory to build the project with bazel
+BAZEL_REMOTE_EXECUTOR = "grpc://192.168.8.120:8980" # If enable the remote executor and remote executor address, leave empty if you want to disable it.
+BAZEL_GIT_CLONE_URL = "http://localhost:8000" # Tell bazel to clone the project from the specified git url

.env.example

@@ -5,8 +5,20 @@ MEGA_DB_MYSQL_URL = "mysql://${MYSQL_USERNAME}:${MYSQL_SECRET}@${MYSQL_HOST}/meg
 MEGA_DB_MAX_CONNECTIONS = 32
 MEGA_DB_MIN_CONNECTIONS = 16
 
+
+# If the object file size exceeds the threshold value, it will be stored in the specified location instead of the database.
+MEGA_BIG_OBJ_THRESHOLD_SIZE = 1024 # Unit KB
+MEGA_BIG_OBJ_STORAGR_PATH = "/tmp/.mega/objects"
+
+
 GIT_INTERNAL_DECODE_CACHE_SIZE = 1000 # Maximum number of git objects in LRU cache
 GIT_INTERNAL_DECODE_STORAGE_BATCH_SIZE = 10000 # The maximum number of git object in a "INSERT" SQL database operation
 GIT_INTERNAL_DECODE_STORAGE_TQUEUE_SIZE = 10 # The maximum number of parallel insertion threads in the database operation queue
 GIT_INTERNAL_DECODE_CACHE_TYEP = "lru" #{lru,redis}
-REDIS_CONFIG = "redis://127.0.0.1:6379"
+REDIS_CONFIG = "redis://127.0.0.1:6379"
+
+
+# Bazel build config, you can use service like buildfarm to enable RBE(remote build execution)
+BAZEL_BUILDP_PATH = "/tmp/.mega/bazel_build_projects" # Specify a temporary directory to build the project with bazel
+BAZEL_REMOTE_EXECUTOR = "grpc://localhost:8980" # If enable the remote executor and remote executor address, leave empty if you want to disable it.
+BAZEL_GIT_CLONE_URL = "http://localhost:8000" # Tell bazel to clone the project from the specified git url

WORKSPACE

@@ -1,8 +1,8 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 http_archive(
     name = "rules_rust",
-    sha256 = "c46bdafc582d9bd48a6f97000d05af4829f62d5fee10a2a3edddf2f3d9a232c1",
-    urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.28.0/rules_rust-v0.28.0.tar.gz"],
+    sha256 = "814680e1ab535f799fd10e8739ddca901351ceb4d2d86dd8126c22d36e9fcbd9",
+    urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.29.0/rules_rust-v0.29.0.tar.gz"],
 )
 
 load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains")
@@ -33,7 +33,8 @@ crates_repository(
         "//:p2p/Cargo.toml",
         "//:mda/Cargo.toml",
         "//:kvcache/Cargo.toml",
-        "//:sync/Cargo.toml"
+        "//:sync/Cargo.toml",
+        "//:build-tool/Cargo.toml"
     ],
 )
 

build-tool/BUILD

@@ -0,0 +1,16 @@
+
+load("@crate_index//:defs.bzl", "aliases", "all_crate_deps")
+load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test", "rust_doc_test")
+
+rust_library(
+    name = "build-tool",
+    srcs = glob([
+        "src/**/*.rs",
+    ]),
+    aliases = aliases(),
+    deps = all_crate_deps(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    visibility = ["//visibility:public"],
+)

build-tool/src/bazel_build.rs

@@ -7,13 +7,13 @@ use std::{
 };
 use url::Url;
 
-pub fn build() {
-    let repo_path = PathBuf::from("/projects/mega");
+pub fn build(repo_path: PathBuf) {
+    let mut temp = PathBuf::from(env::var("BAZEL_BUILDP_PATH").unwrap());
+
     let project_name = repo_path.file_name().unwrap();
 
-    let mut temp = PathBuf::from("/tmp/.mega/bazel_build_projects");
     temp.push(project_name);
-    let mut project_url = Url::parse("http://localhost:8000").unwrap();
+    let mut project_url = Url::parse(&env::var("BAZEL_GIT_CLONE_URL").unwrap()).unwrap();
     project_url.set_path(repo_path.to_str().unwrap());
     if temp.exists() {
         if let Err(err) = fs::remove_dir_all(&temp) {
@@ -22,7 +22,7 @@ pub fn build() {
             tracing::info!("repo removed successfully: {:?}", project_name);
         }
     }
-    Repository::clone(project_url.as_ref(), &temp).expect("failed to clone");
+    Repository::clone(project_url.as_ref(), &temp).expect("failed to clone project");
 
     if let Err(err) = env::set_current_dir(&temp) {
         tracing::error!("Failed to change the working directory: {}", err);
@@ -50,8 +50,12 @@ pub fn build() {
         }
 
         // Execute bazel build
+        let mut remote_executor = String::new();
+        if let Ok(remote_exec) = env::var("BAZEL_REMOTE_EXECUTOR") {
+            remote_executor = format!("--remote_executor={}", remote_exec);
+        }
         let mut bazel_build_child = Command::new("bazel")
-            .args(["build", "//:mega"])
+            .args(["build", &remote_executor, "//:mega"])
             .stdout(Stdio::piped())
             .spawn()
             .expect("Failed to start the bazel build");

git/BUILD

@@ -13,6 +13,7 @@ rust_library(
         "//database",
         "//database/entity",
         "//kvcache",
+        "//build-tool"
     ],
     proc_macro_deps = all_crate_deps(
         proc_macro = True,

git/src/protocol/pack.rs

@@ -199,7 +199,8 @@ impl PackProtocol {
                 // save project directory
                 self.handle_directory().await.unwrap();
                 // start building
-                thread::spawn(build_tool::bazel_build::build);
+                let repo_path = self.path.clone();
+                thread::spawn(|| build_tool::bazel_build::build(repo_path));
             } else {
                 tracing::error!("{}", parse_obj_result.err().unwrap());
                 command.failed(String::from("db operation failed"));