refactor(grpc): 🔥 try to use cfg for a single grpc crate

backend/build.rs

@@ -29,15 +29,13 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             "oj.backend.AnnouncementSortBy",
             "#[derive(serde::Serialize, serde::Deserialize)]",
         )
-        .compile(&["../proto/backend.proto"], &["../proto"])?;
-    // tonic_build::compile_protos("../proto/backend.proto")?;
-    // tonic_build::compile_protos("../proto/judger.proto")?;
+        .compile(&["../grpc/proto/backend.proto"], &["../grpc/proto"])?;
     tonic_build::configure()
         .build_server(false)
         .type_attribute(
             "oj.backend.SortBy",
             "#[derive(serde::Serialize, serde::Deserialize)]",
         )
-        .compile(&["../proto/judger.proto"], &["../proto"])?;
+        .compile(&["../grpc/proto/judger.proto"], &["../grpc/proto"])?;
     Ok(())
 }

frontend/Cargo.toml

@@ -62,7 +62,7 @@ features = ["rt-multi-thread", "fs", "sync"]
 
 [dependencies.grpc]
 path = "../grpc"
-features = ["serde"]
+features = ["wkt"]
 
 [build-dependencies.tonic-build]
 features = ["prost"]

frontend/build.rs

@@ -1,6 +1,6 @@
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     tonic_build::configure()
         // .type_attribute(".", "#[derive(Serialize,Deserialize)]")
-        .compile(&["../proto/backend.proto"], &["../proto/"])?;
+        .compile(&["../grpc/proto/backend.proto"], &["../grpc/proto/"])?;
     Ok(())
 }

grpc/Cargo.toml

@@ -27,5 +27,8 @@ features = ["prost"]
 workspace = true
 
 [features]
+default = ["wkt"]
+wkt = ["serde"]
+backend = []
 serde = ["dep:serde"]
-transport = ["tonic-build/transport"]
+transport = ["tonic-build/transport"]

grpc/build.rs

@@ -4,7 +4,10 @@ use tonic_build::configure;
 fn main() {
     let out = PathBuf::from(env::var("OUT_DIR").unwrap());
     let descriptor_file = out.join("descriptors.bin");
+    #[cfg(feature = "wkt")]
     configure()
+        // FIXME: this line tell us how build.rs works
+        // .build_server(false)
         .type_attribute(
             ".",
             r#"#[cfg_attr(feature = "serde", derive(serde::Serialize,serde::Deserialize))]"#,
@@ -13,12 +16,20 @@ fn main() {
         .extern_path(".google.protobuf.Timestamp", "::prost_wkt_types::Timestamp")
         .extern_path(".google.protobuf.Value", "::prost_wkt_types::Value")
         .file_descriptor_set_path(&descriptor_file)
-        .compile(
-            &["../proto/backend.proto", "../proto/judger.proto"],
-            &["../proto"],
+        .compile(&["proto/backend.proto", "proto/judger.proto"], &["proto"])
+        .unwrap();
+    #[cfg(feature = "backend")]
+    #[cfg(not(feature = "wkt"))]
+    configure()
+        .type_attribute(
+            ".",
+            r#"#[cfg_attr(feature = "serde", derive(serde::Serialize,serde::Deserialize))]"#,
         )
+        .file_descriptor_set_path(&descriptor_file)
+        .compile(&["proto/backend.proto", "proto/judger.proto"], &["proto"])
         .unwrap();
     #[cfg(feature = "serde")]
+    #[cfg(feature = "wkt")]
     {
         use prost_wkt_build::*;
         let descriptor_bytes = std::fs::read(descriptor_file).unwrap();

grpc/src/lib.rs

@@ -1,7 +1,15 @@
+#[cfg(feature = "wkt")]
 pub mod backend {
     tonic::include_proto!("oj.backend");
 }
 
+#[cfg(feature = "backend")]
+#[cfg(not(feature = "wkt"))]
+pub mod backend {
+    tonic::include_proto!("oj.backend");
+}
+
+#[cfg(feature = "backend")]
 pub mod judger {
     tonic::include_proto!("oj.judger");
 }

judger/Cargo.toml

@@ -15,12 +15,16 @@ thiserror = "1.0.40"
 toml = { workspace = true }
 derive_builder = { workspace = true }
 tar = "0.4.40"
-grpc = { path = "../grpc" }
 lazy_static = "1.4.0"
 libc = "0.2.154"
 bytes = "1.6.0"
 async-stream = "0.3.5"
 
+[dependencies.grpc]
+path = "../grpc"
+features = ["backend"]
+default-features = false
+
 [dependencies.log]
 version = "0.4.17"
 features = ["release_max_level_debug"]

judger/dev.md

@@ -0,0 +1,3 @@
+# MDOJ judger
+
+MODJ judger is a grpc server the provide `JudgeService`, see `/proto`.

judger/justfile

@@ -1,5 +1,5 @@
 build-plugin:
-    cd plugins && sh build-all.sh
+    cd plugins && sudo sh build-all.sh
     mkdir -p config plugins-out
     mv plugins/*.lang plugins-out/
 

judger/plugins/readme.md

@@ -1,76 +1,55 @@
-# Develop a module for new language support
+# Develop a module for new language support(plugin)
 
-`rlua-54` and `rlua-54` is the example plugin
+`rlua-54` and `c-11` is the example plugin
 
-## How it works?
+|name|application| 
+|:-:|:-|
+|rlua-54|interpreter language|
+|c-11|compile language|
 
-Compile Stage:
+an language support it's just a tar file containing a whole filesystem, including `spec.toml` at root of tar file.
 
-1. Start compile command in jail
-2. Daemon write source code to jail's stdout, and close pipe
-3. ``Compile`` subprocess exit
+## How to generate such tar file?
 
-Execute Stage:
+use `docker export`
 
-4. Start execute command in jail
-5. Daemon write input to jail's stdin
-6. ``Execute`` subprocess exit
-7. Daemon exam the exit status of subprocess
-8. Daemon exam the stdout of ``Execute`` subprocess.
-
-If you would like to log, write following string(utf-8 only) to stderr in compile stage:
-
-```log
-<Level><Message>
+```
+docker export ___ > c-11.lang
 ```
 
-Don't append ``\n`` in the end of the stdout, and it should never be ``\r\n``.
-
-LEVEL should be number 1-6, would be interpreted as TRACE, DEBUG, INFO, WARN, ERROR.
-
-The return code of non-zero would be considered Compile Error.
-
-## How to develop a container for language support
+## spec.toml
 
-One of the recommanded way is with ``docker export``.
+Not all field is required, the minimal field required is show as example in `rula-54`.
 
-Program your own docker image and run:
-
-```shell
-# if docker is not in rootless mode you need add `sudo` in front of docker command
-docker export ${id of your docker container} | tar -C plugins/${plugin name}/rootfs -xvf -
-```
+the full spec.toml is show below:
 
-finish spec.toml like this
-```ini
-# memory in byte, time in microsecond
-info = "A Lua 5.4 runtime build both real workloads and sandbox tests"
-extension = "lua" # same extension means same language
-name = "rlua-54" # must be same as dictionary name
-uid = "1c41598f-e253-4f81-9ef5-d50bf1e4e74f" # be sure it's unique
+```toml
+file = "/code.c"
+fs_limit = 3145728 # number of byte the whole process(compile+judge) is allowed to write
+info = "gcc 13.2.0 (G++)"
+extension = "c"
+name = "c-11"
+id = "7daff707-26b5-4153-90ae-9858b9fd9619" # you can generate it randomly(https://www.uuidgenerator.net)
 
 [compile]
-lockdown = true
-command = ["/rlua-54","compile"]
-kernel_mem = 67108864
-user_mem = 268435456
-rt_time = 1000000
-cpu_time = 10000000
-total_time = 10000000
+command = ["/usr/bin/cc","-x", "c", "code.c", "-lm", "-o", "/execute"]
+kernel_mem = 1 # number of kernel space memory limit in byte
+memory = 1 # number of total memory limit in byte
+user_mem = 1 # number of userspace memory limit in byte
+rt_time = 1 # number of non-preemptible execution time in nanosecond
+cpu_time = 1 # number of preemptible execution time in nanosecond
+total_time = 1 # # number of non-preemptible execution time in nanosecond
+walltime = 1 # number of time in **milliseconds**(realtime, it count even scheduler didn't dispatch any time for the task)/
 
 [judge]
-command = ["/rlua-54","execute"]
-kernel_mem = 67108864
-multiplier_memory = 6 # user_mem
-rt_time = 1000000
-multiplier_cpu = 3 # cpu_time
-```
-
-## Troubleshooting
-
-### Execute jail recieve SIGKILL with no reason
-
-Possiblility:
-
-1. Execute jail was lockdown(cannot write anything) for security reason, and the process in which try to open a file with write flag('w' in name of chmod).
+command = ["/execute"]
+kernel_mem = 1 # number of kernel space memory limit in byte
+rt_time = 1 # number of non-preemptible execution time in nanosecond
+cpu_time = 1 # number of preemptible execution time in nanosecond
+total_time = 1 # # number of non-preemptible execution time in nanosecond
+memory_multiplier = 1.0 # multiplier for total memory limit in byte
+cpu_multiplier = 1.0  # multiplier for total cpu execution limit in nanosecond
+output = 1 # max output in byte
+walltime = 1 # number of time in **milliseconds**(realtime, it count even scheduler didn't dispatch any time for the task)/
 
+```

judger/plugins/rlua-54/Dockerfile

@@ -17,6 +17,6 @@ FROM scratch
 WORKDIR /
 COPY --from=builder /usr/local/cargo/bin/rlua-54 /
 
-COPY sepc.toml /
+COPY spec.toml /
 
 CMD ["/rlua-54/rlua-54"]

judger/plugins/rlua-54/build.sh

@@ -1,4 +1,4 @@
 mkdir -p rootfs
-docker build -t rlua-54-mdoj-plugin .
+docker build --build-arg ARCH=$(uname -m) -t rlua-54-mdoj-plugin .
 docker export $(docker create rlua-54-mdoj-plugin) > rlua-54.lang
 mv rlua-54.lang ..

testsuit/build.rs

@@ -1,4 +1,4 @@
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    tonic_build::compile_protos("../proto/backend.proto")?;
+    tonic_build::compile_protos("../grpc/proto/backend.proto")?;
     Ok(())
 }