Improve setup

ansible/jjs.yaml

@@ -1,69 +1,34 @@
 ---
-- remote_user: root
-  hosts: all
+- hosts: all
   vars:
     setup_profile: "{{ lookup('template', './profile-template.yaml.j2') }}"
   vars_prompt:
     - name: pkg_path
       prompt: "Where on local host jjs package is located?"
       private: false
       default: "/opt/jjs/pkg/jjs.deb"
+    - name: db_addr
+      prompt: "Database address (must include port; can be urlencoded unix socket path instead)"
+      private: false
+      default: "%2Fvar%2Frun%2Fpostgresql"
+  roles:
+    - db
+    - role: jjs_common
   tasks:
     - name: Install required APT packages
       apt:
-        name:
-          [
-            "postgresql-client-10",
-            "postgresql-10",
-            "g++",
-            "unzip",
-            "python3-setuptools",
-            "python3-pip",
-            "libpq-dev",
-          ]
+        name: ["g++"]
         update_cache: true
-    - name: Install python packages
-      pip:
-        name: psycopg2
-    - name: Copy JJS package
-      copy:
-        dest: /tmp/
-        src: "{{ pkg_path }}"
-    - name: "TODO: hack"
-      apt:
-        name: jjs
-        state: absent
-    - name: Install JJS package
-      apt:
-        deb: /tmp/jjs.deb
-        state: present
-    - name: Setup JJS user
-      user:
-        create_home: true
-        name: jjs
-    - name: Create JJS database
-      postgresql_db:
-        name: jjs
-      become: true
-      become_user: postgres
-    - name: Setup JJS postgres user
-      postgresql_user:
-        db: jjs
-        name: jjs
-        # TODO: take variable 
-        password: internal
-        role_attr_flags: SUPERUSER
-      become: true
-      become_user: postgres
     - name: Setup JJS
       command:
         # Note that "-" is passed as profile path. It tells `jjs-setup` to read profile from stdin.
-        cmd: jjs-setup - upgrade 
+        cmd: jjs-setup - upgrade
         stdin: "{{ setup_profile }}"
       become: true
       become_user: jjs
     - name: Reload SystemD
-      command: systemctl daemon-reload
+      systemd:
+        daemon_reload: true
     - name: Start JJS invoker
       service:
         name: jjs-invoker

ansible/profile-template.yaml.j2

@@ -2,7 +2,7 @@ install-dir: '/opt/jjs'
 data-dir: '/home/jjs'
 pg:
   db-name: jjs
-  conn-string: 'postgresql://jjs@%2Fvar%2Frun%2Fpostgresql/postgres'
+  conn-string: 'postgresql://jjs@{{ db_addr }}/postgres'
 toolchains: {}
 problems:
   tasks:

ansible/roles/db/tasks/main.yaml

@@ -0,0 +1,33 @@
+- name: Install APT packages
+  apt:
+    name:
+      - "libpq-dev"
+      - "python3-setuptools"
+      - "python3-pip"
+    update_cache: true
+- block:
+    - name: Install PG 11 APT packages
+      apt:
+        name:
+          - "postgresql-client-11"
+          - "postgresql-11"
+  rescue:
+    - name: Install PG 10 APT packages
+      apt:
+        name:
+          - "postgresql-client-10"
+          - "postgresql-10"
+- name: Create JJS database
+  postgresql_db:
+    name: jjs
+  become: true
+  become_user: postgres
+- name: Setup JJS postgres user
+  postgresql_user:
+    db: jjs
+    name: jjs
+    # TODO: take variable
+    password: internal
+    role_attr_flags: "SUPERUSER"
+  become: true
+  become_user: postgres

ansible/roles/jjs_common/tasks/main.yaml

@@ -0,0 +1,19 @@
+- name: Install APT packages
+  apt:
+    name: libpq-dev
+- name: Copy JJS package
+  copy:
+    dest: /tmp/
+    src: "{{ pkg_path }}"
+- name: "TODO: hack"
+  apt:
+    name: jjs
+    state: absent
+- name: Install JJS package
+  apt:
+    deb: /tmp/jjs.deb
+    state: present
+- name: Setup JJS user
+  user:
+    create_home: true
+    name: jjs

basic-setup-profile.yaml

@@ -5,10 +5,6 @@ pg:
   conn-string: "postgresql://jjs:internal@localhost:5432/jjs"
 toolchains: {}
 problems:
-  tasks:
-    - source:
-        path: "/opt/jjs/example-problems/a-plus-b"
-    - source:
-        path: "/opt/jjs/example-problems/array-sum"
-    - source:
-        path: "/opt/jjs/example-problems/sqrt"
+  archive:
+    sources:
+      - path: "/opt/jjs/pkg/problems.tgz"

ci-data/test-e2e-profile.yaml

@@ -5,10 +5,6 @@ pg:
   conn-string: "postgresql://jjs:internal@localhost:5432/jjs"
 toolchains: {}
 problems:
-  tasks:
-    - source:
-        path: "/opt/jjs/example-problems/a-plus-b"
-    - source:
-        path: "/opt/jjs/example-problems/array-sum"
-    - source:
-        path: "/opt/jjs/example-problems/sqrt"
+  archive:
+    sources:
+      - path: "/opt/jjs/pkg/problems.tgz"

src/deploy/Cargo.toml

@@ -24,3 +24,5 @@ shell-escape = "0.1.4"
 tera = "1.0.2"
 util = {path = "../util"}
 log = "0.4.8"
+anyhow = "1.0.28"
+flate2 = "1.0.14"

src/deploy/bin/configure.rs

@@ -78,6 +78,9 @@ struct Opt {
     /// If enabled, json schemas will be emitted
     #[structopt(long = "enable-json-schema")]
     json_schema: bool,
+    /// If enabled, trial contest problems will be compiled and packaged into archive
+    #[structopt(long = "enable-compile-trial-contest")]
+    compile_trial_contest: bool,
 }
 
 impl Opt {
@@ -222,6 +225,7 @@ fn main() {
         extras: opt.extras,
         api_doc: opt.apidoc,
         json_schema: opt.json_schema,
+        example_problems: opt.compile_trial_contest,
     };
     let packaging = cfg::PackagingConfig {
         deb: if opt.deb {

src/deploy/src/cfg.rs

@@ -35,6 +35,7 @@ pub struct ComponentsConfig {
     pub core: bool,
     pub extras: bool,
     pub json_schema: bool,
+    pub example_problems: bool,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

src/deploy/src/lib.rs

@@ -21,6 +21,7 @@ use crate::{
     util::print_section,
 };
 use ::util::cmd::{CommandExt, Runner};
+use anyhow::Context as _;
 use std::{ffi::OsStr, fs, path::PathBuf, process::Command};
 
 pub struct Params {
@@ -141,6 +142,12 @@ pub fn package(params: &Params, runner: &Runner) {
     if params.cfg.components.json_schema {
         generate_json_schema(params);
     }
+    if params.cfg.components.example_problems {
+        if let Err(err) = compile_sample_contest(params) {
+            eprintln!("error: {:#}", err);
+            runner.error();
+        }
+    }
     runner.exit_if_errors();
 
     generate_envscript(params);
@@ -292,6 +299,33 @@ fn generate_json_schema(params: &Params) {
         .expect("failed to write schema");
 }
 
+fn compile_sample_contest(params: &Params) -> anyhow::Result<()> {
+    print_section("Compiling sample contest");
+    let items = std::fs::read_dir(params.src.join("example-problems"))?;
+    let intermediate_problems_dir = params.build.join("compiled-problems");
+    for item in items {
+        let item = item?;
+        let mut cmd = std::process::Command::new(params.artifacts.join("bin/jjs-ppc"));
+        cmd.arg("compile");
+        cmd.arg("--pkg").arg(item.path());
+        let dir = intermediate_problems_dir.join(item.file_name());
+
+        std::fs::remove_dir_all(&dir).ok();
+        std::fs::create_dir_all(&dir).ok();
+        cmd.env("JJS_PATH", &params.artifacts);
+        cmd.arg("--out").arg(dir);
+        cmd.try_exec().context("can not execute jjs-ppc")?;
+    }
+    let out_file = std::fs::File::create(params.artifacts.join("pkg/problems.tgz"))?;
+    let out_file = flate2::write::GzEncoder::new(out_file, flate2::Compression::best());
+    let mut tarball_builder = tar::Builder::new(out_file);
+    tarball_builder
+        .append_dir_all("", intermediate_problems_dir)
+        .context("write problem packages to archive")?;
+    tarball_builder.into_inner()?;
+    Ok(())
+}
+
 fn env_add(var_name: &str, prepend: &str) -> String {
     format!("export {}={}:${}", var_name, prepend, var_name)
 }

src/devtool/src/build.rs

@@ -93,6 +93,9 @@ pub(crate) fn task_build(opts: RawBuildOpts, runner: &Runner) -> anyhow::Result<
     } else {
         cmd.arg("--disable-man");
     }
+    if opts.raw().setup || detect_build_type().is_pr_e2e() {
+        cmd.arg("--enable-compile-trial-contest");
+    }
 
     for opt in &opts.raw().configure {
         cmd.arg(opt);
@@ -105,7 +108,7 @@ pub(crate) fn task_build(opts: RawBuildOpts, runner: &Runner) -> anyhow::Result<
 
     if opts.raw().setup {
         println!("running setup");
-        std::fs::remove_dir_all("/tmp/jjs").ok();
+        // std::fs::remove_dir_all("/tmp/jjs").ok();
         Command::new("/opt/jjs/bin/jjs-setup")
             .arg("./basic-setup-profile.yaml")
             .arg("upgrade")

src/setup/Cargo.toml

@@ -24,3 +24,5 @@ serde_yaml = "0.8.11"
 serde_json = "1.0.49"
 toml = "0.5.6"
 futures = "0.3.4"
+flate2 = {version = "1.0.14", features = ["tokio"]}
+tar = "0.4.26"

src/setup/src/data.rs

@@ -62,6 +62,7 @@ impl<'a> crate::Component for DataLayout<'a> {
         tokio::fs::create_dir(base.join("etc/objects/toolchains")).await?;
         tokio::fs::create_dir(base.join("etc/objects/contests")).await?;
         tokio::fs::create_dir(base.join("opt")).await?;
+        tokio::fs::create_dir(base.join("tmp")).await?;
         Ok(())
     }
 }

src/setup/src/lib.rs

@@ -1,3 +1,4 @@
+#![feature(backtrace)]
 // for async-trait
 #![allow(clippy::needless_lifetimes)]
 pub mod config;