From 43a7025514f9bb027a92553d1c9405176407cb8f Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Thu, 3 Sep 2020 00:08:35 +0300 Subject: [PATCH 01/11] Rename invoker -> judge --- Cargo.lock | 50 +++++++++++++++++-- Cargo.toml | 4 +- .../templates/{invoker.yaml => judge.yaml} | 30 +++++------ k8s/jjs/templates/network-policy.yaml | 4 +- src/dist-builder/src/main.rs | 2 +- src/{invoker => judge}/Cargo.toml | 4 +- src/{invoker => judge}/Dockerfile | 4 +- src/{invoker => judge}/src/api.rs | 0 src/{invoker => judge}/src/config.rs | 2 +- src/{invoker => judge}/src/controller.rs | 10 ++-- .../src/controller/notify.rs | 2 +- .../src/controller/task_loading.rs | 0 .../src/controller/toolchains.rs | 0 src/{invoker => judge}/src/init.rs | 0 src/{invoker => judge}/src/lib.rs | 0 src/{invoker => judge}/src/main.rs | 46 ++++++++--------- src/{invoker => judge}/src/scheduler.rs | 6 +-- src/{invoker => judge}/src/sources.rs | 0 .../src/sources/api_source.rs | 8 +-- .../src/sources/cli_source.rs | 10 ++-- src/{invoker => judge}/src/worker.rs | 30 +++++------ src/{invoker => judge}/src/worker/compiler.rs | 4 +- .../src/worker/exec_test.rs | 4 +- .../src/worker/exec_test/checker_proto.rs | 0 .../src/worker/invoke_util.rs | 2 +- src/{invoker => judge}/src/worker/os_util.rs | 0 .../src/worker/transform_judge_log.rs | 4 +- src/{invoker => judge}/src/worker/valuer.rs | 2 +- .../tests/separated_feedback.rs | 0 src/{invoker-api => judging-apis}/Cargo.toml | 2 +- .../src/judge_log.rs | 0 src/{invoker-api => judging-apis}/src/lib.rs | 0 .../src/valuer_proto.rs | 0 src/svaluer/Cargo.toml | 2 +- src/svaluer/src/fiber.rs | 6 +-- src/svaluer/src/fiber/group.rs | 8 +-- src/svaluer/src/lib.rs | 15 +++--- src/svaluer/src/main.rs | 16 +++--- src/svaluer/src/tests.rs | 2 +- 39 files changed, 159 insertions(+), 120 deletions(-) rename k8s/jjs/templates/{invoker.yaml => judge.yaml} (76%) rename src/{invoker => judge}/Cargo.toml (96%) rename src/{invoker => judge}/Dockerfile (73%) rename src/{invoker => judge}/src/api.rs (100%) rename src/{invoker => judge}/src/config.rs (98%) rename src/{invoker => judge}/src/controller.rs (96%) rename src/{invoker => judge}/src/controller/notify.rs (98%) rename src/{invoker => judge}/src/controller/task_loading.rs (100%) rename src/{invoker => judge}/src/controller/toolchains.rs (100%) rename src/{invoker => judge}/src/init.rs (100%) rename src/{invoker => judge}/src/lib.rs (100%) rename src/{invoker => judge}/src/main.rs (80%) rename src/{invoker => judge}/src/scheduler.rs (98%) rename src/{invoker => judge}/src/sources.rs (100%) rename src/{invoker => judge}/src/sources/api_source.rs (96%) rename src/{invoker => judge}/src/sources/cli_source.rs (93%) rename src/{invoker => judge}/src/worker.rs (92%) rename src/{invoker => judge}/src/worker/compiler.rs (96%) rename src/{invoker => judge}/src/worker/exec_test.rs (98%) rename src/{invoker => judge}/src/worker/exec_test/checker_proto.rs (100%) rename src/{invoker => judge}/src/worker/invoke_util.rs (99%) rename src/{invoker => judge}/src/worker/os_util.rs (100%) rename src/{invoker => judge}/src/worker/transform_judge_log.rs (98%) rename src/{invoker => judge}/src/worker/valuer.rs (97%) rename src/{invoker => judge}/tests/separated_feedback.rs (100%) rename src/{invoker-api => judging-apis}/Cargo.toml (93%) rename src/{invoker-api => judging-apis}/src/judge_log.rs (100%) rename src/{invoker-api => judging-apis}/src/lib.rs (100%) rename src/{invoker-api => judging-apis}/src/valuer_proto.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index b06636a9..48541785 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1679,7 +1679,49 @@ dependencies = [ ] [[package]] -name = "invoker" +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "js-sys" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "judge" version = "0.1.0" dependencies = [ "actix-rt", @@ -1695,7 +1737,7 @@ dependencies = [ "dkregistry", "dotenv", "fs_extra", - "invoker-api", + "judging-apis", "k8s-openapi", "kube", "libc", @@ -1723,7 +1765,7 @@ dependencies = [ ] [[package]] -name = "invoker-api" +name = "judging-apis" version = "0.1.0" dependencies = [ "bitflags", @@ -3586,7 +3628,7 @@ dependencies = [ "anyhow", "crossbeam-channel", "either", - "invoker-api", + "judging-apis", "log", "pom", "serde", diff --git a/Cargo.toml b/Cargo.toml index d433987c..404cd4cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,6 @@ opt-level = 0 [workspace] # TODO: add other as they are fixed members=[ "src/devtool", "src/gen-api-client", "src/cli", "src/client", - "src/problem-loader", "src/invoker", "src/dist-files-generator", - "src/dist-builder", "src/svaluer", "src/invoker-api", "src/pps/api", + "src/problem-loader", "src/judge", "src/dist-files-generator", + "src/dist-builder", "src/svaluer", "src/judging-apis", "src/pps/api", "src/pps/cli", "src/pps/server" ] diff --git a/k8s/jjs/templates/invoker.yaml b/k8s/jjs/templates/judge.yaml similarity index 76% rename from k8s/jjs/templates/invoker.yaml rename to k8s/jjs/templates/judge.yaml index 83fb8a89..5569f89a 100644 --- a/k8s/jjs/templates/invoker.yaml +++ b/k8s/jjs/templates/judge.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: invoker + name: judge rules: - apiGroups: [""] resources: ["configmaps"] @@ -10,27 +10,27 @@ rules: apiVersion: v1 kind: ServiceAccount metadata: - name: invoker + name: judge --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: invoker + name: judge subjects: - kind: ServiceAccount - name: invoker + name: judge apiGroup: "" roleRef: kind: Role - name: invoker + name: judge apiGroup: rbac.authorization.k8s.io --- apiVersion: apps/v1 kind: Deployment metadata: - name: invoker + name: judge labels: - app: invoker + app: judge # {{- if .Values.dev.kubeScore }} annotations: kube-score/ignore: container-security-context @@ -39,21 +39,21 @@ spec: replicas: 1 selector: matchLabels: - app: invoker + app: judge template: metadata: labels: - app: invoker + app: judge spec: - serviceAccountName: invoker + serviceAccountName: judge containers: - - name: invoker + - name: judge env: - name: RUST_LOG - value: info,invoker=trace,problem_loader=trace,puller=trace + value: info,judger=trace,problem_loader=trace,puller=trace - name: JJS_AUTH_DATA_INLINE value: '{"endpoint": "http://apiserver:1779/", "auth": {"byToken": {"token": "Dev::root"}}}' - image: "{{ .Values.image.repositoryPrefix }}invoker:{{ .Values.image.tag }}" + image: "{{ .Values.image.repositoryPrefix }}judge:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: privileged: true @@ -69,7 +69,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: invoker + name: judge spec: type: ClusterIP ports: @@ -78,4 +78,4 @@ spec: protocol: TCP name: http selector: - app: invoker + app: judge diff --git a/k8s/jjs/templates/network-policy.yaml b/k8s/jjs/templates/network-policy.yaml index 8c2a0c8b..e6198fc1 100644 --- a/k8s/jjs/templates/network-policy.yaml +++ b/k8s/jjs/templates/network-policy.yaml @@ -16,11 +16,11 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: builtin-invoker + name: builtin-judge spec: podSelector: matchLabels: - app: invoker + app: judge policyTypes: - Ingress ingress: [] diff --git a/src/dist-builder/src/main.rs b/src/dist-builder/src/main.rs index 06de58cc..e2033d16 100644 --- a/src/dist-builder/src/main.rs +++ b/src/dist-builder/src/main.rs @@ -193,7 +193,7 @@ fn make_rust_package_list() -> Vec { add("pps-cli", "jjs-pps", Section::Tool); //add("userlist", "jjs-userlist", Section::Tool); add("cli", "jjs-cli", Section::Tool); - add("invoker", "jjs-invoker", Section::Daemon); + add("judge", "jjs-judge", Section::Daemon); add("svaluer", "jjs-svaluer", Section::Tool); /*add( "configure-toolchains", diff --git a/src/invoker/Cargo.toml b/src/judge/Cargo.toml similarity index 96% rename from src/invoker/Cargo.toml rename to src/judge/Cargo.toml index 4b48a856..5c81831b 100644 --- a/src/invoker/Cargo.toml +++ b/src/judge/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "invoker" +name = "judge" version = "0.1.0" authors = ["Mikail Bagishov "] edition = "2018" @@ -11,7 +11,7 @@ serde = { version = "1.0.117", features = ["derive"] } serde_json = "1.0.59" dotenv = "0.15.0" aho-corasick = "0.7.14" -invoker-api = {path = "../invoker-api"} +judging-apis = {path = "../judging-apis"} pom = {path = "../pom"} libc = "0.2.80" nix = "0.19.0" diff --git a/src/invoker/Dockerfile b/src/judge/Dockerfile similarity index 73% rename from src/invoker/Dockerfile rename to src/judge/Dockerfile index 402e9c5f..8292b04c 100644 --- a/src/invoker/Dockerfile +++ b/src/judge/Dockerfile @@ -3,5 +3,5 @@ RUN apt-get update -y && apt-get install -y libssl-dev ca-certificates ENV JJS_DATA=/data ENV JJS_AUTH_DATA=/auth/authdata.yaml ENV RUST_BACKTRACE=1 -COPY jjs-invoker /bin/jjs-invoker -ENTRYPOINT ["jjs-invoker"] +COPY jjs-judge /bin/jjs-judge +ENTRYPOINT ["jjs-judge"] diff --git a/src/invoker/src/api.rs b/src/judge/src/api.rs similarity index 100% rename from src/invoker/src/api.rs rename to src/judge/src/api.rs diff --git a/src/invoker/src/config.rs b/src/judge/src/config.rs similarity index 98% rename from src/invoker/src/config.rs rename to src/judge/src/config.rs index ec26b428..edbbddd2 100644 --- a/src/invoker/src/config.rs +++ b/src/judge/src/config.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug)] #[serde(deny_unknown_fields)] #[serde(rename_all = "kebab-case")] -pub struct InvokerConfig { +pub struct JudgeConfig { /// How many workers should be spawned /// By default equal to processor count #[serde(default)] diff --git a/src/invoker/src/controller.rs b/src/judge/src/controller.rs similarity index 96% rename from src/invoker/src/controller.rs rename to src/judge/src/controller.rs index 19d1243d..caf3fdf6 100644 --- a/src/invoker/src/controller.rs +++ b/src/judge/src/controller.rs @@ -36,7 +36,7 @@ pub enum InvocationFinishReason { /// Contains both judging task and back address. /// Each task source is represented as mpsc channel of `TaskInfo`s pub struct JudgeRequestAndCallbacks { - pub request: invoker_api::JudgeRequest, + pub request: judging_apis::JudgeRequest, pub callbacks: Arc, } @@ -62,13 +62,13 @@ pub trait JudgeResponseCallbacks: Send + Sync { async fn add_outcome_header( &self, invocation_id: Uuid, - header: invoker_api::JudgeOutcomeHeader, + header: judging_apis::JudgeOutcomeHeader, ) -> anyhow::Result<()>; async fn deliver_live_status_update( &self, invocation_id: Uuid, - lsu: invoker_api::LiveStatusUpdate, + lsu: judging_apis::LiveStatusUpdate, ) -> anyhow::Result<()>; } @@ -77,7 +77,7 @@ pub struct Controller { scheduler: Arc, problem_loader: Arc, toolchains_dir: Arc, - _config: Arc, + _config: Arc, // used as RAII resource owner _temp_dir: Arc, toolchain_loader: Arc, @@ -98,7 +98,7 @@ fn get_num_cpus() -> usize { impl Controller { pub async fn new( cfg_data: util::cfg::CfgData, - config: Arc, + config: Arc, ) -> anyhow::Result { let worker_count = match config.workers { Some(cnt) => cnt, diff --git a/src/invoker/src/controller/notify.rs b/src/judge/src/controller/notify.rs similarity index 98% rename from src/invoker/src/controller/notify.rs rename to src/judge/src/controller/notify.rs index 14b06743..a4980c9f 100644 --- a/src/invoker/src/controller/notify.rs +++ b/src/judge/src/controller/notify.rs @@ -69,7 +69,7 @@ impl Notifier { #[instrument(skip(self), fields(judge_request_id=%self.judge_request_id))] async fn drain(&mut self) { - let event = invoker_api::LiveStatusUpdate { + let event = judging_apis::LiveStatusUpdate { score: self.score.take().map(|x| x as i32), current_test: self.test.take(), }; diff --git a/src/invoker/src/controller/task_loading.rs b/src/judge/src/controller/task_loading.rs similarity index 100% rename from src/invoker/src/controller/task_loading.rs rename to src/judge/src/controller/task_loading.rs diff --git a/src/invoker/src/controller/toolchains.rs b/src/judge/src/controller/toolchains.rs similarity index 100% rename from src/invoker/src/controller/toolchains.rs rename to src/judge/src/controller/toolchains.rs diff --git a/src/invoker/src/init.rs b/src/judge/src/init.rs similarity index 100% rename from src/invoker/src/init.rs rename to src/judge/src/init.rs diff --git a/src/invoker/src/lib.rs b/src/judge/src/lib.rs similarity index 100% rename from src/invoker/src/lib.rs rename to src/judge/src/lib.rs diff --git a/src/invoker/src/main.rs b/src/judge/src/main.rs similarity index 80% rename from src/invoker/src/main.rs rename to src/judge/src/main.rs index de0f7aaa..a857d8b8 100644 --- a/src/invoker/src/main.rs +++ b/src/judge/src/main.rs @@ -1,6 +1,6 @@ #![type_length_limit = "4323264"] use anyhow::Context; -use invoker::controller::JudgeRequestAndCallbacks; +use judge::controller::JudgeRequestAndCallbacks; use std::sync::Arc; use tracing::{info, instrument, warn}; fn is_cli_mode() -> bool { @@ -13,12 +13,12 @@ async fn start_request_providers( ) -> anyhow::Result<()> { if is_cli_mode() { info!("spawning CliSource"); - tokio::task::spawn(invoker::sources::cli_source::run(chan, cancel)); + tokio::task::spawn(judge::sources::cli_source::run(chan, cancel)); } else { info!("Establishing apiserver connection"); let api = client::infer().await.context("API connection failed")?; info!("Spawning ApiSource"); - let api_source = invoker::sources::ApiSource::new(api, chan); + let api_source = judge::sources::ApiSource::new(api, chan); tokio::task::spawn(async move { api_source.run(cancel).await; }); @@ -43,7 +43,7 @@ fn main() -> anyhow::Result<()> { dotenv::dotenv().ok(); util::log::setup(); if is_worker() { - invoker::init::init().context("failed to initialize")?; + judge::init::init().context("failed to initialize")?; worker_self_isolate()?; } let mut rt = tokio::runtime::Builder::new(); @@ -63,12 +63,12 @@ fn main() -> anyhow::Result<()> { } #[instrument(skip(judge_requests))] async fn start_controller( - config: Arc, + config: Arc, system_config_data: util::cfg::CfgData, judge_requests: async_mpmc::Receiver, ) -> anyhow::Result<()> { info!("Starting controller"); - let controller = invoker::controller::Controller::new(system_config_data, config) + let controller = judge::controller::Controller::new(system_config_data, config) .await .context("failed to start controller")?; controller.exec_on(judge_requests); @@ -77,14 +77,14 @@ async fn start_controller( async fn real_main(cancel_token: tokio::sync::CancellationToken) -> anyhow::Result<()> { if is_worker() { - return invoker::worker::main().await; + return judge::worker::main().await; } let system_config_data = util::cfg::load_cfg_data()?; // now we should fetch InvokerConfig // we have generic `get_config_from_fs` and specific `get_config_from_k8s` - let invoker_config = { + let judge_config = { if let Some(cfg) = get_config_from_k8s().await? { info!("Got config from Kubernetes"); cfg @@ -94,14 +94,14 @@ async fn real_main(cancel_token: tokio::sync::CancellationToken) -> anyhow::Resu } }; // TODO probably broken for IPv6 - let bind_address = format!("{}:{}", invoker_config.api.address, invoker_config.api.port); + let bind_address = format!("{}:{}", judge_config.api.address, judge_config.api.port); let bind_address = bind_address .parse() .with_context(|| format!("invalid bind address {}", bind_address))?; let (judge_request_tx, judge_request_rx) = async_mpmc::channel(); - invoker::api::start(cancel_token.clone(), bind_address, judge_request_tx.clone()) + judge::api::start(cancel_token.clone(), bind_address, judge_request_tx.clone()) .await .context("failed to start api")?; @@ -109,13 +109,9 @@ async fn real_main(cancel_token: tokio::sync::CancellationToken) -> anyhow::Resu start_request_providers(cancel_token.clone(), judge_request_tx) .await .context("failed to initialize request providers")?; - start_controller( - Arc::new(invoker_config), - system_config_data, - judge_request_rx, - ) - .await - .context("can not start controller")?; + start_controller(Arc::new(judge_config), system_config_data, judge_request_rx) + .await + .context("can not start controller")?; { let cancel_token = cancel_token.clone(); tokio::task::spawn(async move { @@ -141,26 +137,26 @@ async fn real_main(cancel_token: tokio::sync::CancellationToken) -> anyhow::Resu pub async fn get_config_from_fs( cfg_data: &util::cfg::CfgData, -) -> anyhow::Result { - let invoker_config_file_path = cfg_data.data_dir.join("etc/invoker.yaml"); - let invoker_config_data = tokio::fs::read(&invoker_config_file_path) +) -> anyhow::Result { + let judge_config_file_path = cfg_data.data_dir.join("etc/judge.yaml"); + let judge_config_data = tokio::fs::read(&judge_config_file_path) .await .with_context(|| { format!( "unable to read config from {}", - invoker_config_file_path.display() + judge_config_file_path.display() ) })?; - info!(path=%invoker_config_file_path.display(), "Found config"); + info!(path=%judge_config_file_path.display(), "Found config"); - serde_yaml::from_slice(&invoker_config_data).context("config parse error") + serde_yaml::from_slice(&judge_config_data).context("config parse error") } /// Fetches config from Kubernetes ConfigMap. /// Returns Ok(Some(config)) on success, Err(err) on error /// and Ok(None) if not running inside kubernetes -pub async fn get_config_from_k8s() -> anyhow::Result> { +pub async fn get_config_from_k8s() -> anyhow::Result> { #[cfg(feature = "k8s")] return get_config_from_k8s_inner().await; #[cfg(not(feature = "k8s"))] @@ -168,7 +164,7 @@ pub async fn get_config_from_k8s() -> anyhow::Result anyhow::Result> { +async fn get_config_from_k8s_inner() -> anyhow::Result> { let incluster_config = match kube::Config::from_cluster_env() { Ok(conf) => conf, Err(err) => { diff --git a/src/invoker/src/scheduler.rs b/src/judge/src/scheduler.rs similarity index 98% rename from src/invoker/src/scheduler.rs rename to src/judge/src/scheduler.rs index 75f57333..831138da 100644 --- a/src/invoker/src/scheduler.rs +++ b/src/judge/src/scheduler.rs @@ -1,7 +1,7 @@ // maybe it's overengineered but it's fun use crate::{ - config::InvokerConfig, + config::JudgeConfig, worker::{Request, Response}, }; use anyhow::Context as _; @@ -23,8 +23,8 @@ pub struct Scheduler { impl Scheduler { /// Creates new Scheduler with empty `workers` set - pub fn new(config: &InvokerConfig) -> anyhow::Result { - let config = serde_json::to_string(&config).context("failed to serialize InvokerConfig")?; + pub fn new(config: &JudgeConfig) -> anyhow::Result { + let config = serde_json::to_string(&config).context("failed to serialize JudgeConfig")?; Ok(Scheduler { workers: vec![], config, diff --git a/src/invoker/src/sources.rs b/src/judge/src/sources.rs similarity index 100% rename from src/invoker/src/sources.rs rename to src/judge/src/sources.rs diff --git a/src/invoker/src/sources/api_source.rs b/src/judge/src/sources/api_source.rs similarity index 96% rename from src/invoker/src/sources/api_source.rs rename to src/judge/src/sources/api_source.rs index 78ba1755..3e337c55 100644 --- a/src/invoker/src/sources/api_source.rs +++ b/src/judge/src/sources/api_source.rs @@ -54,7 +54,7 @@ impl JudgeResponseCallbacks for Callbacks { async fn add_outcome_header( &self, invocation_id: uuid::Uuid, - header: invoker_api::JudgeOutcomeHeader, + header: judging_apis::JudgeOutcomeHeader, ) -> anyhow::Result<()> { let run_id = self .inner @@ -85,7 +85,7 @@ impl JudgeResponseCallbacks for Callbacks { async fn deliver_live_status_update( &self, invocation_id: Uuid, - _lsu: invoker_api::LiveStatusUpdate, + _lsu: judging_apis::LiveStatusUpdate, ) -> anyhow::Result<()> { let mapping = self.inner.run_mapping.lock().await; let run_id = match mapping.get(&invocation_id) { @@ -120,7 +120,7 @@ impl ApiSource { }) } - async fn get_tasks_from_api(&self) -> anyhow::Result> { + async fn get_tasks_from_api(&self) -> anyhow::Result> { let runs = client::models::Run::pop_run_from_queue() .limit(1_i64) .send(&self.inner.api) @@ -147,7 +147,7 @@ impl ApiSource { .await .context("toolchain resolution failed")?; - let task = invoker_api::JudgeRequest { + let task = judging_apis::JudgeRequest { problem_id: run.problem_name, request_id, revision: 0, diff --git a/src/invoker/src/sources/cli_source.rs b/src/judge/src/sources/cli_source.rs similarity index 93% rename from src/invoker/src/sources/cli_source.rs rename to src/judge/src/sources/cli_source.rs index 5f7da209..6f888315 100644 --- a/src/invoker/src/sources/cli_source.rs +++ b/src/judge/src/sources/cli_source.rs @@ -1,6 +1,6 @@ use crate::controller::{InvocationFinishReason, JudgeRequestAndCallbacks, JudgeResponseCallbacks}; use anyhow::Context as _; -use invoker_api::{CliJudgeRequest, JudgeRequest}; +use judging_apis::{CliJudgeRequest, JudgeRequest}; use std::sync::Arc; use tokio::io::AsyncBufReadExt; use tracing::debug; @@ -34,13 +34,13 @@ pub struct FinishedMessage { #[derive(serde::Serialize)] pub struct ProgressMessage { invocation_id: Uuid, - header: invoker_api::JudgeOutcomeHeader, + header: judging_apis::JudgeOutcomeHeader, } #[derive(serde::Serialize)] pub struct LsuMessage { invocation_id: Uuid, - update: invoker_api::LiveStatusUpdate, + update: judging_apis::LiveStatusUpdate, } struct Callbacks; @@ -67,7 +67,7 @@ impl JudgeResponseCallbacks for Callbacks { async fn add_outcome_header( &self, invocation_id: Uuid, - header: invoker_api::JudgeOutcomeHeader, + header: judging_apis::JudgeOutcomeHeader, ) -> anyhow::Result<()> { print_message(Message::Progress(ProgressMessage { invocation_id, @@ -79,7 +79,7 @@ impl JudgeResponseCallbacks for Callbacks { async fn deliver_live_status_update( &self, invocation_id: Uuid, - update: invoker_api::LiveStatusUpdate, + update: judging_apis::LiveStatusUpdate, ) -> anyhow::Result<()> { print_message(Message::LiveStatusUpdate(LsuMessage { invocation_id, diff --git a/src/invoker/src/worker.rs b/src/judge/src/worker.rs similarity index 92% rename from src/invoker/src/worker.rs rename to src/judge/src/worker.rs index 9dabcee5..da447f2c 100644 --- a/src/invoker/src/worker.rs +++ b/src/judge/src/worker.rs @@ -12,7 +12,7 @@ mod valuer; use anyhow::Context; use compiler::{BuildOutcome, Compiler}; use exec_test::{ExecRequest, TestExecutor}; -use invoker_api::{ +use judging_apis::{ valuer_proto::{TestDoneNotification, ValuerResponse}, Status, }; @@ -84,7 +84,7 @@ pub(crate) enum Request { #[derive(Debug, Deserialize, Serialize)] pub(crate) enum Response { JudgeDone(JudgeOutcome), - OutcomeHeader(invoker_api::JudgeOutcomeHeader), + OutcomeHeader(judging_apis::JudgeOutcomeHeader), LiveTest(u32), LiveScore(u32), } @@ -92,12 +92,12 @@ pub(crate) enum Response { pub(crate) struct Worker { /// Minion backend to use for invocations minion: Arc, - /// Invoker configuration - config: crate::config::InvokerConfig, + /// Judge configuration + config: crate::config::JudgeConfig, } impl Worker { - pub(crate) fn new(config: crate::config::InvokerConfig) -> anyhow::Result { + pub(crate) fn new(config: crate::config::JudgeConfig) -> anyhow::Result { Ok(Worker { minion: minion::erased::setup() .context("minion initialization failed")? @@ -142,9 +142,9 @@ impl Worker { error!("Invoke failed: {:#}", err); self.create_fake_protocols( &judge_req, - &invoker_api::Status { - kind: invoker_api::StatusKind::InternalError, - code: invoker_api::status_codes::JUDGE_FAULT.to_string(), + &judging_apis::Status { + kind: judging_apis::StatusKind::InternalError, + code: judging_apis::status_codes::JUDGE_FAULT.to_string(), }, ) .await @@ -198,10 +198,10 @@ impl Worker { async fn create_fake_protocols( &mut self, req: &LoweredJudgeRequest, - status: &invoker_api::Status, + status: &judging_apis::Status, ) -> anyhow::Result<()> { - for kind in invoker_api::judge_log::JudgeLogKind::list() { - let pseudo_valuer_proto = invoker_api::valuer_proto::JudgeLog { + for kind in judging_apis::judge_log::JudgeLogKind::list() { + let pseudo_valuer_proto = judging_apis::valuer_proto::JudgeLog { kind, tests: vec![], subtasks: vec![], @@ -218,10 +218,10 @@ impl Worker { async fn put_outcome( &mut self, score: u32, - status: invoker_api::Status, - kind: invoker_api::judge_log::JudgeLogKind, + status: judging_apis::Status, + kind: judging_apis::judge_log::JudgeLogKind, ) { - let header = invoker_api::JudgeOutcomeHeader { + let header = judging_apis::JudgeOutcomeHeader { score: Some(score), status, kind, @@ -232,7 +232,7 @@ impl Worker { async fn put_protocol( &mut self, req: &LoweredJudgeRequest, - protocol: invoker_api::judge_log::JudgeLog, + protocol: judging_apis::judge_log::JudgeLog, ) -> anyhow::Result<()> { let protocol_file_name = format!("protocol-{}.json", protocol.kind.as_str()); let protocol_path = req.out_dir.join(protocol_file_name); diff --git a/src/invoker/src/worker/compiler.rs b/src/judge/src/worker/compiler.rs similarity index 96% rename from src/invoker/src/worker/compiler.rs rename to src/judge/src/worker/compiler.rs index e874c6af..8fe7bed8 100644 --- a/src/invoker/src/worker/compiler.rs +++ b/src/judge/src/worker/compiler.rs @@ -1,6 +1,6 @@ use crate::worker::{invoke_util, LoweredJudgeRequest}; use anyhow::Context; -use invoker_api::{status_codes, Status, StatusKind}; +use judging_apis::{status_codes, Status, StatusKind}; use std::fs; pub(crate) enum BuildOutcome { @@ -12,7 +12,7 @@ pub(crate) enum BuildOutcome { pub(crate) struct Compiler<'a> { pub(crate) req: &'a LoweredJudgeRequest, pub(crate) minion: &'a dyn minion::erased::Backend, - pub(crate) config: &'a crate::config::InvokerConfig, + pub(crate) config: &'a crate::config::JudgeConfig, } impl<'a> Compiler<'a> { diff --git a/src/invoker/src/worker/exec_test.rs b/src/judge/src/worker/exec_test.rs similarity index 98% rename from src/invoker/src/worker/exec_test.rs rename to src/judge/src/worker/exec_test.rs index 9110ceca..62b0a76b 100644 --- a/src/invoker/src/worker/exec_test.rs +++ b/src/judge/src/worker/exec_test.rs @@ -2,7 +2,7 @@ mod checker_proto; use crate::worker::{invoke_util, os_util, LoweredJudgeRequest}; use anyhow::Context; -use invoker_api::{status_codes, Status, StatusKind}; +use judging_apis::{status_codes, Status, StatusKind}; use std::{fs, io::Write, path::PathBuf}; use tracing::{debug, error}; pub(crate) struct ExecRequest<'a> { @@ -21,7 +21,7 @@ pub(crate) struct TestExecutor<'a> { pub(crate) exec: ExecRequest<'a>, pub(crate) req: &'a LoweredJudgeRequest, pub(crate) minion: &'a dyn minion::erased::Backend, - pub(crate) config: &'a crate::config::InvokerConfig, + pub(crate) config: &'a crate::config::JudgeConfig, } enum RunOutcomeVar { diff --git a/src/invoker/src/worker/exec_test/checker_proto.rs b/src/judge/src/worker/exec_test/checker_proto.rs similarity index 100% rename from src/invoker/src/worker/exec_test/checker_proto.rs rename to src/judge/src/worker/exec_test/checker_proto.rs diff --git a/src/invoker/src/worker/invoke_util.rs b/src/judge/src/worker/invoke_util.rs similarity index 99% rename from src/invoker/src/worker/invoke_util.rs rename to src/judge/src/worker/invoke_util.rs index 0423184b..c34b4b9d 100644 --- a/src/invoker/src/worker/invoke_util.rs +++ b/src/judge/src/worker/invoke_util.rs @@ -39,7 +39,7 @@ pub(crate) fn create_sandbox( req: &LoweredJudgeRequest, test_id: Option, backend: &dyn minion::erased::Backend, - config: &crate::config::InvokerConfig, + config: &crate::config::JudgeConfig, ) -> anyhow::Result { let mut shared_dirs = vec![]; if config.host_toolchains { diff --git a/src/invoker/src/worker/os_util.rs b/src/judge/src/worker/os_util.rs similarity index 100% rename from src/invoker/src/worker/os_util.rs rename to src/judge/src/worker/os_util.rs diff --git a/src/invoker/src/worker/transform_judge_log.rs b/src/judge/src/worker/transform_judge_log.rs similarity index 98% rename from src/invoker/src/worker/transform_judge_log.rs rename to src/judge/src/worker/transform_judge_log.rs index b19bc0bf..dc9fd8c1 100644 --- a/src/invoker/src/worker/transform_judge_log.rs +++ b/src/judge/src/worker/transform_judge_log.rs @@ -1,6 +1,6 @@ use crate::worker::{LoweredJudgeRequest, Worker}; use anyhow::Context; -use invoker_api::{ +use judging_apis::{ judge_log, status_codes, valuer_proto::TestVisibleComponents, Status, StatusKind, }; use std::io::Read; @@ -11,7 +11,7 @@ impl Worker { #[allow(clippy::verbose_file_reads)] pub(super) fn process_judge_log( &self, - valuer_log: &invoker_api::valuer_proto::JudgeLog, + valuer_log: &judging_apis::valuer_proto::JudgeLog, req: &LoweredJudgeRequest, test_results: &[(pom::TestId, crate::worker::exec_test::ExecOutcome)], ) -> anyhow::Result { diff --git a/src/invoker/src/worker/valuer.rs b/src/judge/src/worker/valuer.rs similarity index 97% rename from src/invoker/src/worker/valuer.rs rename to src/judge/src/worker/valuer.rs index c2f753be..13b7122e 100644 --- a/src/invoker/src/worker/valuer.rs +++ b/src/judge/src/worker/valuer.rs @@ -1,6 +1,6 @@ use crate::worker::LoweredJudgeRequest; use anyhow::{bail, Context}; -use invoker_api::valuer_proto::{ProblemInfo, TestDoneNotification, ValuerResponse}; +use judging_apis::valuer_proto::{ProblemInfo, TestDoneNotification, ValuerResponse}; use std::os::unix::io::IntoRawFd; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter}; use tracing::warn; diff --git a/src/invoker/tests/separated_feedback.rs b/src/judge/tests/separated_feedback.rs similarity index 100% rename from src/invoker/tests/separated_feedback.rs rename to src/judge/tests/separated_feedback.rs diff --git a/src/invoker-api/Cargo.toml b/src/judging-apis/Cargo.toml similarity index 93% rename from src/invoker-api/Cargo.toml rename to src/judging-apis/Cargo.toml index 6e9e4f8b..dec30df8 100644 --- a/src/invoker-api/Cargo.toml +++ b/src/judging-apis/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "invoker-api" +name = "judging-apis" version = "0.1.0" authors = ["Mikail Bagishov "] edition = "2018" diff --git a/src/invoker-api/src/judge_log.rs b/src/judging-apis/src/judge_log.rs similarity index 100% rename from src/invoker-api/src/judge_log.rs rename to src/judging-apis/src/judge_log.rs diff --git a/src/invoker-api/src/lib.rs b/src/judging-apis/src/lib.rs similarity index 100% rename from src/invoker-api/src/lib.rs rename to src/judging-apis/src/lib.rs diff --git a/src/invoker-api/src/valuer_proto.rs b/src/judging-apis/src/valuer_proto.rs similarity index 100% rename from src/invoker-api/src/valuer_proto.rs rename to src/judging-apis/src/valuer_proto.rs diff --git a/src/svaluer/Cargo.toml b/src/svaluer/Cargo.toml index cf5a549b..50a3c06f 100644 --- a/src/svaluer/Cargo.toml +++ b/src/svaluer/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Mikail Bagishov "] edition = "2018" [dependencies] -invoker-api = {path = "../invoker-api"} +judging-apis = { path = "../judging-apis" } anyhow = "1.0.33" pom = {path = "../pom"} crossbeam-channel = "0.5.0" diff --git a/src/svaluer/src/fiber.rs b/src/svaluer/src/fiber.rs index a8bcfadf..a115b52a 100644 --- a/src/svaluer/src/fiber.rs +++ b/src/svaluer/src/fiber.rs @@ -2,7 +2,7 @@ mod group; use crate::cfg::Config; use group::Group; -use invoker_api::{ +use judging_apis::{ valuer_proto::{ JudgeLog, JudgeLogKind, ProblemInfo, SubtaskVisibleComponents, TestVisibleComponents, }, @@ -126,7 +126,7 @@ impl Fiber { } } - pub(crate) fn add(&mut self, notification: &invoker_api::valuer_proto::TestDoneNotification) { + pub(crate) fn add(&mut self, notification: &judging_apis::valuer_proto::TestDoneNotification) { if !self.visible_tests.contains(¬ification.test_id) { return; } @@ -276,7 +276,7 @@ impl Fiber { #[cfg(test)] mod tests { use super::*; - use invoker_api::valuer_proto::{ + use judging_apis::valuer_proto::{ JudgeLogSubtaskRow, JudgeLogTestRow, SubtaskId, SubtaskVisibleComponents, TestVisibleComponents, }; diff --git a/src/svaluer/src/fiber/group.rs b/src/svaluer/src/fiber/group.rs index 84c1d5c8..2ff9a9c9 100644 --- a/src/svaluer/src/fiber/group.rs +++ b/src/svaluer/src/fiber/group.rs @@ -1,5 +1,5 @@ use either::{Left, Right}; -use invoker_api::{ +use judging_apis::{ valuer_proto::{ JudgeLog, JudgeLogSubtaskRow, JudgeLogTestRow, SubtaskId, SubtaskVisibleComponents, TestVisibleComponents, @@ -100,7 +100,7 @@ impl Group { pub(crate) fn set_tests_vis( &mut self, - vis: invoker_api::valuer_proto::TestVisibleComponents, + vis: judging_apis::valuer_proto::TestVisibleComponents, ) -> &mut Self { self.check_mutable(); self.test_vis_flags = vis; @@ -109,7 +109,7 @@ impl Group { pub(crate) fn set_group_vis( &mut self, - vis: invoker_api::valuer_proto::SubtaskVisibleComponents, + vis: judging_apis::valuer_proto::SubtaskVisibleComponents, ) -> &mut Self { self.check_mutable(); self.subtask_vis_flags = vis; @@ -346,7 +346,7 @@ mod tests { fn simple() { simple_logger::SimpleLogger::new().init().ok(); let st = || Status { - kind: invoker_api::StatusKind::Accepted, + kind: judging_apis::StatusKind::Accepted, code: "MOCK_OK".to_string(), }; let mut g = Group::new(); diff --git a/src/svaluer/src/lib.rs b/src/svaluer/src/lib.rs index 608e61c8..ce3ba3fa 100644 --- a/src/svaluer/src/lib.rs +++ b/src/svaluer/src/lib.rs @@ -11,7 +11,7 @@ pub use cfg::Config; use anyhow::{Context, Result}; use fiber::{Fiber, FiberReply}; -use invoker_api::valuer_proto::{JudgeLogKind, ProblemInfo, TestDoneNotification, ValuerResponse}; +use judging_apis::valuer_proto::{JudgeLogKind, ProblemInfo, TestDoneNotification, ValuerResponse}; use log::debug; use pom::TestId; use std::collections::HashSet; @@ -169,17 +169,18 @@ impl<'a> SimpleValuer<'a> { } pub mod status_util { - pub fn make_ok_status() -> invoker_api::Status { - invoker_api::Status { + use judging_apis::{Status, StatusKind}; + pub fn make_ok_status() -> Status { + Status { code: "OK".to_string(), - kind: invoker_api::StatusKind::Accepted, + kind: StatusKind::Accepted, } } - pub fn make_err_status() -> invoker_api::Status { - invoker_api::Status { + pub fn make_err_status() -> Status { + Status { code: "NOT_OK".to_string(), - kind: invoker_api::StatusKind::Rejected, + kind: StatusKind::Rejected, } } } diff --git a/src/svaluer/src/main.rs b/src/svaluer/src/main.rs index 2b4d6189..36930948 100644 --- a/src/svaluer/src/main.rs +++ b/src/svaluer/src/main.rs @@ -8,13 +8,13 @@ use std::collections::HashSet; #[derive(Debug)] struct TermDriver { current_tests: HashSet, - full_judge_log: Option, + full_judge_log: Option, } mod term_driver { use super::TermDriver; use anyhow::{Context, Result}; - use invoker_api::valuer_proto; + use judging_apis::valuer_proto; use pom::TestId; use std::{ io::{stdin, stdout, Write}, @@ -88,7 +88,7 @@ mod term_driver { } fn poll_notification(&mut self) -> Result> { - fn create_status(ok: bool) -> invoker_api::Status { + fn create_status(ok: bool) -> judging_apis::Status { if ok { svaluer::status_util::make_ok_status() } else { @@ -148,8 +148,8 @@ mod json_driver { #[derive(Deserialize)] #[serde(untagged)] enum Message { - ProblemInfo(invoker_api::valuer_proto::ProblemInfo), - TestDoneNotify(invoker_api::valuer_proto::TestDoneNotification), + ProblemInfo(judging_apis::valuer_proto::ProblemInfo), + TestDoneNotify(judging_apis::valuer_proto::TestDoneNotification), } fn json_driver_thread_func(chan: crossbeam_channel::Sender) { let mut buf = String::new(); @@ -194,7 +194,7 @@ mod json_driver { } impl ValuerDriver for JsonDriver { - fn problem_info(&mut self) -> Result { + fn problem_info(&mut self) -> Result { let begin_time = Instant::now(); const TIMEOUT: Duration = Duration::from_secs(1); let message = loop { @@ -213,7 +213,7 @@ mod json_driver { Ok(problem_info) } - fn send_command(&mut self, cmd: &invoker_api::valuer_proto::ValuerResponse) -> Result<()> { + fn send_command(&mut self, cmd: &judging_apis::valuer_proto::ValuerResponse) -> Result<()> { let cmd = serde_json::to_string(cmd).context("failed to serialize")?; println!("{}", cmd); std::io::stdout().flush().context("failed to flush")?; @@ -222,7 +222,7 @@ mod json_driver { fn poll_notification( &mut self, - ) -> Result> { + ) -> Result> { match self.poll() { None => Ok(None), Some(msg) => match msg { diff --git a/src/svaluer/src/tests.rs b/src/svaluer/src/tests.rs index bd7cc9bd..cc5faa75 100644 --- a/src/svaluer/src/tests.rs +++ b/src/svaluer/src/tests.rs @@ -1,5 +1,5 @@ use super::*; -use invoker_api::{ +use judging_apis::{ valuer_proto::{ JudgeLog, JudgeLogSubtaskRow, JudgeLogTestRow, SubtaskId, SubtaskVisibleComponents, TestVisibleComponents, From 4183203da518381a639c6903996fc235f8719914 Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Sun, 13 Sep 2020 00:32:27 +0300 Subject: [PATCH 02/11] Change boundaries Rename: Worker -> Invoker Invoker is now responsible for executing opaque series of commands. Currently invoker is always child of judge; this can be changed in future. Judge interacts with invoker using RPCs. --- Cargo.lock | 122 ++++++--- Cargo.toml | 2 +- src/invoker/Cargo.toml | 9 + .../src/worker => invoker/src}/invoke_util.rs | 0 src/invoker/src/main.rs | 3 + src/judge/Cargo.toml | 8 +- src/judge/Readme.md | 9 + src/judge/src/api.rs | 6 +- src/judge/src/config.rs | 18 +- src/judge/src/controller.rs | 53 ++-- src/judge/src/controller/task_loading.rs | 9 +- src/judge/src/invoker_set.rs | 232 ++++++++++++++++ src/judge/src/lib.rs | 3 +- src/judge/src/request_handler.rs | 102 +++++++ .../{worker => request_handler}/compiler.rs | 2 +- src/judge/src/request_handler/exec_test.rs | 177 ++++++++++++ .../exec_test/checker_proto.rs | 0 .../transform_judge_log.rs | 0 .../src/{worker => request_handler}/valuer.rs | 0 src/judge/src/scheduler.rs | 242 ----------------- src/judge/src/worker.rs | 81 +----- src/judge/src/worker/exec_test.rs | 255 ------------------ src/judging-apis/Cargo.toml | 5 + src/judging-apis/src/invoke.rs | 148 ++++++++++ src/judging-apis/src/lib.rs | 1 + 25 files changed, 837 insertions(+), 650 deletions(-) create mode 100644 src/invoker/Cargo.toml rename src/{judge/src/worker => invoker/src}/invoke_util.rs (100%) create mode 100644 src/invoker/src/main.rs create mode 100644 src/judge/Readme.md create mode 100644 src/judge/src/invoker_set.rs create mode 100644 src/judge/src/request_handler.rs rename src/judge/src/{worker => request_handler}/compiler.rs (98%) create mode 100644 src/judge/src/request_handler/exec_test.rs rename src/judge/src/{worker => request_handler}/exec_test/checker_proto.rs (100%) rename src/judge/src/{worker => request_handler}/transform_judge_log.rs (100%) rename src/judge/src/{worker => request_handler}/valuer.rs (100%) delete mode 100644 src/judge/src/scheduler.rs delete mode 100644 src/judge/src/worker/exec_test.rs create mode 100644 src/judging-apis/src/invoke.rs diff --git a/Cargo.lock b/Cargo.lock index 48541785..ffb3bd8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -344,6 +344,17 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "async-channel" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21279cfaa4f47df10b1816007e738ca3747ef2ee53ffc51cdbf57a8bb266fee3" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + [[package]] name = "async-compression" version = "0.3.5" @@ -591,6 +602,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + [[package]] name = "cc" version = "1.0.61" @@ -737,6 +754,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "console" version = "0.13.0" @@ -1152,6 +1178,34 @@ dependencies = [ "version_check", ] +[[package]] +name = "event-listener" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cd41440ae7e4734bbd42302f63eaba892afc93a3912dad84006247f0dedb0e" + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.38", + "synstructure 0.12.4", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -1678,6 +1732,10 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "invoker" +version = "0.1.0" + [[package]] name = "iovec" version = "0.1.4" @@ -1728,7 +1786,7 @@ dependencies = [ "actix-web", "aho-corasick", "anyhow", - "async-mpmc", + "async-channel", "async-trait", "base64 0.13.0", "bitflags", @@ -1736,13 +1794,15 @@ dependencies = [ "client", "dkregistry", "dotenv", + "event-listener", "fs_extra", + "futures-util", + "hyper", "judging-apis", "k8s-openapi", "kube", "libc", "minion", - "multiwake", "nix 0.19.0", "num_cpus", "once_cell", @@ -1750,6 +1810,7 @@ dependencies = [ "pom", "problem-loader", "puller", + "rpc 0.1.0 (git+https://github.com/jjs-dev/commons?branch=rpc-box-engine)", "serde", "serde_json", "serde_yaml", @@ -1758,6 +1819,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tower-service", "tracing", "tracing-futures", "util", @@ -1768,8 +1830,13 @@ dependencies = [ name = "judging-apis" version = "0.1.0" dependencies = [ + "anyhow", + "base64 0.12.3", "bitflags", + "futures-util", + "hyper", "pom", + "rpc 0.1.0 (git+https://github.com/jjs-dev/commons)", "serde", "strum", "strum_macros", @@ -2204,11 +2271,6 @@ dependencies = [ "webpki-roots 0.18.0", ] -[[package]] -name = "multiwake" -version = "0.1.0" -source = "git+https://github.com/jjs-dev/commons#f25926f9adecfb364ceb2e791867d4a7afa11fab" - [[package]] name = "native-tls" version = "0.2.4" @@ -2263,12 +2325,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - [[package]] name = "num-integer" version = "0.1.44" @@ -2621,7 +2677,7 @@ version = "0.1.0" dependencies = [ "anyhow", "reqwest", - "rpc", + "rpc 0.1.0 (git+https://github.com/jjs-dev/commons)", "serde", "serde_json", "tokio", @@ -2637,7 +2693,7 @@ dependencies = [ "pps-api", "pps-server", "rand 0.7.3", - "rpc", + "rpc 0.1.0 (git+https://github.com/jjs-dev/commons)", "serde", "serde_json", "tokio", @@ -2665,7 +2721,7 @@ dependencies = [ "pom", "pps-api", "roxmltree", - "rpc", + "rpc 0.1.0 (git+https://github.com/jjs-dev/commons)", "serde", "serde_json", "serde_yaml", @@ -3127,6 +3183,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "rpc" +version = "0.1.0" +source = "git+https://github.com/jjs-dev/commons?branch=rpc-box-engine#a64c6d3ee7d415ceabe1f03d6f28ae8f8f54d0da" +dependencies = [ + "anyhow", + "futures-util", + "hyper", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", + "tower-util", + "tracing", +] + [[package]] name = "rust-argon2" version = "0.8.2" @@ -3505,12 +3578,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "standback" version = "0.2.11" @@ -4073,17 +4140,6 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "triomphe" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda3a502873984dadbac264124e561c7bda47319733644c2c53c6e67155a53ba" -dependencies = [ - "nodrop", - "serde", - "stable_deref_trait", -] - [[package]] name = "trust-dns-proto" version = "0.19.5" diff --git a/Cargo.toml b/Cargo.toml index 404cd4cd..35799da6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,5 +37,5 @@ opt-level = 0 # TODO: add other as they are fixed members=[ "src/devtool", "src/gen-api-client", "src/cli", "src/client", "src/problem-loader", "src/judge", "src/dist-files-generator", - "src/dist-builder", "src/svaluer", "src/judging-apis", "src/pps/api", + "src/dist-builder", "src/svaluer", "src/judging-apis", "src/invoker", "src/pps/api", "src/pps/cli", "src/pps/server" ] diff --git a/src/invoker/Cargo.toml b/src/invoker/Cargo.toml new file mode 100644 index 00000000..b6dd09a6 --- /dev/null +++ b/src/invoker/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "invoker" +version = "0.1.0" +authors = ["Mikail Bagishov "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/src/judge/src/worker/invoke_util.rs b/src/invoker/src/invoke_util.rs similarity index 100% rename from src/judge/src/worker/invoke_util.rs rename to src/invoker/src/invoke_util.rs diff --git a/src/invoker/src/main.rs b/src/invoker/src/main.rs new file mode 100644 index 00000000..e7a11a96 --- /dev/null +++ b/src/invoker/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/src/judge/Cargo.toml b/src/judge/Cargo.toml index 5c81831b..29ae223a 100644 --- a/src/judge/Cargo.toml +++ b/src/judge/Cargo.toml @@ -41,9 +41,13 @@ k8s-openapi = { version = "0.9.0", optional = true, features = ["v1_17"], defaul puller = { git = "https://github.com/jjs-dev/commons" } tracing = "0.1.21" tracing-futures = "0.2.4" -async-mpmc = { git = "https://github.com/jjs-dev/commons" } -multiwake = { git = "https://github.com/jjs-dev/commons" } dkregistry = { git = "https://github.com/mikailbag/dkregistry-rs", branch = "all" } +rpc = { git = "https://github.com/jjs-dev/commons", branch = "rpc-box-engine" } +tower-service = "0.3.0" +hyper = "0.13.7" +futures-util = "0.3.5" +event-listener = "2.4.0" +async-channel = "1.4.2" [features] k8s = ["kube", "k8s-openapi"] diff --git a/src/judge/Readme.md b/src/judge/Readme.md new file mode 100644 index 00000000..78d17017 --- /dev/null +++ b/src/judge/Readme.md @@ -0,0 +1,9 @@ +# JJS Judge +Judge is program that can actually evaluate user submissions and value them. +## Design +### Request lifecycle +On input, Judge receives `JudgeRequest`s different _request providers_. For example, it can be +Judge HTTP RPC API. Judge loads toolchain and problem, mentioned in request, using `ToolchainLoader` +and `ProblemLoader` respectively. Judge then starts special program called `Valuer`. Usually it will +be `svaluer` from the JJS distribution. Valuer determines tests submission should be tested on. +It also scores the run. Score and judging details are returned. diff --git a/src/judge/src/api.rs b/src/judge/src/api.rs index cb933d32..e32c4545 100644 --- a/src/judge/src/api.rs +++ b/src/judge/src/api.rs @@ -11,7 +11,7 @@ use tracing::instrument; #[derive(Clone)] struct State { - task_tx: async_mpmc::Sender, + task_tx: async_channel::Sender, cancel_token: tokio::sync::CancellationToken, } @@ -36,7 +36,7 @@ async fn route_shutdown(state: web::Data) -> impl Responder { async fn exec( cancel_token: tokio::sync::CancellationToken, bind_addr: std::net::SocketAddr, - task_tx: async_mpmc::Sender, + task_tx: async_channel::Sender, ) -> anyhow::Result<()> { let state = State { task_tx, @@ -65,7 +65,7 @@ async fn exec( pub async fn start( cancel_token: tokio::sync::CancellationToken, bind_addr: std::net::SocketAddr, - task_tx: async_mpmc::Sender, + task_tx: async_channel::Sender, ) -> Result<(), anyhow::Error> { tokio::task::spawn_blocking(move || { if let Err(err) = exec(cancel_token, bind_addr, task_tx) { diff --git a/src/judge/src/config.rs b/src/judge/src/config.rs index edbbddd2..f91d31e2 100644 --- a/src/judge/src/config.rs +++ b/src/judge/src/config.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::path::PathBuf; #[derive(Deserialize, Serialize, Debug)] #[serde(deny_unknown_fields)] @@ -7,7 +8,12 @@ pub struct JudgeConfig { /// How many workers should be spawned /// By default equal to processor count #[serde(default)] - pub workers: Option, + pub managed_invokers: Option, + /// Path to invoker binary. + /// By default deduced from path to judge. + /// If `managed_invokers` set to 0, value does not matter + #[serde(default = "JudgeConfig::default_invoker_path")] + pub invoker_path: PathBuf, /// API service config #[serde(default)] pub api: ApiSvcConfig, @@ -27,6 +33,16 @@ pub struct JudgeConfig { pub problems: problem_loader::LoaderConfig, } +impl JudgeConfig { + fn default_invoker_path() -> PathBuf { + let self_path = std::env::current_exe().expect("failed to get path to self"); + let parent = self_path + .parent() + .expect("path to file must contain at least one component"); + parent.join("jjs-invoker") + } +} + #[derive(Deserialize, Serialize, Debug)] #[serde(deny_unknown_fields)] pub struct ApiSvcConfig { diff --git a/src/judge/src/controller.rs b/src/judge/src/controller.rs index caf3fdf6..22d11acb 100644 --- a/src/judge/src/controller.rs +++ b/src/judge/src/controller.rs @@ -8,8 +8,8 @@ mod task_loading; mod toolchains; use crate::{ - scheduler::Scheduler, - worker::{JudgeOutcome, Request, Response}, + invoker_set::InvokerSet, + request_handler::{Event, JudgeContext, JudgeOutcome}, }; use anyhow::Context; use notify::Notifier; @@ -74,7 +74,7 @@ pub trait JudgeResponseCallbacks: Send + Sync { #[derive(Clone)] pub struct Controller { - scheduler: Arc, + invoker_set: Arc, problem_loader: Arc, toolchains_dir: Arc, _config: Arc, @@ -84,15 +84,12 @@ pub struct Controller { } fn get_num_cpus() -> usize { - static CACHE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); - let old = CACHE.load(std::sync::atomic::Ordering::Relaxed); - if old != 0 { - return old; - } - let corr = num_cpus::get(); - assert_ne!(corr, 0); - CACHE.store(corr, std::sync::atomic::Ordering::Relaxed); - corr + static NUM_CPUS: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| { + let cnt = num_cpus::get(); + assert_ne!(cnt, 0); + cnt + }); + *NUM_CPUS } impl Controller { @@ -100,19 +97,20 @@ impl Controller { cfg_data: util::cfg::CfgData, config: Arc, ) -> anyhow::Result { - let worker_count = match config.workers { + let worker_count = match config.managed_invokers { Some(cnt) => cnt, None => get_num_cpus(), }; info!("Using {} workers", worker_count); - let mut scheduler = Scheduler::new(&config).context("failed to initialize Scheduler")?; + let mut invoker_set = + InvokerSet::new(&config).context("failed to initialize InvokerSet")?; for _ in 0..worker_count { - scheduler - .add_worker() + invoker_set + .add_managed_worker() .await .context("failed to start a worker")?; } - let scheduler = Arc::new(scheduler); + let invoker_set = Arc::new(invoker_set); let temp_dir = tempfile::TempDir::new().context("can not find temporary dir")?; @@ -126,7 +124,7 @@ impl Controller { .context("toolchain loader initialization error")?, ); Ok(Controller { - scheduler, + invoker_set, problem_loader: Arc::new(problem_loader), toolchains_dir: cfg_data.data_dir.join("opt").into(), _config: config, @@ -136,19 +134,18 @@ impl Controller { } #[instrument(skip(self, chan))] - pub fn exec_on(self, chan: async_mpmc::Receiver) { - chan.process_all(move |req| { + pub async fn exec(self, chan: async_channel::Receiver) { + while let Ok(req) = chan.recv().await { let this = self.clone(); - - async move { - let request_id = req.request.request_id; + let request_id = req.request.request_id; + tokio::task::spawn(async move { if let Err(err) = this.process_request(req).await { tracing::warn!(request_id = %request_id, - err = %format_args!("{:#}", err), - "Failed to process a judge request"); + err = %format_args!("{:#}", err), + "Failed to process a judge request"); } - } - }); + }); + } } /// This function drives lifecycle of single judge request. @@ -164,7 +161,7 @@ impl Controller { // TODO currently the process of finding a worker is unfair // we should fix it e.g. using a semaphore which permits finding // worker. - let worker = self.scheduler.find_free_worker().await; + let worker = self.invoker_set.find_free_worker().await; // TODO: can we split into LoweredJudgeRequest and Extensions? let mut responses = worker .send(Request::Judge(low_req)) diff --git a/src/judge/src/controller/task_loading.rs b/src/judge/src/controller/task_loading.rs index d7fe5b50..66329436 100644 --- a/src/judge/src/controller/task_loading.rs +++ b/src/judge/src/controller/task_loading.rs @@ -2,7 +2,10 @@ use super::{ notify::Notifier, Controller, JudgeRequestAndCallbacks, LoweredJudgeRequestExtensions, }; -use crate::worker::{self, LoweredJudgeRequest}; +use crate::{ + request_handler::{Command, LoweredJudgeRequest}, + worker, +}; use anyhow::Context; use std::{ collections::{HashMap, HashSet}, @@ -72,8 +75,8 @@ fn interpolate_command( command: &super::toolchains::Command, dict: &HashMap, toolchain_spec: &super::toolchains::ToolchainSpec, -) -> Result { - let mut res: worker::Command = Default::default(); +) -> Result { + let mut res: Command = Default::default(); for arg in &command.argv { let interp = interpolate_string(arg, dict)?; res.argv.push(interp); diff --git a/src/judge/src/invoker_set.rs b/src/judge/src/invoker_set.rs new file mode 100644 index 00000000..8a834935 --- /dev/null +++ b/src/judge/src/invoker_set.rs @@ -0,0 +1,232 @@ +/// Implements `InvokerSet` +use crate::config::JudgeConfig; +use anyhow::Context as _; +use futures_util::{ + future::{FutureExt, TryFutureExt}, + stream::StreamExt, +}; +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicU8, Ordering::SeqCst}, + Arc, + }, +}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; +use tracing::{debug, instrument}; +/// InvokerSet manages internal invoker and connects to external. +pub struct InvokerSet { + /// Information abount spawned invokers + managed: Vec, + /// Path to invoker binary + invoker_path: PathBuf, + /// these field is used to signal that a worker is reclaimed + worker_reclamation: event_listener::Event, +} + +impl InvokerSet { + /// Creates new Invoker with empty `managed` set + pub fn new(config: &JudgeConfig) -> anyhow::Result { + Ok(InvokerSet { + managed: vec![], + invoker_path: config.invoker_path, + worker_reclamation: event_listener::Event::new(), + }) + } + + /// Starts new invoker process and adds it to this InvokerSet + #[instrument(skip(self))] + pub async fn add_managed_worker(&mut self) -> anyhow::Result<()> { + let mut child = tokio::process::Command::new(&self.invoker_path) + .arg("serve") + .arg("--address=cli") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .context("failed to spawn worker")?; + let info = WorkerInfo { + state: WorkerState::new(WorkerStateKind::Idle), + child_stdin: Mutex::new(child.stdin.take().expect("child stdin was captured")), + child_stdout: Mutex::new(tokio::io::BufReader::new( + child.stdout.take().expect("child stdout was captured"), + )), + }; + self.managed.push(info); + Ok(()) + } + + /// Finds a free invoker (waiting if needed) and executes given InvokeRequest. + #[instrument(skip(self, req))] + pub async fn send_request( + &self, + req: judging_apis::invoke::InvokeRequest, + ) -> anyhow::Result { + let mut attempt_id = 0u32; + loop { + debug!(attempt_id, "scanning all workers"); + attempt_id += 1; + let mut worker_reclaimed = self.worker_reclamation.listen(); + for worker in &self.managed { + if let Some(handle) = worker.try_lock(self.worker_reclamation.0.clone()) { + return handle; + } + } + worker_reclaimed.await; + } + } +} + +impl Drop for FreeWorkerHandle<'_> { + fn drop(&mut self) { + // true if worker can be reused later. + // Theoretically, it should always be the case when handle is dropped. + // However, due to bugs in JJS or problems in environment task, using + // this worker can fail in unexpected manner, leaving worker in + // inconsistent state. This flag implements conservative strategy + // which allows us to avoid such situations. + let reclaimable = matches!( + self.worker.state.load(), + WorkerStateKind::Idle | WorkerStateKind::Locked + ); + if reclaimable { + tracing::debug!("Reclaiming worker"); + self.worker.state.store(WorkerStateKind::Idle); + self.notify.wake(); + } else { + tracing::warn!("Leaking worker because it is not in reclaimable state"); + self.worker.state.store(WorkerStateKind::Crash); + } + } +} + +#[derive(Eq, PartialEq)] +enum WorkerState { + /// Worker is ready for new tasks + Idle, + /// Worker is ready, but it is locked by a WorkerHandle + Locked, + /// Worker is juding run + Judge, + /// Worker has crashed + Crash, +} + +struct WorkerDataInner { + io: std::sync::Mutex<( + tokio::io::BufWriter, + tokio::io::BufReader, + )>, + // could be AtomicU8, but mutex is simpler + state: std::sync::Mutex, + request_done: event_listener::Event, +} + +/// Implements "http client" on top of child stdio +struct WorkerData { + inner: Arc, +} + +impl WorkerData { + fn subscribe(&self) -> event_listener::EventListener { + self.inner.request_done.listen() + } + + fn call( + &mut self, + req: hyper::Request, + ) -> Option< + futures_util::future::BoxFuture<'static, anyhow::Result>>, + > { + let mut lock = self.inner.state.lock().unwrap(); + if *lock != WorkerState::Idle { + return None; + } + *lock = WorkerState::Locked; + drop(lock); + let wake = { + let inner = self.inner.clone(); + move |_future_res: &anyhow::Result>| { + inner.request_done.notify_additional(1); + } + }; + let inner = self.inner.clone(); + let mut io = inner.io.lock().unwrap(); + let (stdin, stdout) = &mut *io; + + Some( + (async move { + let mut uri = req.uri().to_string(); + uri.push('\n'); + // tokio::pin!(stdout); + // tokio::pin!(stdin); + stdin.write_all(uri.as_bytes()).await?; + let req_body = hyper::body::to_bytes(req.into_body()).await?; + let cnt = req_body.len(); + stdin.write_all(&cnt.to_ne_bytes()).await?; + stdin.write_all(&req_body).await?; + stdin.flush().await?; + + Result::, anyhow::Error>::Ok(todo!()) + }) + .inspect_ok(wake) + .boxed(), + ) + } +} + +impl tower_service::Service> for InvokerSet { + type Error = anyhow::Error; + type Future = futures_util::future::BoxFuture<'static, Result>; + type Response = hyper::Response; + + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, req: hyper::Request) -> Self::Future { + if Arc::strong_count(&self.inner) != 1 { + panic!("") + } + self.in_flight = true; + Box::pin(async move {}) + } +} + +struct WorkerInfo { + state: WorkerState, + child_stdout: Mutex>, + child_stdin: Mutex, +} + +impl WorkerInfo { + pub async fn recv(&self) -> anyhow::Result { + let mut line = String::new(); + let mut child_stdout = self.child_stdout.lock().await; + + child_stdout.read_line(&mut line).await?; + Ok(serde_json::from_str(&line).context("parse error")?) + } + + pub async fn send(&self, req: Request) -> anyhow::Result<()> { + let mut data = serde_json::to_vec(&req)?; + data.push(b'\n'); + self.child_stdin.lock().await.write_all(&data).await?; + Ok(()) + } + + /// If this worker is idle, returns a handle to it. + /// Otherwise, returns None + pub fn try_lock(&self, notify: multiwake::Sender) -> Option { + if self.state.lock() { + Some(FreeWorkerHandle { + worker: self, + notify, + }) + } else { + None + } + } +} diff --git a/src/judge/src/lib.rs b/src/judge/src/lib.rs index 6b17d7f1..d6c75b3c 100644 --- a/src/judge/src/lib.rs +++ b/src/judge/src/lib.rs @@ -3,6 +3,7 @@ pub mod api; pub mod config; pub mod controller; pub mod init; -mod scheduler; +mod invoker_set; pub mod sources; pub mod worker; +mod request_handler; \ No newline at end of file diff --git a/src/judge/src/request_handler.rs b/src/judge/src/request_handler.rs new file mode 100644 index 00000000..ff7ae9dc --- /dev/null +++ b/src/judge/src/request_handler.rs @@ -0,0 +1,102 @@ +//! This module and its children are responsible for creating requests to Worker. +mod compiler; +mod exec_test; +mod transform_judge_log; +mod valuer; + +use self::{ + compiler::{BuildOutcome, Compiler}, + exec_test::{exec, ExecRequest}, +}; +use judging_apis::Status; +use serde::{Deserialize, Serialize}; +use std::{ + borrow::Cow, + future::Future, + path::{Path, PathBuf}, + pin::Pin, +}; +use tracing::debug; + +/// Allows sending requests to invoker +// TODO upstream BoxedEngine to jjs-commons +pub type InvokerClient = rpc::Client; + +pub(crate) struct JudgeContext { + /// Can be used to send requests to invoker + invoker: InvokerClient, + /// Channel that should be used for sending updates + events_tx: tokio::sync::mpsc::Sender +} + +/// Note: this is not `judging_apis::invoke::Command`, it is higher-level. +#[derive(Debug)] +pub(crate) struct Command { + pub argv: Vec, + pub env: Vec, + pub cwd: String, +} + +/// Submission information, sufficient for judging +#[derive(Debug)] +pub(crate) struct LoweredJudgeRequest { + pub(crate) compile_commands: Vec, + pub(crate) execute_command: Command, + pub(crate) compile_limits: pom::Limits, + pub(crate) problem: pom::Problem, + /// Path to problem dir + pub(crate) problem_dir: PathBuf, + /// Path to file containing run source + pub(crate) run_source: PathBuf, + /// Name of source file in sandbox. E.g., `source.cpp` for C++. + pub(crate) source_file_name: String, + /// Directory for emitting files (source, build, judge log) + pub(crate) out_dir: PathBuf, + /// Toolchain directory (i.e. sysroot for command execution) + pub(crate) toolchain_dir: PathBuf, + /// UUID of request + pub(crate) judge_request_id: uuid::Uuid, +} + +impl LoweredJudgeRequest { + pub(crate) fn resolve_asset(&self, short_path: &pom::FileRef) -> PathBuf { + let root: Cow = match short_path.root { + pom::FileRefRoot::Problem => self.problem_dir.join("assets").into(), + pom::FileRefRoot::Root => Path::new("/").into(), + }; + + debug!( + "full checker path: {}", + root.join(&short_path.path).to_str().unwrap() + ); + + root.join(&short_path.path) + } + + pub(crate) fn step_dir(&self, test_id: Option) -> PathBuf { + match test_id { + Some(t) => self.out_dir.join(format!("t-{}", t)), + None => self.out_dir.join("compile"), + } + } +} + +#[derive(Debug)] +pub(crate) enum Event { + JudgeDone(JudgeOutcome), + OutcomeHeader(judging_apis::JudgeOutcomeHeader), + LiveTest(u32), + LiveScore(u32), +} + +#[derive(Debug, Clone)] +pub(crate) enum JudgeOutcome { + /// Compilation failed + CompileError(Status), + /// Run was executed on some tests successfully (i.e. without judge faults) + /// All protocols were sent already + TestingDone, + /// Run was not judged, because of invocation fault + /// Maybe, several protocols were emitted, but results are neither precise nor complete + Fault, +} diff --git a/src/judge/src/worker/compiler.rs b/src/judge/src/request_handler/compiler.rs similarity index 98% rename from src/judge/src/worker/compiler.rs rename to src/judge/src/request_handler/compiler.rs index 8fe7bed8..e768636d 100644 --- a/src/judge/src/worker/compiler.rs +++ b/src/judge/src/request_handler/compiler.rs @@ -1,4 +1,4 @@ -use crate::worker::{invoke_util, LoweredJudgeRequest}; +use crate::request_handler::{invoke_util, LoweredJudgeRequest}; use anyhow::Context; use judging_apis::{status_codes, Status, StatusKind}; use std::fs; diff --git a/src/judge/src/request_handler/exec_test.rs b/src/judge/src/request_handler/exec_test.rs new file mode 100644 index 00000000..3e151137 --- /dev/null +++ b/src/judge/src/request_handler/exec_test.rs @@ -0,0 +1,177 @@ +mod checker_proto; + +use super::{invoke_util, JudgeContext, LoweredJudgeRequest}; +use anyhow::Context; +use judging_apis::{ + invoke::{Command, Action, Step, Stdio, FileId}, + status_codes, Status, StatusKind, +}; +use std::{fs, io::Write, path::PathBuf}; +use tracing::{debug, error}; +pub(crate) struct ExecRequest<'a> { + pub(crate) test_id: u32, + pub(crate) test: &'a pom::Test, +} + +#[derive(Debug, Clone)] +pub(crate) struct ExecOutcome { + pub(crate) status: Status, + pub(crate) resource_usage: minion::ResourceUsageData, +} + +enum RunOutcomeVar { + Success { out_data_path: PathBuf }, + Fail(Status), +} + +struct RunOutcome { + var: RunOutcomeVar, + resource_usage: minion::ResourceUsageData, +} + +fn map_checker_outcome_to_status(out: checker_proto::Output) -> Status { + match out.outcome { + checker_proto::Outcome::Ok => Status { + kind: StatusKind::Accepted, + code: status_codes::TEST_PASSED.to_string(), + }, + checker_proto::Outcome::BadChecker => Status { + kind: StatusKind::InternalError, + code: status_codes::JUDGE_FAULT.to_string(), + }, + checker_proto::Outcome::PresentationError => Status { + kind: StatusKind::Rejected, + code: status_codes::PRESENTATION_ERROR.to_string(), + }, + checker_proto::Outcome::WrongAnswer => Status { + kind: StatusKind::Rejected, + code: status_codes::WRONG_ANSWER.to_string(), + }, + } +} + +/// Runs Artifact on one test and produces output +pub fn exec(judge_req: &LoweredJudgeRequest, ctx: &JudgeContext) -> anyhow::Result { + let mut invoke_request = judging_apis::invoke::InvokeRequest { + steps: vec![], + inputs: vec![], + outputs: vec![], + }; + const EXEC_SOLUTION_GENERATION: u32 = 0; + const EXEC_SOLUTION_OUTPUT_FILE: FileId = FileId(0); + const EXEC_SOLUTION_ERROR_FILE: FileId = FileId(1); + + const EXEC_CHECKER_GENERATION: u32 = 1; + // produce a step for executing solution + { + let exec_solution_step = Step { + generation: EXEC_SOLUTION_GENERATION, + action: Action::ExecuteCommand(Command { + argv: judge_req.execute_command.argv.clone(), + env: judge_req.execute_command.env.clone(), + cwd: judge_req.execute_command.cwd.clone(), + stdio: Stdio { + + } + }), + }; + } + let input_file = self.req.resolve_asset(&self.exec.test.path); + let test_data = std::fs::read(input_file).context("failed to read test")?; + let run_outcome = self.run_solution(&test_data, self.exec.test_id)?; + let sol_file_path = match run_outcome.var { + RunOutcomeVar::Success { out_data_path } => out_data_path, + RunOutcomeVar::Fail(status) => { + return Ok(ExecOutcome { + status, + resource_usage: run_outcome.resource_usage, + }); + } + }; + // run checker + let step_dir = self.req.step_dir(Some(self.exec.test_id)); + let sol_file = fs::File::open(sol_file_path).context("failed to open run's answer")?; + let sol_handle = os_util::handle_inherit(sol_file.into_raw_fd().into(), true); + let full_checker_path = self.req.resolve_asset(&self.req.problem.checker_exe); + let mut cmd = std::process::Command::new(full_checker_path.clone()); + debug!( + "full checker path: {}, short path: {}", + full_checker_path.to_str().unwrap(), + &self.req.problem.checker_exe.path + ); + cmd.current_dir(&self.req.problem_dir); + + for arg in &self.req.problem.checker_cmd { + cmd.arg(arg); + } + + let test_cfg = self.exec.test; + + let corr_handle = if let Some(corr_path) = &test_cfg.correct { + let full_path = self.req.resolve_asset(corr_path); + let data = fs::read(full_path).context("failed to read correct answer")?; + os_util::buffer_to_file(&data, "invoker-correct-data") + } else { + os_util::buffer_to_file(&[], "invoker-correct-data") + }; + let test_handle = os_util::buffer_to_file(&test_data, "invoker-test-data"); + + cmd.env("JJS_CORR", corr_handle.to_string()); + cmd.env("JJS_SOL", sol_handle.to_string()); + cmd.env("JJS_TEST", test_handle.to_string()); + + let (out_judge_side, out_checker_side) = os_util::make_pipe(); + cmd.env("JJS_CHECKER_OUT", out_checker_side.to_string()); + let (comments_judge_side, comments_checker_side) = os_util::make_pipe(); + cmd.env("JJS_CHECKER_COMMENT", comments_checker_side.to_string()); + let st = cmd.output().context("failed to execute checker")?; + os_util::close(out_checker_side); + os_util::close(comments_checker_side); + os_util::close(corr_handle); + os_util::close(test_handle); + os_util::close(sol_handle); + // TODO: capture comments + os_util::close(comments_judge_side); + + let checker_out = std::fs::File::create(step_dir.join("check-log.txt"))?; + let mut checker_out = std::io::BufWriter::new(checker_out); + checker_out.write_all(b" --- stdout ---\n")?; + checker_out.write_all(&st.stdout)?; + checker_out.write_all(b"--- stderr ---\n")?; + checker_out.write_all(&st.stderr)?; + let return_value_for_judge_fault = Ok(ExecOutcome { + status: Status { + kind: StatusKind::InternalError, + code: status_codes::JUDGE_FAULT.to_string(), + }, + resource_usage: Default::default(), + }); + + let succ = st.status.success(); + if !succ { + error!("Judge fault: checker returned non-zero: {}", st.status); + os_util::close(out_judge_side); + return return_value_for_judge_fault; + } + let checker_out = match String::from_utf8(os_util::handle_read_all(out_judge_side)) { + Ok(c) => c, + Err(_) => { + error!("checker produced non-utf8 output"); + return return_value_for_judge_fault; + } + }; + let parsed_out = match checker_proto::parse(&checker_out) { + Ok(o) => o, + Err(err) => { + error!("checker output couldn't be parsed: {}", err); + return return_value_for_judge_fault; + } + }; + + let status = map_checker_outcome_to_status(parsed_out); + + Ok(ExecOutcome { + status, + resource_usage: run_outcome.resource_usage, + }) +} diff --git a/src/judge/src/worker/exec_test/checker_proto.rs b/src/judge/src/request_handler/exec_test/checker_proto.rs similarity index 100% rename from src/judge/src/worker/exec_test/checker_proto.rs rename to src/judge/src/request_handler/exec_test/checker_proto.rs diff --git a/src/judge/src/worker/transform_judge_log.rs b/src/judge/src/request_handler/transform_judge_log.rs similarity index 100% rename from src/judge/src/worker/transform_judge_log.rs rename to src/judge/src/request_handler/transform_judge_log.rs diff --git a/src/judge/src/worker/valuer.rs b/src/judge/src/request_handler/valuer.rs similarity index 100% rename from src/judge/src/worker/valuer.rs rename to src/judge/src/request_handler/valuer.rs diff --git a/src/judge/src/scheduler.rs b/src/judge/src/scheduler.rs deleted file mode 100644 index 831138da..00000000 --- a/src/judge/src/scheduler.rs +++ /dev/null @@ -1,242 +0,0 @@ -// maybe it's overengineered but it's fun - -use crate::{ - config::JudgeConfig, - worker::{Request, Response}, -}; -use anyhow::Context as _; -use std::sync::atomic::{AtomicU8, Ordering::SeqCst}; -use tokio::{ - io::{AsyncBufReadExt, AsyncWriteExt}, - sync::Mutex, -}; -use tracing::{debug, instrument}; -/// Scheduler is responsible for finding a suitable worker for a task -pub struct Scheduler { - workers: Vec, - /// We need it because we must pass it to a worker. - // TODO: ideally we do not want to pass any configs to a worker - config: String, - /// these field is used to signal that a worker is reclaimed - worker_reclamation: (multiwake::Sender, multiwake::Receiver), -} - -impl Scheduler { - /// Creates new Scheduler with empty `workers` set - pub fn new(config: &JudgeConfig) -> anyhow::Result { - let config = serde_json::to_string(&config).context("failed to serialize JudgeConfig")?; - Ok(Scheduler { - workers: vec![], - config, - worker_reclamation: multiwake::multiwake(), - }) - } - - /// Starts new worker process and adds it to this scheduler - #[instrument(skip(self))] - pub async fn add_worker(&mut self) -> anyhow::Result<()> { - let mut child = tokio::process::Command::new(std::env::current_exe()?) - .env("__JJS_WORKER", "1") - .env("__JJS_WORKER_INVOKER_CONFIG", &self.config) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn worker")?; - let info = WorkerInfo { - state: WorkerState::new(WorkerStateKind::Idle), - child_stdin: Mutex::new(child.stdin.take().expect("child stdin was captured")), - child_stdout: Mutex::new(tokio::io::BufReader::new( - child.stdout.take().expect("child stdout was captured"), - )), - }; - self.workers.push(info); - Ok(()) - } - - /// Tries to find a free worker. On success, returns `FreeWorkerHandle`, - /// which can be used to send requests to that worker. - #[instrument(skip(self))] - pub async fn find_free_worker(&self) -> FreeWorkerHandle<'_> { - let mut receiver = self.worker_reclamation.1.clone(); - let mut attempt_id = 0u32; - loop { - debug!(attempt_id, "scanning all workers"); - attempt_id += 1; - for worker in &self.workers { - if let Some(handle) = worker.try_lock(self.worker_reclamation.0.clone()) { - return handle; - } - } - receiver.wait().await; - } - } -} - -/// This handle logically owns free worker and can be used to send it requests. -pub struct FreeWorkerHandle<'a> { - /// reference to the worker - worker: &'a WorkerInfo, - /// Used to notify that worker is reclaimed - notify: multiwake::Sender, -} - -impl Drop for FreeWorkerHandle<'_> { - fn drop(&mut self) { - // true if worker can be reused later. - // Theoretically, it should always be the case when handle is dropped. - // However, due to bugs in JJS or problems in environment task, using - // this worker can fail in unexpected manner, leaving worker in - // inconsistent state. This flag implements conservative strategy - // which allows us to avoid such situations. - let reclaimable = matches!( - self.worker.state.load(), - WorkerStateKind::Idle | WorkerStateKind::Locked - ); - if reclaimable { - tracing::debug!("Reclaiming worker"); - self.worker.state.store(WorkerStateKind::Idle); - self.notify.wake(); - } else { - tracing::warn!("Leaking worker because it is not in reclaimable state"); - self.worker.state.store(WorkerStateKind::Crash); - } - } -} - -impl<'a> FreeWorkerHandle<'a> { - /// Sends request to worker, returning "stream" of responses - pub(crate) async fn send(self, req: Request) -> anyhow::Result> { - self.worker.state.store(WorkerStateKind::Judge); - self.worker - .send(req) - .await - .map_err(|err| { - tracing::warn!("request not delivered, marking worker as crashed"); - self.worker.state.store(WorkerStateKind::Crash); - err - }) - .context("failed to send request")?; - Ok(WorkerResponses { handle: Some(self) }) - } -} - -/// Provides access to worker responses -pub struct WorkerResponses<'a> { - handle: Option>, -} - -impl WorkerResponses<'_> { - /// Returns next response. - /// If returned response is JudgeDone or error, must not be polled again. - pub(crate) async fn next(&mut self) -> anyhow::Result { - let handle = self - .handle - .as_ref() - .expect("WorkerResponses is polled after finish"); - let res = handle.worker.recv().await; - let is_eos = match &res { - Ok(ok) => matches!(ok, Response::JudgeDone(_)), - Err(_) => true, - }; - if is_eos { - if res.is_ok() { - handle.worker.state.store(WorkerStateKind::Locked); - } else { - handle.worker.state.store(WorkerStateKind::Crash); - } - self.handle.take(); - } - res - } -} - -/// Contains WorkerStateKind -struct WorkerState(AtomicU8); -const WORKER_STATE_IDLE: u8 = 0; -const WORKER_STATE_LOCKED: u8 = 1; -const WORKER_STATE_CRASH: u8 = 2; -const WORKER_STATE_JUDGE: u8 = 3; -impl WorkerState { - fn new(kind: WorkerStateKind) -> Self { - let this = WorkerState(AtomicU8::new(0)); - this.store(kind); - this - } - - fn store(&self, kind: WorkerStateKind) { - let value = match kind { - WorkerStateKind::Idle => WORKER_STATE_IDLE, - WorkerStateKind::Locked => WORKER_STATE_LOCKED, - WorkerStateKind::Crash => WORKER_STATE_CRASH, - WorkerStateKind::Judge => WORKER_STATE_JUDGE, - }; - self.0.store(value, SeqCst); - } - - fn load(&self) -> WorkerStateKind { - let value = self.0.load(SeqCst); - match value { - WORKER_STATE_IDLE => WorkerStateKind::Idle, - WORKER_STATE_LOCKED => WorkerStateKind::Locked, - WORKER_STATE_CRASH => WorkerStateKind::Crash, - WORKER_STATE_JUDGE => WorkerStateKind::Judge, - other => unreachable!("unexpected worker state {}", other), - } - } - - /// Tries to atomically lock this worker state. - /// I.e., this functions succeeds if state was `Idle` and it was - /// successfully CASed to `Locked`. - fn lock(&self) -> bool { - self.0 - .compare_and_swap(WORKER_STATE_IDLE, WORKER_STATE_LOCKED, SeqCst) - == WORKER_STATE_IDLE - } -} - -enum WorkerStateKind { - /// Worker is ready for new tasks - Idle, - /// Worker is ready, but it is locked by a WorkerHandle - Locked, - /// Worker is juding run - Judge, - /// Worker has crashed - Crash, -} - -struct WorkerInfo { - state: WorkerState, - child_stdout: Mutex>, - child_stdin: Mutex, -} - -impl WorkerInfo { - pub async fn recv(&self) -> anyhow::Result { - let mut line = String::new(); - let mut child_stdout = self.child_stdout.lock().await; - - child_stdout.read_line(&mut line).await?; - Ok(serde_json::from_str(&line).context("parse error")?) - } - - pub async fn send(&self, req: Request) -> anyhow::Result<()> { - let mut data = serde_json::to_vec(&req)?; - data.push(b'\n'); - self.child_stdin.lock().await.write_all(&data).await?; - Ok(()) - } - - /// If this worker is idle, returns a handle to it. - /// Otherwise, returns None - pub fn try_lock(&self, notify: multiwake::Sender) -> Option { - if self.state.lock() { - Some(FreeWorkerHandle { - worker: self, - notify, - }) - } else { - None - } - } -} diff --git a/src/judge/src/worker.rs b/src/judge/src/worker.rs index da447f2c..ed859b68 100644 --- a/src/judge/src/worker.rs +++ b/src/judge/src/worker.rs @@ -2,16 +2,10 @@ //! //! Worker is responsible for processing `InvokeRequest`s -mod compiler; -mod exec_test; -mod invoke_util; + mod os_util; -mod transform_judge_log; -mod valuer; use anyhow::Context; -use compiler::{BuildOutcome, Compiler}; -use exec_test::{ExecRequest, TestExecutor}; use judging_apis::{ valuer_proto::{TestDoneNotification, ValuerResponse}, Status, @@ -25,69 +19,7 @@ use std::{ use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; use tracing::{debug, error}; use valuer::Valuer; -#[derive(Default, Debug, Clone, Deserialize, Serialize)] -pub(crate) struct Command { - pub(crate) argv: Vec, - pub(crate) env: Vec, - pub(crate) cwd: String, -} -/// Submission information, sufficient for judging -#[derive(Debug, Deserialize, Serialize)] -pub(crate) struct LoweredJudgeRequest { - pub(crate) compile_commands: Vec, - pub(crate) execute_command: Command, - pub(crate) compile_limits: pom::Limits, - pub(crate) problem: pom::Problem, - /// Path to problem dir - pub(crate) problem_dir: PathBuf, - /// Path to file containing run source - pub(crate) run_source: PathBuf, - /// Name of source file in sandbox. E.g., `source.cpp` for C++. - pub(crate) source_file_name: String, - /// Directory for emitting files (source, build, judge log) - pub(crate) out_dir: PathBuf, - /// Toolchain directory (i.e. sysroot for command execution) - pub(crate) toolchain_dir: PathBuf, - /// UUID of request - pub(crate) judge_request_id: uuid::Uuid, -} - -impl LoweredJudgeRequest { - pub(crate) fn resolve_asset(&self, short_path: &pom::FileRef) -> PathBuf { - let root: Cow = match short_path.root { - pom::FileRefRoot::Problem => self.problem_dir.join("assets").into(), - pom::FileRefRoot::Root => Path::new("/").into(), - }; - - debug!( - "full checker path: {}", - root.join(&short_path.path).to_str().unwrap() - ); - - root.join(&short_path.path) - } - - pub(crate) fn step_dir(&self, test_id: Option) -> PathBuf { - match test_id { - Some(t) => self.out_dir.join(format!("t-{}", t)), - None => self.out_dir.join("compile"), - } - } -} - -#[derive(Deserialize, Serialize)] -pub(crate) enum Request { - Judge(LoweredJudgeRequest), -} - -#[derive(Debug, Deserialize, Serialize)] -pub(crate) enum Response { - JudgeDone(JudgeOutcome), - OutcomeHeader(judging_apis::JudgeOutcomeHeader), - LiveTest(u32), - LiveScore(u32), -} pub(crate) struct Worker { /// Minion backend to use for invocations @@ -309,17 +241,6 @@ impl Worker { } } -#[derive(Serialize, Debug, Clone, Deserialize)] -pub(crate) enum JudgeOutcome { - /// Compilation failed - CompileError(Status), - /// Run was executed on some tests successfully (i.e. without judge faults) - /// All protocols were sent already - TestingDone, - /// Run was not judged, because of invocation fault - /// Maybe, several protocols were emitted, but results are neither precise nor complete - Fault, -} pub async fn main() -> anyhow::Result<()> { let config_data = std::env::var("__JJS_WORKER_INVOKER_CONFIG") diff --git a/src/judge/src/worker/exec_test.rs b/src/judge/src/worker/exec_test.rs deleted file mode 100644 index 62b0a76b..00000000 --- a/src/judge/src/worker/exec_test.rs +++ /dev/null @@ -1,255 +0,0 @@ -mod checker_proto; - -use crate::worker::{invoke_util, os_util, LoweredJudgeRequest}; -use anyhow::Context; -use judging_apis::{status_codes, Status, StatusKind}; -use std::{fs, io::Write, path::PathBuf}; -use tracing::{debug, error}; -pub(crate) struct ExecRequest<'a> { - pub(crate) test_id: u32, - pub(crate) test: &'a pom::Test, -} - -#[derive(Debug, Clone)] -pub(crate) struct ExecOutcome { - pub(crate) status: Status, - pub(crate) resource_usage: minion::ResourceUsageData, -} - -/// Runs Artifact on one test and produces output -pub(crate) struct TestExecutor<'a> { - pub(crate) exec: ExecRequest<'a>, - pub(crate) req: &'a LoweredJudgeRequest, - pub(crate) minion: &'a dyn minion::erased::Backend, - pub(crate) config: &'a crate::config::JudgeConfig, -} - -enum RunOutcomeVar { - Success { out_data_path: PathBuf }, - Fail(Status), -} - -struct RunOutcome { - var: RunOutcomeVar, - resource_usage: minion::ResourceUsageData, -} - -fn map_checker_outcome_to_status(out: checker_proto::Output) -> Status { - match out.outcome { - checker_proto::Outcome::Ok => Status { - kind: StatusKind::Accepted, - code: status_codes::TEST_PASSED.to_string(), - }, - checker_proto::Outcome::BadChecker => Status { - kind: StatusKind::InternalError, - code: status_codes::JUDGE_FAULT.to_string(), - }, - checker_proto::Outcome::PresentationError => Status { - kind: StatusKind::Rejected, - code: status_codes::PRESENTATION_ERROR.to_string(), - }, - checker_proto::Outcome::WrongAnswer => Status { - kind: StatusKind::Rejected, - code: status_codes::WRONG_ANSWER.to_string(), - }, - } -} - -impl<'a> TestExecutor<'a> { - fn run_solution(&self, test_data: &[u8], test_id: u32) -> anyhow::Result { - let step_dir = self.req.step_dir(Some(test_id)); - - let sandbox = - invoke_util::create_sandbox(self.req, Some(test_id), self.minion, self.config)?; - - fs::copy(self.req.out_dir.join("build"), step_dir.join("data/build")) - .context("failed to copy build artifact to share dir")?; - - let stdout_path = step_dir.join("stdout.txt"); - let stderr_path = step_dir.join("stderr.txt"); - let command = &self.req.execute_command; - invoke_util::log_execute_command(command); - - let mut native_command = minion::Command::new(); - - invoke_util::command_set_from_judge_req(&mut native_command, &command); - invoke_util::command_set_stdio(&mut native_command, &stdout_path, &stderr_path); - - native_command.sandbox(sandbox.sandbox.clone()); - - // capture child input - native_command.stdin(minion::InputSpecification::pipe()); - - let mut child = match native_command.spawn(&*self.minion) { - Ok(child) => child, - Err(err) => { - let is_internal_error = match err.downcast_ref::() { - Some(e) => e.is_system(), - None => true, - }; - if is_internal_error { - return Err(err).context("failed to spawn solution"); - } else { - let run_outcome_var = RunOutcomeVar::Fail(Status { - kind: StatusKind::Rejected, - code: status_codes::LAUNCH_ERROR.to_string(), - }); - return Ok(RunOutcome { - var: run_outcome_var, - resource_usage: Default::default(), - }); - } - } - }; - let mut stdin = child.stdin().unwrap(); - stdin.write_all(test_data).ok(); - std::mem::drop(stdin); // close pipe - - let wait_result = child - .wait_for_exit(None) - .context("failed to wait for child")?; - - let resource_usage = sandbox - .sandbox - .resource_usage() - .context("cannot get resource usage")?; - - match wait_result { - minion::WaitOutcome::Timeout => { - return Ok(RunOutcome { - var: RunOutcomeVar::Fail(Status { - kind: StatusKind::Rejected, - code: status_codes::TIME_LIMIT_EXCEEDED.to_string(), - }), - resource_usage, - }); - } - minion::WaitOutcome::AlreadyFinished => unreachable!("not expected other to wait"), - minion::WaitOutcome::Exited => { - if child - .get_exit_code() - .context("failed to get exit code")? - .unwrap() - != 0 - { - return Ok(RunOutcome { - var: RunOutcomeVar::Fail(Status { - kind: StatusKind::Rejected, - code: status_codes::RUNTIME_ERROR.to_string(), - }), - resource_usage, - }); - } - } - } - - Ok(RunOutcome { - var: RunOutcomeVar::Success { - out_data_path: stdout_path, - }, - resource_usage, - }) - } - - pub fn exec(self) -> anyhow::Result { - use std::os::unix::io::IntoRawFd; - let input_file = self.req.resolve_asset(&self.exec.test.path); - let test_data = std::fs::read(input_file).context("failed to read test")?; - let run_outcome = self.run_solution(&test_data, self.exec.test_id)?; - let sol_file_path = match run_outcome.var { - RunOutcomeVar::Success { out_data_path } => out_data_path, - RunOutcomeVar::Fail(status) => { - return Ok(ExecOutcome { - status, - resource_usage: run_outcome.resource_usage, - }); - } - }; - // run checker - let step_dir = self.req.step_dir(Some(self.exec.test_id)); - let sol_file = fs::File::open(sol_file_path).context("failed to open run's answer")?; - let sol_handle = os_util::handle_inherit(sol_file.into_raw_fd().into(), true); - let full_checker_path = self.req.resolve_asset(&self.req.problem.checker_exe); - let mut cmd = std::process::Command::new(full_checker_path.clone()); - debug!( - "full checker path: {}, short path: {}", - full_checker_path.to_str().unwrap(), - &self.req.problem.checker_exe.path - ); - cmd.current_dir(&self.req.problem_dir); - - for arg in &self.req.problem.checker_cmd { - cmd.arg(arg); - } - - let test_cfg = self.exec.test; - - let corr_handle = if let Some(corr_path) = &test_cfg.correct { - let full_path = self.req.resolve_asset(corr_path); - let data = fs::read(full_path).context("failed to read correct answer")?; - os_util::buffer_to_file(&data, "invoker-correct-data") - } else { - os_util::buffer_to_file(&[], "invoker-correct-data") - }; - let test_handle = os_util::buffer_to_file(&test_data, "invoker-test-data"); - - cmd.env("JJS_CORR", corr_handle.to_string()); - cmd.env("JJS_SOL", sol_handle.to_string()); - cmd.env("JJS_TEST", test_handle.to_string()); - - let (out_judge_side, out_checker_side) = os_util::make_pipe(); - cmd.env("JJS_CHECKER_OUT", out_checker_side.to_string()); - let (comments_judge_side, comments_checker_side) = os_util::make_pipe(); - cmd.env("JJS_CHECKER_COMMENT", comments_checker_side.to_string()); - let st = cmd.output().context("failed to execute checker")?; - os_util::close(out_checker_side); - os_util::close(comments_checker_side); - os_util::close(corr_handle); - os_util::close(test_handle); - os_util::close(sol_handle); - // TODO: capture comments - os_util::close(comments_judge_side); - - let checker_out = std::fs::File::create(step_dir.join("check-log.txt"))?; - let mut checker_out = std::io::BufWriter::new(checker_out); - checker_out.write_all(b" --- stdout ---\n")?; - checker_out.write_all(&st.stdout)?; - checker_out.write_all(b"--- stderr ---\n")?; - checker_out.write_all(&st.stderr)?; - let return_value_for_judge_fault = Ok(ExecOutcome { - status: Status { - kind: StatusKind::InternalError, - code: status_codes::JUDGE_FAULT.to_string(), - }, - resource_usage: Default::default(), - }); - - let succ = st.status.success(); - if !succ { - error!("Judge fault: checker returned non-zero: {}", st.status); - os_util::close(out_judge_side); - return return_value_for_judge_fault; - } - let checker_out = match String::from_utf8(os_util::handle_read_all(out_judge_side)) { - Ok(c) => c, - Err(_) => { - error!("checker produced non-utf8 output"); - return return_value_for_judge_fault; - } - }; - let parsed_out = match checker_proto::parse(&checker_out) { - Ok(o) => o, - Err(err) => { - error!("checker output couldn't be parsed: {}", err); - return return_value_for_judge_fault; - } - }; - - let status = map_checker_outcome_to_status(parsed_out); - - Ok(ExecOutcome { - status, - resource_usage: run_outcome.resource_usage, - }) - } -} diff --git a/src/judging-apis/Cargo.toml b/src/judging-apis/Cargo.toml index dec30df8..f021965e 100644 --- a/src/judging-apis/Cargo.toml +++ b/src/judging-apis/Cargo.toml @@ -11,3 +11,8 @@ strum_macros = "0.19.4" bitflags = "1.2.1" pom = {path = "../pom"} uuid = { version = "0.8.1", features = ["serde"] } +rpc = { git = "https://github.com/jjs-dev/commons" } +hyper = "0.13.7" +anyhow = "1.0.32" +base64 = "0.12.3" +futures-util = "0.3.5" diff --git a/src/judging-apis/src/invoke.rs b/src/judging-apis/src/invoke.rs new file mode 100644 index 00000000..d3cef31c --- /dev/null +++ b/src/judging-apis/src/invoke.rs @@ -0,0 +1,148 @@ +//! Defines Invoker API. +//! You can use invoker to securely executed untrusted programs. +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +pub struct Invoke(std::convert::Infallible); + +/// Requests invoker to execute commands, specified in +/// `steps` field in request. +/// # Execution order +/// Each step has assigned `generation`. +/// Steps with equal generation will be executed in the same time. +/// Such steps can share pipes. Sharing pipes between steps with +/// different generations results in error. For each generation, +/// Steps creating new IPC stuff are executed first. +/// Step will not be executed until all steps with less `generation` +/// will be finished. +/// # Data +/// `InvokeRequest` can specify input data items, that can be further used +/// as stdin for executed commands (input data item can be used several times). +/// # DataRequest +/// `InvokeRequest` can specify output data requests, which will be populated +/// from some files, created by `CreateFile` action. +impl rpc::Route for Invoke { + type Request = rpc::Unary; + type Response = rpc::Unary; + + const ENDPOINT: &'static str = "/invoke"; +} + +#[derive(Serialize, Deserialize)] +pub struct InvokeRequest { + /// Set of commands that must be executed + pub steps: Vec, + /// Binary data used for executing commands + pub inputs: Vec, + /// Binary data produced by executing commands + pub outputs: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct InvokeResponse {} + +#[derive(Serialize, Deserialize)] +pub struct OutputRequest { + /// File id that will later receive the data + pub id: FileId, +} + +#[derive(Serialize, Deserialize)] +pub struct Input { + /// File id that must be assigned to this input + pub id: FileId, + /// Data source + pub source: InputSource, +} + +#[derive(Serialize, Deserialize)] +pub enum InputSource { + /// Data available as file on FS + LocalFile { path: PathBuf }, + /// Data provided inline + Inline { data: Vec }, +} + +pub struct Output {} + +#[derive(Serialize, Deserialize)] +pub struct Step { + pub generation: u32, + pub action: Action, +} + +/// Newtype identifier of file-like object, e.g. real file or pipe. +#[derive(Serialize, Deserialize, Copy, Clone, Debug)] +pub struct FileId(pub usize); + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Command { + pub argv: Vec, + pub env: Vec, + pub cwd: String, + pub stdio: Stdio, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Stdio { + pub stdin: FileId, + pub stdout: FileId, + pub stderr: FileId, +} + +/// Single action of execution plan. +#[derive(Serialize, Deserialize)] +pub enum Action { + /// Specifies that a pipe must be allocated + CreatePipe { + /// Will be associated with pipe's read half + read: FileId, + /// Will be associated with pipe's write half + write: FileId, + }, + /// Specifies that a file must be created + CreateFile { + /// Will be associated with the file + id: FileId, + }, + /// Associates file on local fs with a FileId + OpenFile { + /// Path to the file + path: PathBuf, + /// Id to associate with file + id: FileId, + }, + /// Associates file id with empty file, e.g. `/dev/null` + OpenNullFile { id: FileId }, + /// Specifies that command should be executed + ExecuteCommand(Command), +} + +/// Wrapper for hyper::Request that can be sent over CLI +#[derive(Serialize, Deserialize)] +pub struct RequestWrapper { + uri: String, + body: String, +} + +impl RequestWrapper { + pub async fn from_request(req: hyper::Request) -> anyhow::Result { + let uri = req.uri().to_string(); + let body = req.into_body(); + let body = hyper::body::to_bytes(body).await?; + let body = base64::encode(&body); + Ok(RequestWrapper { uri, body }) + } + + pub fn into_request(self) -> anyhow::Result> { + let mut req = hyper::Request::builder(); + req = req.method(hyper::Method::POST).uri(self.uri); + let body = base64::decode(&self.body)?; + let body = futures_util::stream::once(async { Ok::<_, std::convert::Infallible>(body) }); + let req = req.body(hyper::Body::wrap_stream(body))?; + Ok(req) + } +} + +/// Wrapper for hyper::Response that can be sent over CLI +pub struct ResponseWrapper {} diff --git a/src/judging-apis/src/lib.rs b/src/judging-apis/src/lib.rs index fb28fdf6..1a577b35 100644 --- a/src/judging-apis/src/lib.rs +++ b/src/judging-apis/src/lib.rs @@ -1,5 +1,6 @@ pub mod judge_log; pub mod valuer_proto; +pub mod invoke; use serde::{Deserialize, Serialize}; use std::path::PathBuf; From ad7c15092f2c63322dce29a14bc715742d33443c Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Tue, 6 Oct 2020 20:43:31 +0300 Subject: [PATCH 03/11] invoker set --- Cargo.lock | 1 + src/judge/Cargo.toml | 1 + src/judge/src/controller.rs | 33 ++-- src/judge/src/invoker_set.rs | 274 ++++++++++++--------------- src/judge/src/invoker_set/channel.rs | 101 ++++++++++ src/judge/src/request_handler.rs | 4 +- src/judging-apis/src/invoke.rs | 29 --- 7 files changed, 242 insertions(+), 201 deletions(-) create mode 100644 src/judge/src/invoker_set/channel.rs diff --git a/Cargo.lock b/Cargo.lock index ffb3bd8e..db04ba26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1807,6 +1807,7 @@ dependencies = [ "num_cpus", "once_cell", "openssl", + "parking_lot 0.11.0", "pom", "problem-loader", "puller", diff --git a/src/judge/Cargo.toml b/src/judge/Cargo.toml index 29ae223a..2fe8395b 100644 --- a/src/judge/Cargo.toml +++ b/src/judge/Cargo.toml @@ -48,6 +48,7 @@ hyper = "0.13.7" futures-util = "0.3.5" event-listener = "2.4.0" async-channel = "1.4.2" +parking_lot = "0.11.0" [features] k8s = ["kube", "k8s-openapi"] diff --git a/src/judge/src/controller.rs b/src/judge/src/controller.rs index 22d11acb..232203d5 100644 --- a/src/judge/src/controller.rs +++ b/src/judge/src/controller.rs @@ -74,7 +74,7 @@ pub trait JudgeResponseCallbacks: Send + Sync { #[derive(Clone)] pub struct Controller { - invoker_set: Arc, + invoker_set: InvokerSet, problem_loader: Arc, toolchains_dir: Arc, _config: Arc, @@ -102,15 +102,17 @@ impl Controller { None => get_num_cpus(), }; info!("Using {} workers", worker_count); - let mut invoker_set = - InvokerSet::new(&config).context("failed to initialize InvokerSet")?; - for _ in 0..worker_count { - invoker_set - .add_managed_worker() - .await - .context("failed to start a worker")?; - } - let invoker_set = Arc::new(invoker_set); + + let invoker_set = { + let mut builder = InvokerSet::builder(&config); + for _ in 0..worker_count { + builder + .add_managed_worker() + .await + .context("failed to start a worker")?; + } + builder.build() + }; let temp_dir = tempfile::TempDir::new().context("can not find temporary dir")?; @@ -158,10 +160,13 @@ impl Controller { debug!(lowered_judge_request = ?low_req, "created a lowered judge request"); - // TODO currently the process of finding a worker is unfair - // we should fix it e.g. using a semaphore which permits finding - // worker. - let worker = self.invoker_set.find_free_worker().await; + let (judge_events_tx, judge_events_rx) = async_channel::bounded(1); + let engine = self.invoker_set.clone(); + let judge_cx = crate::request_handler::JudgeContext { + events_tx: judge_events_tx, + invoker: rpc::Client::new(rpc::box_engine(engine), "http://does-not-matter".to_string()), + }; + // TODO: can we split into LoweredJudgeRequest and Extensions? let mut responses = worker .send(Request::Judge(low_req)) diff --git a/src/judge/src/invoker_set.rs b/src/judge/src/invoker_set.rs index 8a834935..44b189f9 100644 --- a/src/judge/src/invoker_set.rs +++ b/src/judge/src/invoker_set.rs @@ -1,39 +1,28 @@ +mod channel; + /// Implements `InvokerSet` use crate::config::JudgeConfig; use anyhow::Context as _; -use futures_util::{ - future::{FutureExt, TryFutureExt}, - stream::StreamExt, -}; -use std::{ - path::PathBuf, - sync::{ - atomic::{AtomicU8, Ordering::SeqCst}, - Arc, - }, -}; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; +use parking_lot::Mutex; +use std::{path::PathBuf, sync::Arc}; use tracing::{debug, instrument}; /// InvokerSet manages internal invoker and connects to external. +#[derive(Clone)] pub struct InvokerSet { /// Information abount spawned invokers - managed: Vec, - /// Path to invoker binary - invoker_path: PathBuf, + managed: Arc<[Arc]>, /// these field is used to signal that a worker is reclaimed - worker_reclamation: event_listener::Event, + worker_reclamation: Arc, } -impl InvokerSet { - /// Creates new Invoker with empty `managed` set - pub fn new(config: &JudgeConfig) -> anyhow::Result { - Ok(InvokerSet { - managed: vec![], - invoker_path: config.invoker_path, - worker_reclamation: event_listener::Event::new(), - }) - } +pub struct InvokerSetBuilder { + /// Path to invoker binary + invoker_path: PathBuf, + /// workers spawned so far + managed: Vec>, +} +impl InvokerSetBuilder { /// Starts new invoker process and adds it to this InvokerSet #[instrument(skip(self))] pub async fn add_managed_worker(&mut self) -> anyhow::Result<()> { @@ -44,57 +33,64 @@ impl InvokerSet { .stdout(std::process::Stdio::piped()) .spawn() .context("failed to spawn worker")?; + // we use 1 as capacity because in fact we never send request to invoker before it + // responded to previous + let (req_tx, req_rx) = async_channel::bounded(1); + let (res_tx, res_rx) = async_channel::bounded(1); + tokio::task::spawn(channel::serve( + child.stdin.take().expect("child stdin was captured"), + child.stdout.take().expect("child stdout was captured"), + req_rx, + res_tx, + )); let info = WorkerInfo { - state: WorkerState::new(WorkerStateKind::Idle), - child_stdin: Mutex::new(child.stdin.take().expect("child stdin was captured")), - child_stdout: Mutex::new(tokio::io::BufReader::new( - child.stdout.take().expect("child stdout was captured"), - )), + state: Mutex::new(WorkerState::Idle), + send_request: req_tx, + recv_response: res_rx, }; - self.managed.push(info); + self.managed.push(Arc::new(info)); Ok(()) } - /// Finds a free invoker (waiting if needed) and executes given InvokeRequest. + /// Finalizes InvokerSet construction + pub fn build(self) -> InvokerSet { + InvokerSet { + managed: self.managed.into(), + worker_reclamation: Arc::new(event_listener::Event::new()), + } + } +} + +impl InvokerSet { + /// Creates new builder + pub fn builder(config: &JudgeConfig) -> InvokerSetBuilder { + InvokerSetBuilder { + invoker_path: config.invoker_path, + managed: Vec::new(), + } + } + + /// Finds a free worker (waiting if needed) and sends http request. #[instrument(skip(self, req))] - pub async fn send_request( - &self, - req: judging_apis::invoke::InvokeRequest, - ) -> anyhow::Result { + async fn send_request(&self, req: hyper::Request) -> hyper::Response { let mut attempt_id = 0u32; loop { debug!(attempt_id, "scanning all workers"); attempt_id += 1; - let mut worker_reclaimed = self.worker_reclamation.listen(); - for worker in &self.managed { - if let Some(handle) = worker.try_lock(self.worker_reclamation.0.clone()) { - return handle; + let worker_reclaimed = self.worker_reclamation.listen(); + for worker in &*self.managed { + if let Some(handle) = worker.try_lock(self.worker_reclamation.clone()) { + handle.send_request.send(req).await.expect("worker died"); + let resp = handle + .recv_response + .recv() + .await + .expect("worker died before responding"); + return resp; } } - worker_reclaimed.await; - } - } -} -impl Drop for FreeWorkerHandle<'_> { - fn drop(&mut self) { - // true if worker can be reused later. - // Theoretically, it should always be the case when handle is dropped. - // However, due to bugs in JJS or problems in environment task, using - // this worker can fail in unexpected manner, leaving worker in - // inconsistent state. This flag implements conservative strategy - // which allows us to avoid such situations. - let reclaimable = matches!( - self.worker.state.load(), - WorkerStateKind::Idle | WorkerStateKind::Locked - ); - if reclaimable { - tracing::debug!("Reclaiming worker"); - self.worker.state.store(WorkerStateKind::Idle); - self.notify.wake(); - } else { - tracing::warn!("Leaking worker because it is not in reclaimable state"); - self.worker.state.store(WorkerStateKind::Crash); + worker_reclaimed.await; } } } @@ -110,72 +106,77 @@ enum WorkerState { /// Worker has crashed Crash, } - -struct WorkerDataInner { - io: std::sync::Mutex<( - tokio::io::BufWriter, - tokio::io::BufReader, - )>, +struct WorkerInfo { // could be AtomicU8, but mutex is simpler - state: std::sync::Mutex, - request_done: event_listener::Event, + state: Mutex, + // Danger: must not be used concurrently, otherwise + // we can receive wrong response + send_request: async_channel::Sender>, + recv_response: async_channel::Receiver>, +} +struct LockedWorker { + send_request: async_channel::Sender>, + recv_response: async_channel::Receiver>, + notify_on_drop: Arc, + worker: Arc, } -/// Implements "http client" on top of child stdio -struct WorkerData { - inner: Arc, +impl LockedWorker { + async fn call(self, req: hyper::Request) -> hyper::Response { + let wake = { + let ev = self.notify_on_drop.clone(); + move || { + ev.notify_additional(1); + } + }; + + let result = async move { + self.send_request + .send(req) + .await + .expect("unexpected contention"); + + self.recv_response + .recv() + .await + .expect("response should be sent and non-stolen") + } + .await; + wake(); + result + } } -impl WorkerData { - fn subscribe(&self) -> event_listener::EventListener { - self.inner.request_done.listen() +impl Drop for LockedWorker { + fn drop(&mut self) { + // mark Worker as idle + *self.worker.state.lock() = WorkerState::Idle; + // trigger event + self.notify_on_drop.notify_additional(1); } +} - fn call( - &mut self, - req: hyper::Request, - ) -> Option< - futures_util::future::BoxFuture<'static, anyhow::Result>>, - > { - let mut lock = self.inner.state.lock().unwrap(); +impl WorkerInfo { + fn try_lock( + self: &Arc, + notify_on_drop: Arc, + ) -> Option { + let mut lock = self.state.lock(); if *lock != WorkerState::Idle { return None; } *lock = WorkerState::Locked; - drop(lock); - let wake = { - let inner = self.inner.clone(); - move |_future_res: &anyhow::Result>| { - inner.request_done.notify_additional(1); - } - }; - let inner = self.inner.clone(); - let mut io = inner.io.lock().unwrap(); - let (stdin, stdout) = &mut *io; - - Some( - (async move { - let mut uri = req.uri().to_string(); - uri.push('\n'); - // tokio::pin!(stdout); - // tokio::pin!(stdin); - stdin.write_all(uri.as_bytes()).await?; - let req_body = hyper::body::to_bytes(req.into_body()).await?; - let cnt = req_body.len(); - stdin.write_all(&cnt.to_ne_bytes()).await?; - stdin.write_all(&req_body).await?; - stdin.flush().await?; - - Result::, anyhow::Error>::Ok(todo!()) - }) - .inspect_ok(wake) - .boxed(), - ) + Some(LockedWorker { + send_request: self.send_request.clone(), + recv_response: self.recv_response.clone(), + worker: self.clone(), + notify_on_drop, + }) } } -impl tower_service::Service> for InvokerSet { - type Error = anyhow::Error; +impl hyper::service::Service> for InvokerSet { + type Error = std::convert::Infallible; type Future = futures_util::future::BoxFuture<'static, Result>; type Response = hyper::Response; @@ -187,46 +188,7 @@ impl tower_service::Service> for InvokerSet { } fn call(&mut self, req: hyper::Request) -> Self::Future { - if Arc::strong_count(&self.inner) != 1 { - panic!("") - } - self.in_flight = true; - Box::pin(async move {}) - } -} - -struct WorkerInfo { - state: WorkerState, - child_stdout: Mutex>, - child_stdin: Mutex, -} - -impl WorkerInfo { - pub async fn recv(&self) -> anyhow::Result { - let mut line = String::new(); - let mut child_stdout = self.child_stdout.lock().await; - - child_stdout.read_line(&mut line).await?; - Ok(serde_json::from_str(&line).context("parse error")?) - } - - pub async fn send(&self, req: Request) -> anyhow::Result<()> { - let mut data = serde_json::to_vec(&req)?; - data.push(b'\n'); - self.child_stdin.lock().await.write_all(&data).await?; - Ok(()) - } - - /// If this worker is idle, returns a handle to it. - /// Otherwise, returns None - pub fn try_lock(&self, notify: multiwake::Sender) -> Option { - if self.state.lock() { - Some(FreeWorkerHandle { - worker: self, - notify, - }) - } else { - None - } + let this = self.clone(); + Box::pin(async move { Ok(this.send_request(req).await) }) } } diff --git a/src/judge/src/invoker_set/channel.rs b/src/judge/src/invoker_set/channel.rs new file mode 100644 index 00000000..cb74bf2d --- /dev/null +++ b/src/judge/src/invoker_set/channel.rs @@ -0,0 +1,101 @@ +//! Raw interface to invoker +use std::{ + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; +use tracing::instrument; +/// Means for communicating with invoker +struct Pipes { + stdin: tokio::io::BufWriter, + stdout: tokio::io::BufReader, +} + +struct CoherenceWrapper(tokio::sync::OwnedMutexGuard); + +impl hyper::client::connect::Connection for CoherenceWrapper { + fn connected(&self) -> hyper::client::connect::Connected { + hyper::client::connect::Connected::new() + } +} + +impl tokio::io::AsyncRead for CoherenceWrapper { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> std::task::Poll> { + Pin::new(&mut self.0.stdout).poll_read(cx, buf) + } +} + +impl tokio::io::AsyncWrite for CoherenceWrapper { + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + Pin::new(&mut self.0.stdin).poll_write(cx, buf) + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + Pin::new(&mut self.0.stdin).poll_flush(cx) + } + + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + Pin::new(&mut self.0.stdin).poll_shutdown(cx) + } +} + +#[derive(Clone)] +struct Connector { + pipes: Arc>, +} + +impl hyper::service::Service for Connector { + type Error = std::convert::Infallible; + type Future = futures_util::future::BoxFuture<'static, Result>; + type Response = CoherenceWrapper; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _uri: hyper::Uri) -> Self::Future { + let pipes = self.pipes.clone(); + Box::pin(async move { Ok(CoherenceWrapper(pipes.lock_owned().await)) }) + } +} + +/// Takes child's input and output and abstracts it as a channel. +/// Should be spawned on background task. +#[instrument(skip(stdin, stdout, tx, rx))] +pub(crate) async fn serve( + stdin: tokio::process::ChildStdin, + stdout: tokio::process::ChildStdout, + rx: async_channel::Receiver>, + tx: async_channel::Sender>, +) { + let stdin = tokio::io::BufWriter::new(stdin); + let stdout = tokio::io::BufReader::new(stdout); + let pipes = Arc::new(tokio::sync::Mutex::new(Pipes { stdin, stdout })); + let connector = Connector { pipes }; + let client = hyper::client::Client::builder().build::<_, hyper::Body>(connector); + while let Ok(req) = rx.recv().await { + let res = match client.request(req).await { + Ok(response) => response, + Err(error) => hyper::Response::builder() + .status(418 /* why? because */) + .body(format!("{:#}", error).into()) + .unwrap(), + }; + tx.send(res).await.ok(); + } + tracing::info!("Sender dropped, exiting"); +} diff --git a/src/judge/src/request_handler.rs b/src/judge/src/request_handler.rs index ff7ae9dc..7d00afdc 100644 --- a/src/judge/src/request_handler.rs +++ b/src/judge/src/request_handler.rs @@ -24,9 +24,9 @@ pub type InvokerClient = rpc::Client; pub(crate) struct JudgeContext { /// Can be used to send requests to invoker - invoker: InvokerClient, + pub(crate) invoker: InvokerClient, /// Channel that should be used for sending updates - events_tx: tokio::sync::mpsc::Sender + pub(crate) events_tx: async_channel::Sender, } /// Note: this is not `judging_apis::invoke::Command`, it is higher-level. diff --git a/src/judging-apis/src/invoke.rs b/src/judging-apis/src/invoke.rs index d3cef31c..3063aa16 100644 --- a/src/judging-apis/src/invoke.rs +++ b/src/judging-apis/src/invoke.rs @@ -117,32 +117,3 @@ pub enum Action { /// Specifies that command should be executed ExecuteCommand(Command), } - -/// Wrapper for hyper::Request that can be sent over CLI -#[derive(Serialize, Deserialize)] -pub struct RequestWrapper { - uri: String, - body: String, -} - -impl RequestWrapper { - pub async fn from_request(req: hyper::Request) -> anyhow::Result { - let uri = req.uri().to_string(); - let body = req.into_body(); - let body = hyper::body::to_bytes(body).await?; - let body = base64::encode(&body); - Ok(RequestWrapper { uri, body }) - } - - pub fn into_request(self) -> anyhow::Result> { - let mut req = hyper::Request::builder(); - req = req.method(hyper::Method::POST).uri(self.uri); - let body = base64::decode(&self.body)?; - let body = futures_util::stream::once(async { Ok::<_, std::convert::Infallible>(body) }); - let req = req.body(hyper::Body::wrap_stream(body))?; - Ok(req) - } -} - -/// Wrapper for hyper::Response that can be sent over CLI -pub struct ResponseWrapper {} From 79dfcbf494ffacf71d4e172226a617450daf6aa8 Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Thu, 8 Oct 2020 01:04:36 +0300 Subject: [PATCH 04/11] work --- src/judge/src/controller.rs | 20 +- src/judge/src/controller/task_loading.rs | 1 - src/judge/src/lib.rs | 1 - src/judge/src/request_handler.rs | 194 +++++++++++++- src/judge/src/request_handler/compiler.rs | 13 +- src/judge/src/request_handler/exec_test.rs | 51 +++- .../request_handler/transform_judge_log.rs | 4 +- src/judge/src/request_handler/valuer.rs | 2 +- src/judge/src/worker.rs | 252 ------------------ src/judging-apis/src/invoke.rs | 4 +- 10 files changed, 247 insertions(+), 295 deletions(-) delete mode 100644 src/judge/src/worker.rs diff --git a/src/judge/src/controller.rs b/src/judge/src/controller.rs index 232203d5..1691d523 100644 --- a/src/judge/src/controller.rs +++ b/src/judge/src/controller.rs @@ -164,21 +164,21 @@ impl Controller { let engine = self.invoker_set.clone(); let judge_cx = crate::request_handler::JudgeContext { events_tx: judge_events_tx, - invoker: rpc::Client::new(rpc::box_engine(engine), "http://does-not-matter".to_string()), + invoker: rpc::Client::new( + rpc::box_engine(engine), + "http://does-not-matter".to_string(), + ), }; // TODO: can we split into LoweredJudgeRequest and Extensions? - let mut responses = worker - .send(Request::Judge(low_req)) - .await - .context("failed to submit lowered judge request")?; + crate::request_handler::do_judge(judge_cx, low_req); loop { - let message = responses + let message = judge_events_rx .next() .await .context("failed to receive next worker message")?; match message { - Response::JudgeDone(judge_outcome) => { + Event::JudgeDone(judge_outcome) => { debug!("Publising: JudgeOutcome {:?}", &judge_outcome); let reason = match judge_outcome { JudgeOutcome::Fault => InvocationFinishReason::Fault, @@ -191,13 +191,13 @@ impl Controller { .context("failed to set run outcome in DB")?; break; } - Response::LiveScore(score) => { + Event::LiveScore(score) => { exts.notifier.set_score(score).await; } - Response::LiveTest(test) => { + Event::LiveTest(test) => { exts.notifier.set_test(test).await; } - Response::OutcomeHeader(header) => { + Event::OutcomeHeader(header) => { req.callbacks .add_outcome_header(req.request.request_id, header) .await?; diff --git a/src/judge/src/controller/task_loading.rs b/src/judge/src/controller/task_loading.rs index 66329436..86296e12 100644 --- a/src/judge/src/controller/task_loading.rs +++ b/src/judge/src/controller/task_loading.rs @@ -4,7 +4,6 @@ use super::{ }; use crate::{ request_handler::{Command, LoweredJudgeRequest}, - worker, }; use anyhow::Context; use std::{ diff --git a/src/judge/src/lib.rs b/src/judge/src/lib.rs index d6c75b3c..1eeefa05 100644 --- a/src/judge/src/lib.rs +++ b/src/judge/src/lib.rs @@ -5,5 +5,4 @@ pub mod controller; pub mod init; mod invoker_set; pub mod sources; -pub mod worker; mod request_handler; \ No newline at end of file diff --git a/src/judge/src/request_handler.rs b/src/judge/src/request_handler.rs index 7d00afdc..106fad5c 100644 --- a/src/judge/src/request_handler.rs +++ b/src/judge/src/request_handler.rs @@ -8,7 +8,8 @@ use self::{ compiler::{BuildOutcome, Compiler}, exec_test::{exec, ExecRequest}, }; -use judging_apis::Status; +use anyhow::Context; +use judging_apis::{valuer_proto::ValuerResponse, Status}; use serde::{Deserialize, Serialize}; use std::{ borrow::Cow, @@ -16,21 +17,13 @@ use std::{ path::{Path, PathBuf}, pin::Pin, }; -use tracing::debug; - +use tracing::{debug, error, instrument}; /// Allows sending requests to invoker // TODO upstream BoxedEngine to jjs-commons pub type InvokerClient = rpc::Client; -pub(crate) struct JudgeContext { - /// Can be used to send requests to invoker - pub(crate) invoker: InvokerClient, - /// Channel that should be used for sending updates - pub(crate) events_tx: async_channel::Sender, -} - /// Note: this is not `judging_apis::invoke::Command`, it is higher-level. -#[derive(Debug)] +#[derive(Debug, Default)] pub(crate) struct Command { pub argv: Vec, pub env: Vec, @@ -100,3 +93,182 @@ pub(crate) enum JudgeOutcome { /// Maybe, several protocols were emitted, but results are neither precise nor complete Fault, } + +pub(crate) struct JudgeContext { + /// Can be used to send requests to invoker + pub(crate) invoker: InvokerClient, + /// Channel that should be used for sending updates + pub(crate) events_tx: async_channel::Sender, +} + +#[instrument(skip(cx, judge_req), fields(id = %judge_req.judge_request_id))] +pub fn do_judge(mut cx: JudgeContext, judge_req: LoweredJudgeRequest) { + debug!("Got LoweredJudgeRequest: {:?}", &judge_req); + tokio::task::spawn(async move { + let outcome = match cx.judge(&judge_req).await { + Ok(o) => o, + Err(err) => { + error!("Invoke failed: {:#}", err); + cx.create_fake_protocols( + &judge_req, + &judging_apis::Status { + kind: judging_apis::StatusKind::InternalError, + code: judging_apis::status_codes::JUDGE_FAULT.to_string(), + }, + ) + .await + .ok(); + JudgeOutcome::Fault + } + }; + debug!("JudgeOutcome: {:?}", &outcome); + cx.events_tx.send(Event::JudgeDone(outcome)).await; + }); +} + +impl JudgeContext { + async fn judge(&mut self, req: &LoweredJudgeRequest) -> anyhow::Result { + let compiler = Compiler { req }; + + if !req.run_source.exists() { + anyhow::bail!("Run source file not exists"); + } + + if !req.out_dir.exists() { + anyhow::bail!("Run output dir not exists"); + } + + let compiler_response = compiler.compile(); + + let outcome; + + match compiler_response { + Err(err) => return Err(err).context("compilation error"), + Ok(BuildOutcome::Error(st)) => { + self.create_fake_protocols(req, &st).await?; + outcome = JudgeOutcome::CompileError(st); + } + Ok(BuildOutcome::Success) => { + self.run_tests(req).await.context("failed to run tests")?; + + outcome = JudgeOutcome::TestingDone; + } + }; + Ok(outcome) + } + + /// Used when we are unable to produce protocols, i.e. on compilation errors + /// and judge faults. + async fn create_fake_protocols( + &mut self, + req: &LoweredJudgeRequest, + status: &judging_apis::Status, + ) -> anyhow::Result<()> { + for kind in judging_apis::judge_log::JudgeLogKind::list() { + let pseudo_valuer_proto = judging_apis::valuer_proto::JudgeLog { + kind, + tests: vec![], + subtasks: vec![], + score: 0, + is_full: false, + }; + let mut protocol = self.process_judge_log(&pseudo_valuer_proto, req, &[])?; + protocol.status = status.clone(); + self.put_protocol(req, protocol).await?; + } + Ok(()) + } + + async fn put_outcome( + &mut self, + score: u32, + status: judging_apis::Status, + kind: judging_apis::judge_log::JudgeLogKind, + ) { + let header = judging_apis::JudgeOutcomeHeader { + score: Some(score), + status, + kind, + }; + self.events_tx.send(Event::OutcomeHeader(header)).await.ok(); + } + + async fn put_protocol( + &mut self, + req: &LoweredJudgeRequest, + protocol: judging_apis::judge_log::JudgeLog, + ) -> anyhow::Result<()> { + let protocol_file_name = format!("protocol-{}.json", protocol.kind.as_str()); + let protocol_path = req.out_dir.join(protocol_file_name); + debug!("Writing protocol to {}", protocol_path.display()); + let protocol_file = std::fs::File::create(&protocol_path)?; + let protocol_file = std::io::BufWriter::new(protocol_file); + serde_json::to_writer(protocol_file, &protocol) + .context("failed to write judge log to file")?; + self.put_outcome(protocol.score, protocol.status, protocol.kind) + .await; + Ok(()) + } + + async fn run_tests(&mut self, req: &LoweredJudgeRequest) -> anyhow::Result<()> { + let mut test_results = vec![]; + + let mut valuer = valuer::Valuer::new(req).context("failed to init valuer")?; + valuer + .write_problem_data(req) + .await + .context("failed to send problem data")?; + loop { + match valuer.poll().await? { + ValuerResponse::Test { test_id: tid, live } => { + if live { + self.events_tx.send(Event::LiveTest(tid.get())).await; + } + let tid_u32: u32 = tid.into(); + let test = &req.problem.tests[(tid_u32 - 1u32) as usize]; + let exec_request = ExecRequest { + test, + test_id: tid.into(), + }; + + let judge_response = exec_test::exec(&req, exec_request, self) + .with_context(|| format!("failed to judge solution on test {}", tid))?; + test_results.push((tid, judge_response.clone())); + valuer + .notify_test_done(judging_apis::valuer_proto::TestDoneNotification { + test_id: tid, + test_status: judge_response.status, + }) + .await + .with_context(|| { + format!("failed to notify valuer that test {} is done", tid) + })?; + } + ValuerResponse::Finish => { + break; + } + ValuerResponse::LiveScore { score } => { + self.events_tx.send(Event::LiveScore(score)).await; + } + ValuerResponse::JudgeLog(judge_log) => { + let converted_judge_log = self + .process_judge_log(&judge_log, req, &test_results) + .context("failed to convert valuer judge log to invoker judge log")?; + self.put_protocol(req, converted_judge_log) + .await + .context("failed to save protocol")?; + } + } + } + + Ok(()) + } + + /// Creates `InputSource` that can be further sent to invoker + async fn intern(&self, data: &[u8]) -> anyhow::Result { + // TODO: optimize + Ok(judging_apis::invoke::InputSource::Inline { + data: data.to_vec(), + }) + } +} diff --git a/src/judge/src/request_handler/compiler.rs b/src/judge/src/request_handler/compiler.rs index e768636d..4d6d08e9 100644 --- a/src/judge/src/request_handler/compiler.rs +++ b/src/judge/src/request_handler/compiler.rs @@ -1,4 +1,4 @@ -use crate::request_handler::{invoke_util, LoweredJudgeRequest}; +use crate::request_handler::LoweredJudgeRequest; use anyhow::Context; use judging_apis::{status_codes, Status, StatusKind}; use std::fs; @@ -11,14 +11,17 @@ pub(crate) enum BuildOutcome { /// Compiler turns SubmissionInfo into Artifact pub(crate) struct Compiler<'a> { pub(crate) req: &'a LoweredJudgeRequest, - pub(crate) minion: &'a dyn minion::erased::Backend, - pub(crate) config: &'a crate::config::JudgeConfig, + // pub(crate) config: &'a crate::config::JudgeConfig, } impl<'a> Compiler<'a> { pub(crate) fn compile(&self) -> anyhow::Result { - let sandbox = invoke_util::create_sandbox(self.req, None, self.minion, self.config) - .context("failed to create sandbox")?; + let mut graph = judging_apis::invoke::InvokeRequest { + inputs: vec![], + outputs: vec![], + steps: vec![], + }; + let step_dir = self.req.step_dir(None); fs::copy( &self.req.run_source, diff --git a/src/judge/src/request_handler/exec_test.rs b/src/judge/src/request_handler/exec_test.rs index 3e151137..5be3f329 100644 --- a/src/judge/src/request_handler/exec_test.rs +++ b/src/judge/src/request_handler/exec_test.rs @@ -1,9 +1,9 @@ mod checker_proto; -use super::{invoke_util, JudgeContext, LoweredJudgeRequest}; +use super::{JudgeContext, LoweredJudgeRequest}; use anyhow::Context; use judging_apis::{ - invoke::{Command, Action, Step, Stdio, FileId}, + invoke::{Action, Command, FileId, Input, InputSource, Stdio, Step}, status_codes, Status, StatusKind, }; use std::{fs, io::Write, path::PathBuf}; @@ -51,17 +51,47 @@ fn map_checker_outcome_to_status(out: checker_proto::Output) -> Status { } /// Runs Artifact on one test and produces output -pub fn exec(judge_req: &LoweredJudgeRequest, ctx: &JudgeContext) -> anyhow::Result { +pub async fn exec( + judge_req: &LoweredJudgeRequest, + exec_req: ExecRequest<'_>, + cx: &JudgeContext, +) -> anyhow::Result { let mut invoke_request = judging_apis::invoke::InvokeRequest { steps: vec![], inputs: vec![], outputs: vec![], }; + let input_file = judge_req.resolve_asset(&exec_req.test.path); + let test_data = std::fs::read(input_file).context("failed to read test")?; const EXEC_SOLUTION_GENERATION: u32 = 0; - const EXEC_SOLUTION_OUTPUT_FILE: FileId = FileId(0); - const EXEC_SOLUTION_ERROR_FILE: FileId = FileId(1); - + const TEST_DATA_INPUT_FILE: &str = "test-data"; + const EXEC_SOLUTION_OUTPUT_FILE: &str = "solution-output"; + const EXEC_SOLUTION_ERROR_FILE: &str = "solution-error"; + const EXEC_CHECKER_GENERATION: u32 = 1; + // create an input with the test date + { + let test_data_input = Input { + id: FileId(TEST_DATA_INPUT_FILE.to_string()), + source: cx.intern(&test_data).await?, + }; + invoke_request.inputs.push(test_data_input); + } + // prepare files for stdout & stderr + { + invoke_request.steps.push(Step { + generation: EXEC_SOLUTION_GENERATION, + action: Action::CreateFile { + id: FileId(EXEC_SOLUTION_OUTPUT_FILE.to_string()), + }, + }); + invoke_request.steps.push(Step { + generation: EXEC_SOLUTION_GENERATION, + action: Action::CreateFile { + id: FileId(EXEC_SOLUTION_ERROR_FILE.to_string()), + }, + }) + } // produce a step for executing solution { let exec_solution_step = Step { @@ -71,13 +101,14 @@ pub fn exec(judge_req: &LoweredJudgeRequest, ctx: &JudgeContext) -> anyhow::Resu env: judge_req.execute_command.env.clone(), cwd: judge_req.execute_command.cwd.clone(), stdio: Stdio { - - } + stdin: FileId(TEST_DATA_INPUT_FILE.to_string()), + stdout: FileId(EXEC_SOLUTION_OUTPUT_FILE.to_string()), + stderr: FileId(EXEC_SOLUTION_ERROR_FILE.to_string()), + }, }), }; + invoke_request.steps.push(exec_solution_step); } - let input_file = self.req.resolve_asset(&self.exec.test.path); - let test_data = std::fs::read(input_file).context("failed to read test")?; let run_outcome = self.run_solution(&test_data, self.exec.test_id)?; let sol_file_path = match run_outcome.var { RunOutcomeVar::Success { out_data_path } => out_data_path, diff --git a/src/judge/src/request_handler/transform_judge_log.rs b/src/judge/src/request_handler/transform_judge_log.rs index dc9fd8c1..a7336611 100644 --- a/src/judge/src/request_handler/transform_judge_log.rs +++ b/src/judge/src/request_handler/transform_judge_log.rs @@ -1,11 +1,11 @@ -use crate::worker::{LoweredJudgeRequest, Worker}; +use crate::request_handler::{LoweredJudgeRequest, JudgeContext}; use anyhow::Context; use judging_apis::{ judge_log, status_codes, valuer_proto::TestVisibleComponents, Status, StatusKind, }; use std::io::Read; -impl Worker { +impl JudgeContext { /// Go from valuer judge log to invoker judge log // Bug in clippy: https://github.com/rust-lang/rust-clippy/issues/5368 #[allow(clippy::verbose_file_reads)] diff --git a/src/judge/src/request_handler/valuer.rs b/src/judge/src/request_handler/valuer.rs index 13b7122e..bac9c083 100644 --- a/src/judge/src/request_handler/valuer.rs +++ b/src/judge/src/request_handler/valuer.rs @@ -1,4 +1,4 @@ -use crate::worker::LoweredJudgeRequest; +use crate::request_handler::LoweredJudgeRequest; use anyhow::{bail, Context}; use judging_apis::valuer_proto::{ProblemInfo, TestDoneNotification, ValuerResponse}; use std::os::unix::io::IntoRawFd; diff --git a/src/judge/src/worker.rs b/src/judge/src/worker.rs deleted file mode 100644 index ed859b68..00000000 --- a/src/judge/src/worker.rs +++ /dev/null @@ -1,252 +0,0 @@ -//! Implements Invoker Worker. -//! -//! Worker is responsible for processing `InvokeRequest`s - - -mod os_util; - -use anyhow::Context; -use judging_apis::{ - valuer_proto::{TestDoneNotification, ValuerResponse}, - Status, -}; -use serde::{Deserialize, Serialize}; -use std::{ - borrow::Cow, - path::{Path, PathBuf}, - sync::Arc, -}; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; -use tracing::{debug, error}; -use valuer::Valuer; - - -pub(crate) struct Worker { - /// Minion backend to use for invocations - minion: Arc, - /// Judge configuration - config: crate::config::JudgeConfig, -} - -impl Worker { - pub(crate) fn new(config: crate::config::JudgeConfig) -> anyhow::Result { - Ok(Worker { - minion: minion::erased::setup() - .context("minion initialization failed")? - .into(), - config, - }) - } - - async fn recv(&self, stdin: &mut (impl AsyncBufReadExt + Unpin)) -> Option { - let mut buf = String::new(); - match stdin.read_line(&mut buf).await { - Ok(_) => { - if buf.trim().is_empty() { - return None; - } - - Some(serde_json::from_str(&buf).expect("parse error")) - } - Err(_) => None, - } - } - - async fn send(&self, resp: Response) { - let mut stdout = tokio::io::stdout(); - let mut msg = serde_json::to_vec(&resp).expect("failed to serialize Response"); - msg.push(b'\n'); - stdout - .write_all(&msg) - .await - .expect("Failed to print Response"); - } - - pub(crate) async fn main_loop(mut self) { - let mut stdin = tokio::io::BufReader::new(tokio::io::stdin()); - while let Some(req) = self.recv(&mut stdin).await { - match req { - Request::Judge(judge_req) => { - debug!("Got LoweredJudgeRequest: {:?}", &judge_req); - let outcome = match self.judge(&judge_req).await { - Ok(o) => o, - Err(err) => { - error!("Invoke failed: {:#}", err); - self.create_fake_protocols( - &judge_req, - &judging_apis::Status { - kind: judging_apis::StatusKind::InternalError, - code: judging_apis::status_codes::JUDGE_FAULT.to_string(), - }, - ) - .await - .ok(); - JudgeOutcome::Fault - } - }; - debug!("JudgeOutcome: {:?}", &outcome); - self.send(Response::JudgeDone(outcome)).await; - } - } - } - } - - async fn judge(&mut self, req: &LoweredJudgeRequest) -> anyhow::Result { - let compiler = Compiler { - req, - minion: &*self.minion, - config: &self.config, - }; - - if !req.run_source.exists() { - anyhow::bail!("Run source file not exists"); - } - - if !req.out_dir.exists() { - anyhow::bail!("Run output dir not exists"); - } - - let compiler_response = compiler.compile(); - - let outcome; - - match compiler_response { - Err(err) => return Err(err).context("compilation error"), - Ok(BuildOutcome::Error(st)) => { - self.create_fake_protocols(req, &st).await?; - outcome = JudgeOutcome::CompileError(st); - } - Ok(BuildOutcome::Success) => { - self.run_tests(req).await.context("failed to run tests")?; - - outcome = JudgeOutcome::TestingDone; - } - }; - Ok(outcome) - } - - /// Used when we are unable to produce protocols, i.e. on compilation errors - /// and judge faults. - async fn create_fake_protocols( - &mut self, - req: &LoweredJudgeRequest, - status: &judging_apis::Status, - ) -> anyhow::Result<()> { - for kind in judging_apis::judge_log::JudgeLogKind::list() { - let pseudo_valuer_proto = judging_apis::valuer_proto::JudgeLog { - kind, - tests: vec![], - subtasks: vec![], - score: 0, - is_full: false, - }; - let mut protocol = self.process_judge_log(&pseudo_valuer_proto, req, &[])?; - protocol.status = status.clone(); - self.put_protocol(req, protocol).await?; - } - Ok(()) - } - - async fn put_outcome( - &mut self, - score: u32, - status: judging_apis::Status, - kind: judging_apis::judge_log::JudgeLogKind, - ) { - let header = judging_apis::JudgeOutcomeHeader { - score: Some(score), - status, - kind, - }; - self.send(Response::OutcomeHeader(header)).await; - } - - async fn put_protocol( - &mut self, - req: &LoweredJudgeRequest, - protocol: judging_apis::judge_log::JudgeLog, - ) -> anyhow::Result<()> { - let protocol_file_name = format!("protocol-{}.json", protocol.kind.as_str()); - let protocol_path = req.out_dir.join(protocol_file_name); - debug!("Writing protocol to {}", protocol_path.display()); - let protocol_file = std::fs::File::create(&protocol_path)?; - let protocol_file = std::io::BufWriter::new(protocol_file); - serde_json::to_writer(protocol_file, &protocol) - .context("failed to write judge log to file")?; - self.put_outcome(protocol.score, protocol.status, protocol.kind) - .await; - Ok(()) - } - - async fn run_tests(&mut self, req: &LoweredJudgeRequest) -> anyhow::Result<()> { - let mut test_results = vec![]; - - let mut valuer = Valuer::new(req).context("failed to init valuer")?; - valuer - .write_problem_data(req) - .await - .context("failed to send problem data")?; - loop { - match valuer.poll().await? { - ValuerResponse::Test { test_id: tid, live } => { - if live { - self.send(Response::LiveTest(tid.get())).await; - } - let tid_u32: u32 = tid.into(); - let test = &req.problem.tests[(tid_u32 - 1u32) as usize]; - let judge_request = ExecRequest { - test, - test_id: tid.into(), - }; - - let test_exec = TestExecutor { - exec: judge_request, - req, - minion: &*self.minion, - config: &self.config, - }; - - let judge_response = test_exec - .exec() - .with_context(|| format!("failed to judge solution on test {}", tid))?; - test_results.push((tid, judge_response.clone())); - valuer - .notify_test_done(TestDoneNotification { - test_id: tid, - test_status: judge_response.status, - }) - .await - .with_context(|| { - format!("failed to notify valuer that test {} is done", tid) - })?; - } - ValuerResponse::Finish => { - break; - } - ValuerResponse::LiveScore { score } => { - self.send(Response::LiveScore(score)).await; - } - ValuerResponse::JudgeLog(judge_log) => { - let converted_judge_log = self - .process_judge_log(&judge_log, req, &test_results) - .context("failed to convert valuer judge log to invoker judge log")?; - self.put_protocol(req, converted_judge_log) - .await - .context("failed to save protocol")?; - } - } - } - - Ok(()) - } -} - - -pub async fn main() -> anyhow::Result<()> { - let config_data = std::env::var("__JJS_WORKER_INVOKER_CONFIG") - .context("__JJS_WORKER_INVOKER_CONFIG missing")?; - let config = serde_json::from_str(&config_data)?; - let w = Worker::new(config).context("worker initialization failed")?; - w.main_loop().await; - Ok(()) -} diff --git a/src/judging-apis/src/invoke.rs b/src/judging-apis/src/invoke.rs index 3063aa16..d050d7c2 100644 --- a/src/judging-apis/src/invoke.rs +++ b/src/judging-apis/src/invoke.rs @@ -72,8 +72,8 @@ pub struct Step { } /// Newtype identifier of file-like object, e.g. real file or pipe. -#[derive(Serialize, Deserialize, Copy, Clone, Debug)] -pub struct FileId(pub usize); +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FileId(pub String); #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Command { From ee1297ff56fc351c1ab749742770899cfcbe08bd Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Thu, 8 Oct 2020 19:14:54 +0300 Subject: [PATCH 05/11] generation -> stage --- src/judging-apis/src/invoke.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/judging-apis/src/invoke.rs b/src/judging-apis/src/invoke.rs index d050d7c2..06ecc4ed 100644 --- a/src/judging-apis/src/invoke.rs +++ b/src/judging-apis/src/invoke.rs @@ -8,12 +8,12 @@ pub struct Invoke(std::convert::Infallible); /// Requests invoker to execute commands, specified in /// `steps` field in request. /// # Execution order -/// Each step has assigned `generation`. -/// Steps with equal generation will be executed in the same time. -/// Such steps can share pipes. Sharing pipes between steps with -/// different generations results in error. For each generation, -/// Steps creating new IPC stuff are executed first. -/// Step will not be executed until all steps with less `generation` +/// Each step has assigned `stage`. +/// Steps with equal stage will be executed in the same time. +/// Such steps can share pipes. Sharing pipes between steps from +/// different stages results in error. For each stage, +/// steps creating new IPC stuff are executed first. +/// Step will not be executed until all steps with less `stage` /// will be finished. /// # Data /// `InvokeRequest` can specify input data items, that can be further used @@ -67,7 +67,7 @@ pub struct Output {} #[derive(Serialize, Deserialize)] pub struct Step { - pub generation: u32, + pub stage: u32, pub action: Action, } From 79d7b4006ecf4cc99143f89d6aafbaf043142ac9 Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Thu, 8 Oct 2020 20:34:36 +0300 Subject: [PATCH 06/11] finish graph for exec test --- src/judge/src/controller/task_loading.rs | 4 +- src/judge/src/invoker_set.rs | 64 ++++----- src/judge/src/request_handler.rs | 7 +- src/judge/src/request_handler/exec_test.rs | 156 +++++++++++++-------- src/judging-apis/src/invoke.rs | 19 ++- 5 files changed, 149 insertions(+), 101 deletions(-) diff --git a/src/judge/src/controller/task_loading.rs b/src/judge/src/controller/task_loading.rs index 86296e12..43fbab02 100644 --- a/src/judge/src/controller/task_loading.rs +++ b/src/judge/src/controller/task_loading.rs @@ -84,13 +84,13 @@ fn interpolate_command( for (name, val) in &command.env { let name = interpolate_string(name, dict)?; let val = interpolate_string(val, dict)?; - res.env.push(format!("{}={}", name, val)); + res.env.push((name, val)); used_env_vars.insert(name); } res.cwd = interpolate_string(&command.cwd, dict)?; for (default_key, default_val) in &toolchain_spec.env { if !used_env_vars.contains(default_key) { - res.env.push(format!("{}={}", default_key, default_val)); + res.env.push((default_key.clone(), default_val.clone())); } } Ok(res) diff --git a/src/judge/src/invoker_set.rs b/src/judge/src/invoker_set.rs index 44b189f9..05eb905e 100644 --- a/src/judge/src/invoker_set.rs +++ b/src/judge/src/invoker_set.rs @@ -10,22 +10,22 @@ use tracing::{debug, instrument}; #[derive(Clone)] pub struct InvokerSet { /// Information abount spawned invokers - managed: Arc<[Arc]>, - /// these field is used to signal that a worker is reclaimed - worker_reclamation: Arc, + managed: Arc<[Arc]>, + /// these field is used to signal that an invoker lock was released + invoker_released: Arc, } pub struct InvokerSetBuilder { /// Path to invoker binary invoker_path: PathBuf, - /// workers spawned so far - managed: Vec>, + /// Managed invokers spawned so far + managed: Vec>, } impl InvokerSetBuilder { /// Starts new invoker process and adds it to this InvokerSet #[instrument(skip(self))] - pub async fn add_managed_worker(&mut self) -> anyhow::Result<()> { + pub async fn add_managed_invoker(&mut self) -> anyhow::Result<()> { let mut child = tokio::process::Command::new(&self.invoker_path) .arg("serve") .arg("--address=cli") @@ -43,8 +43,8 @@ impl InvokerSetBuilder { req_rx, res_tx, )); - let info = WorkerInfo { - state: Mutex::new(WorkerState::Idle), + let info = Invoker { + state: Mutex::new(InvokerState::Idle), send_request: req_tx, recv_response: res_rx, }; @@ -56,7 +56,7 @@ impl InvokerSetBuilder { pub fn build(self) -> InvokerSet { InvokerSet { managed: self.managed.into(), - worker_reclamation: Arc::new(event_listener::Event::new()), + invoker_released: Arc::new(event_listener::Event::new()), } } } @@ -77,9 +77,9 @@ impl InvokerSet { loop { debug!(attempt_id, "scanning all workers"); attempt_id += 1; - let worker_reclaimed = self.worker_reclamation.listen(); - for worker in &*self.managed { - if let Some(handle) = worker.try_lock(self.worker_reclamation.clone()) { + let released = self.invoker_released.listen(); + for invoker in &*self.managed { + if let Some(handle) = invoker.try_lock(self.invoker_released.clone()) { handle.send_request.send(req).await.expect("worker died"); let resp = handle .recv_response @@ -90,38 +90,36 @@ impl InvokerSet { } } - worker_reclaimed.await; + released.await; } } } #[derive(Eq, PartialEq)] -enum WorkerState { - /// Worker is ready for new tasks +enum InvokerState { + /// Ready for new tasks Idle, - /// Worker is ready, but it is locked by a WorkerHandle + /// Ready, but it is locked by a WorkerHandle Locked, - /// Worker is juding run - Judge, - /// Worker has crashed + /// Crashed Crash, } -struct WorkerInfo { +struct Invoker { // could be AtomicU8, but mutex is simpler - state: Mutex, + state: Mutex, // Danger: must not be used concurrently, otherwise // we can receive wrong response send_request: async_channel::Sender>, recv_response: async_channel::Receiver>, } -struct LockedWorker { +struct LockedInvoker { send_request: async_channel::Sender>, recv_response: async_channel::Receiver>, notify_on_drop: Arc, - worker: Arc, + raw: Arc, } -impl LockedWorker { +impl LockedInvoker { async fn call(self, req: hyper::Request) -> hyper::Response { let wake = { let ev = self.notify_on_drop.clone(); @@ -147,29 +145,29 @@ impl LockedWorker { } } -impl Drop for LockedWorker { +impl Drop for LockedInvoker { fn drop(&mut self) { - // mark Worker as idle - *self.worker.state.lock() = WorkerState::Idle; + // mark invoker as idle + *self.raw.state.lock() = InvokerState::Idle; // trigger event self.notify_on_drop.notify_additional(1); } } -impl WorkerInfo { +impl Invoker { fn try_lock( self: &Arc, notify_on_drop: Arc, - ) -> Option { + ) -> Option { let mut lock = self.state.lock(); - if *lock != WorkerState::Idle { + if *lock != InvokerState::Idle { return None; } - *lock = WorkerState::Locked; - Some(LockedWorker { + *lock = InvokerState::Locked; + Some(LockedInvoker { send_request: self.send_request.clone(), recv_response: self.recv_response.clone(), - worker: self.clone(), + raw: self.clone(), notify_on_drop, }) } diff --git a/src/judge/src/request_handler.rs b/src/judge/src/request_handler.rs index 106fad5c..edc24ef4 100644 --- a/src/judge/src/request_handler.rs +++ b/src/judge/src/request_handler.rs @@ -26,7 +26,7 @@ pub type InvokerClient = rpc::Client; #[derive(Debug, Default)] pub(crate) struct Command { pub argv: Vec, - pub env: Vec, + pub env: Vec<(String, String)>, pub cwd: String, } @@ -58,11 +58,6 @@ impl LoweredJudgeRequest { pom::FileRefRoot::Root => Path::new("/").into(), }; - debug!( - "full checker path: {}", - root.join(&short_path.path).to_str().unwrap() - ); - root.join(&short_path.path) } diff --git a/src/judge/src/request_handler/exec_test.rs b/src/judge/src/request_handler/exec_test.rs index 5be3f329..892f6c32 100644 --- a/src/judge/src/request_handler/exec_test.rs +++ b/src/judge/src/request_handler/exec_test.rs @@ -3,10 +3,10 @@ mod checker_proto; use super::{JudgeContext, LoweredJudgeRequest}; use anyhow::Context; use judging_apis::{ - invoke::{Action, Command, FileId, Input, InputSource, Stdio, Step}, + invoke::{Action, Command, EnvVarValue, Expose, FileId, Input, InputSource, Stdio, Step}, status_codes, Status, StatusKind, }; -use std::{fs, io::Write, path::PathBuf}; +use std::{io::Write, path::PathBuf}; use tracing::{debug, error}; pub(crate) struct ExecRequest<'a> { pub(crate) test_id: u32, @@ -63,13 +63,16 @@ pub async fn exec( }; let input_file = judge_req.resolve_asset(&exec_req.test.path); let test_data = std::fs::read(input_file).context("failed to read test")?; - const EXEC_SOLUTION_GENERATION: u32 = 0; + const PREPARE_STAGE: u32 = 0; + const EXEC_SOLUTION_STAGE: u32 = 1; const TEST_DATA_INPUT_FILE: &str = "test-data"; const EXEC_SOLUTION_OUTPUT_FILE: &str = "solution-output"; const EXEC_SOLUTION_ERROR_FILE: &str = "solution-error"; + const CORRECT_ANSWER_FILE: &str = "correct"; + const EMPTY_FILE: &str = "empty"; - const EXEC_CHECKER_GENERATION: u32 = 1; - // create an input with the test date + const EXEC_CHECKER_STAGE: u32 = 2; + // create an input with the test data { let test_data_input = Input { id: FileId(TEST_DATA_INPUT_FILE.to_string()), @@ -77,16 +80,25 @@ pub async fn exec( }; invoke_request.inputs.push(test_data_input); } + // prepare empty input + { + invoke_request.steps.push(Step { + stage: PREPARE_STAGE, + action: Action::OpenNullFile { + id: FileId(EMPTY_FILE.to_string()), + }, + }); + } // prepare files for stdout & stderr { invoke_request.steps.push(Step { - generation: EXEC_SOLUTION_GENERATION, + stage: EXEC_SOLUTION_STAGE, action: Action::CreateFile { id: FileId(EXEC_SOLUTION_OUTPUT_FILE.to_string()), }, }); invoke_request.steps.push(Step { - generation: EXEC_SOLUTION_GENERATION, + stage: EXEC_SOLUTION_STAGE, action: Action::CreateFile { id: FileId(EXEC_SOLUTION_ERROR_FILE.to_string()), }, @@ -95,74 +107,100 @@ pub async fn exec( // produce a step for executing solution { let exec_solution_step = Step { - generation: EXEC_SOLUTION_GENERATION, + stage: EXEC_SOLUTION_STAGE, action: Action::ExecuteCommand(Command { argv: judge_req.execute_command.argv.clone(), - env: judge_req.execute_command.env.clone(), + env: judge_req + .execute_command + .env + .iter() + .cloned() + .map(|(k, v)| (k, EnvVarValue::Plain(v))) + .collect(), cwd: judge_req.execute_command.cwd.clone(), stdio: Stdio { stdin: FileId(TEST_DATA_INPUT_FILE.to_string()), stdout: FileId(EXEC_SOLUTION_OUTPUT_FILE.to_string()), stderr: FileId(EXEC_SOLUTION_ERROR_FILE.to_string()), }, + expose: Vec::new(), }), }; invoke_request.steps.push(exec_solution_step); } - let run_outcome = self.run_solution(&test_data, self.exec.test_id)?; - let sol_file_path = match run_outcome.var { - RunOutcomeVar::Success { out_data_path } => out_data_path, - RunOutcomeVar::Fail(status) => { - return Ok(ExecOutcome { - status, - resource_usage: run_outcome.resource_usage, - }); - } - }; - // run checker - let step_dir = self.req.step_dir(Some(self.exec.test_id)); - let sol_file = fs::File::open(sol_file_path).context("failed to open run's answer")?; - let sol_handle = os_util::handle_inherit(sol_file.into_raw_fd().into(), true); - let full_checker_path = self.req.resolve_asset(&self.req.problem.checker_exe); - let mut cmd = std::process::Command::new(full_checker_path.clone()); - debug!( - "full checker path: {}, short path: {}", - full_checker_path.to_str().unwrap(), - &self.req.problem.checker_exe.path - ); - cmd.current_dir(&self.req.problem_dir); - - for arg in &self.req.problem.checker_cmd { - cmd.arg(arg); + // provide a correct answer if requested + { + let source = if let Some(corr_path) = &exec_req.test.correct { + let full_path = judge_req.resolve_asset(corr_path); + let data = tokio::fs::read(full_path) + .await + .context("failed to read correct answer")?; + cx.intern(&data).await? + } else { + cx.intern(&[]).await? + }; } + // generate checker feedback files + const CHECKER_DECISION: &str = "checker-decision"; + const CHECKER_COMMENTS: &str = "checker-comment"; + { + invoke_request.steps.push(Step { + stage: EXEC_CHECKER_STAGE, + action: Action::CreateFile { + id: FileId(CHECKER_DECISION.to_string()), + }, + }); + invoke_request.steps.push(Step { + stage: EXEC_CHECKER_STAGE, + action: Action::CreateFile { + id: FileId(CHECKER_COMMENTS.to_string()), + }, + }) + } + // produce a step for executing checker + { + let exec_checker_step = Step { + stage: EXEC_CHECKER_STAGE, + action: Action::ExecuteCommand(Command { + argv: judge_req.problem.checker_cmd.clone(), + env: vec![ + ( + "JJS_CORR".to_string(), + EnvVarValue::File(FileId(CORRECT_ANSWER_FILE.to_string())), + ), + ( + "JJS_SOL".to_string(), + EnvVarValue::File(FileId(EXEC_SOLUTION_OUTPUT_FILE.to_string())), + ), + ( + "JJS_TEST".to_string(), + EnvVarValue::File(FileId(TEST_DATA_INPUT_FILE.to_string())), + ), + ( + "JJS_CHECKER_OUT".to_string(), + EnvVarValue::File(FileId(CHECKER_DECISION.to_string())), + ), + ( + "JJS_CHECKER_COMMENT".to_string(), + EnvVarValue::File(FileId(CHECKER_COMMENTS.to_string())), + ), + ] + .into_iter() + .collect(), + cwd: "/".to_string(), + stdio: Stdio { + stdin: FileId(EMPTY_FILE.to_string()), + stdout: FileId(EMPTY_FILE.to_string()), + stderr: FileId(EMPTY_FILE.to_string()), + }, + expose: vec![Expose::Problem], + }), + }; - let test_cfg = self.exec.test; - - let corr_handle = if let Some(corr_path) = &test_cfg.correct { - let full_path = self.req.resolve_asset(corr_path); - let data = fs::read(full_path).context("failed to read correct answer")?; - os_util::buffer_to_file(&data, "invoker-correct-data") - } else { - os_util::buffer_to_file(&[], "invoker-correct-data") - }; - let test_handle = os_util::buffer_to_file(&test_data, "invoker-test-data"); - - cmd.env("JJS_CORR", corr_handle.to_string()); - cmd.env("JJS_SOL", sol_handle.to_string()); - cmd.env("JJS_TEST", test_handle.to_string()); + invoke_request.steps.push(exec_checker_step); + } - let (out_judge_side, out_checker_side) = os_util::make_pipe(); - cmd.env("JJS_CHECKER_OUT", out_checker_side.to_string()); - let (comments_judge_side, comments_checker_side) = os_util::make_pipe(); - cmd.env("JJS_CHECKER_COMMENT", comments_checker_side.to_string()); let st = cmd.output().context("failed to execute checker")?; - os_util::close(out_checker_side); - os_util::close(comments_checker_side); - os_util::close(corr_handle); - os_util::close(test_handle); - os_util::close(sol_handle); - // TODO: capture comments - os_util::close(comments_judge_side); let checker_out = std::fs::File::create(step_dir.join("check-log.txt"))?; let mut checker_out = std::io::BufWriter::new(checker_out); diff --git a/src/judging-apis/src/invoke.rs b/src/judging-apis/src/invoke.rs index 06ecc4ed..7ec15db3 100644 --- a/src/judging-apis/src/invoke.rs +++ b/src/judging-apis/src/invoke.rs @@ -78,9 +78,26 @@ pub struct FileId(pub String); #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Command { pub argv: Vec, - pub env: Vec, + pub env: Vec<(String, EnvVarValue)>, pub cwd: String, pub stdio: Stdio, + pub expose: Vec, +} + +/// What should be exposed to command +#[derive(Debug, Clone, Deserialize, Serialize)] +pub enum Expose { + /// Problem files (e.g. we expose it to checker) + Problem +} + +/// Value of environment value +#[derive(Debug, Clone, Deserialize, Serialize)] +pub enum EnvVarValue { + /// Use this string as a value + Plain(String), + /// Pass handle (aka fd) of this file as a value + File(FileId) } #[derive(Debug, Clone, Deserialize, Serialize)] From 453b5183d3bdf6c7f20d7bd96c2765659e28cd24 Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Sat, 31 Oct 2020 23:29:29 +0300 Subject: [PATCH 07/11] work --- Cargo.lock | 178 +++++++++++----------------------------- src/invoker/Cargo.toml | 6 +- src/invoker/src/main.rs | 11 ++- src/judge/Cargo.toml | 2 - 4 files changed, 64 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db04ba26..722110ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,15 +316,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1fd36ffbb1fb7c834eac128ea8d0e310c5aeb635548f9d58861e1308d46e71c" - -[[package]] -name = "arc-swap" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" +checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" [[package]] name = "array_tool" @@ -346,9 +340,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "async-channel" -version = "1.4.2" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21279cfaa4f47df10b1816007e738ca3747ef2ee53ffc51cdbf57a8bb266fee3" +checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9" dependencies = [ "concurrent-queue", "event-listener", @@ -368,17 +362,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-mpmc" -version = "0.1.0" -source = "git+https://github.com/jjs-dev/commons#f25926f9adecfb364ceb2e791867d4a7afa11fab" -dependencies = [ - "tokio", - "tracing", - "tracing-futures", - "triomphe", -] - [[package]] name = "async-stream" version = "0.3.0" @@ -505,9 +488,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "blake2b_simd" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" dependencies = [ "arrayref", "arrayvec", @@ -781,9 +764,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" +checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" [[package]] name = "constant_time_eq" @@ -1088,11 +1071,11 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -1180,31 +1163,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "2.4.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cd41440ae7e4734bbd42302f63eaba892afc93a3912dad84006247f0dedb0e" - -[[package]] -name = "failure" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" -dependencies = [ - "backtrace", - "failure_derive", -] - -[[package]] -name = "failure_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" -dependencies = [ - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.38", - "synstructure 0.12.4", -] +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" [[package]] name = "fake-simd" @@ -1226,11 +1187,11 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee" +checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "crc32fast", "libc", "miniz_oxide", @@ -1627,9 +1588,9 @@ checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" [[package]] name = "hyper" -version = "0.13.8" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" dependencies = [ "bytes", "futures-channel", @@ -1641,7 +1602,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 0.4.27", + "pin-project 1.0.1", "socket2", "tokio", "tower-service", @@ -1735,6 +1696,12 @@ dependencies = [ [[package]] name = "invoker" version = "0.1.0" +dependencies = [ + "hyper", + "judging-apis", + "minion", + "rpc 0.1.0 (git+https://github.com/jjs-dev/commons)", +] [[package]] name = "iovec" @@ -1771,13 +1738,26 @@ checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonpath_lib" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8727f6987896c010ec9add275f59de2ae418b672fafa77bc3673b4cee1f09ca" +dependencies = [ + "array_tool", + "env_logger 0.7.1", + "log", + "serde", + "serde_json", +] + [[package]] name = "judge" version = "0.1.0" @@ -1797,12 +1777,10 @@ dependencies = [ "event-listener", "fs_extra", "futures-util", - "hyper", "judging-apis", "k8s-openapi", "kube", "libc", - "minion", "nix 0.19.0", "num_cpus", "once_cell", @@ -1844,61 +1822,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - -[[package]] -name = "ipconfig" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" -dependencies = [ - "socket2", - "widestring", - "winapi 0.3.9", - "winreg 0.6.2", -] - -[[package]] -name = "ipnet" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" - -[[package]] -name = "itoa" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" - -[[package]] -name = "js-sys" -version = "0.3.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "jsonpath_lib" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8727f6987896c010ec9add275f59de2ae418b672fafa77bc3673b4cee1f09ca" -dependencies = [ - "array_tool", - "env_logger 0.7.1", - "log", - "serde", - "serde_json", -] - [[package]] name = "k8s-openapi" version = "0.9.0" @@ -2736,9 +2659,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "problem-loader" @@ -3052,9 +2975,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" dependencies = [ "aho-corasick", "memchr", @@ -3074,9 +2997,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" [[package]] name = "remove_dir_all" @@ -3519,11 +3442,10 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e12110bc539e657a646068aaf5eb5b63af9d0c1f7b29c97113fad80e15f035" +checksum = "ce32ea0c6c56d5eacaeb814fbed9960547021d3edd010ded1425f180536b20ab" dependencies = [ - "arc-swap", "libc", ] @@ -4121,9 +4043,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2810660b9d5b18895d140caba6401765749a6a162e5d0736cfc44ea50db9d79d" +checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401" dependencies = [ "ansi_term", "chrono", diff --git a/src/invoker/Cargo.toml b/src/invoker/Cargo.toml index b6dd09a6..aeff9748 100644 --- a/src/invoker/Cargo.toml +++ b/src/invoker/Cargo.toml @@ -4,6 +4,8 @@ version = "0.1.0" authors = ["Mikail Bagishov "] edition = "2018" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] +judging-apis = { path = "../judging-apis" } +rpc = { git = "https://github.com/jjs-dev/commons" } +minion = {git = "https://github.com/jjs-dev/minion"} +hyper = "0.13.8" \ No newline at end of file diff --git a/src/invoker/src/main.rs b/src/invoker/src/main.rs index e7a11a96..ea9aa46b 100644 --- a/src/invoker/src/main.rs +++ b/src/invoker/src/main.rs @@ -1,3 +1,12 @@ +struct RpcHandler; + +impl rpc::Handler for RpcHandler { + +} + fn main() { - println!("Hello, world!"); + let mut server = rpc::RouterBuilder::new(); + server.add_route(RpcHandler); + let server = server.build().as_make_service(); + let server = hyper } diff --git a/src/judge/Cargo.toml b/src/judge/Cargo.toml index 2fe8395b..89f04b9a 100644 --- a/src/judge/Cargo.toml +++ b/src/judge/Cargo.toml @@ -6,7 +6,6 @@ authors = ["Mikail Bagishov "] edition = "2018" [dependencies] -minion = {git = "https://github.com/jjs-dev/minion"} serde = { version = "1.0.117", features = ["derive"] } serde_json = "1.0.59" dotenv = "0.15.0" @@ -44,7 +43,6 @@ tracing-futures = "0.2.4" dkregistry = { git = "https://github.com/mikailbag/dkregistry-rs", branch = "all" } rpc = { git = "https://github.com/jjs-dev/commons", branch = "rpc-box-engine" } tower-service = "0.3.0" -hyper = "0.13.7" futures-util = "0.3.5" event-listener = "2.4.0" async-channel = "1.4.2" From 1afe3451ea09f56066a711a1d1876e900e501990 Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Tue, 3 Nov 2020 19:45:00 +0300 Subject: [PATCH 08/11] create_sandbox --- Cargo.lock | 7 +++ src/invoker/Cargo.toml | 9 +++- src/invoker/src/config.rs | 22 ++++++++++ src/{judge => invoker}/src/init.rs | 0 src/invoker/src/invoke_util.rs | 52 +++++++++++------------ src/invoker/src/main.rs | 25 +++++++++-- src/invoker/src/transport.rs | 68 ++++++++++++++++++++++++++++++ src/judge/src/config.rs | 12 ------ src/judging-apis/src/invoke.rs | 17 +++++++- 9 files changed, 166 insertions(+), 46 deletions(-) create mode 100644 src/invoker/src/config.rs rename src/{judge => invoker}/src/init.rs (100%) create mode 100644 src/invoker/src/transport.rs diff --git a/Cargo.lock b/Cargo.lock index 722110ce..bb4e67bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1697,10 +1697,17 @@ dependencies = [ name = "invoker" version = "0.1.0" dependencies = [ + "anyhow", + "futures-util", "hyper", "judging-apis", "minion", + "nix 0.19.0", + "once_cell", "rpc 0.1.0 (git+https://github.com/jjs-dev/commons)", + "serde", + "tokio", + "tracing", ] [[package]] diff --git a/src/invoker/Cargo.toml b/src/invoker/Cargo.toml index aeff9748..b331479a 100644 --- a/src/invoker/Cargo.toml +++ b/src/invoker/Cargo.toml @@ -8,4 +8,11 @@ edition = "2018" judging-apis = { path = "../judging-apis" } rpc = { git = "https://github.com/jjs-dev/commons" } minion = {git = "https://github.com/jjs-dev/minion"} -hyper = "0.13.8" \ No newline at end of file +hyper = "0.13.8" +tokio = { version = "0.2.22", features = ["macros", "io-std"] } +anyhow = "1.0.34" +futures-util = "0.3.7" +tracing = "0.1.21" +nix = "0.19.0" +once_cell = "1.4.1" +serde = { version = "1.0.117", features = ["derive"] } diff --git a/src/invoker/src/config.rs b/src/invoker/src/config.rs new file mode 100644 index 00000000..ddef8edc --- /dev/null +++ b/src/invoker/src/config.rs @@ -0,0 +1,22 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Deserialize, Serialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "kebab-case")] +pub struct Config { + /// If enabled, invoker will directly mount host filesystem instead of + /// toolchain image. + #[serde(default)] + pub host_toolchains: bool, + /// Override directories that will be mounted into sandbox. + /// E.g. if `expose-host-dirs = ["lib64", "usr/lib"]`, + /// then invoker will mount: + /// - `$SANDBOX_ROOT/lib64` -> `/lib64` + /// - `$SANDBOX_ROOT/usr/lib` -> `/usr/lib` + /// As usual, all mounts will be no-suid and read-only. + #[serde(default)] + pub expose_host_dirs: Option>, + /// Directory which will contain temporary invocation data. + pub work_root: PathBuf +} diff --git a/src/judge/src/init.rs b/src/invoker/src/init.rs similarity index 100% rename from src/judge/src/init.rs rename to src/invoker/src/init.rs diff --git a/src/invoker/src/invoke_util.rs b/src/invoker/src/invoke_util.rs index c34b4b9d..f9cf182e 100644 --- a/src/invoker/src/invoke_util.rs +++ b/src/invoker/src/invoke_util.rs @@ -1,10 +1,10 @@ -use crate::worker::{Command, LoweredJudgeRequest}; use anyhow::Context; +use judging_apis::invoke::{Command, InvokeRequest}; use std::{ - fs, path::{Path, PathBuf}, time::Duration, }; +use tokio::fs; use tracing::{debug, error}; pub(crate) struct Sandbox { @@ -35,11 +35,12 @@ static DEFAULT_HOST_MOUNTS: once_cell::sync::Lazy> = once_cell::sync ] }); -pub(crate) fn create_sandbox( - req: &LoweredJudgeRequest, - test_id: Option, +pub(crate) async fn create_sandbox( + config: &crate::config::Config, + req: &InvokeRequest, + req_id: &str, backend: &dyn minion::erased::Backend, - config: &crate::config::JudgeConfig, + settings: judging_apis::invoke::Sandbox, ) -> anyhow::Result { let mut shared_dirs = vec![]; if config.host_toolchains { @@ -58,10 +59,10 @@ pub(crate) fn create_sandbox( } } else { let toolchain_dir = &req.toolchain_dir; - let opt_items = - fs::read_dir(&toolchain_dir).context("failed to list toolchains sysroot")?; - for item in opt_items { - let item = item.context("failed to stat toolchains sysroot item")?; + let opt_items = fs::read_dir(&toolchain_dir) + .await + .context("failed to list toolchains sysroot")?; + while let Some(item) = opt_items.next_entry().await? { let name = item.file_name(); let shared_dir = minion::SharedDir { src: toolchain_dir.join(&name), @@ -72,40 +73,37 @@ pub(crate) fn create_sandbox( } } - let limits = if let Some(test_id) = test_id { - req.problem.tests[(test_id - 1) as usize].limits - } else { - req.compile_limits - }; - let out_dir = req.step_dir(test_id); - std::fs::create_dir_all(&out_dir).context("failed to create step directory")?; + let work_dir = config.work_root.join(req_id); + tokio::fs::create_dir_all(&work_dir) + .await + .context("failed to create working directory")?; let umount_path; #[cfg(target_os = "linux")] { - let quota = limits.work_dir_size(); + let quota = settings.limits.work_dir_size(); let quota = minion::linux::ext::Quota::bytes(quota); - minion::linux::ext::make_tmpfs(&out_dir.join("data"), quota) + minion::linux::ext::make_tmpfs(&work_dir.join("data"), quota) .context("failed to set size limit on shared directory")?; - umount_path = Some(out_dir.join("data")); + umount_path = Some(work_dir.join("data")); } #[cfg(not(target_os = "linux"))] { umount_path = None; } shared_dirs.push(minion::SharedDir { - src: out_dir.join("data"), + src: work_dir.join("data"), dest: PathBuf::from("/jjs"), kind: minion::SharedDirKind::Full, }); - let cpu_time_limit = Duration::from_millis(limits.time() as u64); - let real_time_limit = Duration::from_millis(limits.time() * 3 as u64); - std::fs::create_dir(out_dir.join("root")).context("failed to create chroot dir")?; + let cpu_time_limit = Duration::from_millis(settings.limits.time() as u64); + let real_time_limit = Duration::from_millis(settings.limits.time() * 3 as u64); + tokio::fs::create_dir(work_dir.join("root")).await.context("failed to create chroot dir")?; // TODO adjust integer types let sandbox_options = minion::SandboxOptions { - max_alive_process_count: limits.process_count() as _, - memory_limit: limits.memory() as _, + max_alive_process_count: settings.limits.process_count() as _, + memory_limit: settings.limits.memory() as _, exposed_paths: shared_dirs, - isolation_root: out_dir.join("root"), + isolation_root: work_dir.join("root"), cpu_time_limit, real_time_limit, }; diff --git a/src/invoker/src/main.rs b/src/invoker/src/main.rs index ea9aa46b..ff847f2f 100644 --- a/src/invoker/src/main.rs +++ b/src/invoker/src/main.rs @@ -1,12 +1,29 @@ +mod invoke_util; +mod transport; +mod config; + +#[derive(Clone)] struct RpcHandler; -impl rpc::Handler for RpcHandler { - +impl rpc::Handler for RpcHandler { + type Error = anyhow::Error; + type Fut = futures_util::future::BoxFuture<'static, Result<(), Self::Error>>; + + fn handle( + self, + request: rpc::UnaryRx, + response: rpc::UnaryTx, + ) -> Self::Fut { + todo!() + } } -fn main() { +#[tokio::main] +async fn main() -> anyhow::Result<()> { let mut server = rpc::RouterBuilder::new(); server.add_route(RpcHandler); let server = server.build().as_make_service(); - let server = hyper + let incoming = transport::IncomingStdio::new(); + hyper::Server::builder(incoming).serve(server).await; + Ok(()) } diff --git a/src/invoker/src/transport.rs b/src/invoker/src/transport.rs new file mode 100644 index 00000000..f5095d8e --- /dev/null +++ b/src/invoker/src/transport.rs @@ -0,0 +1,68 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, +}; +use tokio::io::{AsyncRead, AsyncWrite}; + +struct StdioClient { + stdin: tokio::io::Stdin, + stdout: tokio::io::Stdout, +} + +impl AsyncRead for StdioClient { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + Pin::new(&mut self.stdin).poll_read(cx, buf) + } +} + +impl AsyncWrite for StdioClient { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.stdout).poll_write(cx, buf) + } + + fn poll_flush( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.stdout).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.stdout).poll_shutdown(cx) + } +} + +pub(crate) struct IncomingStdio(Option); + +impl hyper::server::accept::Accept for IncomingStdio { + type Conn = StdioClient; + type Error = std::convert::Infallible; + + fn poll_accept( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll>> { + let this = Pin::into_inner(self); + Poll::Ready(this.0.take().map(Ok)) + } +} + +impl IncomingStdio { + pub(crate) fn new() -> Self { + IncomingStdio(Some(StdioClient { + stdin: tokio::io::stdin(), + stdout: tokio::io::stdout(), + })) + } +} diff --git a/src/judge/src/config.rs b/src/judge/src/config.rs index f91d31e2..3cc0d791 100644 --- a/src/judge/src/config.rs +++ b/src/judge/src/config.rs @@ -17,18 +17,6 @@ pub struct JudgeConfig { /// API service config #[serde(default)] pub api: ApiSvcConfig, - /// If enabled, invoker will directly mount host filesystem instead of - /// toolchain image. - #[serde(default)] - pub host_toolchains: bool, - /// Override directories that will be mounted into sandbox. - /// E.g. if `expose-host-dirs = ["lib64", "usr/lib"]`, - /// then invoker will mount: - /// - `$SANDBOX_ROOT/lib64` -> `/lib64` - /// - `$SANDBOX_ROOT/usr/lib` -> `/usr/lib` - /// As usual, all mounts will be no-suid and read-only. - #[serde(default)] - pub expose_host_dirs: Option>, /// Configures how invoker should resolve problems pub problems: problem_loader::LoaderConfig, } diff --git a/src/judging-apis/src/invoke.rs b/src/judging-apis/src/invoke.rs index 7ec15db3..7d9e6c3f 100644 --- a/src/judging-apis/src/invoke.rs +++ b/src/judging-apis/src/invoke.rs @@ -36,6 +36,8 @@ pub struct InvokeRequest { pub inputs: Vec, /// Binary data produced by executing commands pub outputs: Vec, + /// Toolchain to use + pub toolchain_dir: PathBuf, } #[derive(Serialize, Deserialize)] @@ -88,7 +90,7 @@ pub struct Command { #[derive(Debug, Clone, Deserialize, Serialize)] pub enum Expose { /// Problem files (e.g. we expose it to checker) - Problem + Problem, } /// Value of environment value @@ -97,7 +99,7 @@ pub enum EnvVarValue { /// Use this string as a value Plain(String), /// Pass handle (aka fd) of this file as a value - File(FileId) + File(FileId), } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -107,6 +109,15 @@ pub struct Stdio { pub stderr: FileId, } +/// Sandbox settings +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Sandbox { + /// Limits enforced for processes in the sandbox. + pub limits: pom::Limits, + /// Sandbox name. + pub name: String, +} + /// Single action of execution plan. #[derive(Serialize, Deserialize)] pub enum Action { @@ -133,4 +144,6 @@ pub enum Action { OpenNullFile { id: FileId }, /// Specifies that command should be executed ExecuteCommand(Command), + /// Specifies that sandbox should be created + CreateSandbox(Sandbox), } From 3711a606369b3f24d3209aa66d5a3fe63e60443e Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Tue, 3 Nov 2020 20:35:34 +0300 Subject: [PATCH 09/11] gra[h creation for compiler --- Cargo.lock | 1 + src/invoker/src/graph_exec.rs | 12 +++ src/invoker/src/invoke_util.rs | 38 +++++-- src/invoker/src/main.rs | 3 + src/invoker/src/print_invoke_request.rs | 11 ++ src/invoker/src/transport.rs | 2 +- src/judge/Cargo.toml | 1 + src/judge/src/controller.rs | 2 +- src/judge/src/lib.rs | 1 - src/judge/src/request_handler.rs | 12 +-- src/judge/src/request_handler/compiler.rs | 120 ++++++++++------------ src/judging-apis/src/invoke.rs | 3 +- 12 files changed, 120 insertions(+), 86 deletions(-) create mode 100644 src/invoker/src/graph_exec.rs create mode 100644 src/invoker/src/print_invoke_request.rs diff --git a/Cargo.lock b/Cargo.lock index bb4e67bc..ea7a1a80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1784,6 +1784,7 @@ dependencies = [ "event-listener", "fs_extra", "futures-util", + "hyper", "judging-apis", "k8s-openapi", "kube", diff --git a/src/invoker/src/graph_exec.rs b/src/invoker/src/graph_exec.rs new file mode 100644 index 00000000..dbaf4196 --- /dev/null +++ b/src/invoker/src/graph_exec.rs @@ -0,0 +1,12 @@ +//! Interprets given request graph +use judging_apis::invoke::InvokeRequest; + +pub struct Interpreter<'a> { + req: &'a InvokeRequest, +} + +impl<'a> Interpreter<'a> { + pub fn new(req: &'a InvokeRequest) -> Self { + Interpreter { req } + } +} diff --git a/src/invoker/src/invoke_util.rs b/src/invoker/src/invoke_util.rs index f9cf182e..07257f07 100644 --- a/src/invoker/src/invoke_util.rs +++ b/src/invoker/src/invoke_util.rs @@ -1,5 +1,5 @@ use anyhow::Context; -use judging_apis::invoke::{Command, InvokeRequest}; +use judging_apis::invoke::{Command, EnvVarValue, InvokeRequest}; use std::{ path::{Path, PathBuf}, time::Duration, @@ -59,7 +59,7 @@ pub(crate) async fn create_sandbox( } } else { let toolchain_dir = &req.toolchain_dir; - let opt_items = fs::read_dir(&toolchain_dir) + let mut opt_items = fs::read_dir(&toolchain_dir) .await .context("failed to list toolchains sysroot")?; while let Some(item) = opt_items.next_entry().await? { @@ -97,7 +97,9 @@ pub(crate) async fn create_sandbox( }); let cpu_time_limit = Duration::from_millis(settings.limits.time() as u64); let real_time_limit = Duration::from_millis(settings.limits.time() * 3 as u64); - tokio::fs::create_dir(work_dir.join("root")).await.context("failed to create chroot dir")?; + tokio::fs::create_dir(work_dir.join("root")) + .await + .context("failed to create chroot dir")?; // TODO adjust integer types let sandbox_options = minion::SandboxOptions { max_alive_process_count: settings.limits.process_count() as _, @@ -123,17 +125,35 @@ pub(crate) fn log_execute_command(command_interp: &Command) { pub(crate) fn command_set_from_judge_req(cmd: &mut minion::Command, command: &Command) { cmd.path(&command.argv[0]); cmd.args(&command.argv[1..]); - cmd.envs(&command.env); + cmd.envs( + command + .env + .iter() + .map(|(name, value)| -> std::ffi::OsString { + match value { + EnvVarValue::Plain(p) => format!("{}={}", name, p).into(), + EnvVarValue::File(_) => unreachable!(), + } + }), + ); } -pub(crate) fn command_set_stdio(cmd: &mut minion::Command, stdout_path: &Path, stderr_path: &Path) { - let stdout_file = fs::File::create(stdout_path).expect("io error"); +pub(crate) async fn command_set_stdio( + cmd: &mut minion::Command, + stdout_path: &Path, + stderr_path: &Path, +) { + let stdout_file = fs::File::create(stdout_path).await.expect("io error"); - let stderr_file = fs::File::create(stderr_path).expect("io error"); + let stderr_file = fs::File::create(stderr_path).await.expect("io error"); // Safety: std::fs::File owns it's handle unsafe { - cmd.stdout(minion::OutputSpecification::handle_of(stdout_file)); + cmd.stdout(minion::OutputSpecification::handle_of( + stdout_file.into_std().await, + )); - cmd.stderr(minion::OutputSpecification::handle_of(stderr_file)); + cmd.stderr(minion::OutputSpecification::handle_of( + stderr_file.into_std().await, + )); } } diff --git a/src/invoker/src/main.rs b/src/invoker/src/main.rs index ff847f2f..ed4373c9 100644 --- a/src/invoker/src/main.rs +++ b/src/invoker/src/main.rs @@ -1,6 +1,9 @@ mod invoke_util; mod transport; mod config; +mod print_invoke_request; +mod init; +mod graph_exec; #[derive(Clone)] struct RpcHandler; diff --git a/src/invoker/src/print_invoke_request.rs b/src/invoker/src/print_invoke_request.rs new file mode 100644 index 00000000..d675c463 --- /dev/null +++ b/src/invoker/src/print_invoke_request.rs @@ -0,0 +1,11 @@ +//! Implements pretty-printing of invocation request +use judging_apis::invoke::InvokeRequest; +use std::fmt::{self, Display, Formatter}; + +pub struct Request<'a>(pub &'a InvokeRequest); + +impl Request<'_> { + pub fn print(&self) -> String { + todo!() + } +} \ No newline at end of file diff --git a/src/invoker/src/transport.rs b/src/invoker/src/transport.rs index f5095d8e..74fd8e5e 100644 --- a/src/invoker/src/transport.rs +++ b/src/invoker/src/transport.rs @@ -4,7 +4,7 @@ use std::{ }; use tokio::io::{AsyncRead, AsyncWrite}; -struct StdioClient { +pub(crate) struct StdioClient { stdin: tokio::io::Stdin, stdout: tokio::io::Stdout, } diff --git a/src/judge/Cargo.toml b/src/judge/Cargo.toml index 89f04b9a..59fe6bd9 100644 --- a/src/judge/Cargo.toml +++ b/src/judge/Cargo.toml @@ -47,6 +47,7 @@ futures-util = "0.3.5" event-listener = "2.4.0" async-channel = "1.4.2" parking_lot = "0.11.0" +hyper = "0.13.8" [features] k8s = ["kube", "k8s-openapi"] diff --git a/src/judge/src/controller.rs b/src/judge/src/controller.rs index 1691d523..609b81fc 100644 --- a/src/judge/src/controller.rs +++ b/src/judge/src/controller.rs @@ -107,7 +107,7 @@ impl Controller { let mut builder = InvokerSet::builder(&config); for _ in 0..worker_count { builder - .add_managed_worker() + .add_managed_invoker() .await .context("failed to start a worker")?; } diff --git a/src/judge/src/lib.rs b/src/judge/src/lib.rs index 1eeefa05..1b213b11 100644 --- a/src/judge/src/lib.rs +++ b/src/judge/src/lib.rs @@ -2,7 +2,6 @@ pub mod api; pub mod config; pub mod controller; -pub mod init; mod invoker_set; pub mod sources; mod request_handler; \ No newline at end of file diff --git a/src/judge/src/request_handler.rs b/src/judge/src/request_handler.rs index edc24ef4..92db1821 100644 --- a/src/judge/src/request_handler.rs +++ b/src/judge/src/request_handler.rs @@ -60,13 +60,6 @@ impl LoweredJudgeRequest { root.join(&short_path.path) } - - pub(crate) fn step_dir(&self, test_id: Option) -> PathBuf { - match test_id { - Some(t) => self.out_dir.join(format!("t-{}", t)), - None => self.out_dir.join("compile"), - } - } } #[derive(Debug)] @@ -123,7 +116,7 @@ pub fn do_judge(mut cx: JudgeContext, judge_req: LoweredJudgeRequest) { impl JudgeContext { async fn judge(&mut self, req: &LoweredJudgeRequest) -> anyhow::Result { - let compiler = Compiler { req }; + let compiler = Compiler { req, cx: self }; if !req.run_source.exists() { anyhow::bail!("Run source file not exists"); @@ -133,7 +126,7 @@ impl JudgeContext { anyhow::bail!("Run output dir not exists"); } - let compiler_response = compiler.compile(); + let compiler_response = compiler.compile().await; let outcome; @@ -227,6 +220,7 @@ impl JudgeContext { }; let judge_response = exec_test::exec(&req, exec_request, self) + .await .with_context(|| format!("failed to judge solution on test {}", tid))?; test_results.push((tid, judge_response.clone())); valuer diff --git a/src/judge/src/request_handler/compiler.rs b/src/judge/src/request_handler/compiler.rs index 4d6d08e9..d80b5092 100644 --- a/src/judge/src/request_handler/compiler.rs +++ b/src/judge/src/request_handler/compiler.rs @@ -1,6 +1,9 @@ -use crate::request_handler::LoweredJudgeRequest; +use crate::request_handler::{JudgeContext, LoweredJudgeRequest}; use anyhow::Context; -use judging_apis::{status_codes, Status, StatusKind}; +use judging_apis::{ + invoke::{Action, Command, EnvVarValue, FileId, Input, InvokeRequest, Stdio, Step}, + status_codes, Status, StatusKind, +}; use std::fs; pub(crate) enum BuildOutcome { @@ -11,82 +14,71 @@ pub(crate) enum BuildOutcome { /// Compiler turns SubmissionInfo into Artifact pub(crate) struct Compiler<'a> { pub(crate) req: &'a LoweredJudgeRequest, - // pub(crate) config: &'a crate::config::JudgeConfig, + pub(crate) cx: &'a JudgeContext, // pub(crate) config: &'a crate::config::JudgeConfig, } +const FILE_ID_SOURCE: &str = "run-source"; +const FILE_ID_EMPTY: &str = "empty"; + impl<'a> Compiler<'a> { - pub(crate) fn compile(&self) -> anyhow::Result { - let mut graph = judging_apis::invoke::InvokeRequest { + pub(crate) async fn compile(&self) -> anyhow::Result { + let mut graph = InvokeRequest { inputs: vec![], outputs: vec![], steps: vec![], + toolchain_dir: self.req.toolchain_dir.clone(), }; - let step_dir = self.req.step_dir(None); - fs::copy( - &self.req.run_source, - step_dir.join("data").join(&self.req.source_file_name), - ) - .context("failed to copy source")?; - - for (i, command) in self.req.compile_commands.iter().enumerate() { - let stdout_path = step_dir.join(&format!("stdout-{}.txt", i)); - let stderr_path = step_dir.join(&format!("stderr-{}.txt", i)); - - invoke_util::log_execute_command(&command); + let run_source = tokio::fs::read(&self.req.run_source).await?; - let mut native_command = minion::Command::new(); - invoke_util::command_set_from_judge_req(&mut native_command, &command); - invoke_util::command_set_stdio(&mut native_command, &stdout_path, &stderr_path); + graph.inputs.push(Input { + id: FileId(FILE_ID_SOURCE.to_string()), + source: self.cx.intern(&run_source).await?, + }); - native_command.sandbox(sandbox.sandbox.clone()); + graph.steps.push(Step { + stage: 0, + action: judging_apis::invoke::Action::OpenNullFile { + id: FileId(FILE_ID_EMPTY.to_string()), + }, + }); - let child = match native_command.spawn(self.minion) { - Ok(child) => child, - Err(err) => { - let is_internal_error = match err.downcast_ref::() { - Some(e) => e.is_system(), - None => true, - }; - if is_internal_error { - return Err(err.context("failed to launch child")); - } else { - return Ok(BuildOutcome::Error(Status { - kind: StatusKind::Rejected, - code: status_codes::LAUNCH_ERROR.to_string(), - })); - } - } + for (i, command) in self.req.compile_commands.iter().enumerate() { + let stdout_file_id = format!("{}-stdout", i); + let stderr_file_id = format!("{}-stderr", i); + graph.steps.push(Step { + stage: i as u32, + action: judging_apis::invoke::Action::CreateFile { + id: FileId(stdout_file_id.clone()), + }, + }); + graph.steps.push(Step { + stage: i as u32, + action: judging_apis::invoke::Action::CreateFile { + id: FileId(stderr_file_id.clone()), + }, + }); + let inv_cmd = Command { + argv: command.argv.clone(), + env: command + .env + .clone() + .into_iter() + .map(|(k, v)| (k, EnvVarValue::Plain(v))) + .collect(), + cwd: "/jjs".to_string(), + stdio: Stdio { + stdin: FileId(FILE_ID_EMPTY.to_string()), + stdout: FileId(stdout_file_id), + stderr: FileId(stderr_file_id), + }, }; - let wait_result = child - .wait_for_exit(None) - .context("failed to wait for compiler")?; - match wait_result { - minion::WaitOutcome::Timeout => { - return Ok(BuildOutcome::Error(Status { - kind: StatusKind::Rejected, - code: status_codes::COMPILATION_TIMED_OUT.to_string(), - })); - } - minion::WaitOutcome::AlreadyFinished => unreachable!("not expected other to wait"), - minion::WaitOutcome::Exited => { - if child - .get_exit_code() - .context("failed to get compiler exit code")? - .unwrap() - != 0 - { - return Ok(BuildOutcome::Error(Status { - kind: StatusKind::Rejected, - code: status_codes::COMPILER_FAILED.to_string(), - })); - } - } - }; + graph.steps.push(Step { + stage: i as u32, + action: Action::ExecuteCommand(inv_cmd), + }); } - fs::copy(step_dir.join("data/build"), self.req.out_dir.join("build")) - .context("failed to copy artifact to run dir")?; Ok(BuildOutcome::Success) } } diff --git a/src/judging-apis/src/invoke.rs b/src/judging-apis/src/invoke.rs index 7d9e6c3f..f65f4d4f 100644 --- a/src/judging-apis/src/invoke.rs +++ b/src/judging-apis/src/invoke.rs @@ -83,7 +83,6 @@ pub struct Command { pub env: Vec<(String, EnvVarValue)>, pub cwd: String, pub stdio: Stdio, - pub expose: Vec, } /// What should be exposed to command @@ -116,6 +115,8 @@ pub struct Sandbox { pub limits: pom::Limits, /// Sandbox name. pub name: String, + /// Paths to mount into sandbox + pub expose: Vec, } /// Single action of execution plan. From 8eeeba13c8284baf2dd856ef102f8383ee66e3be Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Tue, 3 Nov 2020 20:37:45 +0300 Subject: [PATCH 10/11] fmt --- src/invoker/src/config.rs | 2 +- src/invoker/src/main.rs | 8 ++++---- src/invoker/src/print_invoke_request.rs | 2 +- src/judge/src/controller/task_loading.rs | 4 +--- src/judge/src/lib.rs | 2 +- src/judge/src/request_handler/transform_judge_log.rs | 2 +- src/judging-apis/src/lib.rs | 2 +- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/invoker/src/config.rs b/src/invoker/src/config.rs index ddef8edc..652a8e5e 100644 --- a/src/invoker/src/config.rs +++ b/src/invoker/src/config.rs @@ -18,5 +18,5 @@ pub struct Config { #[serde(default)] pub expose_host_dirs: Option>, /// Directory which will contain temporary invocation data. - pub work_root: PathBuf + pub work_root: PathBuf, } diff --git a/src/invoker/src/main.rs b/src/invoker/src/main.rs index ed4373c9..02e75790 100644 --- a/src/invoker/src/main.rs +++ b/src/invoker/src/main.rs @@ -1,9 +1,9 @@ -mod invoke_util; -mod transport; mod config; -mod print_invoke_request; -mod init; mod graph_exec; +mod init; +mod invoke_util; +mod print_invoke_request; +mod transport; #[derive(Clone)] struct RpcHandler; diff --git a/src/invoker/src/print_invoke_request.rs b/src/invoker/src/print_invoke_request.rs index d675c463..74ff5a93 100644 --- a/src/invoker/src/print_invoke_request.rs +++ b/src/invoker/src/print_invoke_request.rs @@ -8,4 +8,4 @@ impl Request<'_> { pub fn print(&self) -> String { todo!() } -} \ No newline at end of file +} diff --git a/src/judge/src/controller/task_loading.rs b/src/judge/src/controller/task_loading.rs index 43fbab02..78e3b425 100644 --- a/src/judge/src/controller/task_loading.rs +++ b/src/judge/src/controller/task_loading.rs @@ -2,9 +2,7 @@ use super::{ notify::Notifier, Controller, JudgeRequestAndCallbacks, LoweredJudgeRequestExtensions, }; -use crate::{ - request_handler::{Command, LoweredJudgeRequest}, -}; +use crate::request_handler::{Command, LoweredJudgeRequest}; use anyhow::Context; use std::{ collections::{HashMap, HashSet}, diff --git a/src/judge/src/lib.rs b/src/judge/src/lib.rs index 1b213b11..e1f34db8 100644 --- a/src/judge/src/lib.rs +++ b/src/judge/src/lib.rs @@ -3,5 +3,5 @@ pub mod api; pub mod config; pub mod controller; mod invoker_set; +mod request_handler; pub mod sources; -mod request_handler; \ No newline at end of file diff --git a/src/judge/src/request_handler/transform_judge_log.rs b/src/judge/src/request_handler/transform_judge_log.rs index a7336611..3f66d2b3 100644 --- a/src/judge/src/request_handler/transform_judge_log.rs +++ b/src/judge/src/request_handler/transform_judge_log.rs @@ -1,4 +1,4 @@ -use crate::request_handler::{LoweredJudgeRequest, JudgeContext}; +use crate::request_handler::{JudgeContext, LoweredJudgeRequest}; use anyhow::Context; use judging_apis::{ judge_log, status_codes, valuer_proto::TestVisibleComponents, Status, StatusKind, diff --git a/src/judging-apis/src/lib.rs b/src/judging-apis/src/lib.rs index 1a577b35..f0e4fdb2 100644 --- a/src/judging-apis/src/lib.rs +++ b/src/judging-apis/src/lib.rs @@ -1,6 +1,6 @@ +pub mod invoke; pub mod judge_log; pub mod valuer_proto; -pub mod invoke; use serde::{Deserialize, Serialize}; use std::path::PathBuf; From f89c55d25e91bde5e734f2b6f1fd4185e8562186 Mon Sep 17 00:00:00 2001 From: Mikail Bagishov Date: Tue, 3 Nov 2020 20:41:39 +0300 Subject: [PATCH 11/11] microwork --- src/judge/src/request_handler/exec_test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/judge/src/request_handler/exec_test.rs b/src/judge/src/request_handler/exec_test.rs index 892f6c32..8ed3efe1 100644 --- a/src/judge/src/request_handler/exec_test.rs +++ b/src/judge/src/request_handler/exec_test.rs @@ -60,6 +60,7 @@ pub async fn exec( steps: vec![], inputs: vec![], outputs: vec![], + toolchain_dir: judge_req.toolchain_dir.clone(), }; let input_file = judge_req.resolve_asset(&exec_req.test.path); let test_data = std::fs::read(input_file).context("failed to read test")?;