From 6e99c5d81cf577c9a9581d7ea68c4ecb24b8ae3e Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 10 Nov 2025 20:03:38 -0800 Subject: [PATCH 01/20] Add compositional analysis code and example --- examples/compositional_analysis/README.md | 14 + examples/compositional_analysis/analyze.py | 45 +++ examples/compositional_analysis/test.py | 149 ++++++++++ examples/compositional_analysis/train.py | 107 +++++++ src/verifai/compositional_analysis.py | 319 +++++++++++++++++++++ 5 files changed, 634 insertions(+) create mode 100644 examples/compositional_analysis/README.md create mode 100644 examples/compositional_analysis/analyze.py create mode 100644 examples/compositional_analysis/test.py create mode 100644 examples/compositional_analysis/train.py create mode 100644 src/verifai/compositional_analysis.py diff --git a/examples/compositional_analysis/README.md b/examples/compositional_analysis/README.md new file mode 100644 index 0000000..4227376 --- /dev/null +++ b/examples/compositional_analysis/README.md @@ -0,0 +1,14 @@ +# Compositional Analysis for Markovian Specifications + +This directory provides an example use case of **compositional analysis** for *Markovian* (i.e., memoryless) specifications. + +The core idea is to perform **Statistical Model Checking (SMC)** or **falsification** on *primitive scenarios* — scenarios that serve as building blocks for defining more complex *composite scenarios*. The analysis traces generated from these primitives are stored in a `ScenarioBase` object. + +These traces can then be supplied to an instance of `CompositionalAnalysisEngine`, which supports querying over composite scenario structures to perform compositional analysis based on the primitive scenario traces. + +This example uses the [MetaDrive](https://metadriverse.github.io/metadrive/) simulator. + +* Train a reinforcement learning policy using `train.py`. +* Test the policy and save generated traces using `test.py`. +* See `analyze.py` for an example of how to perform compositional analysis on generated traces. + diff --git a/examples/compositional_analysis/analyze.py b/examples/compositional_analysis/analyze.py new file mode 100644 index 0000000..3c686d8 --- /dev/null +++ b/examples/compositional_analysis/analyze.py @@ -0,0 +1,45 @@ +import pandas as pd +from verifai.compositional_analysis import CompositionalAnalysisEngine, ScenarioBase + + +if __name__ == "__main__": + logs = { + "S": "storage/traces/S/traces.csv", + "X": "storage/traces/X/traces.csv", + "O": "storage/traces/O/traces.csv", + "C": "storage/traces/C/traces.csv", + "SX": "storage/traces/SX/traces.csv", + "SO": "storage/traces/SO/traces.csv", + "SC": "storage/traces/SC/traces.csv", + "SXS": "storage/traces/SXS/traces.csv", + "SOS": "storage/traces/SOS/traces.csv", + "SCS": "storage/traces/SCS/traces.csv", + } + scenario_base = ScenarioBase(logs) + + print("SMC") + for s in logs: + print(f"{s}: rho = {scenario_base.get_success_rate(s):.4f} ± {scenario_base.get_success_rate_uncertainty(s):.4f}") + + engine = CompositionalAnalysisEngine(scenario_base) + + pd.set_option('display.max_rows', None) # Display all rows + pd.set_option('display.max_columns', None) # Display all columns + pd.set_option('display.width', 1000) # Ensure enough width to prevent wrapping + + print("Compositional SMC") + for s in logs: + rho, uncertainty = engine.check( + s, + features=["x", "y", "heading", "speed"], + norm_feat_idx=[0, 1], + ) + print(f"Estimated {s}: rho = {rho:.4f} ± {uncertainty:.4f}") + cex = engine.falsify( + s, + features=["x", "y", "heading", "speed"], + norm_feat_idx=[0, 1], + align_feat_idx=[0, 1], + ) + print(f"Counterexample = {cex}") + diff --git a/examples/compositional_analysis/test.py b/examples/compositional_analysis/test.py new file mode 100644 index 0000000..145d179 --- /dev/null +++ b/examples/compositional_analysis/test.py @@ -0,0 +1,149 @@ +import warnings +warnings.filterwarnings("ignore", message="pkg_resources is deprecated") + +import os +import csv +import argparse +import numpy as np +import gymnasium as gym +from functools import partial +import matplotlib.pyplot as plt +from stable_baselines3 import PPO +from metadrive.envs import MetaDriveEnv +from IPython.display import Image, clear_output +from metadrive.utils.doc_utils import generate_gif +from metadrive.component.map.base_map import BaseMap +from stable_baselines3.common.utils import set_random_seed +from metadrive.component.map.pg_map import MapGenerateMethod +from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv +from metadrive.utils.draw_top_down_map import draw_top_down_map +from train import make_env + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Test policy in MetaDrive") + parser.add_argument( + "--seed", + type=int, + default=0, + help="Random seed for reproducibility") + parser.add_argument( + "--save-dir", + type=str, + default="storage", + help="Directory to save the trained model") + parser.add_argument( + "--model", + type=str, + required=True, + help="Saved model zip") + parser.add_argument( + "--n", + type=int, + default=10, + help="Number of test samples") + parser.add_argument( + "--scenario", + type=str, + default="XX", + help="Scenario string") + parser.add_argument( + "--gif", + action="store_true", + help="Generate gifs" + ) + args = parser.parse_args() + + # while True: + # env=make_env(scenario=args.scenario, monitor=False) + # env.reset() + # ret = draw_top_down_map(env.current_map) + # # ret = env.render(mode="topdown", window=False) + # # ret = env.render(mode="topdown", + # # window=False, + # # # screen_size=(600, 600), + # # # camera_position=(50, 50) + # # ) + # env.close() + # plt.axis("off") + # plt.imshow(ret) + # plt.show() + # clear_output() + + set_random_seed(args.seed) + + scenario = int(args.scenario) if args.scenario.isdigit() else args.scenario + env = make_env(scenario=scenario, monitor=False) + + model = PPO.load(args.model) + + all_traces = [] + trace_id = 0 + + if not args.gif: + csv_path = os.path.join(args.save_dir, "traces.csv") + f = open(csv_path, "w", newline="") + writer = csv.DictWriter(f, fieldnames=["trace_id", "step", "x", "y", "heading", + "speed", "action", "reward", "label"]) + + for ep in range(args.n): + obs, _ = env.reset() + + initial_speed = np.random.uniform(low=70/3.6, high=80/3.6) + initial_velocity = env.vehicle.lane.direction * initial_speed + env.vehicle.set_velocity(initial_velocity) + + done = False + total_reward = 0.0 + step = 0 + label = False + + print(f"\n=== Episode {ep+1}/{args.n} ===") + while not done and step <= env.config.horizon: + action, _states = model.predict(obs, deterministic=True) + obs, reward, done, truncated, info = env.step(action) + total_reward += reward + label = not done or info.get("arrive_dest") + + if args.gif: + env.render( + mode="topdown", + screen_record=True, + window=False + ) + else: + agent = env.agent + pos = agent.position + heading = agent.heading_theta + vel = agent.speed + + row = { + "trace_id": trace_id, + "step": step, + "x": pos[0], + "y": pos[1], + "heading": heading, + "speed": vel, + "action": action.tolist() if hasattr(action, "tolist") else action, + "reward": reward, + "label" : label + } + + writer.writerow(row) + + step += 1 + + print(f"Label: {label}") + print(f"Episode reward: {total_reward:.2f}") + + if args.gif: + gif_path = os.path.join(args.save_dir, f"trace_{trace_id:03d}.gif") + env.top_down_renderer.generate_gif(gif_path) + print(f"Saved gif to {gif_path}") + + trace_id += 1 + + if not args.gif: + f.close() + + env.close() + diff --git a/examples/compositional_analysis/train.py b/examples/compositional_analysis/train.py new file mode 100644 index 0000000..bdedf3b --- /dev/null +++ b/examples/compositional_analysis/train.py @@ -0,0 +1,107 @@ +import warnings +warnings.filterwarnings("ignore", message="pkg_resources is deprecated") + +import os +import argparse +import gymnasium as gym +from functools import partial +import matplotlib.pyplot as plt +from stable_baselines3 import PPO +from metadrive.envs import MetaDriveEnv +from IPython.display import Image, clear_output +from metadrive.utils.doc_utils import generate_gif +from metadrive.component.map.base_map import BaseMap +from stable_baselines3.common.monitor import Monitor +from stable_baselines3.common.utils import set_random_seed +from metadrive.component.map.pg_map import MapGenerateMethod +from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv +from metadrive.utils.draw_top_down_map import draw_top_down_map + + +def make_env(scenario, monitor=False): + config = MetaDriveEnv.default_config() + config.map = scenario + config.discrete_action=False + config.horizon=2000 + config.num_scenarios=1000 + config.start_seed=1000 + config.traffic_density=0.05 + config.need_inverse_traffic=True + config.accident_prob=0.0 + config.random_lane_width=False + config.random_agent_model=False + config.random_lane_num=False + if monitor: + return Monitor(MetaDriveEnv(config)) + else: + return MetaDriveEnv(config) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Train policy in MetaDrive") + parser.add_argument( + "--seed", + type=int, + default=0, + help="Random seed for reproducibility") + parser.add_argument( + "--save-dir", + type=str, + default="storage", + help="Directory to save the trained model") + parser.add_argument( + "--model", + type=str, + default=None, + help="Model zip name" + ) + parser.add_argument( + "--n-envs", + type=int, + default=16, + help="Number of parallel environments") + parser.add_argument( + "--timesteps", + type=int, + default=1_000_000, + help="Number of environment steps") + parser.add_argument( + "--scenario", + type=str, + default="2", + help="Scenario string") + args = parser.parse_args() + + # while True: + # env=make_env(monitor=False) + # env.reset() + # ret = draw_top_down_map(env.current_map) + # # ret = env.render(mode="topdown", window=False) + # # ret = env.render(mode="topdown", + # # window=False, + # # # screen_size=(600, 600), + # # # camera_position=(50, 50) + # # ) + # env.close() + # plt.axis("off") + # plt.imshow(ret) + # plt.show() + # clear_output() + + set_random_seed(args.seed) + + scenario = int(args.scenario) if args.scenario.isdigit() else args.scenario + env = SubprocVecEnv([partial(make_env, scenario, True) for _ in range(args.n_envs)]) + + model = PPO("MlpPolicy", env=env, n_steps=4096, verbose=1) + model.learn(total_timesteps=args.timesteps, log_interval=1) + env.close() + clear_output() + + if args.model is None: + arg_str = "_".join(f"{k}={v}" for k, v in vars(args).items() if k != "model") + safe_arg_str = arg_str.replace("/", "_").replace(" ", "_") + args.model = os.path.join(args.save_dir, f"model_{safe_arg_str}.zip") + + model.save(args.model) + print("Training is finished.") + diff --git a/src/verifai/compositional_analysis.py b/src/verifai/compositional_analysis.py new file mode 100644 index 0000000..cf96f92 --- /dev/null +++ b/src/verifai/compositional_analysis.py @@ -0,0 +1,319 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Sequence, Union + +import numpy as np +import pandas as pd +from scipy.stats import gaussian_kde + + +@dataclass +class ScenarioStats: + rho: float + uncertainty: float + + +class ScenarioBase: + """ + Handles loading and basic statistics of scenario trace data. + Computes per-scenario success rates and Hoeffding uncertainty. + """ + + REQUIRED_COLUMNS = {"trace_id", "step", "label"} + + def __init__(self, logbase: Dict[str, str], delta: float = 0.05): + """ + Args: + logbase: Dict mapping scenario names to CSV file paths + delta: Confidence level for Hoeffding bound (default 0.05 → 95% CI) + """ + self.logbase = logbase + self.delta = delta + self.data: Dict[str, pd.DataFrame] = {} + + # Load CSVs + for name, path in logbase.items(): + path_obj = Path(path) + if not path_obj.exists(): + raise FileNotFoundError(f"CSV file for scenario '{name}' not found: {path}") + df = pd.read_csv(path) + missing = self.REQUIRED_COLUMNS - set(df.columns) + if missing: + raise ValueError(f"CSV for scenario '{name}' missing columns: {missing}") + df["trace_id"] = df["trace_id"].astype(str) + self.data[name] = df + + self.success_stats: Dict[str, ScenarioStats] = {} + self._compute_success_stats() + + def _compute_success_stats(self): + for name, df in self.data.items(): + last_steps = df.sort_values("step").groupby("trace_id").tail(1) + labels = last_steps["label"].astype(float).to_numpy() + rho = labels.mean() if len(labels) > 0 else 0.0 + epsilon = np.sqrt(np.log(2 / self.delta) / (2 * len(labels))) if len(labels) > 0 else 0.0 + self.success_stats[name] = ScenarioStats(rho=rho, uncertainty=epsilon) + + def get_success_rate(self, scenario: str) -> float: + return self.success_stats[scenario].rho + + def get_success_rate_uncertainty(self, scenario: str) -> float: + return self.success_stats[scenario].uncertainty + + +class CompositionalAnalysisEngine: + """ + Computes importance-sampled success probabilities across sequential scenarios + using Gaussian KDE and Hoeffding uncertainty propagation. + """ + + def __init__(self, scenario_base: ScenarioBase): + self.scenario_base = scenario_base + + @staticmethod + def _normalize_features(features: np.ndarray) -> np.ndarray: + """Standardize features along each column (mean=0, std=1).""" + mean = np.mean(features, axis=0) + std = np.std(features, axis=0) + std[std == 0] = 1.0 # Avoid division by zero + return (features - mean) / std + + def check( + self, + scenario: List[str], + features: Optional[List[str]] = None, + norm_feat_idx: Optional[List[int]] = None, + ) -> Tuple[float, float]: + """ + Computes importance-sampled success probability and propagated uncertainty. + + Args: + scenario: Ordered list of scenario names + features: Optional list of features to include in KDE + norm_feat_idx: Optional indices of features to normalize + + Returns: + Tuple of (rho_estimate, uncertainty) + """ + if len(scenario) == 0: + raise ValueError("Scenario list must contain at least one scenario.") + + rho = 1.0 + rho_bounds = [] + + n = len(scenario) + if n == 1: + result = self.scenario_base.success_stats[scenario] + return result.rho, result.uncertainty + + delta = self.scenario_base.delta + per_step_delta = delta / n # union bound + + for i in range(len(scenario) - 1): + s_name, t_name = scenario[i], scenario[i+1] + df_s, df_t = self.scenario_base.data[s_name], self.scenario_base.data[t_name] + + # Select successful endpoints + s_last = df_s.sort_values("step").groupby("trace_id").tail(1) + s_last = s_last[s_last["label"] == True] + t_first = df_t.sort_values("step").groupby("trace_id").head(1) + t_last = df_t.sort_values("step").groupby("trace_id").tail(1) + + # KDE features + if features: + s_last_features = s_last[features].to_numpy() + t_first_features = t_first[features].to_numpy() + if s_last_features.shape[0] < 2 or t_first_features.shape[0] < 2: + return 0.0, 0.0 + if norm_feat_idx: + for j in norm_feat_idx: + s_last_features[:, j] = self._normalize_features(s_last_features[:, j].reshape(-1, 1)).flatten() + t_first_features[:, j] = self._normalize_features(t_first_features[:, j].reshape(-1, 1)).flatten() + else: + raise ValueError("Feature list must be provided for KDE.") + + s_last_features, t_first_features = s_last_features.T, t_first_features.T + kde_s_last = gaussian_kde(s_last_features) + kde_t_first = gaussian_kde(t_first_features) + p_vals = kde_s_last(t_first_features) + q_vals = kde_t_first(t_first_features) + weights = np.nan_to_num(p_vals / q_vals, nan=0.0, posinf=0.0, neginf=0.0) + + labels_t_last = t_last["label"].astype(float).to_numpy() + min_len = min(len(weights), len(labels_t_last)) + weights = weights[:min_len] + labels_t_last = labels_t_last[:min_len] + + rho_step = np.sum(weights * labels_t_last) / np.sum(weights) if np.sum(weights) > 0 else 0.0 + rho *= rho_step + + # Hoeffding absolute bound with effective samples + N_eff = np.sum(weights)**2 / np.sum(weights**2) if np.sum(weights**2) > 0 else 1.0 + epsilon_i = np.sqrt(np.log(2 / per_step_delta) / (2 * N_eff)) + rho_bounds.append(epsilon_i) + + # Provable multiplicative error + prod_factor = np.prod([1 + eps / max(rho_step, 1e-12) for eps in rho_bounds]) + uncertainty = rho * (prod_factor - 1) + + return rho, uncertainty + + def falsify( + self, + scenario: Union[str, Sequence[str]], + features: Optional[List[str]] = None, + norm_feat_idx: Optional[List[int]] = None, + align_feat_idx: Optional[List[int]] = None, + ) -> Tuple[Optional[pd.DataFrame], float]: + """ + Generates a counterexample trace using the given traces. + + Args: + scenario: Ordered list of scenario names + features: Optional list of features to include in KDE + norm_feat_idx: Optional indices of features to normalize + align_feat_idx: Optional indices of features to align + + Returns: + Trace + """ + if len(scenario) == 0: + raise ValueError("Scenario list must contain at least one scenario.") + + cex = None + n = len(scenario) + + if n == 1: + t_name = scenario[0] + df_t = self.scenario_base.data[t_name] + + t_traces = df_t.sort_values("step").groupby("trace_id") + t_first = t_traces.head(1).sort_values("trace_id") + t_last = t_traces.tail(1).sort_values("trace_id") + + fail_idx = (t_last["label"] == False).to_numpy() + t_first = t_first[fail_idx].sort_values("trace_id") + t_last = t_last[fail_idx].sort_values("trace_id") + + if t_first.empty or t_last.empty: + return None + + t_last_features = t_last[features].to_numpy() + if t_last_features.shape[0] < 1: + return None + elif t_last_features.shape[0] == 2: + t_trace_id = t_last_features[0]["trace_id"] + t_trace = t_traces.get_group(t_trace_id) + return t_trace + + if norm_feat_idx: + for j in norm_feat_idx: + t_last_features[:, j] = self._normalize_features(t_last_features[:, j].reshape(-1, 1)).flatten() + + kde_t_last = gaussian_kde(t_last_features.T) + t_last_prob = kde_t_last(t_last_features.T) + t_idx = np.argmax(t_last_prob) + t_trace_id = t_first.iloc[t_idx]["trace_id"] + t_trace = t_traces.get_group(t_trace_id) + return t_trace + + for i in reversed(range(n - 1)): + s_name, t_name = scenario[i], scenario[i+1] + df_s, df_t = self.scenario_base.data[s_name], self.scenario_base.data[t_name] + + s_traces = df_s.sort_values("step").groupby("trace_id") + s_last = s_traces.tail(1).sort_values("trace_id") + s_last = s_last[s_last["label"] == True].sort_values("trace_id") + + if cex is None: + t_traces = df_t.sort_values("step").groupby("trace_id") + t_first = t_traces.head(1).sort_values("trace_id") + t_last = t_traces.tail(1).sort_values("trace_id") + + fail_idx = (t_last["label"] == False).to_numpy() + t_first = t_first[fail_idx].sort_values("trace_id") + t_last = t_last[fail_idx].sort_values("trace_id") + + if t_first.empty or t_last.empty: + continue + + # KDE features + if features: + s_last_features = s_last[features].to_numpy() + if s_last_features.shape[0] < 2: + continue + if cex is None: + t_first_features = t_first[features].to_numpy() + t_last_features = t_last[features].to_numpy() + if t_first_features.shape[0] < 2 or t_last_features.shape[0] < 2: + continue + if norm_feat_idx: + for j in norm_feat_idx: + s_last_features[:, j] = self._normalize_features(s_last_features[:, j].reshape(-1, 1)).flatten() + if cex is None: + t_first_features[:, j] = self._normalize_features(t_first_features[:, j].reshape(-1, 1)).flatten() + t_last_features[:, j] = self._normalize_features(t_last_features[:, j].reshape(-1, 1)).flatten() + else: + raise ValueError("Feature list must be provided for KDE.") + + if cex is None: + kde_s_last = gaussian_kde(s_last_features.T) + kde_t_first = gaussian_kde(t_first_features.T) + + s_last_prob = kde_t_first(s_last_features.T) + t_first_prob = kde_s_last(t_first_features.T) + + s_idx = np.argmax(s_last_prob) + t_idx = np.argmax(t_first_prob) + + s_trace_id = s_last.iloc[s_idx]["trace_id"] + t_trace_id = t_first.iloc[t_idx]["trace_id"] + + s_trace = s_traces.get_group(s_trace_id) + t_trace = t_traces.get_group(t_trace_id) + + if align_feat_idx: + for idx in align_feat_idx: + s_feat = s_trace[features[idx]] + t_feat = t_trace[features[idx]] + offset = s_feat.iloc[-1] - t_feat.iloc[0] + t_trace.loc[:, features[idx]] = t_feat + offset + + cex = t_trace + + else: + if align_feat_idx: + compare_idx = align_feat_idx + else: + compare_idx = list(range(len(features))) + + # build arrays + # s_last_features rows correspond to s_last (they were computed above) + s_feat_mat = s_last_features[:, compare_idx] # shape (num_s_last, k) + cex_first = cex[features].iloc[0].to_numpy()[compare_idx] # shape (k,) + + # compute Euclidean distances + diffs = s_feat_mat - cex_first.reshape(1, -1) + dists = np.linalg.norm(diffs, axis=1) + + # choose the s trace with minimum distance + s_idx = int(np.argmin(dists)) + s_trace_id = s_last.iloc[s_idx]["trace_id"] + s_trace = s_traces.get_group(s_trace_id) + + # Align cex to s_trace if align_feat_idx provided + if align_feat_idx: + for idx in align_feat_idx: + s_feat = s_trace[features[idx]] + cex_feat = cex[features[idx]] + offset = s_feat.iloc[-1] - cex_feat.iloc[0] + cex.loc[:, features[idx]] = cex_feat + offset + + cex = pd.concat([s_trace, cex]) + + if cex is None: + return None + + final_features = [feat for feat in features] + ["label"] + return cex[final_features].reset_index(drop=True) + From faf0b603095718a9ca9938106a905c1b11716b3b Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Fri, 28 Nov 2025 15:18:29 -0800 Subject: [PATCH 02/20] Use expert policy to generate traces if model is not given --- examples/compositional_analysis/test.py | 30 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/examples/compositional_analysis/test.py b/examples/compositional_analysis/test.py index 145d179..eb5382e 100644 --- a/examples/compositional_analysis/test.py +++ b/examples/compositional_analysis/test.py @@ -5,7 +5,7 @@ import csv import argparse import numpy as np -import gymnasium as gym +from train import make_env from functools import partial import matplotlib.pyplot as plt from stable_baselines3 import PPO @@ -13,11 +13,12 @@ from IPython.display import Image, clear_output from metadrive.utils.doc_utils import generate_gif from metadrive.component.map.base_map import BaseMap +from metadrive.policy.expert_policy import ExpertPolicy from stable_baselines3.common.utils import set_random_seed from metadrive.component.map.pg_map import MapGenerateMethod -from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv from metadrive.utils.draw_top_down_map import draw_top_down_map -from train import make_env +from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Test policy in MetaDrive") @@ -34,7 +35,6 @@ parser.add_argument( "--model", type=str, - required=True, help="Saved model zip") parser.add_argument( "--n", @@ -74,20 +74,30 @@ scenario = int(args.scenario) if args.scenario.isdigit() else args.scenario env = make_env(scenario=scenario, monitor=False) - model = PPO.load(args.model) all_traces = [] trace_id = 0 if not args.gif: csv_path = os.path.join(args.save_dir, "traces.csv") + if os.path.exists(csv_path): + os.remove(csv_path) f = open(csv_path, "w", newline="") writer = csv.DictWriter(f, fieldnames=["trace_id", "step", "x", "y", "heading", "speed", "action", "reward", "label"]) + writer.writeheader() + + success_count = 0 for ep in range(args.n): obs, _ = env.reset() + if args.model: + policy = PPO.load(args.model) + else: + policy = ExpertPolicy(env.agent) + assert policy is not None + initial_speed = np.random.uniform(low=70/3.6, high=80/3.6) initial_velocity = env.vehicle.lane.direction * initial_speed env.vehicle.set_velocity(initial_velocity) @@ -99,7 +109,10 @@ print(f"\n=== Episode {ep+1}/{args.n} ===") while not done and step <= env.config.horizon: - action, _states = model.predict(obs, deterministic=True) + if isinstance(policy, ExpertPolicy): + action = policy.act() + else: + action, _ = policy.predict(obs, deterministic=True) obs, reward, done, truncated, info = env.step(action) total_reward += reward label = not done or info.get("arrive_dest") @@ -135,6 +148,9 @@ print(f"Label: {label}") print(f"Episode reward: {total_reward:.2f}") + if label: + success_count += 1 + if args.gif: gif_path = os.path.join(args.save_dir, f"trace_{trace_id:03d}.gif") env.top_down_renderer.generate_gif(gif_path) @@ -147,3 +163,5 @@ env.close() + print(f"\n\nEmpirical success probability of generated traces: {success_count/args.n}") + From ff693da13c819041c1565ce8b6182bd2518bf953 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Sat, 29 Nov 2025 00:23:42 -0800 Subject: [PATCH 03/20] Fix compositional SMC --- examples/compositional_analysis/analyze.py | 5 +- src/verifai/compositional_analysis.py | 54 +++++++++++----------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/examples/compositional_analysis/analyze.py b/examples/compositional_analysis/analyze.py index 3c686d8..e5dee05 100644 --- a/examples/compositional_analysis/analyze.py +++ b/examples/compositional_analysis/analyze.py @@ -19,7 +19,7 @@ print("SMC") for s in logs: - print(f"{s}: rho = {scenario_base.get_success_rate(s):.4f} ± {scenario_base.get_success_rate_uncertainty(s):.4f}") + print(f"{s}: rho = {scenario_base.get_success_prob(s):.4f} ± {scenario_base.get_success_prob_uncertainty(s):.4f}") engine = CompositionalAnalysisEngine(scenario_base) @@ -27,12 +27,11 @@ pd.set_option('display.max_columns', None) # Display all columns pd.set_option('display.width', 1000) # Ensure enough width to prevent wrapping - print("Compositional SMC") for s in logs: rho, uncertainty = engine.check( s, features=["x", "y", "heading", "speed"], - norm_feat_idx=[0, 1], + center_feat_idx=[0, 1], ) print(f"Estimated {s}: rho = {rho:.4f} ± {uncertainty:.4f}") cex = engine.falsify( diff --git a/src/verifai/compositional_analysis.py b/src/verifai/compositional_analysis.py index cf96f92..5a2434a 100644 --- a/src/verifai/compositional_analysis.py +++ b/src/verifai/compositional_analysis.py @@ -16,7 +16,7 @@ class ScenarioStats: class ScenarioBase: """ Handles loading and basic statistics of scenario trace data. - Computes per-scenario success rates and Hoeffding uncertainty. + Computes empirical success probabilities and uncertainty. """ REQUIRED_COLUMNS = {"trace_id", "step", "label"} @@ -54,17 +54,17 @@ def _compute_success_stats(self): epsilon = np.sqrt(np.log(2 / self.delta) / (2 * len(labels))) if len(labels) > 0 else 0.0 self.success_stats[name] = ScenarioStats(rho=rho, uncertainty=epsilon) - def get_success_rate(self, scenario: str) -> float: + def get_success_prob(self, scenario: str) -> float: return self.success_stats[scenario].rho - def get_success_rate_uncertainty(self, scenario: str) -> float: + def get_success_prob_uncertainty(self, scenario: str) -> float: return self.success_stats[scenario].uncertainty class CompositionalAnalysisEngine: """ - Computes importance-sampled success probabilities across sequential scenarios - using Gaussian KDE and Hoeffding uncertainty propagation. + Computes importance-sampled success probabilities across sequential + scenarios using Gaussian KDE and uncertainty propagation. """ def __init__(self, scenario_base: ScenarioBase): @@ -82,7 +82,7 @@ def check( self, scenario: List[str], features: Optional[List[str]] = None, - norm_feat_idx: Optional[List[int]] = None, + center_feat_idx: Optional[List[int]] = None, ) -> Tuple[float, float]: """ Computes importance-sampled success probability and propagated uncertainty. @@ -98,16 +98,18 @@ def check( if len(scenario) == 0: raise ValueError("Scenario list must contain at least one scenario.") - rho = 1.0 - rho_bounds = [] - n = len(scenario) if n == 1: result = self.scenario_base.success_stats[scenario] return result.rho, result.uncertainty + first_scenario_result = self.scenario_base.success_stats[scenario[0]] + + rho = first_scenario_result.rho + eps_rho_ratios = [first_scenario_result.uncertainty/rho] + delta = self.scenario_base.delta - per_step_delta = delta / n # union bound + per_step_delta = delta / n # union bound for i in range(len(scenario) - 1): s_name, t_name = scenario[i], scenario[i+1] @@ -125,36 +127,34 @@ def check( t_first_features = t_first[features].to_numpy() if s_last_features.shape[0] < 2 or t_first_features.shape[0] < 2: return 0.0, 0.0 - if norm_feat_idx: - for j in norm_feat_idx: - s_last_features[:, j] = self._normalize_features(s_last_features[:, j].reshape(-1, 1)).flatten() - t_first_features[:, j] = self._normalize_features(t_first_features[:, j].reshape(-1, 1)).flatten() + if center_feat_idx: + for j in center_feat_idx: + s_last_features[:, j] = s_last_features[:, j] - np.mean(s_last_features[:, j]) # center + t_first_features[:, j] = t_first_features[:, j] - np.mean(t_first_features[:, j]) # center else: raise ValueError("Feature list must be provided for KDE.") s_last_features, t_first_features = s_last_features.T, t_first_features.T - kde_s_last = gaussian_kde(s_last_features) - kde_t_first = gaussian_kde(t_first_features) + + kde_s_last = gaussian_kde(s_last_features, bw_method=10) + kde_t_first = gaussian_kde(t_first_features, bw_method=10) + p_vals = kde_s_last(t_first_features) q_vals = kde_t_first(t_first_features) + weights = np.nan_to_num(p_vals / q_vals, nan=0.0, posinf=0.0, neginf=0.0) labels_t_last = t_last["label"].astype(float).to_numpy() - min_len = min(len(weights), len(labels_t_last)) - weights = weights[:min_len] - labels_t_last = labels_t_last[:min_len] - rho_step = np.sum(weights * labels_t_last) / np.sum(weights) if np.sum(weights) > 0 else 0.0 - rho *= rho_step + rho_step = np.sum(weights * labels_t_last) / np.sum(weights) # P(T | S) + rho *= rho_step # P(T | S) * P(S) = P(S, T) - # Hoeffding absolute bound with effective samples - N_eff = np.sum(weights)**2 / np.sum(weights**2) if np.sum(weights**2) > 0 else 1.0 + N_eff = np.sum(weights)**2 / np.sum(weights**2) epsilon_i = np.sqrt(np.log(2 / per_step_delta) / (2 * N_eff)) - rho_bounds.append(epsilon_i) + eps_rho_ratios.append(epsilon_i / rho_step) - # Provable multiplicative error - prod_factor = np.prod([1 + eps / max(rho_step, 1e-12) for eps in rho_bounds]) - uncertainty = rho * (prod_factor - 1) + # Multiplicative error + uncertainty = rho * np.sqrt(np.sum([eps_rho_ratios**2 for eps_rho_ratios in eps_rho_ratios])) return rho, uncertainty From ffae171ca87c59c4787468ea2bfa2e85daa29b01 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Sat, 29 Nov 2025 14:37:34 -0800 Subject: [PATCH 04/20] Fix compositional falsification --- examples/compositional_analysis/analyze.py | 17 +++--- src/verifai/compositional_analysis.py | 66 +++++++++++++--------- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/examples/compositional_analysis/analyze.py b/examples/compositional_analysis/analyze.py index e5dee05..a7faaac 100644 --- a/examples/compositional_analysis/analyze.py +++ b/examples/compositional_analysis/analyze.py @@ -6,14 +6,14 @@ logs = { "S": "storage/traces/S/traces.csv", "X": "storage/traces/X/traces.csv", - "O": "storage/traces/O/traces.csv", - "C": "storage/traces/C/traces.csv", + # "O": "storage/traces/O/traces.csv", + # "C": "storage/traces/C/traces.csv", "SX": "storage/traces/SX/traces.csv", - "SO": "storage/traces/SO/traces.csv", - "SC": "storage/traces/SC/traces.csv", + # "SO": "storage/traces/SO/traces.csv", + # "SC": "storage/traces/SC/traces.csv", "SXS": "storage/traces/SXS/traces.csv", - "SOS": "storage/traces/SOS/traces.csv", - "SCS": "storage/traces/SCS/traces.csv", + # "SOS": "storage/traces/SOS/traces.csv", + # "SCS": "storage/traces/SCS/traces.csv", } scenario_base = ScenarioBase(logs) @@ -28,16 +28,19 @@ pd.set_option('display.width', 1000) # Ensure enough width to prevent wrapping for s in logs: + print(f"Scenario: {s}") + print(f"Compositional SMC") rho, uncertainty = engine.check( s, features=["x", "y", "heading", "speed"], center_feat_idx=[0, 1], ) print(f"Estimated {s}: rho = {rho:.4f} ± {uncertainty:.4f}") + print(f"Compositional Falsification") cex = engine.falsify( s, features=["x", "y", "heading", "speed"], - norm_feat_idx=[0, 1], + center_feat_idx=[0, 1], align_feat_idx=[0, 1], ) print(f"Counterexample = {cex}") diff --git a/src/verifai/compositional_analysis.py b/src/verifai/compositional_analysis.py index 5a2434a..93f0d46 100644 --- a/src/verifai/compositional_analysis.py +++ b/src/verifai/compositional_analysis.py @@ -83,6 +83,7 @@ def check( scenario: List[str], features: Optional[List[str]] = None, center_feat_idx: Optional[List[int]] = None, + bw_method: str | int = 10, ) -> Tuple[float, float]: """ Computes importance-sampled success probability and propagated uncertainty. @@ -136,8 +137,8 @@ def check( s_last_features, t_first_features = s_last_features.T, t_first_features.T - kde_s_last = gaussian_kde(s_last_features, bw_method=10) - kde_t_first = gaussian_kde(t_first_features, bw_method=10) + kde_s_last = gaussian_kde(s_last_features, bw_method=bw_method) + kde_t_first = gaussian_kde(t_first_features, bw_method=bw_method) p_vals = kde_s_last(t_first_features) q_vals = kde_t_first(t_first_features) @@ -162,8 +163,9 @@ def falsify( self, scenario: Union[str, Sequence[str]], features: Optional[List[str]] = None, - norm_feat_idx: Optional[List[int]] = None, + center_feat_idx: Optional[List[int]] = None, align_feat_idx: Optional[List[int]] = None, + bw_method: str | int = 10, ) -> Tuple[Optional[pd.DataFrame], float]: """ Generates a counterexample trace using the given traces. @@ -201,16 +203,13 @@ def falsify( t_last_features = t_last[features].to_numpy() if t_last_features.shape[0] < 1: return None - elif t_last_features.shape[0] == 2: - t_trace_id = t_last_features[0]["trace_id"] + elif t_last_features.shape[0] <= t_last_features.shape[1]: + random_idx = np.random.randint(t_last_features.shape[0]) + t_trace_id = t_last.iloc[random_idx]["trace_id"] t_trace = t_traces.get_group(t_trace_id) return t_trace - if norm_feat_idx: - for j in norm_feat_idx: - t_last_features[:, j] = self._normalize_features(t_last_features[:, j].reshape(-1, 1)).flatten() - - kde_t_last = gaussian_kde(t_last_features.T) + kde_t_last = gaussian_kde(t_last_features.T, bw_method=bw_method) t_last_prob = kde_t_last(t_last_features.T) t_idx = np.argmax(t_last_prob) t_trace_id = t_first.iloc[t_idx]["trace_id"] @@ -244,33 +243,46 @@ def falsify( continue if cex is None: t_first_features = t_first[features].to_numpy() - t_last_features = t_last[features].to_numpy() - if t_first_features.shape[0] < 2 or t_last_features.shape[0] < 2: + if t_first_features.shape[0] < 2: continue - if norm_feat_idx: - for j in norm_feat_idx: - s_last_features[:, j] = self._normalize_features(s_last_features[:, j].reshape(-1, 1)).flatten() + if center_feat_idx: + for j in center_feat_idx: + s_last_features[:, j] = s_last_features[:, j] - np.mean(s_last_features[:, j]) if cex is None: - t_first_features[:, j] = self._normalize_features(t_first_features[:, j].reshape(-1, 1)).flatten() - t_last_features[:, j] = self._normalize_features(t_last_features[:, j].reshape(-1, 1)).flatten() + t_first_features[:, j] = t_first_features[:, j] - np.mean(t_first_features[:, j]) else: raise ValueError("Feature list must be provided for KDE.") if cex is None: - kde_s_last = gaussian_kde(s_last_features.T) - kde_t_first = gaussian_kde(t_first_features.T) + if t_first_features.shape[0] <= t_first_features.shape[1]: + random_idx = np.random.randint(t_first_features.shape[0]) + t_trace_id = t_first.iloc[random_idx]["trace_id"] + t_trace = t_traces.get_group(t_trace_id) - s_last_prob = kde_t_first(s_last_features.T) - t_first_prob = kde_s_last(t_first_features.T) + # compute Euclidean distances + diffs = s_last_features - t_first_features[random_idx].reshape(1, -1) + dists = np.linalg.norm(diffs, axis=1) - s_idx = np.argmax(s_last_prob) - t_idx = np.argmax(t_first_prob) + # choose the s trace with minimum distance + s_idx = int(np.argmin(dists)) + s_trace_id = s_last.iloc[s_idx]["trace_id"] + s_trace = s_traces.get_group(s_trace_id) - s_trace_id = s_last.iloc[s_idx]["trace_id"] - t_trace_id = t_first.iloc[t_idx]["trace_id"] + else: + kde_s_last = gaussian_kde(s_last_features.T, bw_method=bw_method) + kde_t_first = gaussian_kde(t_first_features.T, bw_method=bw_method) - s_trace = s_traces.get_group(s_trace_id) - t_trace = t_traces.get_group(t_trace_id) + s_last_prob = kde_t_first(s_last_features.T) + t_first_prob = kde_s_last(t_first_features.T) + + s_idx = np.argmax(s_last_prob) + t_idx = np.argmax(t_first_prob) + + s_trace_id = s_last.iloc[s_idx]["trace_id"] + t_trace_id = t_first.iloc[t_idx]["trace_id"] + + s_trace = s_traces.get_group(s_trace_id) + t_trace = t_traces.get_group(t_trace_id) if align_feat_idx: for idx in align_feat_idx: From 54c88bd1e646ce9974a37c528acc10271e256cb6 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 16:43:08 -0800 Subject: [PATCH 05/20] Add exp script --- examples/compositional_analysis/analyze.py | 2 +- examples/compositional_analysis/run_exp.py | 374 +++++++++++++++++++++ examples/compositional_analysis/utils.py | 130 +++++++ 3 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 examples/compositional_analysis/run_exp.py create mode 100644 examples/compositional_analysis/utils.py diff --git a/examples/compositional_analysis/analyze.py b/examples/compositional_analysis/analyze.py index a7faaac..4d0e17b 100644 --- a/examples/compositional_analysis/analyze.py +++ b/examples/compositional_analysis/analyze.py @@ -15,7 +15,7 @@ # "SOS": "storage/traces/SOS/traces.csv", # "SCS": "storage/traces/SCS/traces.csv", } - scenario_base = ScenarioBase(logs) + scenario_base = ScenarioBase(logs, delta=0.01) print("SMC") for s in logs: diff --git a/examples/compositional_analysis/run_exp.py b/examples/compositional_analysis/run_exp.py new file mode 100644 index 0000000..e46e32a --- /dev/null +++ b/examples/compositional_analysis/run_exp.py @@ -0,0 +1,374 @@ +import os +import time +import shutil +import math +import multiprocessing as mp +from verifai.compositional_analysis import ScenarioBase, CompositionalAnalysisEngine +from utils import generate_traces + + +def compute_hoeffding_samples(confidence_level, error_bound): + """ + Compute the number of samples needed using Hoeffding's inequality. + + Hoeffding's inequality: P(|rho_hat - rho| >= epsilon) <= 2 * exp(-2 * n * epsilon^2) + + Setting 2 * exp(-2 * n * epsilon^2) = 1 - confidence_level (delta) + Solving for n: n = ln(2/delta) / (2 * epsilon^2) + + Args: + confidence_level: Desired confidence level (e.g., 0.95 for 95%) + error_bound: Maximum error epsilon (e.g., 0.01 for 1%) + + Returns: + Number of samples needed + """ + delta = 1 - confidence_level + n = math.log(2 / delta) / (2 * error_bound ** 2) + return int(math.ceil(n)) + + +def _worker_generate_traces(save_dir, scenario, n, expert, model_path): + print(f"[PID={os.getpid()}] Starting scenario {scenario}") + # If n is None or inf, use a very large number that generate_traces can handle + if n is None or n == float('inf'): + traces_to_generate = 10**9 # 1 billion (effectively infinite for practical purposes) + else: + traces_to_generate = n + generate_traces( + n=traces_to_generate, + save_dir=save_dir, + scenario=scenario, + model_path=model_path, + expert=expert + ) + print(f"[PID={os.getpid()}] Finished scenario {scenario}") + + +def generate_traces_parallel(n, save_dir, scenarios, time_budget, expert, model_path): + """ + Generate traces in parallel using multiprocessing with HARD STOP. + Terminates all processes when time budget is reached, discarding only the current partial trace. + Completed traces are kept. + """ + # Clear old trace directories + for s in scenarios: + scenario_dir = os.path.join(save_dir, s) + if os.path.exists(scenario_dir): + shutil.rmtree(scenario_dir) + + print("=== Generating Traces (Parallel - HARD STOP) ===") + + # Launch all processes + processes = [] + start_time = time.time() + + for s in scenarios: + print(f"Launching scenario {s}") + p = mp.Process( + target=_worker_generate_traces, + args=(save_dir, s, n, expert, model_path) + ) + p.start() + processes.append((s, p)) + + # Monitor time budget and terminate if exceeded + trace_counts_before_termination = {} + while True: + elapsed = time.time() - start_time + + # Check if time budget exceeded (skip check if time_budget is inf) + if time_budget != float('inf') and elapsed >= time_budget: + print(f"\n[HARD STOP] Time budget ({time_budget}s) reached at {elapsed:.2f}s") + + # Record trace counts RIGHT BEFORE termination + for s in scenarios: + csv_path = os.path.join(save_dir, s, "traces.csv") + if os.path.exists(csv_path): + with open(csv_path, 'r') as f: + lines = f.readlines() + if len(lines) <= 1: # Only header or empty + trace_counts_before_termination[s] = 0 + else: + # Count unique trace_ids (episodes) + trace_ids = set() + for line in lines[1:]: # Skip header + parts = line.split(',') + if parts: + trace_ids.add(parts[0]) # First column is trace_id + trace_counts_before_termination[s] = len(trace_ids) + else: + trace_counts_before_termination[s] = 0 + + print("Terminating all running processes...") + + for scenario_name, proc in processes: + if proc.is_alive(): + print(f"Terminating scenario {scenario_name} (PID={proc.pid})") + proc.terminate() + proc.join(timeout=5) # Wait up to 5 seconds for graceful termination + if proc.is_alive(): + print(f"Force killing scenario {scenario_name}") + proc.kill() + proc.join() + break + + # Check if all processes finished naturally + all_done = all(not proc.is_alive() for _, proc in processes) + if all_done: + print(f"All processes finished naturally (elapsed: {elapsed:.2f}s)") + break + + time.sleep(0.1) # Check every 100ms + + # Build logs dict for scenarios with valid traces + logs = {} + for s, proc in processes: + csv_path = os.path.join(save_dir, s, "traces.csv") + + if os.path.exists(csv_path): + # If process was terminated, restore to pre-termination state + # (remove the last partial trace that was being written) + if proc.exitcode != 0 and s in trace_counts_before_termination: + with open(csv_path, 'r') as f: + lines = f.readlines() + + if len(lines) <= 1: # Only header or empty + current_count = 0 + else: + # Count unique trace_ids in final file + trace_ids = set() + for line in lines[1:]: # Skip header + parts = line.split(',') + if parts: + trace_ids.add(parts[0]) + current_count = len(trace_ids) + + expected_count = trace_counts_before_termination[s] + + if current_count > expected_count: + # There's a partial trace - keep only the completed traces + print(f"[INFO] Scenario {s}: Removing partial episode (had {current_count} episodes, keeping {expected_count})") + + # Keep only rows with trace_id < expected_count + with open(csv_path, 'w') as f: + f.write(lines[0]) # Write header + for line in lines[1:]: + parts = line.split(',') + if parts: + trace_id = int(parts[0]) + if trace_id < expected_count: + f.write(line) + + # Only add to logs if there are any complete traces + if expected_count > 0: + logs[s] = csv_path + print(f"[INFO] Scenario {s} has {expected_count} completed episodes.") + else: + print(f"[INFO] Scenario {s} had no completed episodes.") + else: + # Process completed successfully + # Count episodes in completed file + with open(csv_path, 'r') as f: + lines = f.readlines() + if len(lines) > 1: + trace_ids = set() + for line in lines[1:]: + parts = line.split(',') + if parts: + trace_ids.add(parts[0]) + episode_count = len(trace_ids) + else: + episode_count = 0 + + logs[s] = csv_path + print(f"[INFO] Scenario {s} completed successfully with {episode_count} episodes.") + else: + print(f"[INFO] Scenario {s} produced no traces.") + + if not logs: + print("No traces generated.") + + return logs + + +def run_monolithic_smc(scenario_base): + """ + Run monolithic SMC analysis on generated traces. + """ + if not scenario_base.logbase: + print("No traces to analyze.") + + results = {} + + print("\n=== Monolithic SMC Results ===") + for s in scenario_base.logbase: + rho = scenario_base.get_success_prob(s) + unc = scenario_base.get_success_prob_uncertainty(s) + print(f"{s}: rho = {rho:.4f} ± {unc:.4f}") + results[s] = {"rho": rho, "uncertainty": unc} + + return results + + +def run_SMC_compositional(scenarios, time_budget, scenario_base): + print("\n=== Running Compositional SMC ===") + start_time = time.time() + results = {} + + engine = CompositionalAnalysisEngine(scenario_base) + + for s in scenarios: + elapsed = time.time() - start_time + remaining_time = time_budget - elapsed + if time_budget != float('inf') and remaining_time <= 0: + print(f"Time budget exhausted before scenario {s}") + break + + rho, uncertainty = engine.check( + s, + features=["x", "y", "heading", "speed"], + center_feat_idx=[0, 1], + ) + + print(f"Estimated {s}: rho = {rho:.4f} ± {uncertainty:.4f}") + + results[s] = {"rho": rho, "uncertainty": uncertainty} + + return results + + +def parse_scenario(input_scenario): + scenarios_set = set() + for s in input_scenario: + scenarios_set.add(s) + return scenarios_set + + +def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expert, model_path, ground_truth=False, confidence_level=None, error_bound=None, reuse_traces=False): + """ + Test scenario with hard time budget enforcement. + Terminates all processes when time budget is reached. + + Args: + ground_truth: If True, compute ground truth using Hoeffding's inequality + confidence_level: Confidence level for ground truth (e.g., 0.95) + error_bound: Error bound for ground truth (e.g., 0.01) + reuse_traces: If True, use existing traces from save_dir without generating new ones + """ + # If ground truth mode, compute required samples using Hoeffding's inequality + if ground_truth: + if confidence_level is None or error_bound is None: + raise ValueError("Ground truth mode requires --confidence_level and --error_bound") + + n_required = compute_hoeffding_samples(confidence_level, error_bound) + print(f"\n=== Ground Truth Mode ===") + print(f"Confidence Level: {confidence_level * 100}%") + print(f"Error Bound: {error_bound}") + print(f"Required samples (Hoeffding): {n_required}") + + # Override n and remove time budget for ground truth + n = n_required + time_budget = float('inf') # No time limit for ground truth + + # monolithic trace generation + if not isCompositional: + print("Running MONOLITHIC SMC") + scenarios = [input_scenario] + + if not reuse_traces: + logs = generate_traces_parallel(n=n, save_dir=save_dir, scenarios=scenarios, time_budget=time_budget, expert=expert, model_path=model_path) + else: + # Build logs from existing traces + logs = {} + for s in scenarios: + csv_path = os.path.join(save_dir, s, "traces.csv") + if os.path.exists(csv_path): + logs[s] = csv_path + print(f"[INFO] Using existing traces for scenario {s}") + else: + print(f"[ERROR] No existing traces found for scenario {s} at {csv_path}") + + scenario_base = ScenarioBase(logs, delta=1-confidence_level) + run_monolithic_smc(scenario_base) + + # compositional trace generation + else: + print("Running COMPOSITIONAL SMC on primitive cases (what compositional will use)") + scenarios_set = parse_scenario(input_scenario) + scenarios = list(scenarios_set) + + if not reuse_traces: + logs = generate_traces_parallel(n=n, save_dir=save_dir, scenarios=scenarios, time_budget=time_budget, expert=expert, model_path=model_path) + else: + # Build logs from existing traces + logs = {} + for s in scenarios: + csv_path = os.path.join(save_dir, s, "traces.csv") + if os.path.exists(csv_path): + logs[s] = csv_path + print(f"[INFO] Using existing traces for scenario {s}") + else: + print(f"[ERROR] No existing traces found for scenario {s} at {csv_path}") + + scenario_base = ScenarioBase(logs, delta=1-confidence_level) + + # checking individual rho's + run_monolithic_smc(scenario_base) + + # compositional rho + run_SMC_compositional(scenarios=[input_scenario], time_budget=time_budget, scenario_base=scenario_base) + + # Print summary for easy copy-paste to README + print("\n" + "="*60) + print("SUMMARY") + print("="*60) + print(f"Command: python compare_analysis.py --scenario \"{input_scenario}\" " + f"{'--compositional ' if isCompositional else ''}" + f"{'--expert ' if expert else ''}" + f"{'--reuse_traces ' if reuse_traces else ''}" + f"--time_budget {time_budget if time_budget != float('inf') else 'N/A'} " + f"--save_dir \"{save_dir}\"") + if ground_truth: + print(f"Ground Truth: confidence_level={confidence_level}, error_bound={error_bound}") + print("="*60) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Run SMC tests with compositional or monolithic approaches") + parser.add_argument("--scenario", type=str, default="SXC", help="Input scenario string (default: SXC)") + parser.add_argument("--compositional", action="store_true", help="Use compositional approach (default: False)") + parser.add_argument("--time_budget", type=int, default=None, help="Time budget in seconds (default: None)") + parser.add_argument("--n", type=int, default=None, help="Number of traces to generate. If not specified, runs until time budget is hit (default: None)") + parser.add_argument("--expert", action="store_true", help="Use expert mode (default: False)") + parser.add_argument("--save_dir", type=str, default="storage/new_traces", help="Directory to save traces (default: storage/run1)") + parser.add_argument("--model_path", type=str, default="storage/models/model_map_2.zip", help="Path to model file (default: storage/models/model_map_2.zip)") + + # Ground truth options + parser.add_argument("--ground_truth", action="store_true", help="Compute ground truth using Hoeffding's inequality (default: False)") + parser.add_argument("--confidence_level", type=float, default=0.99, help="Confidence level for ground truth (default: 0.99)") + parser.add_argument("--error_bound", type=float, default=0.001, help="Error bound (epsilon) for ground truth (default: 0.001)") + + # Reuse traces option + parser.add_argument("--reuse_traces", action="store_true", help="Use existing traces from save_dir without generating new ones (default: False)") + + args = parser.parse_args() + + mp.set_start_method("spawn") + + testScenario( + input_scenario=args.scenario, + isCompositional=args.compositional, + time_budget=args.time_budget if args.time_budget is not None else float('inf'), + n=args.n, + save_dir=args.save_dir, + expert=args.expert, + model_path=args.model_path, + ground_truth=args.ground_truth, + confidence_level=args.confidence_level, + error_bound=args.error_bound, + reuse_traces=args.reuse_traces + ) + diff --git a/examples/compositional_analysis/utils.py b/examples/compositional_analysis/utils.py new file mode 100644 index 0000000..3285c9e --- /dev/null +++ b/examples/compositional_analysis/utils.py @@ -0,0 +1,130 @@ +import os +import csv +import numpy as np +from stable_baselines3 import PPO +from stable_baselines3.common.utils import set_random_seed +from train import make_env + + +def generate_traces( + seed: int = 0, + save_dir: str = "storage/run0", + model_path: str = None, + expert: bool = False, + n: int = 50, + scenario: str = "XX", + gif: bool = False +): + """ + Runs MetaDrive simulation using a trained PPO model or expert policy and logs trajectory traces. + + Args: + seed (int): Random seed for reproducibility. + save_dir (str): Directory where traces or gifs will be saved. + model_path (str): Path to the trained PPO model (.zip file). Not used if expert=True. + expert (bool): If True, use expert policy instead of trained model. + n (int): Number of test episodes to run. + scenario (str or int): Scenario string or ID. + gif (bool): If True, generate top-down gifs instead of CSV traces. + """ + + if not expert: + assert model_path is not None, "You must provide a valid model_path (.zip file)" + + set_random_seed(seed) + + scenario_id = int(scenario) if str(scenario).isdigit() else scenario + env = make_env(scenario=scenario_id, monitor=False) + + if expert: + # print("USING EXPERT POLICY") + from metadrive.policy.expert_policy import ExpertPolicy + # Expert policy will be created per episode after reset + model = None + use_expert = True + else: + model = PPO.load(model_path) + use_expert = False + + all_traces = [] + trace_id = 0 + + # Create save dir + os.makedirs(save_dir, exist_ok=True) + + if not gif: + csv_path = os.path.join(save_dir, scenario, "traces.csv") + os.makedirs(os.path.dirname(csv_path), exist_ok=True) + f = open(csv_path, "w", newline="") + + writer = csv.DictWriter( + f, + fieldnames=[ + "trace_id", "step", "x", "y", "heading", + "speed", "action", "reward", "label" + ] + ) + writer.writeheader() + + for ep in range(n): + obs, _ = env.reset() + + # Create expert policy for this episode (needs current vehicle) + if use_expert: + expert_policy = ExpertPolicy(env.agent) + + initial_speed = np.random.uniform(low=70/3.6, high=80/3.6) + initial_velocity = env.agent.lane.direction * initial_speed + env.agent.set_velocity(initial_velocity) + + done = False + total_reward = 0.0 + step = 0 + label = False + + while not done and step <= env.config.horizon: + if use_expert: + action = expert_policy.act() + else: + action, _states = model.predict(obs, deterministic=True) + + obs, reward, done, truncated, info = env.step(action) + total_reward += reward + label = not done or info.get("arrive_dest") + + if gif: + env.render(mode="topdown", screen_record=True, window=False) + else: + agent = env.agent + pos = agent.position + heading = agent.heading_theta + vel = agent.speed + + row = { + "trace_id": trace_id, + "step": step, + "x": pos[0], + "y": pos[1], + "heading": heading, + "speed": vel, + "action": action.tolist() if hasattr(action, "tolist") else action, + "reward": reward, + "label": label + } + writer.writerow(row) + + step += 1 + + if gif: + gif_path = os.path.join(save_dir, f"trace_{trace_id:03d}.gif") + env.top_down_renderer.generate_gif(gif_path) + print(f"Saved gif to {gif_path}") + + trace_id += 1 + + if not gif: + f.close() + + env.close() + return + From 221fbebae52ce6bebae2c0bc3ce49d062d7b27e2 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 17:40:56 -0800 Subject: [PATCH 06/20] Fix exp code --- examples/compositional_analysis/run_exp.py | 43 +++++++--------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/examples/compositional_analysis/run_exp.py b/examples/compositional_analysis/run_exp.py index e46e32a..111f169 100644 --- a/examples/compositional_analysis/run_exp.py +++ b/examples/compositional_analysis/run_exp.py @@ -245,31 +245,16 @@ def parse_scenario(input_scenario): return scenarios_set -def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expert, model_path, ground_truth=False, confidence_level=None, error_bound=None, reuse_traces=False): +def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expert, model_path, confidence_level=None, error_bound=None, reuse_traces=False): """ Test scenario with hard time budget enforcement. Terminates all processes when time budget is reached. Args: - ground_truth: If True, compute ground truth using Hoeffding's inequality confidence_level: Confidence level for ground truth (e.g., 0.95) error_bound: Error bound for ground truth (e.g., 0.01) reuse_traces: If True, use existing traces from save_dir without generating new ones """ - # If ground truth mode, compute required samples using Hoeffding's inequality - if ground_truth: - if confidence_level is None or error_bound is None: - raise ValueError("Ground truth mode requires --confidence_level and --error_bound") - - n_required = compute_hoeffding_samples(confidence_level, error_bound) - print(f"\n=== Ground Truth Mode ===") - print(f"Confidence Level: {confidence_level * 100}%") - print(f"Error Bound: {error_bound}") - print(f"Required samples (Hoeffding): {n_required}") - - # Override n and remove time budget for ground truth - n = n_required - time_budget = float('inf') # No time limit for ground truth # monolithic trace generation if not isCompositional: @@ -312,9 +297,6 @@ def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expe print(f"[ERROR] No existing traces found for scenario {s} at {csv_path}") scenario_base = ScenarioBase(logs, delta=1-confidence_level) - - # checking individual rho's - run_monolithic_smc(scenario_base) # compositional rho run_SMC_compositional(scenarios=[input_scenario], time_budget=time_budget, scenario_base=scenario_base) @@ -329,8 +311,6 @@ def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expe f"{'--reuse_traces ' if reuse_traces else ''}" f"--time_budget {time_budget if time_budget != float('inf') else 'N/A'} " f"--save_dir \"{save_dir}\"") - if ground_truth: - print(f"Ground Truth: confidence_level={confidence_level}, error_bound={error_bound}") print("="*60) @@ -341,32 +321,35 @@ def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expe parser.add_argument("--scenario", type=str, default="SXC", help="Input scenario string (default: SXC)") parser.add_argument("--compositional", action="store_true", help="Use compositional approach (default: False)") parser.add_argument("--time_budget", type=int, default=None, help="Time budget in seconds (default: None)") - parser.add_argument("--n", type=int, default=None, help="Number of traces to generate. If not specified, runs until time budget is hit (default: None)") parser.add_argument("--expert", action="store_true", help="Use expert mode (default: False)") parser.add_argument("--save_dir", type=str, default="storage/new_traces", help="Directory to save traces (default: storage/run1)") parser.add_argument("--model_path", type=str, default="storage/models/model_map_2.zip", help="Path to model file (default: storage/models/model_map_2.zip)") - - # Ground truth options - parser.add_argument("--ground_truth", action="store_true", help="Compute ground truth using Hoeffding's inequality (default: False)") parser.add_argument("--confidence_level", type=float, default=0.99, help="Confidence level for ground truth (default: 0.99)") parser.add_argument("--error_bound", type=float, default=0.001, help="Error bound (epsilon) for ground truth (default: 0.001)") - - # Reuse traces option parser.add_argument("--reuse_traces", action="store_true", help="Use existing traces from save_dir without generating new ones (default: False)") args = parser.parse_args() mp.set_start_method("spawn") + + assert args.confidence_level is not None + assert (args.error_bound is not None) or (args.time_budget is not None) + + n = None + time_budget = float('inf') + if args.error_bound is not None: + n = compute_hoeffding_samples(args.confidence_level, args.error_bound) + elif args.time_budget is not None: + time_budget = args.time_budget testScenario( input_scenario=args.scenario, isCompositional=args.compositional, - time_budget=args.time_budget if args.time_budget is not None else float('inf'), - n=args.n, + time_budget=time_budget, + n=n, save_dir=args.save_dir, expert=args.expert, model_path=args.model_path, - ground_truth=args.ground_truth, confidence_level=args.confidence_level, error_bound=args.error_bound, reuse_traces=args.reuse_traces From 5735c2458134a3d27cc3c5aadb8589513a9735ac Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 20:16:37 -0800 Subject: [PATCH 07/20] Fix exp code --- examples/compositional_analysis/run_exp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/compositional_analysis/run_exp.py b/examples/compositional_analysis/run_exp.py index 111f169..8e0ccaf 100644 --- a/examples/compositional_analysis/run_exp.py +++ b/examples/compositional_analysis/run_exp.py @@ -324,8 +324,8 @@ def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expe parser.add_argument("--expert", action="store_true", help="Use expert mode (default: False)") parser.add_argument("--save_dir", type=str, default="storage/new_traces", help="Directory to save traces (default: storage/run1)") parser.add_argument("--model_path", type=str, default="storage/models/model_map_2.zip", help="Path to model file (default: storage/models/model_map_2.zip)") - parser.add_argument("--confidence_level", type=float, default=0.99, help="Confidence level for ground truth (default: 0.99)") - parser.add_argument("--error_bound", type=float, default=0.001, help="Error bound (epsilon) for ground truth (default: 0.001)") + parser.add_argument("--confidence_level", type=float, default=None, help="Confidence level for ground truth (default: None)") + parser.add_argument("--error_bound", type=float, default=None, help="Error bound (epsilon) for ground truth (default: None)") parser.add_argument("--reuse_traces", action="store_true", help="Use existing traces from save_dir without generating new ones (default: False)") args = parser.parse_args() From b1125857579267125a1009f704537ff42af14020 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 20:40:58 -0800 Subject: [PATCH 08/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 examples/compositional_analysis/exps.sh diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh new file mode 100755 index 0000000..61643c7 --- /dev/null +++ b/examples/compositional_analysis/exps.sh @@ -0,0 +1,25 @@ +# Fixed Error (epsilon = 0.1) +## Monolithic +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &>> results/sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &>> results/cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &>> results/xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &>> results/socxs.txt & +## Compositional +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &>> results/sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &>> results/cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &>> results/xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &>> results/socxs.txt & + +# Fixed Time Budget (2 mins) +## Monolithic +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &>> results/sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &>> results/cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &>> results/xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &>> results/socxs.txt & +## Compositional +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &>> results/sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &>> results/cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &>> results/xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &>> results/socxs.txt & + +wait From 95e3176ab1266fa068492614a551f7c91700bf8d Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 20:44:41 -0800 Subject: [PATCH 09/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index 61643c7..8242c73 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -1,25 +1,25 @@ # Fixed Error (epsilon = 0.1) ## Monolithic -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &>> results/sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &>> results/cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &>> results/xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &>> results/socxs.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &>> results/fixed_error_monolithic_sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &>> results/fixed_error_monolithic_cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &>> results/fixed_error_monolithic_xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &>> results/fixed_error_monolithic_socxs.txt & ## Compositional -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &>> results/sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &>> results/cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &>> results/xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &>> results/socxs.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &>> results/fixed_error_compositional_sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &>> results/fixed_error_compositional_cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &>> results/fixed_error_compositional_xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &>> results/fixed_error_compositional_socxs.txt & # Fixed Time Budget (2 mins) ## Monolithic -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &>> results/sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &>> results/cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &>> results/xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &>> results/socxs.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &>> results/fixed_time_monolithic_sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &>> results/fixed_time_monolithic_cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &>> results/fixed_time_monolithic_xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &>> results/fixed_time_monolithic_socxs.txt & ## Compositional -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &>> results/sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &>> results/cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &>> results/xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &>> results/socxs.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &>> results/fixed_time_compositional_sx.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &>> results/fixed_time_compositional_cos.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &>> results/fixed_time_compositional_xsoc.txt & +{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &>> results/fixed_time_compositional_socxs.txt & wait From 95393524e60703f2c55bebf3e2bb38076e3aba5e Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 21:01:33 -0800 Subject: [PATCH 10/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 34 ++++++++++------------ examples/compositional_analysis/run_exp.py | 2 +- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index 8242c73..d13d67b 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -1,25 +1,23 @@ # Fixed Error (epsilon = 0.1) ## Monolithic -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &>> results/fixed_error_monolithic_sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &>> results/fixed_error_monolithic_cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &>> results/fixed_error_monolithic_xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &>> results/fixed_error_monolithic_socxs.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &>> storage/fixed_error_monolithic/SX/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &>> storage/fixed_error_monolithic/COS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &>> storage/fixed_error_monolithic/XSOC/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &>> storage/fixed_error_monolithic/SOCXS/results.txt & ## Compositional -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &>> results/fixed_error_compositional_sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &>> results/fixed_error_compositional_cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &>> results/fixed_error_compositional_xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &>> results/fixed_error_compositional_socxs.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &>> storage/fixed_error_compositional/SX/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &>> storage/fixed_error_compositional/COS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &>> storage/fixed_error_compositional/XSOC/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &>> storage/fixed_error_compositional/SOCXS/results.txt & # Fixed Time Budget (2 mins) ## Monolithic -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &>> results/fixed_time_monolithic_sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &>> results/fixed_time_monolithic_cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &>> results/fixed_time_monolithic_xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &>> results/fixed_time_monolithic_socxs.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &>> storage/fixed_time_monolithic/SX/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &>> storage/fixed_time_monolithic/COS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &>> storage/fixed_time_monolithic/XSOC/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &>> storage/fixed_time_monolithic/SOCXS/results.txt & ## Compositional -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &>> results/fixed_time_compositional_sx.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &>> results/fixed_time_compositional_cos.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &>> results/fixed_time_compositional_xsoc.txt & -{ time python run_exp.py --expert --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &>> results/fixed_time_compositional_socxs.txt & - -wait +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &>> storage/fixed_time_compositional/SX/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &>> storage/fixed_time_compositional/COS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &>> storage/fixed_time_compositional/XSOC/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &>> storage/fixed_time_compositional/SOCXS/results.txt & diff --git a/examples/compositional_analysis/run_exp.py b/examples/compositional_analysis/run_exp.py index 8e0ccaf..528057e 100644 --- a/examples/compositional_analysis/run_exp.py +++ b/examples/compositional_analysis/run_exp.py @@ -322,7 +322,7 @@ def testScenario(input_scenario, isCompositional, time_budget, n, save_dir, expe parser.add_argument("--compositional", action="store_true", help="Use compositional approach (default: False)") parser.add_argument("--time_budget", type=int, default=None, help="Time budget in seconds (default: None)") parser.add_argument("--expert", action="store_true", help="Use expert mode (default: False)") - parser.add_argument("--save_dir", type=str, default="storage/new_traces", help="Directory to save traces (default: storage/run1)") + parser.add_argument("--save_dir", type=str, default="storage/traces", help="Directory to save traces (default: storage/traces)") parser.add_argument("--model_path", type=str, default="storage/models/model_map_2.zip", help="Path to model file (default: storage/models/model_map_2.zip)") parser.add_argument("--confidence_level", type=float, default=None, help="Confidence level for ground truth (default: None)") parser.add_argument("--error_bound", type=float, default=None, help="Error bound (epsilon) for ground truth (default: None)") From 45c2b39014c4716c76d554384a8e41ec157f6755 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 21:08:23 -0800 Subject: [PATCH 11/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 33 +++++++++++++------------ 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index d13d67b..3b178e7 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -1,23 +1,24 @@ # Fixed Error (epsilon = 0.1) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &>> storage/fixed_error_monolithic/SX/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &>> storage/fixed_error_monolithic/COS/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &>> storage/fixed_error_monolithic/XSOC/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &>> storage/fixed_error_monolithic/SOCXS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/fixed_error_monolithic/SX/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/fixed_error_monolithic/COS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/fixed_error_monolithic/XSOC/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/fixed_error_monolithic/SOCXS/results.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &>> storage/fixed_error_compositional/SX/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &>> storage/fixed_error_compositional/COS/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &>> storage/fixed_error_compositional/XSOC/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &>> storage/fixed_error_compositional/SOCXS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/fixed_error_compositional/SX/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/fixed_error_compositional/COS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/fixed_error_compositional/XSOC/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/fixed_error_compositional/SOCXS/results.txt & # Fixed Time Budget (2 mins) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &>> storage/fixed_time_monolithic/SX/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &>> storage/fixed_time_monolithic/COS/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &>> storage/fixed_time_monolithic/XSOC/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &>> storage/fixed_time_monolithic/SOCXS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/fixed_time_monolithic/SX/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/fixed_time_monolithic/COS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/fixed_time_monolithic/XSOC/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/fixed_time_monolithic/SOCXS/results.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &>> storage/fixed_time_compositional/SX/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &>> storage/fixed_time_compositional/COS/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &>> storage/fixed_time_compositional/XSOC/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &>> storage/fixed_time_compositional/SOCXS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/fixed_time_compositional/SX/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/fixed_time_compositional/COS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/fixed_time_compositional/XSOC/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/fixed_time_compositional/SOCXS/results.txt & +p \ No newline at end of file From 76cd2601a9e020ae61a77b26092bdbd003e6887c Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 21:14:36 -0800 Subject: [PATCH 12/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index 3b178e7..95e945b 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -1,24 +1,24 @@ # Fixed Error (epsilon = 0.1) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/fixed_error_monolithic/SX/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/fixed_error_monolithic/COS/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/fixed_error_monolithic/XSOC/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/fixed_error_monolithic/SOCXS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/fixed_error_monolithic/SX/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/fixed_error_monolithic/COS/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/fixed_error_monolithic/XSOC/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/fixed_error_monolithic/SOCXS/result.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/fixed_error_compositional/SX/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/fixed_error_compositional/COS/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/fixed_error_compositional/XSOC/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/fixed_error_compositional/SOCXS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/fixed_error_compositional/SX/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/fixed_error_compositional/COS/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/fixed_error_compositional/XSOC/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/fixed_error_compositional/SOCXS/result.txt & # Fixed Time Budget (2 mins) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/fixed_time_monolithic/SX/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/fixed_time_monolithic/COS/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/fixed_time_monolithic/XSOC/results.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/fixed_time_monolithic/SOCXS/results.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/fixed_time_monolithic/SX/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/fixed_time_monolithic/COS/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/fixed_time_monolithic/XSOC/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/fixed_time_monolithic/SOCXS/result.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/fixed_time_compositional/SX/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/fixed_time_compositional/COS/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/fixed_time_compositional/XSOC/results.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/fixed_time_compositional/SOCXS/results.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/fixed_time_compositional/SX/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/fixed_time_compositional/COS/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/fixed_time_compositional/XSOC/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/fixed_time_compositional/SOCXS/result.txt & p \ No newline at end of file From 0e3ba2fa56fc803547f528eb254e493e9130aa07 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 21:24:39 -0800 Subject: [PATCH 13/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index 95e945b..4e29844 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -1,24 +1,24 @@ # Fixed Error (epsilon = 0.1) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/fixed_error_monolithic/SX/result.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/fixed_error_monolithic/COS/result.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/fixed_error_monolithic/XSOC/result.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/fixed_error_monolithic/SOCXS/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/results/fixed_error_monolithic_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/results/fixed_error_monolithic_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/results/fixed_error_monolithic_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/results/fixed_error_monolithic_SOCXS.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/fixed_error_compositional/SX/result.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/fixed_error_compositional/COS/result.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/fixed_error_compositional/XSOC/result.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/fixed_error_compositional/SOCXS/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt & # Fixed Time Budget (2 mins) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/fixed_time_monolithic/SX/result.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/fixed_time_monolithic/COS/result.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/fixed_time_monolithic/XSOC/result.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/fixed_time_monolithic/SOCXS/result.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/results/fixed_time_monolithic_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/results/fixed_time_monolithic_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/results/fixed_time_monolithic_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/results/fixed_time_monolithic_SOCXS.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/fixed_time_compositional/SX/result.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/fixed_time_compositional/COS/result.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/fixed_time_compositional/XSOC/result.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/fixed_time_compositional/SOCXS/result.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/results/fixed_time_compositional_SX.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/results/fixed_time_compositional_COS.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/results/fixed_time_compositional_XSOC.txt & +{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/results/fixed_time_compositional_SOCXS.txt & p \ No newline at end of file From 90972cb07a6bae0200f2113ed29df26749aaa590 Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 21:28:13 -0800 Subject: [PATCH 14/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index 4e29844..5099714 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -21,4 +21,3 @@ { time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/results/fixed_time_compositional_COS.txt & { time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/results/fixed_time_compositional_XSOC.txt & { time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/results/fixed_time_compositional_SOCXS.txt & -p \ No newline at end of file From c47d027127149abe1680e9a9b0bae1725826821b Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 21:30:00 -0800 Subject: [PATCH 15/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index 5099714..6bd9a79 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -5,10 +5,10 @@ { time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/results/fixed_error_monolithic_XSOC.txt & { time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/results/fixed_error_monolithic_SOCXS.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt & # Fixed Time Budget (2 mins) ## Monolithic @@ -17,7 +17,7 @@ { time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/results/fixed_time_monolithic_XSOC.txt & { time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/results/fixed_time_monolithic_SOCXS.txt & ## Compositional -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/results/fixed_time_compositional_SX.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/results/fixed_time_compositional_COS.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/results/fixed_time_compositional_XSOC.txt & -{ time python run_exp.py --expert --save-dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/results/fixed_time_compositional_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/results/fixed_time_compositional_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/results/fixed_time_compositional_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/results/fixed_time_compositional_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/results/fixed_time_compositional_SOCXS.txt & From 332b85b914e8bf3d0cbda45ace23c4a1691304eb Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 21:38:02 -0800 Subject: [PATCH 16/20] Add exp bash script --- examples/compositional_analysis/exps.sh | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index 6bd9a79..b559726 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -1,23 +1,23 @@ # Fixed Error (epsilon = 0.1) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/results/fixed_error_monolithic_SX.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/results/fixed_error_monolithic_COS.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/results/fixed_error_monolithic_XSOC.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/results/fixed_error_monolithic_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SX --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/results/fixed_error_monolithic_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_COS --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/results/fixed_error_monolithic_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_XSOC --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/results/fixed_error_monolithic_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SOCXS --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/results/fixed_error_monolithic_SOCXS.txt & ## Compositional -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SX --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_COS --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_XSOC --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SOCXS --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt & # Fixed Time Budget (2 mins) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/results/fixed_time_monolithic_SX.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/results/fixed_time_monolithic_COS.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/results/fixed_time_monolithic_XSOC.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/results/fixed_time_monolithic_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_SX --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/results/fixed_time_monolithic_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_COS --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/results/fixed_time_monolithic_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_XSOC --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/results/fixed_time_monolithic_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_SOCXS --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/results/fixed_time_monolithic_SOCXS.txt & ## Compositional -{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/results/fixed_time_compositional_SX.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/results/fixed_time_compositional_COS.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/results/fixed_time_compositional_XSOC.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/results/fixed_time_compositional_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_SX --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/results/fixed_time_compositional_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_COS --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/results/fixed_time_compositional_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_XSOC --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/results/fixed_time_compositional_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_SOCXS --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/results/fixed_time_compositional_SOCXS.txt & From 113f7432d56850e47b53905f221e210beb2b7ea2 Mon Sep 17 00:00:00 2001 From: beyazit Date: Tue, 16 Dec 2025 05:42:20 +0000 Subject: [PATCH 17/20] Add some results --- .../results/fixed_error_compositional_COS.txt | 34 +++++++++ .../fixed_error_compositional_SOCXS.txt | 46 +++++++++++++ .../results/fixed_error_compositional_SX.txt | 42 +++++++++++ .../fixed_error_compositional_XSOC.txt | 46 +++++++++++++ .../results/fixed_error_monolithic_COS.txt | 12 ++++ .../results/fixed_error_monolithic_SOCXS.txt | 12 ++++ .../results/fixed_error_monolithic_SX.txt | 12 ++++ .../results/fixed_error_monolithic_XSOC.txt | 12 ++++ .../results/fixed_time_compositional_COS.txt | 55 +++++++++++++++ .../fixed_time_compositional_SOCXS.txt | 68 ++++++++++++++++++ .../results/fixed_time_compositional_SX.txt | 43 ++++++++++++ .../results/fixed_time_compositional_XSOC.txt | 69 +++++++++++++++++++ .../results/fixed_time_monolithic_COS.txt | 30 ++++++++ .../results/fixed_time_monolithic_SOCXS.txt | 30 ++++++++ .../results/fixed_time_monolithic_SX.txt | 30 ++++++++ .../results/fixed_time_monolithic_XSOC.txt | 30 ++++++++ 16 files changed, 571 insertions(+) create mode 100644 examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt create mode 100644 examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt new file mode 100644 index 0000000..258e9e3 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt @@ -0,0 +1,34 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario S +Launching scenario C +Launching scenario O +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[PID=1879301] Starting scenario S +[PID=1879301] Finished scenario S diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt new file mode 100644 index 0000000..e8bfd74 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt @@ -0,0 +1,46 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario X +Launching scenario C +Launching scenario S +Launching scenario O +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[PID=1879281] Starting scenario S +[PID=1879281] Finished scenario S +[PID=1879279] Starting scenario X +[PID=1879279] Finished scenario X diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt new file mode 100644 index 0000000..fb44878 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt @@ -0,0 +1,42 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario S +Launching scenario X +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[PID=1879284] Starting scenario S +[PID=1879284] Finished scenario S +[PID=1879285] Starting scenario X +[PID=1879285] Finished scenario X +All processes finished naturally (elapsed: 193.15s) +[INFO] Scenario S completed successfully with 185 episodes. +[INFO] Scenario X completed successfully with 185 episodes. + +=== Running Compositional SMC === +Estimated SX: rho = 0.9500 ± 0.1474 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SX" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_SX" +============================================================ + +real 3m19.564s +user 5m56.348s +sys 0m12.423s diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt new file mode 100644 index 0000000..41ae45f --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt @@ -0,0 +1,46 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario S +Launching scenario X +Launching scenario C +Launching scenario O +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[PID=1879290] Starting scenario S +[PID=1879290] Finished scenario S +[PID=1879291] Starting scenario X +[PID=1879291] Finished scenario X diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt new file mode 100644 index 0000000..7e0a83c --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt @@ -0,0 +1,12 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario COS +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt new file mode 100644 index 0000000..6b2663e --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt @@ -0,0 +1,12 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario SOCXS +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt new file mode 100644 index 0000000..1378c14 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt @@ -0,0 +1,12 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario SX +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt new file mode 100644 index 0000000..9183f31 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt @@ -0,0 +1,12 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario XSOC +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt new file mode 100644 index 0000000..f32377c --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt @@ -0,0 +1,55 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario S +Launching scenario O +Launching scenario C +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.09s +Terminating all running processes... +Terminating scenario S (PID=1879264) +Terminating scenario O (PID=1879265) +Terminating scenario C (PID=1879267) +[INFO] Scenario S has 99 completed episodes. +[INFO] Scenario O: Removing partial episode (had 60 episodes, keeping 59) +[INFO] Scenario O has 59 completed episodes. +[INFO] Scenario C has 50 completed episodes. + +=== Running Compositional SMC === +Estimated COS: rho = 0.6759 ± 0.2614 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "COS" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_COS" +============================================================ + +real 2m8.534s +user 5m59.436s +sys 0m10.350s diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt new file mode 100644 index 0000000..2d12500 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt @@ -0,0 +1,68 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario X +Launching scenario C +Launching scenario O +Launching scenario S +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.09s +Terminating all running processes... +Terminating scenario X (PID=1879257) +Terminating scenario C (PID=1879258) +Terminating scenario O (PID=1879259) +Terminating scenario S (PID=1879260) +[INFO] Scenario X has 74 completed episodes. +[INFO] Scenario C has 50 completed episodes. +[INFO] Scenario O: Removing partial episode (had 54 episodes, keeping 53) +[INFO] Scenario O has 53 completed episodes. +[INFO] Scenario S: Removing partial episode (had 99 episodes, keeping 97) +[INFO] Scenario S has 97 completed episodes. + +=== Running Compositional SMC === +Estimated SOCXS: rho = 0.6790 ± 0.3496 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SOCXS" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_SOCXS" +============================================================ + +real 2m9.156s +user 8m16.055s +sys 0m18.684s diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt new file mode 100644 index 0000000..5e42dcb --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt @@ -0,0 +1,43 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario S +Launching scenario X +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.09s +Terminating all running processes... +Terminating scenario S (PID=1879297) +Terminating scenario X (PID=1879298) +[INFO] Scenario S has 90 completed episodes. +[INFO] Scenario X: Removing partial episode (had 67 episodes, keeping 66) +[INFO] Scenario X has 66 completed episodes. + +=== Running Compositional SMC === +Estimated SX: rho = 0.9527 ± 0.2296 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SX" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_SX" +============================================================ + +real 2m8.718s +user 4m3.019s +sys 0m13.507s diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt new file mode 100644 index 0000000..9736880 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt @@ -0,0 +1,69 @@ +Running COMPOSITIONAL SMC on primitive cases (what compositional will use) +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario O +Launching scenario S +Launching scenario C +Launching scenario X +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.09s +Terminating all running processes... +Terminating scenario O (PID=1879312) +Terminating scenario S (PID=1879313) +Terminating scenario C (PID=1879314) +Terminating scenario X (PID=1879315) +[INFO] Scenario O has 57 completed episodes. +[INFO] Scenario S: Removing partial episode (had 93 episodes, keeping 92) +[INFO] Scenario S has 92 completed episodes. +[INFO] Scenario C: Removing partial episode (had 51 episodes, keeping 50) +[INFO] Scenario C has 50 completed episodes. +[INFO] Scenario X: Removing partial episode (had 71 episodes, keeping 69) +[INFO] Scenario X has 69 completed episodes. + +=== Running Compositional SMC === +Estimated XSOC: rho = 0.5891 ± 0.3120 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "XSOC" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_XSOC" +============================================================ + +real 2m9.137s +user 7m22.034s +sys 0m18.302s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt new file mode 100644 index 0000000..57e9343 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt @@ -0,0 +1,30 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario COS +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.08s +Terminating all running processes... +Terminating scenario COS (PID=1879277) +[INFO] Scenario COS has 25 completed episodes. + +=== Monolithic SMC Results === +COS: rho = 0.7200 ± 0.2716 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "COS" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_COS" +============================================================ + +real 2m6.191s +user 1m55.053s +sys 0m4.688s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt new file mode 100644 index 0000000..9b6f692 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt @@ -0,0 +1,30 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario SOCXS +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.08s +Terminating all running processes... +Terminating scenario SOCXS (PID=1879255) +[INFO] Scenario SOCXS has 16 completed episodes. + +=== Monolithic SMC Results === +SOCXS: rho = 0.8125 ± 0.3395 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SOCXS" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_SOCXS" +============================================================ + +real 2m5.764s +user 1m55.900s +sys 0m3.838s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt new file mode 100644 index 0000000..198f914 --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt @@ -0,0 +1,30 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario SX +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.09s +Terminating all running processes... +Terminating scenario SX (PID=1879295) +[INFO] Scenario SX has 46 completed episodes. + +=== Monolithic SMC Results === +SX: rho = 1.0000 ± 0.2002 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SX" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_SX" +============================================================ + +real 2m6.655s +user 1m52.599s +sys 0m5.580s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt new file mode 100644 index 0000000..7c0f7de --- /dev/null +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt @@ -0,0 +1,30 @@ +Running MONOLITHIC SMC +=== Generating Traces (Parallel - HARD STOP) === +Launching scenario XSOC +[INFO] Environment: MetaDriveEnv +[INFO] MetaDrive version: 0.4.3 +[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] +[INFO] Render Mode: none +[INFO] Horizon (Max steps per agent): 2000 +[INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Use Torch PPO expert. + +[HARD STOP] Time budget (120s) reached at 120.09s +Terminating all running processes... +Terminating scenario XSOC (PID=1879288) +[INFO] Scenario XSOC has 18 completed episodes. + +=== Monolithic SMC Results === +XSOC: rho = 0.7778 ± 0.3201 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "XSOC" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_XSOC" +============================================================ + +real 2m6.538s +user 1m52.206s +sys 0m5.747s From f0dbeecc8482223105bfea718f61fb71df366308 Mon Sep 17 00:00:00 2001 From: beyazit Date: Tue, 16 Dec 2025 05:50:33 +0000 Subject: [PATCH 18/20] Add some results --- .../results/fixed_error_compositional_COS.txt | 21 ++++++++++++++++++ .../fixed_error_compositional_SOCXS.txt | 22 +++++++++++++++++++ .../fixed_error_compositional_XSOC.txt | 22 +++++++++++++++++++ .../results/fixed_error_monolithic_COS.txt | 17 ++++++++++++++ .../results/fixed_error_monolithic_SOCXS.txt | 17 ++++++++++++++ .../results/fixed_error_monolithic_SX.txt | 17 ++++++++++++++ .../results/fixed_error_monolithic_XSOC.txt | 17 ++++++++++++++ 7 files changed, 133 insertions(+) diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt index 258e9e3..50e322a 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt @@ -32,3 +32,24 @@ Launching scenario O [INFO] Use Torch PPO expert. [PID=1879301] Starting scenario S [PID=1879301] Finished scenario S +[PID=1879303] Starting scenario O +[PID=1879303] Finished scenario O +[PID=1879302] Starting scenario C +[PID=1879302] Finished scenario C +All processes finished naturally (elapsed: 249.00s) +[INFO] Scenario S completed successfully with 185 episodes. +[INFO] Scenario C completed successfully with 185 episodes. +[INFO] Scenario O completed successfully with 185 episodes. + +=== Running Compositional SMC === +Estimated COS: rho = 0.7133 ± 0.1591 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "COS" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_COS" +============================================================ + +real 4m15.541s +user 10m47.982s +sys 0m18.648s diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt index e8bfd74..514e4d5 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt @@ -44,3 +44,25 @@ Launching scenario O [PID=1879281] Finished scenario S [PID=1879279] Starting scenario X [PID=1879279] Finished scenario X +[PID=1879280] Starting scenario C +[PID=1879280] Finished scenario C +[PID=1879282] Starting scenario O +[PID=1879282] Finished scenario O +All processes finished naturally (elapsed: 245.35s) +[INFO] Scenario X completed successfully with 185 episodes. +[INFO] Scenario C completed successfully with 185 episodes. +[INFO] Scenario S completed successfully with 185 episodes. +[INFO] Scenario O completed successfully with 185 episodes. + +=== Running Compositional SMC === +Estimated SOCXS: rho = 0.6622 ± 0.2027 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SOCXS" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_SOCXS" +============================================================ + +real 4m11.566s +user 13m26.132s +sys 0m14.524s diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt index 41ae45f..b7df49f 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt @@ -44,3 +44,25 @@ Launching scenario O [PID=1879290] Finished scenario S [PID=1879291] Starting scenario X [PID=1879291] Finished scenario X +[PID=1879292] Starting scenario C +[PID=1879292] Finished scenario C +[PID=1879293] Starting scenario O +[PID=1879293] Finished scenario O +All processes finished naturally (elapsed: 245.15s) +[INFO] Scenario S completed successfully with 185 episodes. +[INFO] Scenario X completed successfully with 185 episodes. +[INFO] Scenario C completed successfully with 185 episodes. +[INFO] Scenario O completed successfully with 185 episodes. + +=== Running Compositional SMC === +Estimated XSOC: rho = 0.6563 ± 0.1833 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "XSOC" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_XSOC" +============================================================ + +real 4m11.670s +user 13m41.638s +sys 0m22.259s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt index 7e0a83c..d7a2262 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt @@ -10,3 +10,20 @@ Launching scenario COS [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. +[PID=1879309] Starting scenario COS +[PID=1879309] Finished scenario COS +All processes finished naturally (elapsed: 477.32s) +[INFO] Scenario COS completed successfully with 185 episodes. + +=== Monolithic SMC Results === +COS: rho = 0.8378 ± 0.0998 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "COS" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_COS" +============================================================ + +real 8m3.349s +user 7m43.820s +sys 0m6.999s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt index 6b2663e..9f9f001 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt @@ -10,3 +10,20 @@ Launching scenario SOCXS [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. +[PID=1879262] Starting scenario SOCXS +[PID=1879262] Finished scenario SOCXS +All processes finished naturally (elapsed: 605.60s) +[INFO] Scenario SOCXS completed successfully with 185 episodes. + +=== Monolithic SMC Results === +SOCXS: rho = 0.6649 ± 0.0998 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SOCXS" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_SOCXS" +============================================================ + +real 10m11.484s +user 9m51.459s +sys 0m7.801s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt index 1378c14..a82dbb9 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt @@ -10,3 +10,20 @@ Launching scenario SX [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. +[PID=1879323] Starting scenario SX +[PID=1879323] Finished scenario SX +All processes finished naturally (elapsed: 243.17s) +[INFO] Scenario SX completed successfully with 185 episodes. + +=== Monolithic SMC Results === +SX: rho = 0.9676 ± 0.0998 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "SX" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_SX" +============================================================ + +real 4m9.268s +user 3m52.543s +sys 0m6.364s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt index 9183f31..c70f4f0 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt @@ -10,3 +10,20 @@ Launching scenario XSOC [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. +[PID=1879268] Starting scenario XSOC +[PID=1879268] Finished scenario XSOC +All processes finished naturally (elapsed: 508.30s) +[INFO] Scenario XSOC completed successfully with 185 episodes. + +=== Monolithic SMC Results === +XSOC: rho = 0.7514 ± 0.0998 + +============================================================ +SUMMARY +============================================================ +Command: python compare_analysis.py --scenario "XSOC" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_XSOC" +============================================================ + +real 8m34.106s +user 8m16.839s +sys 0m6.503s From 3493635013ed38dd21b8ab26078449584084889e Mon Sep 17 00:00:00 2001 From: beyazit-y Date: Mon, 15 Dec 2025 22:32:03 -0800 Subject: [PATCH 19/20] Lower error bound in exps --- examples/compositional_analysis/exps.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/compositional_analysis/exps.sh b/examples/compositional_analysis/exps.sh index b559726..53fc722 100755 --- a/examples/compositional_analysis/exps.sh +++ b/examples/compositional_analysis/exps.sh @@ -1,14 +1,14 @@ # Fixed Error (epsilon = 0.1) ## Monolithic -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SX --confidence_level 0.95 --error_bound 0.1 --scenario "SX"; } &> storage/results/fixed_error_monolithic_SX.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_COS --confidence_level 0.95 --error_bound 0.1 --scenario "COS"; } &> storage/results/fixed_error_monolithic_COS.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_XSOC --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC"; } &> storage/results/fixed_error_monolithic_XSOC.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SOCXS --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS"; } &> storage/results/fixed_error_monolithic_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SX --confidence_level 0.95 --error_bound 0.04295 --scenario "SX"; } &> storage/results/fixed_error_monolithic_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_COS --confidence_level 0.95 --error_bound 0.04295 --scenario "COS"; } &> storage/results/fixed_error_monolithic_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_XSOC --confidence_level 0.95 --error_bound 0.04295 --scenario "XSOC"; } &> storage/results/fixed_error_monolithic_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SOCXS --confidence_level 0.95 --error_bound 0.04295 --scenario "SOCXS"; } &> storage/results/fixed_error_monolithic_SOCXS.txt & ## Compositional -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SX --confidence_level 0.95 --error_bound 0.1 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_COS --confidence_level 0.95 --error_bound 0.1 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_XSOC --confidence_level 0.95 --error_bound 0.1 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt & -{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SOCXS --confidence_level 0.95 --error_bound 0.1 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SX --confidence_level 0.95 --error_bound 0.04295 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_COS --confidence_level 0.95 --error_bound 0.04295 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_XSOC --confidence_level 0.95 --error_bound 0.04295 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt & +{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SOCXS --confidence_level 0.95 --error_bound 0.04295 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt & # Fixed Time Budget (2 mins) ## Monolithic From 68da456a2eb72ac48ed9dcbbacd6ba8536278f48 Mon Sep 17 00:00:00 2001 From: beyazit Date: Tue, 16 Dec 2025 07:30:51 +0000 Subject: [PATCH 20/20] Add new results --- .../results/fixed_error_compositional_COS.txt | 34 +++++++------- .../fixed_error_compositional_SOCXS.txt | 44 +++++++++---------- .../results/fixed_error_compositional_SX.txt | 24 +++++----- .../fixed_error_compositional_XSOC.txt | 36 +++++++-------- .../results/fixed_error_monolithic_COS.txt | 16 +++---- .../results/fixed_error_monolithic_SOCXS.txt | 16 +++---- .../results/fixed_error_monolithic_SX.txt | 16 +++---- .../results/fixed_error_monolithic_XSOC.txt | 16 +++---- .../results/fixed_time_compositional_COS.txt | 27 ++++++------ .../fixed_time_compositional_SOCXS.txt | 35 ++++++++------- .../results/fixed_time_compositional_SX.txt | 22 +++++----- .../results/fixed_time_compositional_XSOC.txt | 33 +++++++------- .../results/fixed_time_monolithic_COS.txt | 12 ++--- .../results/fixed_time_monolithic_SOCXS.txt | 8 ++-- .../results/fixed_time_monolithic_SX.txt | 12 ++--- .../results/fixed_time_monolithic_XSOC.txt | 14 +++--- 16 files changed, 183 insertions(+), 182 deletions(-) diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt index 50e322a..d3d84b2 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_COS.txt @@ -1,16 +1,14 @@ Running COMPOSITIONAL SMC on primitive cases (what compositional will use) === Generating Traces (Parallel - HARD STOP) === -Launching scenario S Launching scenario C Launching scenario O +Launching scenario S [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] [INFO] Render Mode: none [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 -[INFO] Known Pipes: glxGraphicsPipe -[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -18,6 +16,8 @@ Launching scenario O [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 [INFO] Known Pipes: glxGraphicsPipe +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 @@ -30,19 +30,19 @@ Launching scenario O [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. -[PID=1879301] Starting scenario S -[PID=1879301] Finished scenario S -[PID=1879303] Starting scenario O -[PID=1879303] Finished scenario O -[PID=1879302] Starting scenario C -[PID=1879302] Finished scenario C -All processes finished naturally (elapsed: 249.00s) -[INFO] Scenario S completed successfully with 185 episodes. -[INFO] Scenario C completed successfully with 185 episodes. -[INFO] Scenario O completed successfully with 185 episodes. +[PID=1889097] Starting scenario S +[PID=1889097] Finished scenario S +[PID=1889096] Starting scenario O +[PID=1889096] Finished scenario O +[PID=1889095] Starting scenario C +[PID=1889095] Finished scenario C +All processes finished naturally (elapsed: 1013.29s) +[INFO] Scenario C completed successfully with 1000 episodes. +[INFO] Scenario O completed successfully with 1000 episodes. +[INFO] Scenario S completed successfully with 1000 episodes. === Running Compositional SMC === -Estimated COS: rho = 0.7133 ± 0.1591 +Estimated COS: rho = 0.7152 ± 0.0685 ============================================================ SUMMARY @@ -50,6 +50,6 @@ SUMMARY Command: python compare_analysis.py --scenario "COS" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_COS" ============================================================ -real 4m15.541s -user 10m47.982s -sys 0m18.648s +real 17m0.943s +user 42m57.621s +sys 0m28.489s diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt index 514e4d5..8812c25 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_SOCXS.txt @@ -1,9 +1,9 @@ Running COMPOSITIONAL SMC on primitive cases (what compositional will use) === Generating Traces (Parallel - HARD STOP) === Launching scenario X -Launching scenario C Launching scenario S Launching scenario O +Launching scenario C [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -18,16 +18,14 @@ Launching scenario O [INFO] Render Mode: none [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] [INFO] Render Mode: none [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 -[INFO] Known Pipes: glxGraphicsPipe -[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 -[INFO] Known Pipes: glxGraphicsPipe -[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -36,26 +34,28 @@ Launching scenario O [INFO] Assets version: 0.4.3 [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. -[PID=1879281] Starting scenario S -[PID=1879281] Finished scenario S -[PID=1879279] Starting scenario X -[PID=1879279] Finished scenario X -[PID=1879280] Starting scenario C -[PID=1879280] Finished scenario C -[PID=1879282] Starting scenario O -[PID=1879282] Finished scenario O -All processes finished naturally (elapsed: 245.35s) -[INFO] Scenario X completed successfully with 185 episodes. -[INFO] Scenario C completed successfully with 185 episodes. -[INFO] Scenario S completed successfully with 185 episodes. -[INFO] Scenario O completed successfully with 185 episodes. +[PID=1889100] Starting scenario S +[PID=1889100] Finished scenario S +[PID=1889099] Starting scenario X +[PID=1889099] Finished scenario X +[PID=1889101] Starting scenario O +[PID=1889101] Finished scenario O +[PID=1889102] Starting scenario C +[PID=1889102] Finished scenario C +All processes finished naturally (elapsed: 1017.08s) +[INFO] Scenario X completed successfully with 1000 episodes. +[INFO] Scenario S completed successfully with 1000 episodes. +[INFO] Scenario O completed successfully with 1000 episodes. +[INFO] Scenario C completed successfully with 1000 episodes. === Running Compositional SMC === -Estimated SOCXS: rho = 0.6622 ± 0.2027 +Estimated SOCXS: rho = 0.6733 ± 0.0873 ============================================================ SUMMARY @@ -63,6 +63,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SOCXS" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_SOCXS" ============================================================ -real 4m11.566s -user 13m26.132s -sys 0m14.524s +real 17m5.056s +user 53m39.961s +sys 0m30.265s diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt index fb44878..8b9f0d5 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_SX.txt @@ -1,7 +1,7 @@ Running COMPOSITIONAL SMC on primitive cases (what compositional will use) === Generating Traces (Parallel - HARD STOP) === -Launching scenario S Launching scenario X +Launching scenario S [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -20,16 +20,16 @@ Launching scenario X [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. -[PID=1879284] Starting scenario S -[PID=1879284] Finished scenario S -[PID=1879285] Starting scenario X -[PID=1879285] Finished scenario X -All processes finished naturally (elapsed: 193.15s) -[INFO] Scenario S completed successfully with 185 episodes. -[INFO] Scenario X completed successfully with 185 episodes. +[PID=1889090] Starting scenario S +[PID=1889090] Finished scenario S +[PID=1889089] Starting scenario X +[PID=1889089] Finished scenario X +All processes finished naturally (elapsed: 719.05s) +[INFO] Scenario X completed successfully with 1000 episodes. +[INFO] Scenario S completed successfully with 1000 episodes. === Running Compositional SMC === -Estimated SX: rho = 0.9500 ± 0.1474 +Estimated SX: rho = 0.9497 ± 0.0640 ============================================================ SUMMARY @@ -37,6 +37,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SX" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_SX" ============================================================ -real 3m19.564s -user 5m56.348s -sys 0m12.423s +real 12m6.153s +user 21m41.154s +sys 0m18.462s diff --git a/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt index b7df49f..0be13bd 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_compositional_XSOC.txt @@ -1,8 +1,8 @@ Running COMPOSITIONAL SMC on primitive cases (what compositional will use) === Generating Traces (Parallel - HARD STOP) === -Launching scenario S Launching scenario X Launching scenario C +Launching scenario S Launching scenario O [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 @@ -40,22 +40,22 @@ Launching scenario O [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. -[PID=1879290] Starting scenario S -[PID=1879290] Finished scenario S -[PID=1879291] Starting scenario X -[PID=1879291] Finished scenario X -[PID=1879292] Starting scenario C -[PID=1879292] Finished scenario C -[PID=1879293] Starting scenario O -[PID=1879293] Finished scenario O -All processes finished naturally (elapsed: 245.15s) -[INFO] Scenario S completed successfully with 185 episodes. -[INFO] Scenario X completed successfully with 185 episodes. -[INFO] Scenario C completed successfully with 185 episodes. -[INFO] Scenario O completed successfully with 185 episodes. +[PID=1889109] Starting scenario S +[PID=1889109] Finished scenario S +[PID=1889107] Starting scenario X +[PID=1889107] Finished scenario X +[PID=1889110] Starting scenario O +[PID=1889110] Finished scenario O +[PID=1889108] Starting scenario C +[PID=1889108] Finished scenario C +All processes finished naturally (elapsed: 1016.35s) +[INFO] Scenario X completed successfully with 1000 episodes. +[INFO] Scenario C completed successfully with 1000 episodes. +[INFO] Scenario S completed successfully with 1000 episodes. +[INFO] Scenario O completed successfully with 1000 episodes. === Running Compositional SMC === -Estimated XSOC: rho = 0.6563 ± 0.1833 +Estimated XSOC: rho = 0.6705 ± 0.0789 ============================================================ SUMMARY @@ -63,6 +63,6 @@ SUMMARY Command: python compare_analysis.py --scenario "XSOC" --compositional --expert --time_budget N/A --save_dir "storage/fixed_error_compositional_XSOC" ============================================================ -real 4m11.670s -user 13m41.638s -sys 0m22.259s +real 17m4.733s +user 54m51.710s +sys 0m36.699s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt index d7a2262..f3b3ea9 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_COS.txt @@ -10,13 +10,13 @@ Launching scenario COS [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. -[PID=1879309] Starting scenario COS -[PID=1879309] Finished scenario COS -All processes finished naturally (elapsed: 477.32s) -[INFO] Scenario COS completed successfully with 185 episodes. +[PID=1889105] Starting scenario COS +[PID=1889105] Finished scenario COS +All processes finished naturally (elapsed: 2133.73s) +[INFO] Scenario COS completed successfully with 1000 episodes. === Monolithic SMC Results === -COS: rho = 0.8378 ± 0.0998 +COS: rho = 0.7560 ± 0.0429 ============================================================ SUMMARY @@ -24,6 +24,6 @@ SUMMARY Command: python compare_analysis.py --scenario "COS" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_COS" ============================================================ -real 8m3.349s -user 7m43.820s -sys 0m6.999s +real 35m40.316s +user 34m57.279s +sys 0m17.355s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt index 9f9f001..fb8e7a3 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SOCXS.txt @@ -10,13 +10,13 @@ Launching scenario SOCXS [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. -[PID=1879262] Starting scenario SOCXS -[PID=1879262] Finished scenario SOCXS -All processes finished naturally (elapsed: 605.60s) -[INFO] Scenario SOCXS completed successfully with 185 episodes. +[PID=1889087] Starting scenario SOCXS +[PID=1889087] Finished scenario SOCXS +All processes finished naturally (elapsed: 2883.29s) +[INFO] Scenario SOCXS completed successfully with 1000 episodes. === Monolithic SMC Results === -SOCXS: rho = 0.6649 ± 0.0998 +SOCXS: rho = 0.6780 ± 0.0429 ============================================================ SUMMARY @@ -24,6 +24,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SOCXS" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_SOCXS" ============================================================ -real 10m11.484s -user 9m51.459s -sys 0m7.801s +real 48m9.998s +user 47m19.091s +sys 0m21.593s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt index a82dbb9..d56e9a0 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_SX.txt @@ -10,13 +10,13 @@ Launching scenario SX [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. -[PID=1879323] Starting scenario SX -[PID=1879323] Finished scenario SX -All processes finished naturally (elapsed: 243.17s) -[INFO] Scenario SX completed successfully with 185 episodes. +[PID=1889068] Starting scenario SX +[PID=1889068] Finished scenario SX +All processes finished naturally (elapsed: 980.50s) +[INFO] Scenario SX completed successfully with 1000 episodes. === Monolithic SMC Results === -SX: rho = 0.9676 ± 0.0998 +SX: rho = 0.9900 ± 0.0429 ============================================================ SUMMARY @@ -24,6 +24,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SX" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_SX" ============================================================ -real 4m9.268s -user 3m52.543s -sys 0m6.364s +real 16m26.267s +user 16m4.159s +sys 0m7.722s diff --git a/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt index c70f4f0..be3bcf4 100644 --- a/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt +++ b/examples/compositional_analysis/storage/results/fixed_error_monolithic_XSOC.txt @@ -10,13 +10,13 @@ Launching scenario XSOC [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. -[PID=1879268] Starting scenario XSOC -[PID=1879268] Finished scenario XSOC -All processes finished naturally (elapsed: 508.30s) -[INFO] Scenario XSOC completed successfully with 185 episodes. +[PID=1889092] Starting scenario XSOC +[PID=1889092] Finished scenario XSOC +All processes finished naturally (elapsed: 2393.20s) +[INFO] Scenario XSOC completed successfully with 1000 episodes. === Monolithic SMC Results === -XSOC: rho = 0.7514 ± 0.0998 +XSOC: rho = 0.7050 ± 0.0429 ============================================================ SUMMARY @@ -24,6 +24,6 @@ SUMMARY Command: python compare_analysis.py --scenario "XSOC" --expert --time_budget N/A --save_dir "storage/fixed_error_monolithic_XSOC" ============================================================ -real 8m34.106s -user 8m16.839s -sys 0m6.503s +real 39m59.756s +user 39m13.651s +sys 0m18.494s diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt index f32377c..538ecbc 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_COS.txt @@ -19,7 +19,6 @@ Launching scenario C [INFO] Assets version: 0.4.3 [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 -[INFO] Use Torch PPO expert. [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -30,19 +29,21 @@ Launching scenario C [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. +[INFO] Use Torch PPO expert. -[HARD STOP] Time budget (120s) reached at 120.09s +[HARD STOP] Time budget (120s) reached at 120.10s Terminating all running processes... -Terminating scenario S (PID=1879264) -Terminating scenario O (PID=1879265) -Terminating scenario C (PID=1879267) -[INFO] Scenario S has 99 completed episodes. -[INFO] Scenario O: Removing partial episode (had 60 episodes, keeping 59) -[INFO] Scenario O has 59 completed episodes. -[INFO] Scenario C has 50 completed episodes. +Terminating scenario S (PID=1889117) +Terminating scenario O (PID=1889118) +Terminating scenario C (PID=1889119) +[INFO] Scenario S has 90 completed episodes. +[INFO] Scenario O: Removing partial episode (had 51 episodes, keeping 50) +[INFO] Scenario O has 50 completed episodes. +[INFO] Scenario C: Removing partial episode (had 48 episodes, keeping 47) +[INFO] Scenario C has 47 completed episodes. === Running Compositional SMC === -Estimated COS: rho = 0.6759 ± 0.2614 +Estimated COS: rho = 0.6896 ± 0.2796 ============================================================ SUMMARY @@ -50,6 +51,6 @@ SUMMARY Command: python compare_analysis.py --scenario "COS" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_COS" ============================================================ -real 2m8.534s -user 5m59.436s -sys 0m10.350s +real 2m8.291s +user 5m46.844s +sys 0m17.067s diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt index 2d12500..c41aa8f 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_SOCXS.txt @@ -1,9 +1,9 @@ Running COMPOSITIONAL SMC on primitive cases (what compositional will use) === Generating Traces (Parallel - HARD STOP) === -Launching scenario X Launching scenario C -Launching scenario O +Launching scenario X Launching scenario S +Launching scenario O [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -18,14 +18,14 @@ Launching scenario S [INFO] Render Mode: none [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 +[INFO] Known Pipes: glxGraphicsPipe +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] [INFO] Render Mode: none [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 -[INFO] Known Pipes: glxGraphicsPipe -[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -43,19 +43,20 @@ Launching scenario S [HARD STOP] Time budget (120s) reached at 120.09s Terminating all running processes... -Terminating scenario X (PID=1879257) -Terminating scenario C (PID=1879258) -Terminating scenario O (PID=1879259) -Terminating scenario S (PID=1879260) +Terminating scenario C (PID=1889073) +Terminating scenario X (PID=1889074) +Terminating scenario S (PID=1889075) +Terminating scenario O (PID=1889076) +[INFO] Scenario C has 60 completed episodes. +[INFO] Scenario X: Removing partial episode (had 75 episodes, keeping 74) [INFO] Scenario X has 74 completed episodes. -[INFO] Scenario C has 50 completed episodes. -[INFO] Scenario O: Removing partial episode (had 54 episodes, keeping 53) -[INFO] Scenario O has 53 completed episodes. -[INFO] Scenario S: Removing partial episode (had 99 episodes, keeping 97) -[INFO] Scenario S has 97 completed episodes. +[INFO] Scenario S: Removing partial episode (had 107 episodes, keeping 105) +[INFO] Scenario S has 105 completed episodes. +[INFO] Scenario O: Removing partial episode (had 59 episodes, keeping 57) +[INFO] Scenario O has 57 completed episodes. === Running Compositional SMC === -Estimated SOCXS: rho = 0.6790 ± 0.3496 +Estimated SOCXS: rho = 0.6843 ± 0.3473 ============================================================ SUMMARY @@ -63,6 +64,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SOCXS" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_SOCXS" ============================================================ -real 2m9.156s -user 8m16.055s -sys 0m18.684s +real 2m8.564s +user 8m5.151s +sys 0m14.412s diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt index 5e42dcb..144005c 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_SX.txt @@ -1,7 +1,7 @@ Running COMPOSITIONAL SMC on primitive cases (what compositional will use) === Generating Traces (Parallel - HARD STOP) === -Launching scenario S Launching scenario X +Launching scenario S [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -21,16 +21,16 @@ Launching scenario X [INFO] Use Torch PPO expert. [INFO] Use Torch PPO expert. -[HARD STOP] Time budget (120s) reached at 120.09s +[HARD STOP] Time budget (120s) reached at 120.01s Terminating all running processes... -Terminating scenario S (PID=1879297) -Terminating scenario X (PID=1879298) -[INFO] Scenario S has 90 completed episodes. -[INFO] Scenario X: Removing partial episode (had 67 episodes, keeping 66) -[INFO] Scenario X has 66 completed episodes. +Terminating scenario X (PID=1889070) +Terminating scenario S (PID=1889071) +[INFO] Scenario X has 77 completed episodes. +[INFO] Scenario S: Removing partial episode (had 105 episodes, keeping 104) +[INFO] Scenario S has 104 completed episodes. === Running Compositional SMC === -Estimated SX: rho = 0.9527 ± 0.2296 +Estimated SX: rho = 0.8978 ± 0.2126 ============================================================ SUMMARY @@ -38,6 +38,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SX" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_SX" ============================================================ -real 2m8.718s -user 4m3.019s -sys 0m13.507s +real 2m6.924s +user 4m10.098s +sys 0m9.144s diff --git a/examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt index 9736880..682887d 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_compositional_XSOC.txt @@ -1,9 +1,9 @@ Running COMPOSITIONAL SMC on primitive cases (what compositional will use) === Generating Traces (Parallel - HARD STOP) === Launching scenario O -Launching scenario S Launching scenario C Launching scenario X +Launching scenario S [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] @@ -27,13 +27,13 @@ Launching scenario X [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 [INFO] Known Pipes: glxGraphicsPipe -[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Environment: MetaDriveEnv [INFO] MetaDrive version: 0.4.3 [INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector()] [INFO] Render Mode: none [INFO] Horizon (Max steps per agent): 2000 [INFO] Assets version: 0.4.3 +[INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Known Pipes: glxGraphicsPipe [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. @@ -43,20 +43,19 @@ Launching scenario X [HARD STOP] Time budget (120s) reached at 120.09s Terminating all running processes... -Terminating scenario O (PID=1879312) -Terminating scenario S (PID=1879313) -Terminating scenario C (PID=1879314) -Terminating scenario X (PID=1879315) -[INFO] Scenario O has 57 completed episodes. -[INFO] Scenario S: Removing partial episode (had 93 episodes, keeping 92) -[INFO] Scenario S has 92 completed episodes. -[INFO] Scenario C: Removing partial episode (had 51 episodes, keeping 50) -[INFO] Scenario C has 50 completed episodes. -[INFO] Scenario X: Removing partial episode (had 71 episodes, keeping 69) -[INFO] Scenario X has 69 completed episodes. +Terminating scenario O (PID=1889078) +Terminating scenario C (PID=1889079) +Terminating scenario X (PID=1889080) +Terminating scenario S (PID=1889081) +[INFO] Scenario O has 49 completed episodes. +[INFO] Scenario C has 54 completed episodes. +[INFO] Scenario X: Removing partial episode (had 69 episodes, keeping 68) +[INFO] Scenario X has 68 completed episodes. +[INFO] Scenario S: Removing partial episode (had 102 episodes, keeping 99) +[INFO] Scenario S has 99 completed episodes. === Running Compositional SMC === -Estimated XSOC: rho = 0.5891 ± 0.3120 +Estimated XSOC: rho = 0.7358 ± 0.3406 ============================================================ SUMMARY @@ -64,6 +63,6 @@ SUMMARY Command: python compare_analysis.py --scenario "XSOC" --compositional --expert --time_budget 120 --save_dir "storage/fixed_time_compositional_XSOC" ============================================================ -real 2m9.137s -user 7m22.034s -sys 0m18.302s +real 2m8.241s +user 7m27.822s +sys 0m15.124s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt index 57e9343..292933c 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_COS.txt @@ -13,11 +13,11 @@ Launching scenario COS [HARD STOP] Time budget (120s) reached at 120.08s Terminating all running processes... -Terminating scenario COS (PID=1879277) -[INFO] Scenario COS has 25 completed episodes. +Terminating scenario COS (PID=1889114) +[INFO] Scenario COS has 20 completed episodes. === Monolithic SMC Results === -COS: rho = 0.7200 ± 0.2716 +COS: rho = 0.8500 ± 0.3037 ============================================================ SUMMARY @@ -25,6 +25,6 @@ SUMMARY Command: python compare_analysis.py --scenario "COS" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_COS" ============================================================ -real 2m6.191s -user 1m55.053s -sys 0m4.688s +real 2m6.928s +user 1m51.736s +sys 0m6.374s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt index 9b6f692..4374a4b 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SOCXS.txt @@ -13,7 +13,7 @@ Launching scenario SOCXS [HARD STOP] Time budget (120s) reached at 120.08s Terminating all running processes... -Terminating scenario SOCXS (PID=1879255) +Terminating scenario SOCXS (PID=1889085) [INFO] Scenario SOCXS has 16 completed episodes. === Monolithic SMC Results === @@ -25,6 +25,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SOCXS" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_SOCXS" ============================================================ -real 2m5.764s -user 1m55.900s -sys 0m3.838s +real 2m6.443s +user 1m55.832s +sys 0m4.647s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt index 198f914..7895b3a 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_SX.txt @@ -13,11 +13,11 @@ Launching scenario SX [HARD STOP] Time budget (120s) reached at 120.09s Terminating all running processes... -Terminating scenario SX (PID=1879295) -[INFO] Scenario SX has 46 completed episodes. +Terminating scenario SX (PID=1889064) +[INFO] Scenario SX has 52 completed episodes. === Monolithic SMC Results === -SX: rho = 1.0000 ± 0.2002 +SX: rho = 1.0000 ± 0.1883 ============================================================ SUMMARY @@ -25,6 +25,6 @@ SUMMARY Command: python compare_analysis.py --scenario "SX" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_SX" ============================================================ -real 2m6.655s -user 1m52.599s -sys 0m5.580s +real 2m5.595s +user 1m57.325s +sys 0m3.691s diff --git a/examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt b/examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt index 7c0f7de..933e6bd 100644 --- a/examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt +++ b/examples/compositional_analysis/storage/results/fixed_time_monolithic_XSOC.txt @@ -11,13 +11,13 @@ Launching scenario XSOC [INFO] Start Scenario Index: 1000, Num Scenarios : 1000 [INFO] Use Torch PPO expert. -[HARD STOP] Time budget (120s) reached at 120.09s +[HARD STOP] Time budget (120s) reached at 120.08s Terminating all running processes... -Terminating scenario XSOC (PID=1879288) -[INFO] Scenario XSOC has 18 completed episodes. +Terminating scenario XSOC (PID=1889066) +[INFO] Scenario XSOC has 22 completed episodes. === Monolithic SMC Results === -XSOC: rho = 0.7778 ± 0.3201 +XSOC: rho = 0.7727 ± 0.2895 ============================================================ SUMMARY @@ -25,6 +25,6 @@ SUMMARY Command: python compare_analysis.py --scenario "XSOC" --expert --time_budget 120 --save_dir "storage/fixed_time_monolithic_XSOC" ============================================================ -real 2m6.538s -user 1m52.206s -sys 0m5.747s +real 2m5.619s +user 2m0.372s +sys 0m3.612s