Env - Dogfight

.gitignore

@@ -162,3 +162,4 @@ pufferlib/ocean/impulse_wars/*-release/
 pufferlib/ocean/impulse_wars/debug-*/
 pufferlib/ocean/impulse_wars/release-*/
 pufferlib/ocean/impulse_wars/benchmark/
+pufferlib/ocean/dogfight/dogfight_test

pufferlib/config/default.ini

@@ -28,24 +28,24 @@ device = cuda
 optimizer = muon
 anneal_lr = True
 precision = float32
-total_timesteps = 10_000_000
+total_timesteps = 100_000_000
 learning_rate = 0.015
 gamma = 0.995
-gae_lambda = 0.90
+gae_lambda = 0.95
 update_epochs = 1
 clip_coef = 0.2
 vf_coef = 2.0
 vf_clip_coef = 0.2
 max_grad_norm = 1.5
-ent_coef = 0.001
+ent_coef = 0.01
 adam_beta1 = 0.95
 adam_beta2 = 0.999
 adam_eps = 1e-12
 
 data_dir = experiments
 checkpoint_interval = 200
 batch_size = auto
-minibatch_size = 8192
+minibatch_size = 16384
 
 # Accumulate gradients above this size
 max_minibatch_size = 32768
@@ -58,7 +58,7 @@ vtrace_rho_clip = 1.0
 vtrace_c_clip = 1.0
 
 prio_alpha = 0.8
-prio_beta0 = 0.2
+prio_beta0 = 0.5
 
 [sweep]
 method = Protein 
@@ -83,16 +83,9 @@ max = 1e10
 mean = 2e8
 scale = time
 
-[sweep.train.bptt_horizon]
-distribution = uniform_pow2
-min = 16
-max = 64
-mean = 64
-scale = auto
-
 [sweep.train.minibatch_size]
 distribution = uniform_pow2
-min = 8192
+min = 16384
 max = 65536
 mean = 32768
 scale = auto

pufferlib/config/ocean/dogfight.ini

@@ -0,0 +1,206 @@
+[base]
+env_name = puffer_dogfight
+package = ocean
+policy_name = Policy
+rnn_name = Recurrent
+
+[policy]
+hidden_size = 128
+
+[rnn]
+input_size = 128
+hidden_size = 128
+
+[vec]
+num_envs = 8
+
+[env]
+alt_max = 2500.0
+curriculum_enabled = 1
+curriculum_randomize = 0
+episodes_per_stage = 25
+max_steps = 3000
+num_envs = 1024
+obs_scheme = 4
+penalty_aileron = 0.004787828722037375
+penalty_bias = 0.019452434551902115
+penalty_neg_g = 0.038022669870406395
+penalty_roll = 0.0019647147422656415
+penalty_rudder = 0.00015276678362861277
+penalty_stall = 0.0007385806553065777
+reward_approach = 0.0065743024460971355
+reward_closing_scale = 0.0011914868978783488
+reward_firing_solution = 0.045721526537090544
+reward_level = 0.025920397927984597
+reward_tail_scale = 0.0009967820532619954
+reward_tracking = 0.031819639401510356
+speed_min = 50.0
+aim_cone_start = 0.3856851533800364
+aim_cone_end = 0.05015228554606438
+aim_anneal_episodes = 500
+
+[train]
+adam_beta1 = 0.9723082880428708
+adam_beta2 = 0.9912225347178505
+adam_eps = 8.183951125996682e-13
+batch_size = auto
+bptt_horizon = 64
+checkpoint_interval = 200
+clip_coef = 0.983341504810378
+ent_coef = 0.03071064008271062
+gae_lambda = 0.9949418302404375
+gamma = 0.9855692943246729
+learning_rate = 0.0003102693135543651
+max_grad_norm = 1.955089159309864
+max_minibatch_size = 32768
+minibatch_size = 65536
+prio_alpha = 0.9022484586887103
+prio_beta0 = 0.8983571008600393
+seed = 42
+total_timesteps = 100_000_000
+update_epochs = 4
+vf_clip_coef = 0.4664481597021223
+vf_coef = 1.3376509584486485
+vtrace_c_clip = 0.4391395812854171
+vtrace_rho_clip = 4.6142582874745
+
+[sweep]
+downsample = 1
+goal = maximize
+method = Protein
+metric = ultimate
+prune_pareto = True
+use_gpu = True
+
+[sweep.env.obs_scheme]
+distribution = int_uniform
+max = 5
+mean = 0
+min = 0
+scale = 1.0
+
+[sweep.env.episodes_per_stage]
+distribution = int_uniform
+min = 20
+max = 75
+mean = 25
+scale = 1.0
+
+[sweep.env.penalty_stall]
+distribution = uniform
+max = 0.005
+mean = 0.0016092406492793122
+min = 0.0
+scale = auto
+
+[sweep.env.penalty_roll]
+distribution = uniform
+max = 0.003
+mean = 0.0021072644960864573
+min = 0.0
+scale = auto
+
+[sweep.env.penalty_neg_g]
+distribution = uniform
+max = 0.1
+mean = 0.05
+min = 0.01
+scale = auto
+
+[sweep.env.penalty_rudder]
+distribution = uniform
+max = 0.001
+mean = 0.0002985792260932028
+min = 0.0001
+scale = auto
+
+[sweep.env.penalty_aileron]
+distribution = uniform
+max = 0.005
+mean = 0.002
+min = 0.0
+scale = auto
+
+[sweep.env.penalty_bias]
+distribution = uniform
+max = 0.02
+mean = 0.008614029763839244
+min = 0.001
+scale = auto
+
+[sweep.env.reward_approach]
+distribution = uniform
+max = 0.02
+mean = 0.003836667464147351
+min = 0.0
+scale = auto
+
+[sweep.env.reward_level]
+distribution = uniform
+max = 0.05
+mean = 0.029797846539013125
+min = 0.0
+scale = auto
+
+[sweep.env.reward_closing_scale]
+distribution = uniform
+max = 0.005
+mean = 0.005
+min = 0.0
+scale = auto
+
+[sweep.env.reward_firing_solution]
+distribution = uniform
+max = 0.1
+mean = 0.01
+min = 0.0
+scale = auto
+
+[sweep.env.reward_tail_scale]
+distribution = uniform
+max = 0.01
+mean = 0.005
+min = 0.0
+scale = auto
+
+[sweep.env.reward_tracking]
+distribution = uniform
+max = 0.05
+mean = 0.005177132307187232
+min = 0.0
+scale = auto
+
+[sweep.env.aim_cone_start]
+distribution = uniform
+max = 0.52
+mean = 0.35
+min = 0.17
+scale = auto
+
+[sweep.env.aim_cone_end]
+distribution = uniform
+max = 0.17
+mean = 0.087
+min = 0.05
+scale = auto
+
+[sweep.env.aim_anneal_episodes]
+distribution = int_uniform
+min = 50
+max = 5000
+mean = 500
+scale = 1.0
+
+[sweep.train.learning_rate]
+distribution = log_normal
+max = 0.0005
+mean = 9.0e-06
+min = 0.000000001
+scale = 0.5
+
+[sweep.train.total_timesteps]
+distribution = log_normal
+max = 1.01e8
+mean = 1.005e8
+min = 1.0e8
+scale = time

pufferlib/environments/mani_skill/torch.py

@@ -64,7 +64,7 @@ def decode_actions(self, hidden):
         '''Decodes a batch of hidden states into (multi)discrete actions.
         Assumes no time dimension (handled by LSTM wrappers).'''
         mean = self.decoder_mean(hidden)
-        logstd = self.decoder_logstd.expand_as(mean)
+        logstd = self.decoder_logstd.expand_as(mean).clamp(min=-20, max=2)
         std = torch.exp(logstd)
         logits = torch.distributions.Normal(mean, std)
         values = self.value(hidden)

pufferlib/models.py

@@ -88,7 +88,7 @@ def decode_actions(self, hidden):
             logits = self.decoder(hidden).split(self.action_nvec, dim=1)
         elif self.is_continuous:
             mean = self.decoder_mean(hidden)
-            logstd = self.decoder_logstd.expand_as(mean)
+            logstd = self.decoder_logstd.expand_as(mean).clamp(min=-20, max=2)
             std = torch.exp(logstd)
             logits = torch.distributions.Normal(mean, std)
         else: