solve conflicts

.github/workflows/build.yml

@@ -15,4 +15,4 @@ jobs:
              --build-arg PYTHON_VERSION=${{ matrix.python-version }} \
              --tag gym-docker .
       - name: Run tests
-        run: docker run gym-docker pytest --forked --import-mode=append
+        run: docker run gym-docker pytest --import-mode=append

README.md

@@ -32,10 +32,10 @@ env.close()
 
 ## Notable Related Libraries
 
-* [Stable Baselines 3](https://github.com/DLR-RM/stable-baselines3) is a learning library based on the Gym API. It is our recommendation for beginners who want to start learning things quickly.
-* [RL Baselines3 Zoo](https://github.com/DLR-RM/rl-baselines3-zoo) builds upon SB3, containing optimal hyperparameters for Gym environments as well as code to easily find new ones. Such tuning is almost always required.
-* The [Autonomous Learning Library](https://github.com/cpnota/autonomous-learning-library) and [Tianshou](https://github.com/thu-ml/tianshou) are two reinforcement learning libraries I like that are generally geared towards more experienced users.
-* [RLlib](https://docs.ray.io/en/latest/rllib/index.html) is a commonly used library that allows for distributed training and inferencing.
+* [Stable Baselines 3](https://github.com/DLR-RM/stable-baselines3) is a learning library based on the Gym API. It is designed to cater to complete beginners in the field who want to start learning things quickly.
+* [RL Baselines3 Zoo](https://github.com/DLR-RM/rl-baselines3-zoo) builds upon SB3, containing optimal hyperparameters for Gym environments as well as code to easily find new ones.
+* [Tianshou](https://github.com/thu-ml/tianshou) is a learning library that's geared towards very experienced users and is design to allow for ease in complex algorithm modifications.
+* [RLlib](https://docs.ray.io/en/latest/rllib/index.html) is a learning library that allows for distributed training and inferencing and supports an extraordinarily large number of features throughout the reinforcement learning space.
 * [PettingZoo](https://github.com/Farama-Foundation/PettingZoo) is like Gym, but for environments with multiple agents.
 
 ## Environment Versioning

gym/__init__.py

@@ -14,15 +14,22 @@
 from gym import vector
 from gym import wrappers
 import os
-
+import sys
 
 __all__ = ["Env", "Space", "Wrapper", "make", "spec", "register"]
 
+# Initializing pygame initializes audio connections through SDL. SDL uses alsa by default on all Linux systems
+# SDL connecting to alsa frequently create these giant lists of warnings every time you import an environment using
+#   pygame
+# DSP is far more benign (and should probably be the default in SDL anyways)
+
+if sys.platform.startswith("linux"):
+    os.environ["SDL_AUDIODRIVER"] = "dsp"
+
 os.environ["PYGAME_HIDE_SUPPORT_PROMPT"] = "hide"
 
 try:
     import gym_notices.notices as notices
-    import sys
 
     # print version warning if necessary
     notice = notices.notices.get(__version__)

gym/envs/box2d/car_racing.py

@@ -169,6 +169,8 @@ def __init__(self, render_mode="human", verbose=1, lap_complete_percent=0.95):
             shape=polygonShape(vertices=[(0, 0), (1, 0), (1, -1), (0, -1)])
         )
 
+        # This will throw a warning in tests/envs/test_envs in utils/env_checker.py as the space is not symmetric
+        #   or normalised however this is not possible here so ignore
         self.action_space = spaces.Box(
             np.array([-1, 0, 0]).astype(np.float32),
             np.array([+1, +1, +1]).astype(np.float32),
@@ -613,9 +615,9 @@ def _create_image_array(self, screen, size):
         )
 
     def close(self):
+        pygame.quit()
         if self.screen is not None:
             pygame.display.quit()
-            pygame.quit()
             self.isopen = False
 
 

gym/envs/classic_control/pendulum.py

@@ -99,6 +99,9 @@ def __init__(self, render_mode="human", g=10.0):
         self.render_list = []
 
         high = np.array([1.0, 1.0, self.max_speed], dtype=np.float32)
+        # This will throw a warning in tests/envs/test_envs in utils/env_checker.py as the space is not symmetric
+        #   or normalised as max_torque == 2 by default. Ignoring the issue here as the default settings are too old
+        #   to update to follow the openai gym api
         self.action_space = spaces.Box(
             low=-self.max_torque, high=self.max_torque, shape=(1,), dtype=np.float32
         )
@@ -202,7 +205,7 @@ def _render(self):
             scale_img = pygame.transform.smoothscale(
                 img, (scale * np.abs(self.last_u) / 2, scale * np.abs(self.last_u) / 2)
             )
-            is_flip = self.last_u > 0
+            is_flip = bool(self.last_u > 0)
             scale_img = pygame.transform.flip(scale_img, is_flip, True)
             self.surf.blit(
                 scale_img,

gym/envs/mujoco/pusher.py

@@ -2,8 +2,132 @@
 from gym import utils
 from gym.envs.mujoco import mujoco_env
 
+import mujoco_py
+
 
 class PusherEnv(mujoco_env.MujocoEnv, utils.EzPickle):
+    """
+    ### Description
+    "Pusher" is a multi-jointed robot arm which is very similar to that of a human.
+     The goal is to move a target cylinder (called *object*) to a goal position using the robot's end effector (called *fingertip*).
+      The robot consists of shoulder, elbow, forearm, and wrist joints.
+
+    ### Action Space
+    The action space is a `Box(-2, 2, (7,), float32)`. An action `(a, b)` represents the torques applied at the hinge joints.
+
+    | Num | Action                                                             | Control Min | Control Max | Name (in corresponding XML file) | Joint | Unit         |
+    |-----|--------------------------------------------------------------------|-------------|-------------|----------------------------------|-------|--------------|
+    | 0    | Rotation of the panning the shoulder                              | -2          | 2           | r_shoulder_pan_joint             | hinge | torque (N m) |
+    | 1    | Rotation of the shoulder lifting joint                            | -2          | 2           | r_shoulder_lift_joint            | hinge | torque (N m) |
+    | 2    | Rotation of the shoulder rolling joint                            | -2          | 2           | r_upper_arm_roll_joint           | hinge | torque (N m) |
+    | 3    | Rotation of hinge joint that flexed the elbow                     | -2          | 2           | r_elbow_flex_joint               | hinge | torque (N m) |
+    | 4    | Rotation of hinge that rolls the forearm                          | -2          | 2           | r_forearm_roll_joint             | hinge | torque (N m) |
+    | 5    | Rotation of flexing the wrist                                     | -2          | 2           | r_wrist_flex_joint               | hinge | torque (N m) |
+    | 6    | Rotation of rolling the wrist                                     | -2          | 2           | r_wrist_roll_joint               | hinge | torque (N m) |
+
+    ### Observation Space
+
+    Observations consist of
+
+    - Angle of rotational joints on the pusher
+    - Anglular velocities of rotational joints on the pusher
+    - The coordinates of the fingertip of the pusher
+    - The coordinates of the object to be moved
+    - The coordinates of the goal position
+
+    The observation is a `ndarray` with shape `(23,)` where the elements correspond to the table below.
+    An analogy can be drawn to a human arm in order to help understand the state space, with the words flex and roll meaning the
+    same as human joints.
+
+    | Num | Observation           | Min                  | Max                | Name (in corresponding XML file) | Joint| Unit |
+    |-----|-----------------------|----------------------|--------------------|----------------------|--------------------|--------------------|
+    | 0    | Rotation of the panning the shoulder                              | -Inf                 | Inf                | r_shoulder_pan_joint | hinge | angle (rad) |
+    | 1    | Rotation of the shoulder lifting joint                            | -Inf                 | Inf                | r_shoulder_lift_joint | hinge | angle (rad) |
+    | 2    | Rotation of the shoulder rolling joint                            | -Inf                 | Inf                | r_upper_arm_roll_joint | hinge | angle (rad) |
+    | 3    | Rotation of hinge joint that flexed the elbow                     | -Inf                 | Inf                | r_elbow_flex_joint | hinge | angle (rad) |
+    | 4    | Rotation of hinge that rolls the forearm                          | -Inf                 | Inf                | r_forearm_roll_joint | hinge | angle (rad) |
+    | 5    | Rotation of flexing the wrist                                     | -Inf                 | Inf                | r_wrist_flex_joint | hinge | angle (rad) |
+    | 6    | Rotation of rolling the wrist                                     | -Inf                 | Inf                | r_wrist_roll_joint | hinge | angle (rad) |
+    | 7    | Rotational velocity of the panning the shoulder                   | -Inf                 | Inf                | r_shoulder_pan_joint | hinge | angular velocity (rad/s) |
+    | 8    | Rotational velocity of the shoulder lifting joint                 | -Inf                 | Inf                | r_shoulder_lift_joint | hinge | angular velocity (rad/s) |
+    | 9    | Rotational velocity of the shoulder rolling joint                 | -Inf                 | Inf                | r_upper_arm_roll_joint | hinge | angular velocity (rad/s) |
+    | 10   | Rotational velocity of hinge joint that flexed the elbow          | -Inf                 | Inf                | r_elbow_flex_joint | hinge | angular velocity (rad/s) |
+    | 11   | Rotational velocity of hinge that rolls the forearm               | -Inf                 | Inf                | r_forearm_roll_joint | hinge | angular velocity (rad/s) |
+    | 12   | Rotational velocity of flexing the wrist                          | -Inf                 | Inf                | r_wrist_flex_joint | hinge | angular velocity (rad/s) |
+    | 13   | Rotational velocity of rolling the wrist                          | -Inf                 | Inf                | r_wrist_roll_joint | hinge | angular velocity (rad/s) |
+    | 14   | x-coordinate of the fingertip of the pusher                       | -Inf                 | Inf                | tips_arm | slide | position (m) |
+    | 15   | y-coordinate of the fingertip of the pusher                       | -Inf                 | Inf                | tips_arm | slide | position (m) |
+    | 16   | z-coordinate of the fingertip of the pusher                       | -Inf                 | Inf                | tips_arm | slide | position (m) |
+    | 17   | x-coordinate of the object to be moved                            | -Inf                 | Inf                | object (obj_slidex) | slide | position (m) |
+    | 18   | y-coordinate of the object to be moved                            | -Inf                 | Inf                | object (obj_slidey) | slide | position (m) |
+    | 19   | z-coordinate of the object to be moved                            | -Inf                 | Inf                | object | cylinder | position (m) |
+    | 20   | x-coordinate of the goal position of the object                   | -Inf                 | Inf                | goal (goal_slidex) | slide | position (m) |
+    | 21   | y-coordinate of the goal position of the object                   | -Inf                 | Inf                | goal (goal_slidey) | slide | position (m) |
+    | 22   | z-coordinate of the goal position of the object                   | -Inf                 | Inf                | goal | sphere | position (m) |
+
+
+    ### Rewards
+    The reward consists of two parts:
+    - *reward_near *: This reward is a measure of how far the *fingertip*
+    of the pusher (the unattached end) is from the object, with a more negative
+    value assigned for when the pusher's *fingertip* is further away from the
+    target. It is calculated as the negative vector norm of (position of
+    the fingertip - position of target), or *-norm("fingertip" - "target")*.
+    - *reward_dist *: This reward is a measure of how far the object is from
+    the target goal position, with a more negative value assigned for object is
+    further away from the target. It is calculated as the negative vector norm of
+    (position of the object - position of goal), or *-norm("object" - "target")*.
+    - *reward_control*: A negative reward for penalising the pusher if
+    it takes actions that are too large. It is measured as the negative squared
+    Euclidean norm of the action, i.e. as *- sum(action<sup>2</sup>)*.
+
+    The total reward returned is ***reward*** *=* *reward_dist + 0.1 * reward_ctrl + 0.5 * reward_near*
+
+    Unlike other environments, Pusher does not allow you to specify weights for the individual reward terms.
+    However, `info` does contain the keys *reward_dist* and *reward_ctrl*. Thus, if you'd like to weight the terms,
+    you should create a wrapper that computes the weighted reward from `info`.
+
+
+    ### Starting State
+    All pusher (not including object and goal) states start in
+    (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0). A uniform noise in the range
+    [-0.005, 0.005] is added to the velocity attributes only. The velocities of
+    the object and goal are permanently set to 0. The object's x-position is selected uniformly
+    between [-0.3, 0] while the y-position is selected uniformly between [-0.2, 0.2], and this
+    process is repeated until the vector norm between the object's (x,y) position and origin is not greater
+    than 0.17. The goal always have the same position of (0.45, -0.05, -0.323).
+
+    The default framerate is 5 with each frame lasting for 0.01, giving rise to a *dt = 5 * 0.01 = 0.05*
+
+    ### Episode Termination
+
+    The episode terminates when any of the following happens:
+
+    1. The episode duration reaches a 100 timesteps.
+    2. Any of the state space values is no longer finite.
+
+    ### Arguments
+
+    No additional arguments are currently supported (in v2 and lower),
+    but modifications can be made to the XML file in the assets folder
+    (or by changing the path to a modified XML file in another folder)..
+
+    ```
+    env = gym.make('Pusher-v2')
+    ```
+
+    There is no v3 for Pusher, unlike the robot environments where a v3 and
+    beyond take gym.make kwargs such as xml_file, ctrl_cost_weight, reset_noise_scale etc.
+
+
+    ### Version History
+
+    * v2: All continuous control environments now use mujoco_py >= 1.50
+    * v1: max_time_steps raised to 1000 for robot based tasks (not including reacher, which has a max_time_steps of 50). Added reward_threshold to environments.
+    * v0: Initial versions release (1.0.0)
+
+    """
+
     def __init__(self, **kwargs):
         utils.EzPickle.__init__(self)
         mujoco_env.MujocoEnv.__init__(self, "pusher.xml", 5, **kwargs)

gym/envs/toy_text/blackjack.py

@@ -292,3 +292,6 @@ def scale_card_img(card_img):
                     np.array(pygame.surfarray.pixels3d(self.screen)), axes=(1, 0, 2)
                 )
             )
+
+
+# Pixel art from Mariia Khmelnytska (https://www.123rf.com/photo_104453049_stock-vector-pixel-art-playing-cards-standart-deck-vector-set.html)