ray\rllib\agents\trainer.py raise NotImplementedError in Tutorial9_Feature_Generation.ipynb

[eromoe]

in Tutorial9_Feature_Generation.ipynb.

import ray
from ray.tune.registry import register_env
from trademaster.environments.portfolio_management.environment import PortfolioManagementEnvironment
def env_creator(env_name):
    if env_name == 'portfolio_management':
        env = PortfolioManagementEnvironment
    else:
        raise NotImplementedError
    return env
ray.init(ignore_reinit_error=True)
register_env("portfolio_management", lambda config: env_creator("portfolio_management")(config))
trainer.train_and_valid()

error

---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
File c:\envs\py39\lib\site-packages\ray\rllib\agents\trainer.py:935, in Trainer.setup(self, config)
    934 try:
--> 935     self._init(self.config, self.env_creator)
    936 # New design: Override `Trainable.setup()` (as indented by Trainable)
    937 # and do or don't call super().setup() from within your override.
    938 # By default, `super().setup()` will create both worker sets:
   (...)
    941 # parallel to training.
    942 # TODO: Deprecate `_init()` and remove this try/except block.

File c:\envs\py39\lib\site-packages\ray\rllib\agents\trainer.py:1074, in Trainer._init(self, config, env_creator)
   1073 def _init(self, config: TrainerConfigDict, env_creator: EnvCreator) -> None:
-> 1074     raise NotImplementedError

NotImplementedError: 

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
File c:\envs\py39\lib\site-packages\ray\rllib\utils\pre_checks\env.py:67, in check_env(env)
     66 elif isinstance(env, gym.Env):
---> 67     check_gym_environments(env)
     68 elif isinstance(env, BaseEnv):

File c:\envs\py39\lib\site-packages\ray\rllib\utils\pre_checks\env.py:193, in check_gym_environments(env)
    192         raise ValueError(error)
--> 193 _check_done(done)
    194 _check_reward(reward)

File c:\envs\py39\lib\site-packages\ray\rllib\utils\pre_checks\env.py:414, in _check_done(done, base_env, agent_ids)
    413                 raise ValueError(error)
--> 414 elif not isinstance(done, (bool, np.bool, np.bool_)):
    415     error = (
    416         "Your step function must return a done that is a boolean. But instead "
    417         f"was a {type(done)}"
    418     )

File c:\envs\py39\lib\site-packages\numpy\__init__.py:284, in __getattr__(attr)
    282     return Tester
--> 284 raise AttributeError("module {!r} has no attribute "
    285                      "{!r}".format(__name__, attr))

AttributeError: module 'numpy' has no attribute 'bool'

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
Cell In[12], line 12
     10 ray.init(ignore_reinit_error=True)
     11 register_env("portfolio_management", lambda config: env_creator("portfolio_management")(config))
---> 12 trainer.train_and_valid()

File e:\Workspace\github_me\TradeMaster\trademaster\trainers\portfolio_management\trainer.py:133, in PortfolioManagementTrainer.train_and_valid(self)
    131 valid_score_list = []
    132 save_dict_list = []
--> 133 self.trainer = self.trainer_name(
    134     env="portfolio_management", config=self.configs)
    136 for epoch in range(1, self.epochs + 1):
    137     ray.get(f.remote("Train Episode: [{}/{}]".format(epoch, self.epochs)))

File c:\envs\py39\lib\site-packages\ray\rllib\agents\trainer.py:870, in Trainer.__init__(self, config, env, logger_creator, remote_checkpoint_dir, sync_function_tpl)
    858 # Initialize common evaluation_metrics to nan, before they become
    859 # available. We want to make sure the metrics are always present
    860 # (although their values may be nan), so that Tune does not complain
    861 # when we use these as stopping criteria.
    862 self.evaluation_metrics = {
    863     "evaluation": {
    864         "episode_reward_max": np.nan,
   (...)
    867     }
    868 }
--> 870 super().__init__(
    871     config, logger_creator, remote_checkpoint_dir, sync_function_tpl
    872 )

File c:\envs\py39\lib\site-packages\ray\tune\trainable.py:156, in Trainable.__init__(self, config, logger_creator, remote_checkpoint_dir, sync_function_tpl)
    154 start_time = time.time()
    155 self._local_ip = self.get_current_ip()
--> 156 self.setup(copy.deepcopy(self.config))
    157 setup_time = time.time() - start_time
    158 if setup_time > SETUP_TIME_THRESHOLD:

File c:\envs\py39\lib\site-packages\ray\rllib\agents\trainer.py:950, in Trainer.setup(self, config)
    936 # New design: Override `Trainable.setup()` (as indented by Trainable)
    937 # and do or don't call super().setup() from within your override.
    938 # By default, `super().setup()` will create both worker sets:
   (...)
    941 # parallel to training.
    942 # TODO: Deprecate `_init()` and remove this try/except block.
    943 except NotImplementedError:
    944     # Only if user did not override `_init()`:
    945     # - Create rollout workers here automatically.
   (...)
    948     # This matches the behavior of using `build_trainer()`, which
    949     # has been deprecated.
--> 950     self.workers = WorkerSet(
    951         env_creator=self.env_creator,
    952         validate_env=self.validate_env,
    953         policy_class=self.get_default_policy_class(self.config),
    954         trainer_config=self.config,
    955         num_workers=self.config["num_workers"],
    956         local_worker=True,
    957         logdir=self.logdir,
    958     )
    959     # By default, collect metrics for all remote workers.
    960     self._remote_workers_for_metrics = self.workers.remote_workers()

File c:\envs\py39\lib\site-packages\ray\rllib\evaluation\worker_set.py:170, in WorkerSet.__init__(self, env_creator, validate_env, policy_class, trainer_config, num_workers, local_worker, logdir, _setup)
    167     spaces = None
    169 if local_worker:
--> 170     self._local_worker = self._make_worker(
    171         cls=RolloutWorker,
    172         env_creator=env_creator,
    173         validate_env=validate_env,
    174         policy_cls=self._policy_class,
    175         worker_index=0,
    176         num_workers=num_workers,
    177         config=self._local_config,
    178         spaces=spaces,
    179     )

File c:\envs\py39\lib\site-packages\ray\rllib\evaluation\worker_set.py:630, in WorkerSet._make_worker(self, cls, env_creator, validate_env, policy_cls, worker_index, num_workers, recreated_worker, config, spaces)
    627 else:
    628     extra_python_environs = config.get("extra_python_environs_for_worker", None)
--> 630 worker = cls(
    631     env_creator=env_creator,
    632     validate_env=validate_env,
    633     policy_spec=policies,
    634     policy_mapping_fn=config["multiagent"]["policy_mapping_fn"],
    635     policies_to_train=config["multiagent"]["policies_to_train"],
    636     tf_session_creator=(session_creator if config["tf_session_args"] else None),
    637     rollout_fragment_length=config["rollout_fragment_length"],
    638     count_steps_by=config["multiagent"]["count_steps_by"],
    639     batch_mode=config["batch_mode"],
    640     episode_horizon=config["horizon"],
    641     preprocessor_pref=config["preprocessor_pref"],
    642     sample_async=config["sample_async"],
    643     compress_observations=config["compress_observations"],
    644     num_envs=config["num_envs_per_worker"],
    645     observation_fn=config["multiagent"]["observation_fn"],
    646     observation_filter=config["observation_filter"],
    647     clip_rewards=config["clip_rewards"],
    648     normalize_actions=config["normalize_actions"],
    649     clip_actions=config["clip_actions"],
    650     env_config=config["env_config"],
    651     policy_config=config,
    652     worker_index=worker_index,
    653     num_workers=num_workers,
    654     recreated_worker=recreated_worker,
    655     record_env=config["record_env"],
    656     log_dir=self._logdir,
    657     log_level=config["log_level"],
    658     callbacks=config["callbacks"],
    659     input_creator=input_creator,
    660     input_evaluation=input_evaluation,
    661     output_creator=output_creator,
    662     remote_worker_envs=config["remote_worker_envs"],
    663     remote_env_batch_wait_ms=config["remote_env_batch_wait_ms"],
    664     soft_horizon=config["soft_horizon"],
    665     no_done_at_end=config["no_done_at_end"],
    666     seed=(config["seed"] + worker_index)
    667     if config["seed"] is not None
    668     else None,
    669     fake_sampler=config["fake_sampler"],
    670     extra_python_environs=extra_python_environs,
    671     spaces=spaces,
    672     disable_env_checking=config["disable_env_checking"],
    673 )
    675 return worker

File c:\envs\py39\lib\site-packages\ray\rllib\evaluation\rollout_worker.py:511, in RolloutWorker.__init__(self, env_creator, validate_env, policy_spec, policy_mapping_fn, policies_to_train, tf_session_creator, rollout_fragment_length, count_steps_by, batch_mode, episode_horizon, preprocessor_pref, sample_async, compress_observations, num_envs, observation_fn, observation_filter, clip_rewards, normalize_actions, clip_actions, env_config, model_config, policy_config, worker_index, num_workers, recreated_worker, record_env, log_dir, log_level, callbacks, input_creator, input_evaluation, output_creator, remote_worker_envs, remote_env_batch_wait_ms, soft_horizon, no_done_at_end, seed, extra_python_environs, fake_sampler, spaces, policy, monitor_path, disable_env_checking)
    508 if self.env is not None:
    509     # Validate environment (general validation function).
    510     if not self._disable_env_checking:
--> 511         check_env(self.env)
    512     # Custom validation function given, typically a function attribute of the
    513     # algorithm trainer.
    514     if validate_env is not None:

File c:\envs\py39\lib\site-packages\ray\rllib\utils\pre_checks\env.py:78, in check_env(env)
     76 except Exception:
     77     actual_error = traceback.format_exc()
---> 78     raise ValueError(
     79         f"{actual_error}\n"
     80         "The above error has been found in your environment! "
     81         "We've added a module for checking your custom environments. It "
     82         "may cause your experiment to fail if your environment is not set up"
     83         "correctly. You can disable this behavior by setting "
     84         "`disable_env_checking=True` in your config "
     85         "dictionary. You can run the environment checking module "
     86         "standalone by calling ray.rllib.utils.check_env([env])."
     87     )

ValueError: Traceback (most recent call last):
  File "c:\envs\py39\lib\site-packages\ray\rllib\agents\trainer.py", line 935, in setup
    self._init(self.config, self.env_creator)
  File "c:\envs\py39\lib\site-packages\ray\rllib\agents\trainer.py", line 1074, in _init
    raise NotImplementedError
NotImplementedError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\envs\py39\lib\site-packages\ray\rllib\utils\pre_checks\env.py", line 67, in check_env
    check_gym_environments(env)
  File "c:\envs\py39\lib\site-packages\ray\rllib\utils\pre_checks\env.py", line 193, in check_gym_environments
    _check_done(done)
  File "c:\envs\py39\lib\site-packages\ray\rllib\utils\pre_checks\env.py", line 414, in _check_done
    elif not isinstance(done, (bool, np.bool, np.bool_)):
  File "c:\envs\py39\lib\site-packages\numpy\__init__.py", line 284, in __getattr__
    raise AttributeError("module {!r} has no attribute "
AttributeError: module 'numpy' has no attribute 'bool'

The above error has been found in your environment! We've added a module for checking your custom environments. It may cause your experiment to fail if your environment is not set upcorrectly. You can disable this behavior by setting `disable_env_checking=True` in your config dictionary. You can run the environment checking module standalone by calling ray.rllib.utils.check_env([env]).

[sherryfans777]

I choose to pip install numpy==1.23.5 and it solves my problem.