From d949a1c9f11c8c57bcd4433d65d84637228f89d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clod=C3=A9ric=20Mars?= Date: Sun, 12 Jun 2022 17:01:06 -0400 Subject: [PATCH 01/64] Single python entry point & hydra based configuration --- .apache-license-checker.yaml | 14 +- .gitignore | 18 +- .gitlab-ci.yml | 50 +- README.md | 105 ++--- .env => _old/.env | 1 + {environment => _old/environment}/.coveragerc | 0 .../cogment_verse_environment/atari.py | 0 .../cogment_verse_environment/base.py | 0 .../cogment_verse_environment/env_spec.py | 0 .../environment_adapter.py | 0 .../generate_specs.py | 0 .../cogment_verse_environment/gym_env.py | 0 .../cogment_verse_environment/minatarenv.py | 0 .../cogment_verse_environment/procgen_env.py | 0 .../pybullet_driving.py | 0 .../envs/simple_driving_env.py | 0 .../resources/__init__.py | 0 .../pybullet_driving_env/resources/car.py | 0 .../pybullet_driving_env/resources/goal.py | 0 .../pybullet_driving_env/resources/plane.py | 0 .../resources/simplecar.urdf | 0 .../resources/simplegoal.urdf | 0 .../resources/simpleplane.urdf | 0 .../cogment_verse_environment/tetris.py | 0 .../cogment_verse_environment/zoo_env.py | 0 {environment => _old/environment}/main.py | 0 .../environment}/pyproject.toml | 0 .../environment}/requirements.txt | 0 .../environment}/tests/conftest.py | 0 .../tests/mock_environment_session.py | 0 .../environment}/tests/test_atari.py | 2 +- .../environment}/tests/test_minatarenv.py | 0 .../environment}/tests/test_pettingzoo.py | 2 +- .../environment}/tests/test_procgen.py | 2 +- .../dashboards/experiment_tracker.json | 0 .../grafana}/dashboards/model_registry.json | 0 {grafana => _old/grafana}/grafana.ini | 0 .../dashboards/cogment_verse_dashboards.yml | 0 .../provisioning/datasources/prometheus.yaml | 0 .../prometheus}/prometheus.yml.tmpl | 0 run.sh => _old/run.sh | 4 +- run_api.proto => _old/run_api.proto | 0 run_params.yaml => _old/run_params.yaml | 0 .../reinforce/model.py | 0 .../reinforce/reinforce.py | 0 .../reinforce/reinforce_agent_adapter.py | 0 .../reinforce/replaybuffer.py | 0 .../reinforce/sample_producer.py | 0 .../reinforce/training_run.py | 0 .../cogment_verse_tf_agents/wrapper.py | 6 +- {tf_agents => _old/tf_agents}/main.py | 0 .../tf_agents}/requirements.txt | 0 .../torch_agents}/.coveragerc | 0 .../cogment_verse_torch_agents/atari_cnn.py | 0 .../hf_sb3/__init__.py | 0 .../hf_sb3/sb3_adapter.py | 2 +- .../hive_adapter/hive_agent_adapter.py | 24 +- .../hive_adapter/sample_producer.py | 6 +- .../hive_adapter/training_run.py | 15 +- .../muzero/adapter.py | 45 +- .../muzero/agent.py | 0 .../cogment_verse_torch_agents/muzero/mcts.py | 0 .../muzero/networks.py | 0 .../muzero/reanalyze_worker.py | 0 .../muzero/replay_buffer.py | 0 .../muzero/replay_worker.py | 0 .../muzero/schedule.py | 0 .../muzero/train_worker.py | 0 .../muzero/trial_worker.py | 13 +- .../muzero/utils.py | 0 .../selfplay_td3/model.py | 0 .../selfplay_td3/replaybuffer.py | 0 .../selfplay_td3/selfplay_agent.py | 0 .../selfplay_td3/selfplay_sample_producer.py | 0 .../selfplay_td3/selfplay_td3.py | 0 .../selfplay_td3/selfplay_training_run.py | 0 .../selfplay_td3/wrapper.py | 2 +- .../third_party/README.md | 0 .../third_party/hive/agent.py | 0 .../third_party/hive/ddpg.py | 0 .../third_party/hive/dqn.py | 0 .../third_party/hive/mlp.py | 45 +- .../third_party/hive/rainbow.py | 0 .../third_party/hive/replay_buffer.py | 18 +- .../third_party/hive/utils/schedule.py | 0 .../third_party/hive/utils/utils.py | 0 .../third_party/td3/README.md | 0 .../third_party/td3/td3.py | 0 .../third_party/td3/td3_mlp.py | 0 .../utils/tensors.py | 2 +- .../utils/throttle.py | 0 .../cogment_verse_torch_agents/wrapper.py | 10 +- {torch_agents => _old/torch_agents}/main.py | 0 .../torch_agents}/pyproject.toml | 0 .../torch_agents}/requirements.txt | 0 .../torch_agents}/tests/test_atari_dqn.py | 0 .../torch_agents}/tests/test_ddpg.py | 0 .../torch_agents}/tests/test_episode.py | 0 .../torch_agents}/tests/test_mcts.py | 0 .../torch_agents}/tests/test_muzero.py | 0 .../torch_agents}/tests/test_simple_a2c.py | 0 .../torch_agents}/tests/test_td3.py | 0 .../tests/test_trial_replay_buffer.py | 0 actors/random_actor.py | 44 ++ actors/simple_a2c.py | 357 +++++++++++++++ actors/tutorial/tutorial_1.py | 143 ++++++ actors/tutorial/tutorial_2.py | 176 +++++++ actors/tutorial/tutorial_3.py | 287 ++++++++++++ actors/tutorial/tutorial_4.py | 319 +++++++++++++ base_python/cogment_verse/agent_adapter.py | 192 -------- base_python/cogment_verse/api/.gitkeep | 0 base_python/cogment_verse/run/run_context.py | 211 --------- .../run/run_sample_producer_session.py | 193 -------- base_python/cogment_verse/run/run_servicer.py | 101 ---- base_python/cogment_verse/run/run_session.py | 378 --------------- base_python/cogment_verse/run/run_stepper.py | 49 -- base_python/cogment_verse/spaces/flatten.py | 36 -- .../cogment_verse/trial_datastore_client.py | 46 -- .../cogment_verse/utils/flatten_dict.py | 31 -- base_python/pyproject.toml | 6 - base_python/requirements.txt | 6 - base_python/setup.cfg | 15 - base_python/tests/test_run_config.py | 72 --- client/main.py | 129 ------ client/requirements.txt | 3 - client/run_controller.py | 83 ---- .../spaces => cogment_verse}/__init__.py | 5 +- cogment_verse/app.py | 169 +++++++ .../constants.py | 10 +- .../mlflow_experiment_tracker.py | 20 +- .../model_registry.py | 131 ++++-- .../processes}/__init__.py | 14 +- cogment_verse/processes/actor.py | 93 ++++ .../processes/cogment_cli_process.py | 78 ++++ .../processes/cogment_py_sdk_process.py | 23 + .../processes/cogment_verse_process.py | 56 +++ cogment_verse/processes/environment.py | 101 ++++ cogment_verse/processes/model_registry.py | 45 ++ cogment_verse/processes/orchestrator.py | 42 ++ cogment_verse/processes/popen_process.py | 76 +++ cogment_verse/processes/run.py | 89 ++++ cogment_verse/processes/trial_datastore.py | 39 ++ cogment_verse/processes/web.py | 80 ++++ .../run/__init__.py | 4 +- cogment_verse/run/run_session.py | 85 ++++ cogment_verse/run/sample_producer_worker.py | 125 +++++ cogment_verse/run/trial_runner_worker.py | 143 ++++++ cogment_verse/services_directory.py | 55 +++ cogment_verse/specs/__init__.py | 37 ++ .../specs/cogment.yaml | 17 +- cogment_verse/specs/data.proto | 126 +++++ cogment_verse/specs/encode_rendered_frame.py | 36 ++ cogment_verse/specs/environment_specs.py | 37 ++ cogment_verse/specs/flatten.py | 97 ++++ .../specs/ndarray.py | 24 +- cogment_verse/specs/sample_space.py | 64 +++ cogment_verse/specs/value.py | 22 + .../utils/__init__.py | 9 +- cogment_verse/utils/download_cogment.py | 141 ++++++ cogment_verse/utils/find_free_port.py | 39 ++ cogment_verse/utils/generate.py | 36 ++ .../utils/get_implementation_name.py | 24 + cogment_verse/utils/import_class.py | 21 + .../utils/lru.py | 2 +- .../utils/sizeof_fmt.py | 2 +- .../web}/__init__.py | 9 +- .../web/components}/.eslintignore | 0 .../web/components}/.prettierignore | 0 .../web/components/build/asset-manifest.json | 15 + .../build}/assets/cogment-splash.png | Bin .../web/components/build}/favicon.ico | Bin cogment_verse/web/components/build/index.html | 1 + .../web/components/build}/logo192.png | Bin .../web/components/build}/logo512.png | Bin .../web/components/build}/manifest.json | 0 .../web/components/build}/robots.txt | 0 .../build/static/css/main.21cabb04.css | 4 + .../build/static/css/main.21cabb04.css.map | 1 + .../build/static/js/787.90542627.chunk.js | 2 + .../build/static/js/787.90542627.chunk.js.map | 1 + .../build/static/js/main.80bd9354.js | 3 + .../static/js/main.80bd9354.js.LICENSE.txt | 326 +++++++++++++ .../build/static/js/main.80bd9354.js.map | 1 + cogment_verse/web/components/cogment.yaml | 32 ++ cogment_verse/web/components/data.proto | 126 +++++ .../web/components}/package-lock.json | 432 +----------------- .../web/components}/package.json | 26 +- .../web/components}/postcss.config.js | 2 +- .../public/assets/cogment-splash.png | Bin 0 -> 5874960 bytes .../web/components/public/favicon.ico | Bin 0 -> 3870 bytes .../web/components}/public/index.html | 14 +- .../web/components/public/logo192.png | Bin 0 -> 5347 bytes .../web/components/public/logo512.png | Bin 0 -> 9664 bytes .../web/components/public/manifest.json | 25 + .../web/components/public/robots.txt | 3 + .../web/components}/src/App.jsx | 14 +- .../web/components}/src/components/Button.jsx | 0 .../components}/src/components/Countdown.jsx | 0 .../web/components}/src/components/DPad.jsx | 0 .../src/components/DPad.module.css | 0 .../components}/src/components/FpsCounter.jsx | 0 .../components}/src/components/Joystick.jsx | 0 .../src/components/Joystick.module.css | 0 .../src/components/KeyboardControlList.jsx | 4 +- .../src/components/RenderedScreen.jsx | 10 +- .../src/components/RenderedScreen.module.css | 0 .../src/controls/AtariPitfallControls.jsx | 59 ++- .../web/components}/src/controls/Controls.jsx | 14 +- .../src/controls/GymCartPoleControls.jsx | 24 +- .../GymLunarLanderContinuousControls.jsx | 28 +- .../src/controls/GymLunarLanderControls.jsx | 38 +- .../src/controls/GymMountainCarControls.jsx | 90 ++++ .../src/controls/ObserverControls.jsx | 0 .../src/controls/TetrisControls.jsx | 32 +- .../web/components}/src/hooks/useActions.ts | 88 ++-- .../src/hooks/useDocumentEventListener.js | 2 +- .../components}/src/hooks/usePressedKeys.jsx | 0 .../src/hooks/useRealTimeUpdate.jsx | 0 .../web/components}/src/index.css | 0 .../web/components}/src/index.tsx | 0 .../web/components}/src/logo.svg | 0 .../web/components}/src/react-app-env.d.ts | 2 +- .../web/components}/src/reportWebVitals.js | 2 +- .../web/components}/src/utils/constants.js | 15 +- .../components}/src/utils/controlLookup.js | 2 +- .../web/components}/tailwind.config.js | 2 +- .../web/components}/tsconfig.json | 0 cogment_verse/web/package-lock.json | 3 + cogment_verse/web/server/dev_server.py | 33 ++ cogment_verse/web/server/server.py | 72 +++ cogment_verse/web/utils/generate.py | 49 ++ cogment_verse/web/utils/npm.py | 31 ++ config/config.yaml | 14 + config/experiment/simple_a2c/cartpole.yaml | 18 + config/run/observe.yaml | 6 + config/run/play.yaml | 5 + config/run/simple_a2c.yaml | 16 + config/run/simple_bc.yaml | 5 + config/services/actor/random.yaml | 2 + config/services/actor/simple_a2c.yaml | 2 + config/services/actor/simple_bc.yaml | 2 + config/services/environment/cartpole.yaml | 3 + config/services/environment/lunar_lander.yaml | 3 + .../environment/lunar_lander_continuous.yaml | 3 + config/services/environment/mountain_car.yaml | 3 + config/services/local_base_services.yaml | 14 + data.proto | 369 --------------- data/mlflow/.gitkeep | 0 data/model-registry/.gitkeep | 0 docs/development_setup.md | 40 ++ docs/results/a2c.md | 2 +- .../base_agent_adapter.py | 210 --------- environments/gym_adapter.py | 186 ++++++++ main.py | 35 ++ pyproject.toml | 40 +- requirements.txt | 20 +- runs/play.py | 173 +++++++ simple_mlflow.py | 44 ++ tests/conftest.py | 24 + {base_python/tests => tests}/test_flatten.py | 47 +- tests/test_sample.py | 91 ++++ .../simple_a2c/simple_a2c_agent.py | 326 ------------- .../simple_bc/__init__.py | 20 - .../simple_bc/tutorial_1.py | 145 ------ .../simple_bc/tutorial_2.py | 168 ------- .../simple_bc/tutorial_3.py | 251 ---------- .../simple_bc/tutorial_4.py | 305 ------------- .../utils/throttle.py | 46 -- web_client/.gitignore | 29 -- .../src/controls/GymMountainCarControls.jsx | 73 --- 270 files changed, 5491 insertions(+), 4366 deletions(-) rename .env => _old/.env (99%) rename {environment => _old/environment}/.coveragerc (100%) rename {environment => _old/environment}/cogment_verse_environment/atari.py (100%) rename {environment => _old/environment}/cogment_verse_environment/base.py (100%) rename {environment => _old/environment}/cogment_verse_environment/env_spec.py (100%) rename {environment => _old/environment}/cogment_verse_environment/environment_adapter.py (100%) rename {environment => _old/environment}/cogment_verse_environment/generate_specs.py (100%) rename {environment => _old/environment}/cogment_verse_environment/gym_env.py (100%) rename {environment => _old/environment}/cogment_verse_environment/minatarenv.py (100%) rename {environment => _old/environment}/cogment_verse_environment/procgen_env.py (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving.py (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/envs/simple_driving_env.py (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/resources/__init__.py (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/resources/car.py (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/resources/goal.py (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/resources/plane.py (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/resources/simplecar.urdf (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/resources/simplegoal.urdf (100%) rename {environment => _old/environment}/cogment_verse_environment/pybullet_driving_env/resources/simpleplane.urdf (100%) rename {environment => _old/environment}/cogment_verse_environment/tetris.py (100%) rename {environment => _old/environment}/cogment_verse_environment/zoo_env.py (100%) rename {environment => _old/environment}/main.py (100%) rename {environment => _old/environment}/pyproject.toml (100%) rename {environment => _old/environment}/requirements.txt (100%) rename {environment => _old/environment}/tests/conftest.py (100%) rename {environment => _old/environment}/tests/mock_environment_session.py (100%) rename {environment => _old/environment}/tests/test_atari.py (97%) rename {environment => _old/environment}/tests/test_minatarenv.py (100%) rename {environment => _old/environment}/tests/test_pettingzoo.py (98%) rename {environment => _old/environment}/tests/test_procgen.py (98%) rename {grafana => _old/grafana}/dashboards/experiment_tracker.json (100%) rename {grafana => _old/grafana}/dashboards/model_registry.json (100%) rename {grafana => _old/grafana}/grafana.ini (100%) rename {grafana => _old/grafana}/provisioning/dashboards/cogment_verse_dashboards.yml (100%) rename {grafana => _old/grafana}/provisioning/datasources/prometheus.yaml (100%) rename {prometheus => _old/prometheus}/prometheus.yml.tmpl (100%) rename run.sh => _old/run.sh (97%) rename run_api.proto => _old/run_api.proto (100%) rename run_params.yaml => _old/run_params.yaml (100%) rename {tf_agents => _old/tf_agents}/cogment_verse_tf_agents/reinforce/model.py (100%) rename {tf_agents => _old/tf_agents}/cogment_verse_tf_agents/reinforce/reinforce.py (100%) rename {tf_agents => _old/tf_agents}/cogment_verse_tf_agents/reinforce/reinforce_agent_adapter.py (100%) rename {tf_agents => _old/tf_agents}/cogment_verse_tf_agents/reinforce/replaybuffer.py (100%) rename {tf_agents => _old/tf_agents}/cogment_verse_tf_agents/reinforce/sample_producer.py (100%) rename {tf_agents => _old/tf_agents}/cogment_verse_tf_agents/reinforce/training_run.py (100%) rename {tf_agents => _old/tf_agents}/cogment_verse_tf_agents/wrapper.py (92%) rename {tf_agents => _old/tf_agents}/main.py (100%) rename {tf_agents => _old/tf_agents}/requirements.txt (100%) rename {torch_agents => _old/torch_agents}/.coveragerc (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/atari_cnn.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/hf_sb3/__init__.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py (98%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py (92%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/hive_adapter/sample_producer.py (97%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/hive_adapter/training_run.py (97%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/adapter.py (93%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/agent.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/mcts.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/networks.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/reanalyze_worker.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/replay_buffer.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/replay_worker.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/schedule.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/train_worker.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/trial_worker.py (85%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/muzero/utils.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/selfplay_td3/model.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/selfplay_td3/replaybuffer.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/selfplay_td3/selfplay_agent.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/selfplay_td3/selfplay_sample_producer.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/selfplay_td3/selfplay_td3.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/selfplay_td3/selfplay_training_run.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/selfplay_td3/wrapper.py (97%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/README.md (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/agent.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/ddpg.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/dqn.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/mlp.py (83%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/rainbow.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/replay_buffer.py (96%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/utils/schedule.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/hive/utils/utils.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/td3/README.md (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/td3/td3.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/third_party/td3/td3_mlp.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/utils/tensors.py (97%) rename {base_python/cogment_verse => _old/torch_agents/cogment_verse_torch_agents}/utils/throttle.py (100%) rename {torch_agents => _old/torch_agents}/cogment_verse_torch_agents/wrapper.py (92%) rename {torch_agents => _old/torch_agents}/main.py (100%) rename {torch_agents => _old/torch_agents}/pyproject.toml (100%) rename {torch_agents => _old/torch_agents}/requirements.txt (100%) rename {torch_agents => _old/torch_agents}/tests/test_atari_dqn.py (100%) rename {torch_agents => _old/torch_agents}/tests/test_ddpg.py (100%) rename {torch_agents => _old/torch_agents}/tests/test_episode.py (100%) rename {torch_agents => _old/torch_agents}/tests/test_mcts.py (100%) rename {torch_agents => _old/torch_agents}/tests/test_muzero.py (100%) rename {torch_agents => _old/torch_agents}/tests/test_simple_a2c.py (100%) rename {torch_agents => _old/torch_agents}/tests/test_td3.py (100%) rename {torch_agents => _old/torch_agents}/tests/test_trial_replay_buffer.py (100%) create mode 100644 actors/random_actor.py create mode 100644 actors/simple_a2c.py create mode 100644 actors/tutorial/tutorial_1.py create mode 100644 actors/tutorial/tutorial_2.py create mode 100644 actors/tutorial/tutorial_3.py create mode 100644 actors/tutorial/tutorial_4.py delete mode 100644 base_python/cogment_verse/agent_adapter.py delete mode 100644 base_python/cogment_verse/api/.gitkeep delete mode 100644 base_python/cogment_verse/run/run_context.py delete mode 100644 base_python/cogment_verse/run/run_sample_producer_session.py delete mode 100644 base_python/cogment_verse/run/run_servicer.py delete mode 100644 base_python/cogment_verse/run/run_session.py delete mode 100644 base_python/cogment_verse/run/run_stepper.py delete mode 100644 base_python/cogment_verse/spaces/flatten.py delete mode 100644 base_python/cogment_verse/trial_datastore_client.py delete mode 100644 base_python/cogment_verse/utils/flatten_dict.py delete mode 100644 base_python/pyproject.toml delete mode 100644 base_python/requirements.txt delete mode 100644 base_python/setup.cfg delete mode 100644 base_python/tests/test_run_config.py delete mode 100644 client/main.py delete mode 100644 client/requirements.txt delete mode 100644 client/run_controller.py rename {base_python/cogment_verse/spaces => cogment_verse}/__init__.py (81%) create mode 100644 cogment_verse/app.py rename {base_python/cogment_verse => cogment_verse}/constants.py (63%) rename {base_python/cogment_verse => cogment_verse}/mlflow_experiment_tracker.py (90%) rename base_python/cogment_verse/model_registry_client.py => cogment_verse/model_registry.py (60%) rename {base_python/cogment_verse/utils => cogment_verse/processes}/__init__.py (54%) create mode 100644 cogment_verse/processes/actor.py create mode 100644 cogment_verse/processes/cogment_cli_process.py create mode 100644 cogment_verse/processes/cogment_py_sdk_process.py create mode 100644 cogment_verse/processes/cogment_verse_process.py create mode 100644 cogment_verse/processes/environment.py create mode 100644 cogment_verse/processes/model_registry.py create mode 100644 cogment_verse/processes/orchestrator.py create mode 100644 cogment_verse/processes/popen_process.py create mode 100644 cogment_verse/processes/run.py create mode 100644 cogment_verse/processes/trial_datastore.py create mode 100644 cogment_verse/processes/web.py rename {base_python/cogment_verse => cogment_verse}/run/__init__.py (83%) create mode 100644 cogment_verse/run/run_session.py create mode 100644 cogment_verse/run/sample_producer_worker.py create mode 100644 cogment_verse/run/trial_runner_worker.py create mode 100644 cogment_verse/services_directory.py create mode 100644 cogment_verse/specs/__init__.py rename cogment.yaml => cogment_verse/specs/cogment.yaml (52%) create mode 100644 cogment_verse/specs/data.proto create mode 100644 cogment_verse/specs/encode_rendered_frame.py create mode 100644 cogment_verse/specs/environment_specs.py create mode 100644 cogment_verse/specs/flatten.py rename environment/cogment_verse_environment/utils/serialization_helpers.py => cogment_verse/specs/ndarray.py (55%) create mode 100644 cogment_verse/specs/sample_space.py create mode 100644 cogment_verse/specs/value.py rename base_python/cogment_verse/utils/get_full_class_name.py => cogment_verse/utils/__init__.py (75%) create mode 100644 cogment_verse/utils/download_cogment.py create mode 100644 cogment_verse/utils/find_free_port.py create mode 100644 cogment_verse/utils/generate.py create mode 100644 cogment_verse/utils/get_implementation_name.py create mode 100644 cogment_verse/utils/import_class.py rename {base_python/cogment_verse => cogment_verse}/utils/lru.py (95%) rename {base_python/cogment_verse => cogment_verse}/utils/sizeof_fmt.py (93%) rename {base_python/cogment_verse => cogment_verse/web}/__init__.py (70%) rename {web_client => cogment_verse/web/components}/.eslintignore (100%) rename {web_client => cogment_verse/web/components}/.prettierignore (100%) create mode 100644 cogment_verse/web/components/build/asset-manifest.json rename {web_client/public => cogment_verse/web/components/build}/assets/cogment-splash.png (100%) rename {web_client/public => cogment_verse/web/components/build}/favicon.ico (100%) create mode 100644 cogment_verse/web/components/build/index.html rename {web_client/public => cogment_verse/web/components/build}/logo192.png (100%) rename {web_client/public => cogment_verse/web/components/build}/logo512.png (100%) rename {web_client/public => cogment_verse/web/components/build}/manifest.json (100%) rename {web_client/public => cogment_verse/web/components/build}/robots.txt (100%) create mode 100644 cogment_verse/web/components/build/static/css/main.21cabb04.css create mode 100644 cogment_verse/web/components/build/static/css/main.21cabb04.css.map create mode 100644 cogment_verse/web/components/build/static/js/787.90542627.chunk.js create mode 100644 cogment_verse/web/components/build/static/js/787.90542627.chunk.js.map create mode 100644 cogment_verse/web/components/build/static/js/main.80bd9354.js create mode 100644 cogment_verse/web/components/build/static/js/main.80bd9354.js.LICENSE.txt create mode 100644 cogment_verse/web/components/build/static/js/main.80bd9354.js.map create mode 100644 cogment_verse/web/components/cogment.yaml create mode 100644 cogment_verse/web/components/data.proto rename {web_client => cogment_verse/web/components}/package-lock.json (97%) rename {web_client => cogment_verse/web/components}/package.json (85%) rename {web_client => cogment_verse/web/components}/postcss.config.js (91%) create mode 100644 cogment_verse/web/components/public/assets/cogment-splash.png create mode 100644 cogment_verse/web/components/public/favicon.ico rename {web_client => cogment_verse/web/components}/public/index.html (76%) create mode 100644 cogment_verse/web/components/public/logo192.png create mode 100644 cogment_verse/web/components/public/logo512.png create mode 100644 cogment_verse/web/components/public/manifest.json create mode 100644 cogment_verse/web/components/public/robots.txt rename {web_client => cogment_verse/web/components}/src/App.jsx (86%) rename {web_client => cogment_verse/web/components}/src/components/Button.jsx (100%) rename {web_client => cogment_verse/web/components}/src/components/Countdown.jsx (100%) rename {web_client => cogment_verse/web/components}/src/components/DPad.jsx (100%) rename {web_client => cogment_verse/web/components}/src/components/DPad.module.css (100%) rename {web_client => cogment_verse/web/components}/src/components/FpsCounter.jsx (100%) rename {web_client => cogment_verse/web/components}/src/components/Joystick.jsx (100%) rename {web_client => cogment_verse/web/components}/src/components/Joystick.module.css (100%) rename {web_client => cogment_verse/web/components}/src/components/KeyboardControlList.jsx (93%) rename {web_client => cogment_verse/web/components}/src/components/RenderedScreen.jsx (79%) rename {web_client => cogment_verse/web/components}/src/components/RenderedScreen.module.css (100%) rename {web_client => cogment_verse/web/components}/src/controls/AtariPitfallControls.jsx (52%) rename {web_client => cogment_verse/web/components}/src/controls/Controls.jsx (80%) rename {web_client => cogment_verse/web/components}/src/controls/GymCartPoleControls.jsx (75%) rename {web_client => cogment_verse/web/components}/src/controls/GymLunarLanderContinuousControls.jsx (83%) rename {web_client => cogment_verse/web/components}/src/controls/GymLunarLanderControls.jsx (80%) create mode 100644 cogment_verse/web/components/src/controls/GymMountainCarControls.jsx rename {web_client => cogment_verse/web/components}/src/controls/ObserverControls.jsx (100%) rename {web_client => cogment_verse/web/components}/src/controls/TetrisControls.jsx (71%) rename {web_client => cogment_verse/web/components}/src/hooks/useActions.ts (65%) rename {web_client => cogment_verse/web/components}/src/hooks/useDocumentEventListener.js (93%) rename {web_client => cogment_verse/web/components}/src/hooks/usePressedKeys.jsx (100%) rename {web_client => cogment_verse/web/components}/src/hooks/useRealTimeUpdate.jsx (100%) rename {web_client => cogment_verse/web/components}/src/index.css (100%) rename {web_client => cogment_verse/web/components}/src/index.tsx (100%) rename {web_client => cogment_verse/web/components}/src/logo.svg (100%) rename {web_client => cogment_verse/web/components}/src/react-app-env.d.ts (91%) rename {web_client => cogment_verse/web/components}/src/reportWebVitals.js (93%) rename {web_client => cogment_verse/web/components}/src/utils/constants.js (54%) rename {web_client => cogment_verse/web/components}/src/utils/controlLookup.js (95%) rename {web_client => cogment_verse/web/components}/tailwind.config.js (91%) rename {web_client => cogment_verse/web/components}/tsconfig.json (100%) create mode 100644 cogment_verse/web/package-lock.json create mode 100644 cogment_verse/web/server/dev_server.py create mode 100644 cogment_verse/web/server/server.py create mode 100644 cogment_verse/web/utils/generate.py create mode 100644 cogment_verse/web/utils/npm.py create mode 100644 config/config.yaml create mode 100644 config/experiment/simple_a2c/cartpole.yaml create mode 100644 config/run/observe.yaml create mode 100644 config/run/play.yaml create mode 100644 config/run/simple_a2c.yaml create mode 100644 config/run/simple_bc.yaml create mode 100644 config/services/actor/random.yaml create mode 100644 config/services/actor/simple_a2c.yaml create mode 100644 config/services/actor/simple_bc.yaml create mode 100644 config/services/environment/cartpole.yaml create mode 100644 config/services/environment/lunar_lander.yaml create mode 100644 config/services/environment/lunar_lander_continuous.yaml create mode 100644 config/services/environment/mountain_car.yaml create mode 100644 config/services/local_base_services.yaml delete mode 100644 data.proto delete mode 100644 data/mlflow/.gitkeep delete mode 100644 data/model-registry/.gitkeep delete mode 100644 environment/cogment_verse_environment/base_agent_adapter.py create mode 100644 environments/gym_adapter.py create mode 100644 main.py create mode 100644 runs/play.py create mode 100644 simple_mlflow.py create mode 100644 tests/conftest.py rename {base_python/tests => tests}/test_flatten.py (64%) create mode 100644 tests/test_sample.py delete mode 100644 torch_agents/cogment_verse_torch_agents/simple_a2c/simple_a2c_agent.py delete mode 100644 torch_agents/cogment_verse_torch_agents/simple_bc/__init__.py delete mode 100644 torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_1.py delete mode 100644 torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_2.py delete mode 100644 torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_3.py delete mode 100644 torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_4.py delete mode 100644 torch_agents/cogment_verse_torch_agents/utils/throttle.py delete mode 100644 web_client/.gitignore delete mode 100644 web_client/src/controls/GymMountainCarControls.jsx diff --git a/.apache-license-checker.yaml b/.apache-license-checker.yaml index 137db624..1b14616e 100644 --- a/.apache-license-checker.yaml +++ b/.apache-license-checker.yaml @@ -1,8 +1,6 @@ ignore: - - "?eggs" - - "**/__pycache__" - "**/.venv" - - "**/cogment_*.yaml" + - "**/_old" - "**/*_pb2*.py" - "**/*.pb.go" - "**/*_pb*.js" @@ -12,12 +10,8 @@ ignore: - "**/CogSettings.d.ts" - "**/CogSettings.js" - "**/CogTypes.d.ts" - - "**/third_party" - - "*/cogment/api" - - "**/htmlcov" - - "web_client/node_modules" - - "web_client/build" - - "**/pybullet_driving_env/*" + - "**/node_modules" + - "**/build" license: - CopyrightYear: 2021 + CopyrightYear: 2022 Author: "AI Redefined Inc. " diff --git a/.gitignore b/.gitignore index a02b1711..0b02cbdd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ # Generated code -/*/*.proto -/*/cogment.yaml -cog_settings.py -*_pb2.py -*_pb2_grpc.py -base_python/cogment_verse/api +CogSettings.* +CogTypes.d.ts +*_pb.d.ts +*_pb.js +cogment_verse/web/cogment.yaml +cogment_verse/web/*.proto # Python stuffs __pycache__/ @@ -26,4 +26,8 @@ node_modules/ /data # Cogment -/.cogment +/.cogment_verse + +# Run outputs generated by Hydra +outputs +multirun diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 474722d2..674b4f1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,44 +1,50 @@ +stages: + - lint + - test + .base: image: python:3.9 variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - NPM_CACHE_DIR: "$CI_PROJECT_DIR/.cache/npm" before_script: - mkdir -p ${PIP_CACHE_DIR} - - mkdir -p ${NPM_CACHE_DIR} - # Installation instructions from https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions - - curl -fsSL https://deb.nodesource.com/setup_14.x | bash - - - apt-get update && apt-get install -y software-properties-common && apt-add-repository non-free && apt-get update - - apt-get install -y nodejs swig unrar python3-opencv - - pip install virtualenv - - npm config set cache ${NPM_CACHE_DIR} --global + - apt-get update + - apt-get install -y python3-opencv + - python -m venv .venv + - source .venv/bin/activate + - pip install -r requirements.txt cache: # pip's cache - paths: - .cache/pip - "**/.venv" - # npm's cache + # .venv - key: files: - - web_client/package-lock.json + - requirements.txt paths: - - .cache/npm/ - # atari roms cache - - key: - files: - - run.sh - - environment/requirements.txr - paths: - - environment/.atari_roms + - .venv + +black: + stage: lint + extends: .base + script: + - black --check --diff . -build_and_test: +pylint: + stage: lint extends: .base script: - - ./run.sh build - - ./run.sh lint - - ./run.sh test + - pylint --recursive=y . apache_licenses_check: + stage: lint image: registry.gitlab.com/ai-r/apache-license-checker:latest script: - apache-license-checker + +pytest: + stage: test + extends: .base + script: + - python -m pytest diff --git a/README.md b/README.md index d84fc329..b13ea473 100644 --- a/README.md +++ b/README.md @@ -41,64 +41,55 @@ Cogment verse includes environments from: ## Getting started -### Setup, Build and Run - 1. Clone this repository -2. Install the following dependencies: - - [Python 3.9](https://www.python.org/) or above, - - [Node.JS v14](https://nodejs.org/) or above, - - `parallel`, on ubuntu it is installable using `apt-get install parallel`, on mac it is available through `brew install parallel`, - - `unrar`, on ubuntu it is installable using `apt-get install unrar`, on mac it is available through `brew install unrar`. -3. `./run.sh build` -4. `./run.sh services_start` -5. In a different terminal, start the trials with `./run.sh client start `. - Different run names can be found in `run_params.yaml` -6. (Optional) To launch webclient, run `./run.sh web_client_start` in a different - terminal. Open http://localhost:8000 to join or visualize trials - -#### Run monitoring - -You can monitor ongoing run using [mlflow](https://mlflow.org). By default a local instance of mlflow is started by cogment-verse and is accessible at . - -#### Human player - -Some of the availabe run involve a human player, -for example `benchmark_lander_hill` enables a human player -to momentarily take control of the lunar lander to help the -AI agents during the training process. - -Then start the run - -```console -./run.sh client start benchmark_lander_hill -``` - -Access the playing interface by launching a webclient with -`./run.sh web_client_start` and navigating to - -#### **Play** - -The `play` run implementation can be used to have any actor play in any environment. 3 example run parameters are provided: - -**`headless_play`** instanciates any agents and start a number of trials. - -```console -./run.sh client start headless_play -``` - -**`observe`** instanciates any agents and start a number of trials with a human observer through the webclient. - -```console -./run.sh client start observe -``` - -**`play`** instanciates let a human player play in a supported environment. - -```console -./run.sh client start play -``` - -They can be inspected and adapted to your needs in `run_params.yaml`: +2. Install [Python 3.9](https://www.python.org/) +3. Create and activate a virtual environment by runnning + ```console + $ python -m venv .venv + $ source .venv/bin/activate + ``` +4. Install the python dependencies by running + ```console + $ pip install -r requirements.txt + ``` +5. In another terminal, launch a mlflow server on port 3000 by running + ```console + $ source .venv/bin/activate + $ python -m simple_mlflow + ``` +6. Start the default Cogment Verse run using `python -m main` +7. Open Chrome (other web browser might work but haven't tested) and navigate to http://localhost:8080/ +8. Play the game! + +That's the basic setup for Cogment Verse, you are now ready to train AI agents. + +### Configuration + +Cogment Verse relies on [hydra](https://hydra.cc) for configuration. This enables easy configuration and composition of configuration directly from yaml files and the command line. + +The configuration files are located in the `config` directory, with defaults defined in `config/config.yaml`. + +Here are a few examples: + +- Launch a Simple Behavior Cloning run with the [Mountain Car Gym environment](https://www.gymlibrary.ml/environments/classic_control/mountain_car/) (which is the default environment) + ```console + $ python -m main services/actor=simple_bc run=simple_bc + ``` +- Launch a Simple Behavior Cloning run with the [Lunar Lander Gym environment](https://www.gymlibrary.ml/environments/box2d/lunar_lander/) + ```console + $ python -m main services/actor=simple_bc services/environment=lunar_lander run=simple_bc + ``` +- Launch and play a single trial of the Lunar Lander Gym environment with continuous controls + ```console + $ python -m main services/environment=lunar_lander_continuous + ``` +- Launch an A2C training run with the [Cartpole Gym environment](https://www.gymlibrary.ml/environments/classic_control/cartpole/) + + ```console + $ python -m main +experiment=simple_a2c/cartpole + ``` + + This one is completely _headless_ (training doens't involve interaction with a human player). It will take a little while to run, you can monitor the progress using mlflow at ## List of publications and submissions using Cogment and/or Cogment Verse diff --git a/.env b/_old/.env similarity index 99% rename from .env rename to _old/.env index 2f11f5b2..f79fce87 100644 --- a/.env +++ b/_old/.env @@ -7,6 +7,7 @@ COGMENT_VERSE_MODEL_REGISTRY_PORT=9002 COGMENT_VERSE_TORCH_AGENTS_PORT=9003 COGMENT_VERSE_TF_AGENTS_PORT=9004 COGMENT_VERSE_ENVIRONMENT_PORT=9005 +COGMENT_VERSE_PRETRIAL_HOOK_PORT=9006 ## Prometheus metrics server ports COGMENT_VERSE_TORCH_AGENTS_PROMETHEUS_PORT=9500 diff --git a/environment/.coveragerc b/_old/environment/.coveragerc similarity index 100% rename from environment/.coveragerc rename to _old/environment/.coveragerc diff --git a/environment/cogment_verse_environment/atari.py b/_old/environment/cogment_verse_environment/atari.py similarity index 100% rename from environment/cogment_verse_environment/atari.py rename to _old/environment/cogment_verse_environment/atari.py diff --git a/environment/cogment_verse_environment/base.py b/_old/environment/cogment_verse_environment/base.py similarity index 100% rename from environment/cogment_verse_environment/base.py rename to _old/environment/cogment_verse_environment/base.py diff --git a/environment/cogment_verse_environment/env_spec.py b/_old/environment/cogment_verse_environment/env_spec.py similarity index 100% rename from environment/cogment_verse_environment/env_spec.py rename to _old/environment/cogment_verse_environment/env_spec.py diff --git a/environment/cogment_verse_environment/environment_adapter.py b/_old/environment/cogment_verse_environment/environment_adapter.py similarity index 100% rename from environment/cogment_verse_environment/environment_adapter.py rename to _old/environment/cogment_verse_environment/environment_adapter.py diff --git a/environment/cogment_verse_environment/generate_specs.py b/_old/environment/cogment_verse_environment/generate_specs.py similarity index 100% rename from environment/cogment_verse_environment/generate_specs.py rename to _old/environment/cogment_verse_environment/generate_specs.py diff --git a/environment/cogment_verse_environment/gym_env.py b/_old/environment/cogment_verse_environment/gym_env.py similarity index 100% rename from environment/cogment_verse_environment/gym_env.py rename to _old/environment/cogment_verse_environment/gym_env.py diff --git a/environment/cogment_verse_environment/minatarenv.py b/_old/environment/cogment_verse_environment/minatarenv.py similarity index 100% rename from environment/cogment_verse_environment/minatarenv.py rename to _old/environment/cogment_verse_environment/minatarenv.py diff --git a/environment/cogment_verse_environment/procgen_env.py b/_old/environment/cogment_verse_environment/procgen_env.py similarity index 100% rename from environment/cogment_verse_environment/procgen_env.py rename to _old/environment/cogment_verse_environment/procgen_env.py diff --git a/environment/cogment_verse_environment/pybullet_driving.py b/_old/environment/cogment_verse_environment/pybullet_driving.py similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving.py rename to _old/environment/cogment_verse_environment/pybullet_driving.py diff --git a/environment/cogment_verse_environment/pybullet_driving_env/envs/simple_driving_env.py b/_old/environment/cogment_verse_environment/pybullet_driving_env/envs/simple_driving_env.py similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/envs/simple_driving_env.py rename to _old/environment/cogment_verse_environment/pybullet_driving_env/envs/simple_driving_env.py diff --git a/environment/cogment_verse_environment/pybullet_driving_env/resources/__init__.py b/_old/environment/cogment_verse_environment/pybullet_driving_env/resources/__init__.py similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/resources/__init__.py rename to _old/environment/cogment_verse_environment/pybullet_driving_env/resources/__init__.py diff --git a/environment/cogment_verse_environment/pybullet_driving_env/resources/car.py b/_old/environment/cogment_verse_environment/pybullet_driving_env/resources/car.py similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/resources/car.py rename to _old/environment/cogment_verse_environment/pybullet_driving_env/resources/car.py diff --git a/environment/cogment_verse_environment/pybullet_driving_env/resources/goal.py b/_old/environment/cogment_verse_environment/pybullet_driving_env/resources/goal.py similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/resources/goal.py rename to _old/environment/cogment_verse_environment/pybullet_driving_env/resources/goal.py diff --git a/environment/cogment_verse_environment/pybullet_driving_env/resources/plane.py b/_old/environment/cogment_verse_environment/pybullet_driving_env/resources/plane.py similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/resources/plane.py rename to _old/environment/cogment_verse_environment/pybullet_driving_env/resources/plane.py diff --git a/environment/cogment_verse_environment/pybullet_driving_env/resources/simplecar.urdf b/_old/environment/cogment_verse_environment/pybullet_driving_env/resources/simplecar.urdf similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/resources/simplecar.urdf rename to _old/environment/cogment_verse_environment/pybullet_driving_env/resources/simplecar.urdf diff --git a/environment/cogment_verse_environment/pybullet_driving_env/resources/simplegoal.urdf b/_old/environment/cogment_verse_environment/pybullet_driving_env/resources/simplegoal.urdf similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/resources/simplegoal.urdf rename to _old/environment/cogment_verse_environment/pybullet_driving_env/resources/simplegoal.urdf diff --git a/environment/cogment_verse_environment/pybullet_driving_env/resources/simpleplane.urdf b/_old/environment/cogment_verse_environment/pybullet_driving_env/resources/simpleplane.urdf similarity index 100% rename from environment/cogment_verse_environment/pybullet_driving_env/resources/simpleplane.urdf rename to _old/environment/cogment_verse_environment/pybullet_driving_env/resources/simpleplane.urdf diff --git a/environment/cogment_verse_environment/tetris.py b/_old/environment/cogment_verse_environment/tetris.py similarity index 100% rename from environment/cogment_verse_environment/tetris.py rename to _old/environment/cogment_verse_environment/tetris.py diff --git a/environment/cogment_verse_environment/zoo_env.py b/_old/environment/cogment_verse_environment/zoo_env.py similarity index 100% rename from environment/cogment_verse_environment/zoo_env.py rename to _old/environment/cogment_verse_environment/zoo_env.py diff --git a/environment/main.py b/_old/environment/main.py similarity index 100% rename from environment/main.py rename to _old/environment/main.py diff --git a/environment/pyproject.toml b/_old/environment/pyproject.toml similarity index 100% rename from environment/pyproject.toml rename to _old/environment/pyproject.toml diff --git a/environment/requirements.txt b/_old/environment/requirements.txt similarity index 100% rename from environment/requirements.txt rename to _old/environment/requirements.txt diff --git a/environment/tests/conftest.py b/_old/environment/tests/conftest.py similarity index 100% rename from environment/tests/conftest.py rename to _old/environment/tests/conftest.py diff --git a/environment/tests/mock_environment_session.py b/_old/environment/tests/mock_environment_session.py similarity index 100% rename from environment/tests/mock_environment_session.py rename to _old/environment/tests/mock_environment_session.py diff --git a/environment/tests/test_atari.py b/_old/environment/tests/test_atari.py similarity index 97% rename from environment/tests/test_atari.py rename to _old/environment/tests/test_atari.py index ff98b864..2b4f33c8 100644 --- a/environment/tests/test_atari.py +++ b/_old/environment/tests/test_atari.py @@ -14,7 +14,7 @@ import pytest from cogment_verse_environment.utils.serialization_helpers import deserialize_img, deserialize_np_array -from data_pb2 import AgentAction, EnvironmentConfig +from data_pb2 import PlayerAction, EnvironmentConfig from mock_environment_session import ActorInfo # pylint doesn't like test fixtures diff --git a/environment/tests/test_minatarenv.py b/_old/environment/tests/test_minatarenv.py similarity index 100% rename from environment/tests/test_minatarenv.py rename to _old/environment/tests/test_minatarenv.py diff --git a/environment/tests/test_pettingzoo.py b/_old/environment/tests/test_pettingzoo.py similarity index 98% rename from environment/tests/test_pettingzoo.py rename to _old/environment/tests/test_pettingzoo.py index 1771a076..2da262e2 100644 --- a/environment/tests/test_pettingzoo.py +++ b/_old/environment/tests/test_pettingzoo.py @@ -15,7 +15,7 @@ import numpy as np import pytest from cogment_verse_environment.utils.serialization_helpers import deserialize_img, deserialize_np_array -from data_pb2 import AgentAction, EnvironmentConfig +from data_pb2 import PlayerAction, EnvironmentConfig from mock_environment_session import ActorInfo # pylint doesn't like test fixtures diff --git a/environment/tests/test_procgen.py b/_old/environment/tests/test_procgen.py similarity index 98% rename from environment/tests/test_procgen.py rename to _old/environment/tests/test_procgen.py index b47a9e4d..664acce9 100644 --- a/environment/tests/test_procgen.py +++ b/_old/environment/tests/test_procgen.py @@ -16,7 +16,7 @@ import pytest from cogment_verse_environment.procgen_env import ENV_NAMES, ProcGenEnv from cogment_verse_environment.utils.serialization_helpers import deserialize_img, deserialize_np_array -from data_pb2 import AgentAction, EnvironmentConfig +from data_pb2 import PlayerAction, EnvironmentConfig from mock_environment_session import ActorInfo # pylint doesn't like test fixtures diff --git a/grafana/dashboards/experiment_tracker.json b/_old/grafana/dashboards/experiment_tracker.json similarity index 100% rename from grafana/dashboards/experiment_tracker.json rename to _old/grafana/dashboards/experiment_tracker.json diff --git a/grafana/dashboards/model_registry.json b/_old/grafana/dashboards/model_registry.json similarity index 100% rename from grafana/dashboards/model_registry.json rename to _old/grafana/dashboards/model_registry.json diff --git a/grafana/grafana.ini b/_old/grafana/grafana.ini similarity index 100% rename from grafana/grafana.ini rename to _old/grafana/grafana.ini diff --git a/grafana/provisioning/dashboards/cogment_verse_dashboards.yml b/_old/grafana/provisioning/dashboards/cogment_verse_dashboards.yml similarity index 100% rename from grafana/provisioning/dashboards/cogment_verse_dashboards.yml rename to _old/grafana/provisioning/dashboards/cogment_verse_dashboards.yml diff --git a/grafana/provisioning/datasources/prometheus.yaml b/_old/grafana/provisioning/datasources/prometheus.yaml similarity index 100% rename from grafana/provisioning/datasources/prometheus.yaml rename to _old/grafana/provisioning/datasources/prometheus.yaml diff --git a/prometheus/prometheus.yml.tmpl b/_old/prometheus/prometheus.yml.tmpl similarity index 100% rename from prometheus/prometheus.yml.tmpl rename to _old/prometheus/prometheus.yml.tmpl diff --git a/run.sh b/_old/run.sh similarity index 97% rename from run.sh rename to _old/run.sh index 1e68008d..9e2dc027 100755 --- a/run.sh +++ b/_old/run.sh @@ -236,7 +236,7 @@ function mlflow_start() { function web_client_build() { _load_dot_env export PORT="${COGMENT_VERSE_WEBCLIENT_PORT}" - export REACT_APP_ORCHESTRATOR_HTTP_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}" + export REACT_APP_ORCHESTRATOR_WEB_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}" cp "${ROOT_DIR}/data.proto" "${ROOT_DIR}/cogment.yaml" "${ROOT_DIR}/web_client" cd "${ROOT_DIR}/web_client" npm install --no-audit @@ -253,7 +253,7 @@ function web_client_start() { function web_client_start_dev() { _load_dot_env export PORT="${COGMENT_VERSE_WEBCLIENT_PORT}" - export REACT_APP_ORCHESTRATOR_HTTP_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}" + export REACT_APP_ORCHESTRATOR_WEB_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}" cd "${ROOT_DIR}/web_client" npm run dev } diff --git a/run_api.proto b/_old/run_api.proto similarity index 100% rename from run_api.proto rename to _old/run_api.proto diff --git a/run_params.yaml b/_old/run_params.yaml similarity index 100% rename from run_params.yaml rename to _old/run_params.yaml diff --git a/tf_agents/cogment_verse_tf_agents/reinforce/model.py b/_old/tf_agents/cogment_verse_tf_agents/reinforce/model.py similarity index 100% rename from tf_agents/cogment_verse_tf_agents/reinforce/model.py rename to _old/tf_agents/cogment_verse_tf_agents/reinforce/model.py diff --git a/tf_agents/cogment_verse_tf_agents/reinforce/reinforce.py b/_old/tf_agents/cogment_verse_tf_agents/reinforce/reinforce.py similarity index 100% rename from tf_agents/cogment_verse_tf_agents/reinforce/reinforce.py rename to _old/tf_agents/cogment_verse_tf_agents/reinforce/reinforce.py diff --git a/tf_agents/cogment_verse_tf_agents/reinforce/reinforce_agent_adapter.py b/_old/tf_agents/cogment_verse_tf_agents/reinforce/reinforce_agent_adapter.py similarity index 100% rename from tf_agents/cogment_verse_tf_agents/reinforce/reinforce_agent_adapter.py rename to _old/tf_agents/cogment_verse_tf_agents/reinforce/reinforce_agent_adapter.py diff --git a/tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py b/_old/tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py similarity index 100% rename from tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py rename to _old/tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py diff --git a/tf_agents/cogment_verse_tf_agents/reinforce/sample_producer.py b/_old/tf_agents/cogment_verse_tf_agents/reinforce/sample_producer.py similarity index 100% rename from tf_agents/cogment_verse_tf_agents/reinforce/sample_producer.py rename to _old/tf_agents/cogment_verse_tf_agents/reinforce/sample_producer.py diff --git a/tf_agents/cogment_verse_tf_agents/reinforce/training_run.py b/_old/tf_agents/cogment_verse_tf_agents/reinforce/training_run.py similarity index 100% rename from tf_agents/cogment_verse_tf_agents/reinforce/training_run.py rename to _old/tf_agents/cogment_verse_tf_agents/reinforce/training_run.py diff --git a/tf_agents/cogment_verse_tf_agents/wrapper.py b/_old/tf_agents/cogment_verse_tf_agents/wrapper.py similarity index 92% rename from tf_agents/cogment_verse_tf_agents/wrapper.py rename to _old/tf_agents/cogment_verse_tf_agents/wrapper.py index b19309b5..90954007 100644 --- a/tf_agents/cogment_verse_tf_agents/wrapper.py +++ b/_old/tf_agents/cogment_verse_tf_agents/wrapper.py @@ -14,7 +14,7 @@ import cv2 import numpy as np -from data_pb2 import AgentAction, ContinuousAction +from data_pb2 import PlayerAction, ContinuousAction # TODO directly use tf tensors @@ -53,11 +53,11 @@ def cog_action_from_tf_action(action): if dtype in (int, np.int32, np.int64): field = "discrete_action" kwargs = {field: action} - return AgentAction(**kwargs) + return PlayerAction(**kwargs) # else # pylint: disable=no-member - agent_action = AgentAction(continuous_action=ContinuousAction()) + agent_action = PlayerAction(continuous_action=ContinuousAction()) action = np.squeeze(action) if action.shape == (): agent_action.continuous_action.data.append(action) diff --git a/tf_agents/main.py b/_old/tf_agents/main.py similarity index 100% rename from tf_agents/main.py rename to _old/tf_agents/main.py diff --git a/tf_agents/requirements.txt b/_old/tf_agents/requirements.txt similarity index 100% rename from tf_agents/requirements.txt rename to _old/tf_agents/requirements.txt diff --git a/torch_agents/.coveragerc b/_old/torch_agents/.coveragerc similarity index 100% rename from torch_agents/.coveragerc rename to _old/torch_agents/.coveragerc diff --git a/torch_agents/cogment_verse_torch_agents/atari_cnn.py b/_old/torch_agents/cogment_verse_torch_agents/atari_cnn.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/atari_cnn.py rename to _old/torch_agents/cogment_verse_torch_agents/atari_cnn.py diff --git a/torch_agents/cogment_verse_torch_agents/hf_sb3/__init__.py b/_old/torch_agents/cogment_verse_torch_agents/hf_sb3/__init__.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/hf_sb3/__init__.py rename to _old/torch_agents/cogment_verse_torch_agents/hf_sb3/__init__.py diff --git a/torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py b/_old/torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py similarity index 98% rename from torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py rename to _old/torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py index 0949b62d..1ab2227f 100644 --- a/torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py +++ b/_old/torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py @@ -21,7 +21,7 @@ from cogment_verse import AgentAdapter from cogment_verse_torch_agents.utils.tensors import tensor_from_cog_obs -from data_pb2 import AgentAction +from data_pb2 import PlayerAction from huggingface_sb3 import load_from_hub from stable_baselines3 import PPO diff --git a/torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py b/_old/torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py similarity index 92% rename from torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py rename to _old/torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py index 85b3cc47..6f98735d 100644 --- a/torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py +++ b/_old/torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py @@ -24,7 +24,11 @@ from cogment_verse_torch_agents.third_party.hive.dqn import DQNAgent from cogment_verse_torch_agents.third_party.hive.rainbow import RainbowDQNAgent from cogment_verse_torch_agents.third_party.td3.td3 import TD3Agent -from cogment_verse_torch_agents.wrapper import cog_action_from_torch_action, format_legal_moves, torch_obs_from_cog_obs +from cogment_verse_torch_agents.wrapper import ( + cog_action_from_torch_action, + format_legal_moves, + torch_obs_from_cog_obs, +) from data_pb2 import RunConfig from prometheus_client import Summary @@ -76,7 +80,15 @@ def _create(self, model_id, impl_name, environment_specs, **kwargs): return model, model_user_data - def _load(self, model_id, version_number, model_user_data, version_user_data, model_data_f, **kwargs): + def _load( + self, + model_id, + version_number, + model_user_data, + version_user_data, + model_data_f, + **kwargs, + ): model = self.agent_class_from_impl_name(model_user_data["agent_implementation"])( id=model_id, obs_dim=int(model_user_data["num_input"]), @@ -143,4 +155,10 @@ async def impl(actor_session): return {impl_name: (create_actor_impl(impl_name), ["agent"]) for impl_name in self._agent_classes} def _create_run_implementations(self): - return {"cogment_verse_run_impl": (sample_producer, create_training_run(self), RunConfig())} + return { + "cogment_verse_run_impl": ( + sample_producer, + create_training_run(self), + RunConfig(), + ) + } diff --git a/torch_agents/cogment_verse_torch_agents/hive_adapter/sample_producer.py b/_old/torch_agents/cogment_verse_torch_agents/hive_adapter/sample_producer.py similarity index 97% rename from torch_agents/cogment_verse_torch_agents/hive_adapter/sample_producer.py rename to _old/torch_agents/cogment_verse_torch_agents/hive_adapter/sample_producer.py index cc52f8f2..9144d792 100644 --- a/torch_agents/cogment_verse_torch_agents/hive_adapter/sample_producer.py +++ b/_old/torch_agents/cogment_verse_torch_agents/hive_adapter/sample_producer.py @@ -15,7 +15,11 @@ from collections import namedtuple from cogment_verse.spaces import flattened_dimensions import cogment.api.common_pb2 as common_api -from cogment_verse_torch_agents.wrapper import format_legal_moves, torch_action_from_cog_action, torch_obs_from_cog_obs +from cogment_verse_torch_agents.wrapper import ( + format_legal_moves, + torch_action_from_cog_action, + torch_obs_from_cog_obs, +) def vectorized_training_sample_from_samples( diff --git a/torch_agents/cogment_verse_torch_agents/hive_adapter/training_run.py b/_old/torch_agents/cogment_verse_torch_agents/hive_adapter/training_run.py similarity index 97% rename from torch_agents/cogment_verse_torch_agents/hive_adapter/training_run.py rename to _old/torch_agents/cogment_verse_torch_agents/hive_adapter/training_run.py index 2eb7f172..c2ce921b 100644 --- a/torch_agents/cogment_verse_torch_agents/hive_adapter/training_run.py +++ b/_old/torch_agents/cogment_verse_torch_agents/hive_adapter/training_run.py @@ -20,7 +20,11 @@ from cogment_verse import MlflowExperimentTracker from cogment_verse.utils import sizeof_fmt, throttle -from cogment_verse.constants import HUMAN_ACTOR_NAME, HUMAN_ACTOR_CLASS, HUMAN_ACTOR_IMPL +from cogment_verse.constants import ( + WEB_ACTOR_NAME, + TEACHER_ACTOR_CLASS, + HUMAN_ACTOR_IMPL, +) from cogment_verse_torch_agents.third_party.hive.utils.schedule import ( CosineSchedule, LinearSchedule, @@ -156,8 +160,8 @@ async def training_run(run_session): if config.demonstration_count > 0: # create the config for the teacher agent teacher_actor_config = ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, + name=WEB_ACTOR_NAME, + actor_class=TEACHER_ACTOR_CLASS, implementation=HUMAN_ACTOR_IMPL, human_config=HumanConfig( run_id=run_id, @@ -317,7 +321,10 @@ async def run_trials(trial_configs, max_parallel_trials): max_parallel_trials=config.max_parallel_trials, ) if self_play_trial_configs: - await run_trials(self_play_trial_configs, max_parallel_trials=config.max_parallel_trials) + await run_trials( + self_play_trial_configs, + max_parallel_trials=config.max_parallel_trials, + ) run_xp_tracker.terminate_success() diff --git a/torch_agents/cogment_verse_torch_agents/muzero/adapter.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/adapter.py similarity index 93% rename from torch_agents/cogment_verse_torch_agents/muzero/adapter.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/adapter.py index de8e7966..18c41b4b 100644 --- a/torch_agents/cogment_verse_torch_agents/muzero/adapter.py +++ b/_old/torch_agents/cogment_verse_torch_agents/muzero/adapter.py @@ -38,7 +38,11 @@ ) from cogment_verse.utils import LRU -from cogment_verse.constants import HUMAN_ACTOR_NAME, HUMAN_ACTOR_CLASS, HUMAN_ACTOR_IMPL +from cogment_verse.constants import ( + WEB_ACTOR_NAME, + TEACHER_ACTOR_CLASS, + HUMAN_ACTOR_IMPL, +) from cogment_verse import AgentAdapter from cogment_verse import MlflowExperimentTracker from cogment_verse.spaces import flattened_dimensions @@ -59,7 +63,10 @@ log = logging.getLogger(__name__) -MuZeroSample = namedtuple("MuZeroSample", ["state", "action", "reward", "next_state", "done", "policy", "value"]) +MuZeroSample = namedtuple( + "MuZeroSample", + ["state", "action", "reward", "next_state", "done", "policy", "value"], +) DEFAULT_MUZERO_RUN_CONFIG = MuZeroRunConfig( @@ -193,7 +200,15 @@ def _load( ): return MuZeroAgent.load(model_data_f, "cpu") - def _save(self, model, model_user_data, model_data_f, epoch_idx=-1, total_samples=0, **kwargs): + def _save( + self, + model, + model_user_data, + model_data_f, + epoch_idx=-1, + total_samples=0, + **kwargs, + ): assert isinstance(model, MuZeroAgent) model.save(model_data_f) return {"epoch_idx": epoch_idx, "total_samples": total_samples} @@ -243,7 +258,10 @@ async def single_agent_muzero_sample_producer_implementation(self, run_sample_pr total_reward += reward run_sample_producer_session.produce_training_sample( - (MuZeroSample(state, action, reward, next_state, done, policy, value), total_reward) + ( + MuZeroSample(state, action, reward, next_state, done, policy, value), + total_reward, + ) ) if done: @@ -318,7 +336,10 @@ async def single_agent_muzero_run_implementation(self, run_session): max_parallel_trials=config.max_parallel_trials, ) - async for _step, timestamp, trial_id, _tick, (sample, total_reward) in sample_generator: + async for _step, timestamp, trial_id, _tick, ( + sample, + total_reward, + ) in sample_generator: replay_buffer.add_sample(trial_id, sample) total_samples += 1 @@ -328,7 +349,10 @@ async def single_agent_muzero_run_implementation(self, run_session): if sample.done: trials_completed += 1 xp_tracker.log_metrics( - timestamp, total_samples, trial_total_reward=total_reward, trials_completed=trials_completed + timestamp, + total_samples, + trial_total_reward=total_reward, + trials_completed=trials_completed, ) run_total_reward += total_reward @@ -337,7 +361,10 @@ async def single_agent_muzero_run_implementation(self, run_session): while not train_worker.results_queue.empty(): try: - info, serialized_model = train_worker.results_queue.get_nowait() + ( + info, + serialized_model, + ) = train_worker.results_queue.get_nowait() if serialized_model is not None: epoch_idx += 1 agent = MuZeroAgent.load(io.BytesIO(serialized_model), "cpu") @@ -414,8 +441,8 @@ def clone_config(config, render, seed): agent_config=actor_config, ) teacher_config = ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, + name=WEB_ACTOR_NAME, + actor_class=TEACHER_ACTOR_CLASS, implementation=HUMAN_ACTOR_IMPL, agent_config=actor_config, # todo: this needs to be modified to HumanConfig ) diff --git a/torch_agents/cogment_verse_torch_agents/muzero/agent.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/agent.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/agent.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/agent.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/mcts.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/mcts.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/mcts.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/mcts.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/networks.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/networks.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/networks.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/networks.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/reanalyze_worker.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/reanalyze_worker.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/reanalyze_worker.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/reanalyze_worker.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/replay_buffer.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/replay_buffer.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/replay_buffer.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/replay_buffer.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/replay_worker.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/replay_worker.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/replay_worker.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/replay_worker.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/schedule.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/schedule.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/schedule.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/schedule.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/train_worker.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/train_worker.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/train_worker.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/train_worker.py diff --git a/torch_agents/cogment_verse_torch_agents/muzero/trial_worker.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/trial_worker.py similarity index 85% rename from torch_agents/cogment_verse_torch_agents/muzero/trial_worker.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/trial_worker.py index 33035006..e0da2d01 100644 --- a/torch_agents/cogment_verse_torch_agents/muzero/trial_worker.py +++ b/_old/torch_agents/cogment_verse_torch_agents/muzero/trial_worker.py @@ -17,9 +17,12 @@ import torch import torch.multiprocessing as mp -from data_pb2 import AgentAction +from data_pb2 import PlayerAction -from cogment_verse_torch_agents.wrapper import np_array_from_proto_array, proto_array_from_np_array +from cogment_verse_torch_agents.wrapper import ( + np_array_from_proto_array, + proto_array_from_np_array, +) from cogment_verse_torch_agents.muzero.utils import MuZeroWorker @@ -56,5 +59,9 @@ async def main(self): observation = event.observation.snapshot.vectorized observation = np_array_from_proto_array(observation) action_int, policy, value = self._agent.act(torch.tensor(observation)) - action = AgentAction(discrete_action=action_int, policy=proto_array_from_np_array(policy), value=value) + action = AgentAction( + discrete_action=action_int, + policy=proto_array_from_np_array(policy), + value=value, + ) self._action_queue.put(action) diff --git a/torch_agents/cogment_verse_torch_agents/muzero/utils.py b/_old/torch_agents/cogment_verse_torch_agents/muzero/utils.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/muzero/utils.py rename to _old/torch_agents/cogment_verse_torch_agents/muzero/utils.py diff --git a/torch_agents/cogment_verse_torch_agents/selfplay_td3/model.py b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/model.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/selfplay_td3/model.py rename to _old/torch_agents/cogment_verse_torch_agents/selfplay_td3/model.py diff --git a/torch_agents/cogment_verse_torch_agents/selfplay_td3/replaybuffer.py b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/replaybuffer.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/selfplay_td3/replaybuffer.py rename to _old/torch_agents/cogment_verse_torch_agents/selfplay_td3/replaybuffer.py diff --git a/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_agent.py b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_agent.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_agent.py rename to _old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_agent.py diff --git a/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_sample_producer.py b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_sample_producer.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_sample_producer.py rename to _old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_sample_producer.py diff --git a/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_td3.py b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_td3.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_td3.py rename to _old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_td3.py diff --git a/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_training_run.py b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_training_run.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_training_run.py rename to _old/torch_agents/cogment_verse_torch_agents/selfplay_td3/selfplay_training_run.py diff --git a/torch_agents/cogment_verse_torch_agents/selfplay_td3/wrapper.py b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/wrapper.py similarity index 97% rename from torch_agents/cogment_verse_torch_agents/selfplay_td3/wrapper.py rename to _old/torch_agents/cogment_verse_torch_agents/selfplay_td3/wrapper.py index ef9cb77c..e18419ad 100644 --- a/torch_agents/cogment_verse_torch_agents/selfplay_td3/wrapper.py +++ b/_old/torch_agents/cogment_verse_torch_agents/selfplay_td3/wrapper.py @@ -15,7 +15,7 @@ import numpy as np import torch -from data_pb2 import AgentAction, ContinuousAction +from data_pb2 import PlayerAction, ContinuousAction def tensor_from_cog_state(cog_obs, dtype=torch.float, device=None): diff --git a/torch_agents/cogment_verse_torch_agents/third_party/README.md b/_old/torch_agents/cogment_verse_torch_agents/third_party/README.md similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/README.md rename to _old/torch_agents/cogment_verse_torch_agents/third_party/README.md diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/agent.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/agent.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/agent.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/agent.py diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/ddpg.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/ddpg.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/ddpg.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/ddpg.py diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/dqn.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/dqn.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/dqn.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/dqn.py diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/mlp.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/mlp.py similarity index 83% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/mlp.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/mlp.py index e7669b3c..90f4632c 100644 --- a/torch_agents/cogment_verse_torch_agents/third_party/hive/mlp.py +++ b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/mlp.py @@ -63,7 +63,9 @@ def forward(self, inp): if self.training: weight_eps, bias_eps = self.sample_noise() return F.linear( - inp, self.weight_mu + self.weight_sigma * weight_eps, self.bias_mu + self.bias_sigma * bias_eps, + inp, + self.weight_mu + self.weight_sigma * weight_eps, + self.bias_mu + self.bias_sigma * bias_eps, ) else: return F.linear(inp, self.weight_mu, self.bias_mu) @@ -100,11 +102,15 @@ def __init__( def init_networks(self): if self._noisy: self.input_layer = nn.Sequential( - NoisyLinear(self._in_dim, self._hidden_units, self._sigma_init), nn.ReLU(), + NoisyLinear(self._in_dim, self._hidden_units, self._sigma_init), + nn.ReLU(), ) self.hidden_layers = nn.Sequential( *[ - nn.Sequential(NoisyLinear(self._hidden_units, self._hidden_units, self._sigma_init), nn.ReLU(),) + nn.Sequential( + NoisyLinear(self._hidden_units, self._hidden_units, self._sigma_init), + nn.ReLU(), + ) for _ in range(self._num_hidden_layers - 1) ] ) @@ -128,26 +134,42 @@ def init_networks(self): self.output_layer_adv = nn.Sequential( NoisyLinear(self._hidden_units, self._hidden_units, self._sigma_init), nn.ReLU(), - NoisyLinear(self._hidden_units, self._out_dim * self._atoms, self._sigma_init,), + NoisyLinear( + self._hidden_units, + self._out_dim * self._atoms, + self._sigma_init, + ), ) self.output_layer_val = nn.Sequential( NoisyLinear(self._hidden_units, self._hidden_units, self._sigma_init), nn.ReLU(), - NoisyLinear(self._hidden_units, 1 * self._atoms, self._sigma_init,), + NoisyLinear( + self._hidden_units, + 1 * self._atoms, + self._sigma_init, + ), ) else: self.output_layer_adv = nn.Sequential( nn.Linear(self._hidden_units, self._hidden_units, self._sigma_init), nn.ReLU(), - nn.Linear(self._hidden_units, self._out_dim * self._atoms, self._sigma_init,), + nn.Linear( + self._hidden_units, + self._out_dim * self._atoms, + self._sigma_init, + ), ) self.output_layer_val = nn.Sequential( nn.Linear(self._hidden_units, self._hidden_units, self._sigma_init), nn.ReLU(), - nn.Linear(self._hidden_units, 1 * self._atoms, self._sigma_init,), + nn.Linear( + self._hidden_units, + 1 * self._atoms, + self._sigma_init, + ), ) else: if self._noisy: @@ -192,7 +214,14 @@ def __init__( atoms=51, ): super().__init__( - in_dim, out_dim, hidden_units, num_hidden_layers, noisy, dueling, sigma_init, atoms, + in_dim, + out_dim, + hidden_units, + num_hidden_layers, + noisy, + dueling, + sigma_init, + atoms, ) self._supports = supports diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/rainbow.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/rainbow.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/rainbow.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/rainbow.py diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/replay_buffer.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/replay_buffer.py similarity index 96% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/replay_buffer.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/replay_buffer.py index 339e3fad..487168ac 100644 --- a/torch_agents/cogment_verse_torch_agents/third_party/hive/replay_buffer.py +++ b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/replay_buffer.py @@ -231,7 +231,10 @@ def __init__( self._stack_size = stack_size self._n_step = n_step self._gamma = gamma - self._discount = np.asarray([self._gamma ** i for i in range(self._n_step)], dtype=self._specs["reward"][0],) + self._discount = np.asarray( + [self._gamma**i for i in range(self._n_step)], + dtype=self._specs["reward"][0], + ) self._episode_start = True self._cursor = 0 self._num_added = 0 @@ -239,7 +242,10 @@ def __init__( def size(self): """Returns the number of transitions stored in the buffer.""" - return max(min(self._num_added, self._capacity) - self._stack_size - self._n_step + 1, 0,) + return max( + min(self._num_added, self._capacity) - self._stack_size - self._n_step + 1, + 0, + ) def _create_storage(self, capacity, specs): """Creates the storage buffer for each type of item in the buffer. @@ -364,7 +370,9 @@ def sample(self, batch_size): for key in self._specs: if key == "observation": batch[key] = self._get_from_storage( - "observation", indices - self._stack_size + 1, num_to_access=self._stack_size, + "observation", + indices - self._stack_size + 1, + num_to_access=self._stack_size, ) elif key == "done": batch["done"] = is_terminal @@ -381,7 +389,9 @@ def sample(self, batch_size): else: batch[key] = self._get_from_storage(key, indices) batch["next_observation"] = self._get_from_storage( - "observation", indices + trajectory_lengths - self._stack_size + 1, num_to_access=self._stack_size, + "observation", + indices + trajectory_lengths - self._stack_size + 1, + num_to_access=self._stack_size, ) return batch diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/utils/schedule.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/utils/schedule.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/utils/schedule.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/utils/schedule.py diff --git a/torch_agents/cogment_verse_torch_agents/third_party/hive/utils/utils.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/hive/utils/utils.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/hive/utils/utils.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/hive/utils/utils.py diff --git a/torch_agents/cogment_verse_torch_agents/third_party/td3/README.md b/_old/torch_agents/cogment_verse_torch_agents/third_party/td3/README.md similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/td3/README.md rename to _old/torch_agents/cogment_verse_torch_agents/third_party/td3/README.md diff --git a/torch_agents/cogment_verse_torch_agents/third_party/td3/td3.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/td3/td3.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/td3/td3.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/td3/td3.py diff --git a/torch_agents/cogment_verse_torch_agents/third_party/td3/td3_mlp.py b/_old/torch_agents/cogment_verse_torch_agents/third_party/td3/td3_mlp.py similarity index 100% rename from torch_agents/cogment_verse_torch_agents/third_party/td3/td3_mlp.py rename to _old/torch_agents/cogment_verse_torch_agents/third_party/td3/td3_mlp.py diff --git a/torch_agents/cogment_verse_torch_agents/utils/tensors.py b/_old/torch_agents/cogment_verse_torch_agents/utils/tensors.py similarity index 97% rename from torch_agents/cogment_verse_torch_agents/utils/tensors.py rename to _old/torch_agents/cogment_verse_torch_agents/utils/tensors.py index 4c71f29e..bbc8a683 100644 --- a/torch_agents/cogment_verse_torch_agents/utils/tensors.py +++ b/_old/torch_agents/cogment_verse_torch_agents/utils/tensors.py @@ -14,7 +14,7 @@ import numpy as np import torch -from data_pb2 import AgentAction +from data_pb2 import PlayerAction def tensor_from_cog_obs(cog_obs, device=None, dtype=torch.float): diff --git a/base_python/cogment_verse/utils/throttle.py b/_old/torch_agents/cogment_verse_torch_agents/utils/throttle.py similarity index 100% rename from base_python/cogment_verse/utils/throttle.py rename to _old/torch_agents/cogment_verse_torch_agents/utils/throttle.py diff --git a/torch_agents/cogment_verse_torch_agents/wrapper.py b/_old/torch_agents/cogment_verse_torch_agents/wrapper.py similarity index 92% rename from torch_agents/cogment_verse_torch_agents/wrapper.py rename to _old/torch_agents/cogment_verse_torch_agents/wrapper.py index bfb9e5cf..9ec9d02c 100644 --- a/torch_agents/cogment_verse_torch_agents/wrapper.py +++ b/_old/torch_agents/cogment_verse_torch_agents/wrapper.py @@ -14,7 +14,7 @@ import cv2 import numpy as np -from data_pb2 import AgentAction, ContinuousAction, Observation, NDArray +from data_pb2 import PlayerAction, ContinuousAction, Observation, NDArray # TODO directly use torch tensors @@ -99,14 +99,14 @@ def cog_action_from_torch_action(action): if dtype in (int, np.int32, np.int64): if isinstance(action, np.ndarray): - return AgentAction(discrete_action=action.item()) + return PlayerAction(discrete_action=action.item()) if isinstance(action, list): - return AgentAction(discrete_action=action[0]) - return AgentAction(discrete_action=action) + return PlayerAction(discrete_action=action[0]) + return PlayerAction(discrete_action=action) # else # pylint: disable=no-member - agent_action = AgentAction(continuous_action=ContinuousAction()) + agent_action = PlayerAction(continuous_action=ContinuousAction()) action = np.squeeze(action) if action.shape == (): agent_action.continuous_action.data.append(action) diff --git a/torch_agents/main.py b/_old/torch_agents/main.py similarity index 100% rename from torch_agents/main.py rename to _old/torch_agents/main.py diff --git a/torch_agents/pyproject.toml b/_old/torch_agents/pyproject.toml similarity index 100% rename from torch_agents/pyproject.toml rename to _old/torch_agents/pyproject.toml diff --git a/torch_agents/requirements.txt b/_old/torch_agents/requirements.txt similarity index 100% rename from torch_agents/requirements.txt rename to _old/torch_agents/requirements.txt diff --git a/torch_agents/tests/test_atari_dqn.py b/_old/torch_agents/tests/test_atari_dqn.py similarity index 100% rename from torch_agents/tests/test_atari_dqn.py rename to _old/torch_agents/tests/test_atari_dqn.py diff --git a/torch_agents/tests/test_ddpg.py b/_old/torch_agents/tests/test_ddpg.py similarity index 100% rename from torch_agents/tests/test_ddpg.py rename to _old/torch_agents/tests/test_ddpg.py diff --git a/torch_agents/tests/test_episode.py b/_old/torch_agents/tests/test_episode.py similarity index 100% rename from torch_agents/tests/test_episode.py rename to _old/torch_agents/tests/test_episode.py diff --git a/torch_agents/tests/test_mcts.py b/_old/torch_agents/tests/test_mcts.py similarity index 100% rename from torch_agents/tests/test_mcts.py rename to _old/torch_agents/tests/test_mcts.py diff --git a/torch_agents/tests/test_muzero.py b/_old/torch_agents/tests/test_muzero.py similarity index 100% rename from torch_agents/tests/test_muzero.py rename to _old/torch_agents/tests/test_muzero.py diff --git a/torch_agents/tests/test_simple_a2c.py b/_old/torch_agents/tests/test_simple_a2c.py similarity index 100% rename from torch_agents/tests/test_simple_a2c.py rename to _old/torch_agents/tests/test_simple_a2c.py diff --git a/torch_agents/tests/test_td3.py b/_old/torch_agents/tests/test_td3.py similarity index 100% rename from torch_agents/tests/test_td3.py rename to _old/torch_agents/tests/test_td3.py diff --git a/torch_agents/tests/test_trial_replay_buffer.py b/_old/torch_agents/tests/test_trial_replay_buffer.py similarity index 100% rename from torch_agents/tests/test_trial_replay_buffer.py rename to _old/torch_agents/tests/test_trial_replay_buffer.py diff --git a/actors/random_actor.py b/actors/random_actor.py new file mode 100644 index 00000000..c4851650 --- /dev/null +++ b/actors/random_actor.py @@ -0,0 +1,44 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cogment + +from cogment_verse.specs import ( + PLAYER_ACTOR_CLASS, + PlayerAction, + sample_space, +) + + +class RandomActor: + def __init__(self, _cfg): + pass + + def get_actor_classes(self): + return [PLAYER_ACTOR_CLASS] + + async def impl(self, actor_session): + actor_session.start() + + config = actor_session.config + + action_space = config.environment_specs.action_space + + # TODO this is something that could be configured + random_seed = 0 + + async for event in actor_session.all_events(): + if event.observation and event.type == cogment.EventType.ACTIVE: + [action_value] = sample_space(action_space, seed=random_seed + actor_session.get_tick_id()) + actor_session.do_action(PlayerAction(value=action_value)) diff --git a/actors/simple_a2c.py b/actors/simple_a2c.py new file mode 100644 index 00000000..ca97b977 --- /dev/null +++ b/actors/simple_a2c.py @@ -0,0 +1,357 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import cogment +import torch + +from cogment_verse.specs import ( + AgentConfig, + cog_settings, + EnvironmentConfig, + flatten, + flattened_dimensions, + PLAYER_ACTOR_CLASS, + PlayerAction, + SpaceValue, +) +from cogment_verse import Model + + +log = logging.getLogger(__name__) + + +class SimpleA2CModel(Model): + def __init__( + self, + model_id, + environment_implementation, + num_input, + num_output, + actor_network_hidden_size=64, + critic_network_hidden_size=64, + dtype=torch.float, + version_number=0, + ): + super().__init__(model_id, version_number) + self._dtype = dtype + self._environment_implementation = environment_implementation + self._num_input = num_input + self._num_output = num_output + self._actor_network_hidden_size = actor_network_hidden_size + self._critic_network_hidden_size = critic_network_hidden_size + + self.actor_network = torch.nn.Sequential( + torch.nn.Linear(self._num_input, self._actor_network_hidden_size, dtype=self._dtype), + torch.nn.Tanh(), + torch.nn.Linear(self._actor_network_hidden_size, self._actor_network_hidden_size, dtype=self._dtype), + torch.nn.Tanh(), + torch.nn.Linear(self._actor_network_hidden_size, self._num_output, dtype=self._dtype), + ) + + self.critic_network = torch.nn.Sequential( + torch.nn.Linear(self._num_input, self._critic_network_hidden_size, dtype=self._dtype), + torch.nn.Tanh(), + torch.nn.Linear(self._critic_network_hidden_size, self._critic_network_hidden_size, dtype=self._dtype), + torch.nn.Tanh(), + torch.nn.Linear(self._critic_network_hidden_size, 1, dtype=self._dtype), + ) + + # version user data + self.epoch_idx = 0 + self.total_samples = 0 + + def get_model_user_data(self): + return { + "environment_implementation": self._environment_implementation, + "num_input": self._num_input, + "num_output": self._num_output, + "actor_network_hidden_size": self._actor_network_hidden_size, + "critic_network_hidden_size": self._critic_network_hidden_size, + } + + def save(self, model_data_f): + torch.save((self.actor_network.state_dict(), self.critic_network.state_dict()), model_data_f) + + return {"epoch_idx": self.epoch_idx, "total_samples": self.total_samples} + + @classmethod + def load(cls, model_id, version_number, model_user_data, version_user_data, model_data_f): + # Create the model instance + model = SimpleA2CModel( + model_id=model_id, + version_number=version_number, + environment_implementation=model_user_data["environment_implementation"], + num_input=int(model_user_data["num_input"]), + num_output=int(model_user_data["num_output"]), + actor_network_hidden_size=int(model_user_data["actor_network_hidden_size"]), + critic_network_hidden_size=int(model_user_data["critic_network_hidden_size"]), + ) + + # Load the saved states + (actor_network_state_dict, critic_network_state_dict) = torch.load(model_data_f) + model.actor_network.load_state_dict(actor_network_state_dict) + model.critic_network.load_state_dict(critic_network_state_dict) + + # Load version data + model.epoch_idx = version_user_data["epoch_idx"] + model.total_samples = version_user_data["total_samples"] + return model + + +class SimpleA2CActor: + def __init__(self, _cfg): + self._dtype = torch.float + + def get_actor_classes(self): + return [PLAYER_ACTOR_CLASS] + + async def impl(self, actor_session): + actor_session.start() + + config = actor_session.config + + assert config.environment_specs.num_players == 1 + assert len(config.environment_specs.action_space.properties) == 1 + assert config.environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + + observation_space = config.environment_specs.observation_space + + model, _, _ = await actor_session.model_registry.retrieve_version( + SimpleA2CModel, config.model_id, config.model_version + ) + model.actor_network.eval() + model.critic_network.eval() + + async for event in actor_session.all_events(): + if event.observation and event.type == cogment.EventType.ACTIVE: + obs_tensor = torch.tensor( + flatten(observation_space, event.observation.observation.value), dtype=self._dtype + ) + probs = torch.softmax(model.actor_network(obs_tensor), dim=-1) + discrete_action_tensor = torch.distributions.Categorical(probs).sample() + action_value = SpaceValue(properties=[SpaceValue.PropertyValue(discrete=discrete_action_tensor.item())]) + actor_session.do_action(PlayerAction(value=action_value)) + + +class SimpleA2CTraining: + default_cfg = { + "environment_config": {"seed": 10}, + "training": { + "epoch_count": 500, + "epoch_trial_count": 10, + "num_parallel_trials": 8, + "discount_factor": 0.99, + "entropy_loss_coef": 0.05, + "value_loss_coef": 0.5, + "action_loss_coef": 1.0, + "learning_rate": 0.01, + }, + "actor_network": {"hidden_size": 64}, + "critic_network": {"hidden_size": 64}, + } + + def __init__(self, environment_specs, cfg): + super().__init__() + self._dtype = torch.float + self._environment_specs = environment_specs + self._cfg = cfg + + async def trial_sample_sequences_producer_impl(self, sample_producer_session): + observation = [] + action = [] + reward = [] + done = [] + + player_actor_params = sample_producer_session.trial_info.parameters.actors[0] + + player_actor_name = player_actor_params.name + player_observation_space = player_actor_params.config.environment_specs.observation_space + + async for sample in sample_producer_session.all_trial_samples(): + if sample.trial_state == cogment.TrialState.ENDED: + # This sample includes the last observation and no action + # The last sample was the last useful one + done[-1] = torch.ones(1, dtype=self._dtype) + break + + actor_sample = sample.actors_data[player_actor_name] + observation.append( + torch.tensor(flatten(player_observation_space, actor_sample.observation.value), dtype=self._dtype) + ) + action_value = actor_sample.action.value + action.append( + torch.tensor( + action_value.properties[0].discrete if len(action_value.properties) > 0 else 0, dtype=self._dtype + ) + ) + reward.append( + torch.tensor(actor_sample.reward if actor_sample.reward is not None else 0, dtype=self._dtype) + ) + done.append(torch.zeros(1, dtype=self._dtype)) + + # Keeping the samples grouped by trial by emitting only one grouped sample at the end of the trial + sample_producer_session.produce_sample((observation, action, reward, done)) + + async def impl(self, run_session): + # Initializing a model + model_id = f"{run_session.run_id}_model" + + assert self._environment_specs.num_players == 1 + assert len(self._environment_specs.action_space.properties) == 1 + assert self._environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + + model = SimpleA2CModel( + model_id, + environment_implementation=self._environment_specs.implementation, + num_input=flattened_dimensions(self._environment_specs.observation_space), + num_output=flattened_dimensions(self._environment_specs.action_space), + actor_network_hidden_size=self._cfg.actor_network.hidden_size, + critic_network_hidden_size=self._cfg.critic_network.hidden_size, + dtype=self._dtype, + ) + _model_info, version_info = await run_session.model_registry.publish_initial_version(model) + + run_session.log_params( + self._cfg.training, + self._cfg.environment_config, + environment_implementation=self._environment_specs.implementation, + actor_network_hidden_size=self._cfg.actor_network.hidden_size, + critic_network_hidden_size=self._cfg.critic_network.hidden_size, + ) + + # Configure the optimizer over the two models + optimizer = torch.optim.Adam( + torch.nn.Sequential(model.actor_network, model.critic_network).parameters(), + lr=self._cfg.training.learning_rate, + ) + + total_samples = 0 + for epoch_idx in range(self._cfg.training.epoch_count): + # Rollout a bunch of trials + observation = [] + action = [] + reward = [] + done = [] + for (_step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + trials_id_and_params=[ + ( + f"{run_session.run_id}_{epoch_idx}_{trial_idx}", + cogment.TrialParameters( + cog_settings, + environment_name="env", + environment_implementation=self._environment_specs.implementation, + environment_config=EnvironmentConfig( + run_id=run_session.run_id, + render=False, + seed=self._cfg.environment_config.seed + + trial_idx + + epoch_idx * self._cfg.training.epoch_trial_count, + ), + actors=[ + cogment.ActorParameters( + cog_settings, + name="player", + class_name=PLAYER_ACTOR_CLASS, + implementation="actors.simple_a2c.SimpleA2CActor", + config=AgentConfig( + run_id=run_session.run_id, + model_id=model_id, + model_version=version_info["version_number"], + environment_specs=self._environment_specs, + ), + ) + ], + ), + ) + for trial_idx in range(self._cfg.training.epoch_trial_count) + ], + sample_producer_impl=self.trial_sample_sequences_producer_impl, + num_parallel_trials=self._cfg.training.num_parallel_trials, + ): + (trial_observation, trial_action, trial_reward, trial_done) = sample + observation.extend(trial_observation) + action.extend(trial_action) + reward.extend(trial_reward) + done.extend(trial_done) + + run_session.log_metrics(total_reward=sum(r.item() for r in trial_reward)) + + if len(observation) == 0: + log.warning( + f"[SimpleA2CTraining/{run_session.run_id}] epoch #{epoch_idx + 1}/{self._cfg.training.epoch_count} finished without generating any sample (every trial ended at the first tick), skipping training." + ) + continue + + total_samples += len(observation) + + # Convert the accumulated observation/action/reward over the epoch to tensors + observation = torch.vstack(observation) + action = torch.vstack(action) + reward = torch.vstack(reward) + done = torch.vstack(done) + + # Compute the action probability and the critic value over the epoch + action_probs = torch.softmax(model.actor_network(observation), dim=-1) + critic = model.critic_network(observation).squeeze(-1) + + # Compute the estimated advantage over the epoch + advantage = ( + reward[1:] + + self._cfg.training.discount_factor * critic[1:].detach() * (1.0 - done[1:].float()) + - critic[:-1] + ) + + # Compute critic loss + value_loss = advantage.pow(2).mean() + + # Compute entropy loss + entropy_loss = torch.distributions.Categorical(action_probs).entropy().mean() + + # Compute A2C loss + action_log_probs = torch.gather(action_probs, -1, action.long()).log() + action_loss = -(action_log_probs[:-1] * advantage.detach()).mean() + + # Compute the complete loss + loss = ( + -self._cfg.training.entropy_loss_coef * entropy_loss + + self._cfg.training.value_loss_coef * value_loss + + self._cfg.training.action_loss_coef * action_loss + ) + + # Backprop! + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # Publish the newly trained version + last_epoch = (epoch_idx + 1) == self._cfg.training.epoch_count + + model.epoch_idx = epoch_idx + model.total_samples = total_samples + + version_info = await run_session.model_registry.publish_version(model, archived=last_epoch) + run_session.log_metrics( + model_version_number=version_info["version_number"], + epoch_idx=epoch_idx, + entropy_loss=entropy_loss.item(), + value_loss=value_loss.item(), + action_loss=action_loss.item(), + loss=loss.item(), + total_samples=total_samples, + ) + log.info( + f"[SimpleA2CTraining/{run_session.run_id}] epoch #{epoch_idx + 1}/{self._cfg.training.epoch_count} finished ({total_samples} samples seen)" + ) diff --git a/actors/tutorial/tutorial_1.py b/actors/tutorial/tutorial_1.py new file mode 100644 index 00000000..0f282eed --- /dev/null +++ b/actors/tutorial/tutorial_1.py @@ -0,0 +1,143 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import cogment + +from cogment_verse.specs import ( + AgentConfig, + cog_settings, + EnvironmentConfig, + HUMAN_ACTOR_IMPL, + PLAYER_ACTOR_CLASS, + PlayerAction, + sample_space, + TEACHER_ACTOR_CLASS, + WEB_ACTOR_NAME, +) + +log = logging.getLogger(__name__) + + +class SimpleBCActor: + def __init__(self, _cfg): + super().__init__() + + def get_actor_classes(self): + return [PLAYER_ACTOR_CLASS] + + async def impl(self, actor_session): + actor_session.start() + + config = actor_session.config + + async for event in actor_session.all_events(): + if event.observation and event.type == cogment.EventType.ACTIVE: + [action_value] = sample_space(config.environment_specs.action_space, seed=actor_session.get_tick_id()) + actor_session.do_action(PlayerAction(value=action_value)) + + +class SimpleBCTraining: + default_cfg = { + "environment_config": {"seed": 12}, + "training": { + "num_trials": 10, + }, + } + + def __init__(self, environment_specs, cfg): + super().__init__() + self._environment_specs = environment_specs + self._cfg = cfg + + async def sample_producer(self, sample_producer_session): + assert len(sample_producer_session.trial_info.parameters.actors) == 2 + + players_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == PLAYER_ACTOR_CLASS + ] + teachers_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == TEACHER_ACTOR_CLASS + ] + assert len(players_params) == 1 + assert len(teachers_params) == 1 + player_params = players_params[0] + teacher_params = teachers_params[0] + + async for sample in sample_producer_session.all_trial_samples(): + teacher_action = sample.actors_data[teacher_params.name].action + + if teacher_action.HasField("value"): + log.info(f"Got raw sample with action override from [{teacher_params.name}]") + else: + log.info(f"Got raw sample with action from [{player_params.name}]") + + async def impl(self, run_session): + assert self._environment_specs.num_players == 1 + + run_session.log_params( + self._cfg.training, + self._cfg.environment_config, + environment_implementation=self._environment_specs.implementation, + ) + + # Helper function to create a trial configuration + def create_trial_params(trial_idx): + agent_actor_params = cogment.ActorParameters( + cog_settings, + name="player", + class_name=PLAYER_ACTOR_CLASS, + implementation="actors.tutorial.tutorial_1.SimpleBCActor", + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + + teacher_actor_params = cogment.ActorParameters( + cog_settings, + name=WEB_ACTOR_NAME, + class_name=TEACHER_ACTOR_CLASS, + implementation=HUMAN_ACTOR_IMPL, + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + + return cogment.TrialParameters( + cog_settings, + environment_name="env", + environment_implementation=self._environment_specs.implementation, + environment_config=EnvironmentConfig( + run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + ), + actors=[agent_actor_params, teacher_actor_params], + ) + + # Rollout a bunch of trials + for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + trials_id_and_params=[ + (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) + for trial_idx in range(self._cfg.training.num_trials) + ], + sample_producer_impl=self.sample_producer, + num_parallel_trials=1, + ): + log.info(f"[{step_idx}] - Got sample [{sample}]") diff --git a/actors/tutorial/tutorial_2.py b/actors/tutorial/tutorial_2.py new file mode 100644 index 00000000..ba767236 --- /dev/null +++ b/actors/tutorial/tutorial_2.py @@ -0,0 +1,176 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import cogment + +############ TUTORIAL STEP 2 ############ +import torch + +######################################### + +from cogment_verse.specs import ( + AgentConfig, + cog_settings, + EnvironmentConfig, + ############ TUTORIAL STEP 2 ############ + flatten, + ######################################### + HUMAN_ACTOR_IMPL, + PLAYER_ACTOR_CLASS, + PlayerAction, + sample_space, + TEACHER_ACTOR_CLASS, + WEB_ACTOR_NAME, +) + +log = logging.getLogger(__name__) + + +class SimpleBCActor: + def __init__(self, _cfg): + super().__init__() + + def get_actor_classes(self): + return [PLAYER_ACTOR_CLASS] + + async def impl(self, actor_session): + actor_session.start() + + config = actor_session.config + + async for event in actor_session.all_events(): + if event.observation and event.type == cogment.EventType.ACTIVE: + [action_value] = sample_space(config.environment_specs.action_space, seed=actor_session.get_tick_id()) + actor_session.do_action(PlayerAction(value=action_value)) + + +class SimpleBCTraining: + default_cfg = { + "environment_config": {"seed": 12}, + "training": { + "num_trials": 10, + }, + } + + def __init__(self, environment_specs, cfg): + super().__init__() + ############ TUTORIAL STEP 2 ############ + self._dtype = torch.float + ######################################### + self._environment_specs = environment_specs + self._cfg = cfg + + async def sample_producer(self, sample_producer_session): + assert len(sample_producer_session.trial_info.parameters.actors) == 2 + + players_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == PLAYER_ACTOR_CLASS + ] + teachers_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == TEACHER_ACTOR_CLASS + ] + assert len(players_params) == 1 + assert len(teachers_params) == 1 + player_params = players_params[0] + teacher_params = teachers_params[0] + + ############ TUTORIAL STEP 2 ############ + environment_specs = player_params.config.environment_specs + ######################################### + + async for sample in sample_producer_session.all_trial_samples(): + ############ TUTORIAL STEP 2 ############ + observation_tensor = torch.tensor( + flatten(environment_specs.observation_space, sample.actors_data[player_params.name].observation.value), + dtype=self._dtype, + ) + ######################################### + + teacher_action = sample.actors_data[teacher_params.name].action + + ############ TUTORIAL STEP 2 ############ + if teacher_action.HasField("value"): + applied_action = teacher_action + demonstration = True + else: + applied_action = sample.actors_data[player_params.name].action + demonstration = False + + action_tensor = torch.tensor( + flatten(environment_specs.action_space, applied_action.value), dtype=self._dtype + ) + sample_producer_session.produce_sample((demonstration, observation_tensor, action_tensor)) + ######################################### + + async def impl(self, run_session): + assert self._environment_specs.num_players == 1 + + run_session.log_params( + self._cfg.training, + self._cfg.environment_config, + environment_implementation=self._environment_specs.implementation, + ) + + # Helper function to create a trial configuration + def create_trial_params(trial_idx): + agent_actor_params = cogment.ActorParameters( + cog_settings, + name="player", + class_name=PLAYER_ACTOR_CLASS, + ############ TUTORIAL STEP 2 ############ + implementation="actors.tutorial.tutorial_2.SimpleBCActor", + ######################################### + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + + teacher_actor_params = cogment.ActorParameters( + cog_settings, + name=WEB_ACTOR_NAME, + class_name=TEACHER_ACTOR_CLASS, + implementation=HUMAN_ACTOR_IMPL, + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + + return cogment.TrialParameters( + cog_settings, + environment_name="env", + environment_implementation=self._environment_specs.implementation, + environment_config=EnvironmentConfig( + run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + ), + actors=[agent_actor_params, teacher_actor_params], + ) + + # Rollout a bunch of trials + for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + trials_id_and_params=[ + (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) + for trial_idx in range(self._cfg.training.num_trials) + ], + sample_producer_impl=self.sample_producer, + num_parallel_trials=1, + ): + log.info(f"[{step_idx}] - Got sample [{sample}]") diff --git a/actors/tutorial/tutorial_3.py b/actors/tutorial/tutorial_3.py new file mode 100644 index 00000000..a1fb4e22 --- /dev/null +++ b/actors/tutorial/tutorial_3.py @@ -0,0 +1,287 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import cogment +import torch + +############ TUTORIAL STEP 3 ############ +from cogment_verse import Model + +######################################### +from cogment_verse.specs import ( + AgentConfig, + cog_settings, + EnvironmentConfig, + flatten, + ############ TUTORIAL STEP 3 ############ + flattened_dimensions, + ######################################### + HUMAN_ACTOR_IMPL, + PLAYER_ACTOR_CLASS, + PlayerAction, + ############ TUTORIAL STEP 3 ############ + SpaceValue, + ######################################### + TEACHER_ACTOR_CLASS, + WEB_ACTOR_NAME, +) + +log = logging.getLogger(__name__) + +############ TUTORIAL STEP 3 ############ +class SimpleBCModel(Model): + def __init__( + self, + model_id, + environment_implementation, + num_input, + num_output, + policy_network_hidden_size=64, + version_number=0, + ): + super().__init__(model_id, version_number) + + self._dtype = torch.float + self._environment_implementation = environment_implementation + self._num_input = num_input + self._num_output = num_output + self._policy_network_hidden_size = policy_network_hidden_size + + self.policy_network = torch.nn.Sequential( + torch.nn.Linear(num_input, policy_network_hidden_size, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.ReLU(), + torch.nn.Linear(policy_network_hidden_size, policy_network_hidden_size, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.ReLU(), + torch.nn.Linear(policy_network_hidden_size, num_output, dtype=self._dtype), + ) + + self.total_samples = 0 + + def get_model_user_data(self): + return { + "environment_implementation": self._environment_implementation, + "num_input": self._num_input, + "num_output": self._num_output, + "policy_network_hidden_size": self._policy_network_hidden_size, + } + + def save(self, model_data_f): + torch.save(self.policy_network.state_dict(), model_data_f) + + return {"total_samples": self.total_samples} + + @classmethod + def load(cls, model_id, version_number, model_user_data, version_user_data, model_data_f): + # Create the model instance + model = SimpleBCModel( + model_id=model_id, + version_number=version_number, + environment_implementation=model_user_data["environment_implementation"], + num_input=int(model_user_data["num_input"]), + num_output=int(model_user_data["num_output"]), + policy_network_hidden_size=int(model_user_data["policy_network_hidden_size"]), + ) + + # Load the saved states + policy_network_state_dict = torch.load(model_data_f) + model.policy_network.load_state_dict(policy_network_state_dict) + + # Load version data + model.total_samples = version_user_data["total_samples"] + return model + + +########################################## + + +class SimpleBCActor: + def __init__(self, _cfg): + super().__init__() + ############ TUTORIAL STEP 3 ############# + self._dtype = torch.float + ########################################## + + def get_actor_classes(self): + return [PLAYER_ACTOR_CLASS] + + async def impl(self, actor_session): + actor_session.start() + + config = actor_session.config + + ############ TUTORIAL STEP 3 ############ + observation_space = config.environment_specs.observation_space + + model, _model_info, version_info = await actor_session.model_registry.retrieve_version( + SimpleBCModel, config.model_id, config.model_version + ) + model_version_number = version_info["version_number"] + log.info(f"Starting trial with model v{model_version_number}") + + model.policy_network.eval() + ######################################### + + async for event in actor_session.all_events(): + if event.observation and event.type == cogment.EventType.ACTIVE: + ############ TUTORIAL STEP 3 ############ + observation_tensor = torch.tensor( + flatten(observation_space, event.observation.observation.value), dtype=self._dtype + ) + scores = model.policy_network(observation_tensor.view(1, -1)) + probs = torch.softmax(scores, dim=-1) + discrete_action_tensor = torch.distributions.Categorical(probs).sample() + action_value = SpaceValue(properties=[SpaceValue.PropertyValue(discrete=discrete_action_tensor.item())]) + ########################################## + actor_session.do_action(PlayerAction(value=action_value)) + + +class SimpleBCTraining: + default_cfg = { + "environment_config": {"seed": 12}, + "training": { + "num_trials": 10, + }, + ############ TUTORIAL STEP 3 ############ + "policy_network": {"hidden_size": 64}, + ########################################## + } + + def __init__(self, environment_specs, cfg): + super().__init__() + self._dtype = torch.float + self._environment_specs = environment_specs + self._cfg = cfg + + async def sample_producer(self, sample_producer_session): + assert len(sample_producer_session.trial_info.parameters.actors) == 2 + + players_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == PLAYER_ACTOR_CLASS + ] + teachers_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == TEACHER_ACTOR_CLASS + ] + assert len(players_params) == 1 + assert len(teachers_params) == 1 + player_params = players_params[0] + teacher_params = teachers_params[0] + + environment_specs = player_params.config.environment_specs + + async for sample in sample_producer_session.all_trial_samples(): + observation_tensor = torch.tensor( + flatten(environment_specs.observation_space, sample.actors_data[player_params.name].observation.value), + dtype=self._dtype, + ) + + teacher_action = sample.actors_data[teacher_params.name].action + if teacher_action.HasField("value"): + applied_action = teacher_action + demonstration = True + else: + applied_action = sample.actors_data[player_params.name].action + demonstration = False + + action_tensor = torch.tensor( + flatten(environment_specs.action_space, applied_action.value), dtype=self._dtype + ) + sample_producer_session.produce_sample((demonstration, observation_tensor, action_tensor)) + + if sample.trial_state == cogment.TrialState.ENDED: + break + + async def impl(self, run_session): + assert self._environment_specs.num_players == 1 + + ############ TUTORIAL STEP 3 ############ + model_id = f"{run_session.run_id}_model" + + # Initializing a model + model = SimpleBCModel( + model_id, + environment_implementation=self._environment_specs.implementation, + num_input=flattened_dimensions(self._environment_specs.observation_space), + num_output=flattened_dimensions(self._environment_specs.action_space), + policy_network_hidden_size=self._cfg.policy_network.hidden_size, + ) + _model_info, _version_info = await run_session.model_registry.publish_initial_version(model) + ########################################## + + run_session.log_params( + self._cfg.training, + self._cfg.environment_config, + environment_implementation=self._environment_specs.implementation, + ############ TUTORIAL STEP 3 ############ + policy_network_hidden_size=self._cfg.policy_network.hidden_size, + ######################################### + ) + + # Helper function to create a trial configuration + def create_trial_params(trial_idx): + agent_actor_params = cogment.ActorParameters( + cog_settings, + name="player", + class_name=PLAYER_ACTOR_CLASS, + ############ TUTORIAL STEP 3 ############ + implementation="actors.tutorial.tutorial_3.SimpleBCActor", + ######################################### + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ############ TUTORIAL STEP 3 ############ + model_id=model_id, + model_version=-1, + ########################################## + ), + ) + + teacher_actor_params = cogment.ActorParameters( + cog_settings, + name=WEB_ACTOR_NAME, + class_name=TEACHER_ACTOR_CLASS, + implementation=HUMAN_ACTOR_IMPL, + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + + return cogment.TrialParameters( + cog_settings, + environment_name="env", + environment_implementation=self._environment_specs.implementation, + environment_config=EnvironmentConfig( + run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + ), + actors=[agent_actor_params, teacher_actor_params], + ) + + # Rollout a bunch of trials + for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + trials_id_and_params=[ + (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) + for trial_idx in range(self._cfg.training.num_trials) + ], + sample_producer_impl=self.sample_producer, + num_parallel_trials=1, + ): + log.info(f"[{step_idx}] - Got sample [{sample}]") diff --git a/actors/tutorial/tutorial_4.py b/actors/tutorial/tutorial_4.py new file mode 100644 index 00000000..cef0e43c --- /dev/null +++ b/actors/tutorial/tutorial_4.py @@ -0,0 +1,319 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import cogment +import torch + +############ TUTORIAL STEP 4 ############ +import numpy as np + +######################################### + +from cogment_verse import Model +from cogment_verse.specs import ( + AgentConfig, + cog_settings, + EnvironmentConfig, + flatten, + flattened_dimensions, + HUMAN_ACTOR_IMPL, + PLAYER_ACTOR_CLASS, + PlayerAction, + SpaceValue, + TEACHER_ACTOR_CLASS, + WEB_ACTOR_NAME, +) + +log = logging.getLogger(__name__) + + +class SimpleBCModel(Model): + def __init__( + self, + model_id, + environment_implementation, + num_input, + num_output, + policy_network_hidden_size=64, + version_number=0, + ): + super().__init__(model_id, version_number) + + self._dtype = torch.float + self._environment_implementation = environment_implementation + self._num_input = num_input + self._num_output = num_output + self._policy_network_hidden_size = policy_network_hidden_size + + self.policy_network = torch.nn.Sequential( + torch.nn.Linear(num_input, policy_network_hidden_size, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.ReLU(), + torch.nn.Linear(policy_network_hidden_size, policy_network_hidden_size, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.ReLU(), + torch.nn.Linear(policy_network_hidden_size, num_output, dtype=self._dtype), + ) + + self.total_samples = 0 + + def get_model_user_data(self): + return { + "environment_implementation": self._environment_implementation, + "num_input": self._num_input, + "num_output": self._num_output, + "policy_network_hidden_size": self._policy_network_hidden_size, + } + + def save(self, model_data_f): + torch.save(self.policy_network.state_dict(), model_data_f) + + return {"total_samples": self.total_samples} + + @classmethod + def load(cls, model_id, version_number, model_user_data, version_user_data, model_data_f): + # Create the model instance + model = SimpleBCModel( + model_id=model_id, + version_number=version_number, + environment_implementation=model_user_data["environment_implementation"], + num_input=int(model_user_data["num_input"]), + num_output=int(model_user_data["num_output"]), + policy_network_hidden_size=int(model_user_data["policy_network_hidden_size"]), + ) + + # Load the saved states + policy_network_state_dict = torch.load(model_data_f) + model.policy_network.load_state_dict(policy_network_state_dict) + + # Load version data + model.total_samples = version_user_data["total_samples"] + return model + + +class SimpleBCActor: + def __init__(self, _cfg): + super().__init__() + self._dtype = torch.float + + def get_actor_classes(self): + return [PLAYER_ACTOR_CLASS] + + async def impl(self, actor_session): + actor_session.start() + + config = actor_session.config + + observation_space = config.environment_specs.observation_space + + model, _model_info, version_info = await actor_session.model_registry.retrieve_version( + SimpleBCModel, config.model_id, config.model_version + ) + model_version_number = version_info["version_number"] + log.info(f"Starting trial with model v{model_version_number}") + + model.policy_network.eval() + + async for event in actor_session.all_events(): + if event.observation and event.type == cogment.EventType.ACTIVE: + observation_tensor = torch.tensor( + flatten(observation_space, event.observation.observation.value), dtype=self._dtype + ) + scores = model.policy_network(observation_tensor.view(1, -1)) + probs = torch.softmax(scores, dim=-1) + discrete_action_tensor = torch.distributions.Categorical(probs).sample() + action_value = SpaceValue(properties=[SpaceValue.PropertyValue(discrete=discrete_action_tensor.item())]) + actor_session.do_action(PlayerAction(value=action_value)) + + +class SimpleBCTraining: + default_cfg = { + "environment_config": {"seed": 12}, + "training": { + "num_trials": 10, + ############ TUTORIAL STEP 4 ############ + "discount_factor": 0.95, + "learning_rate": 0.01, + "batch_size": 32, + "train_only_from_demonstration": False + ######################################### + }, + "policy_network": {"hidden_size": 64}, + } + + def __init__(self, environment_specs, cfg): + super().__init__() + self._dtype = torch.float + self._environment_specs = environment_specs + self._cfg = cfg + + async def sample_producer(self, sample_producer_session): + assert len(sample_producer_session.trial_info.parameters.actors) == 2 + + players_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == PLAYER_ACTOR_CLASS + ] + teachers_params = [ + actor_params + for actor_params in sample_producer_session.trial_info.parameters.actors + if actor_params.class_name == TEACHER_ACTOR_CLASS + ] + assert len(players_params) == 1 + assert len(teachers_params) == 1 + player_params = players_params[0] + teacher_params = teachers_params[0] + + environment_specs = player_params.config.environment_specs + + async for sample in sample_producer_session.all_trial_samples(): + observation_tensor = torch.tensor( + flatten(environment_specs.observation_space, sample.actors_data[player_params.name].observation.value), + dtype=self._dtype, + ) + + teacher_action = sample.actors_data[teacher_params.name].action + if teacher_action.HasField("value"): + applied_action = teacher_action + demonstration = True + else: + applied_action = sample.actors_data[player_params.name].action + demonstration = False + + action_tensor = torch.tensor( + flatten(environment_specs.action_space, applied_action.value), dtype=self._dtype + ) + sample_producer_session.produce_sample((demonstration, observation_tensor, action_tensor)) + + async def impl(self, run_session): + assert self._environment_specs.num_players == 1 + + model_id = f"{run_session.run_id}_model" + + # Initializing a model + model = SimpleBCModel( + model_id, + environment_implementation=self._environment_specs.implementation, + num_input=flattened_dimensions(self._environment_specs.observation_space), + num_output=flattened_dimensions(self._environment_specs.action_space), + policy_network_hidden_size=self._cfg.policy_network.hidden_size, + ) + _model_info, _version_info = await run_session.model_registry.publish_initial_version(model) + + run_session.log_params( + self._cfg.training, + self._cfg.environment_config, + environment_implementation=self._environment_specs.implementation, + policy_network_hidden_size=self._cfg.policy_network.hidden_size, + ) + + # Helper function to create a trial configuration + def create_trial_params(trial_idx): + agent_actor_params = cogment.ActorParameters( + cog_settings, + name="player", + class_name=PLAYER_ACTOR_CLASS, + ############ TUTORIAL STEP 4 ############ + implementation="actors.tutorial.tutorial_4.SimpleBCActor", + ######################################### + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + model_id=model_id, + model_version=-1, + ), + ) + + teacher_actor_params = cogment.ActorParameters( + cog_settings, + name=WEB_ACTOR_NAME, + class_name=TEACHER_ACTOR_CLASS, + implementation=HUMAN_ACTOR_IMPL, + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + + return cogment.TrialParameters( + cog_settings, + environment_name="env", + environment_implementation=self._environment_specs.implementation, + environment_config=EnvironmentConfig( + run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + ), + actors=[agent_actor_params, teacher_actor_params], + ) + + ############ TUTORIAL STEP 4 ############ + # Configure the optimizer + optimizer = torch.optim.Adam( + model.policy_network.parameters(), + lr=self._cfg.training.learning_rate, + ) + + # Keep accumulated observations/actions around + observations = [] + actions = [] + + loss_fn = torch.nn.CrossEntropyLoss() + ########################################## + + # Rollout a bunch of trials + for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + trials_id_and_params=[ + (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) + for trial_idx in range(self._cfg.training.num_trials) + ], + sample_producer_impl=self.sample_producer, + num_parallel_trials=1, + ): + ############ TUTORIAL STEP 4 ############ + (demonstration, observation, action) = sample + if self._cfg.training.train_only_from_demonstration and not demonstration: + continue + + observations.append(observation) + actions.append(action) + + if len(observations) < self._cfg.training.batch_size: + continue + + # Sample a batch of observations/actions + batch_indices = np.random.default_rng().integers(0, len(observations), self._cfg.training.batch_size) + batch_obs = torch.vstack([observations[i] for i in batch_indices]) + batch_act = torch.vstack([actions[i] for i in batch_indices]) + + model.policy_network.train() + pred_policy = model.policy_network(batch_obs) + loss = loss_fn(pred_policy, batch_act) + + # Backprop! + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # Publish the newly trained version every 100 steps + if step_idx % 100 == 0: + version_info = await run_session.model_registry.publish_version(model) + + run_session.log_metrics( + model_version_number=version_info["version_number"], + loss=loss.item(), + total_samples=len(observations), + ) + ########################################## diff --git a/base_python/cogment_verse/agent_adapter.py b/base_python/cogment_verse/agent_adapter.py deleted file mode 100644 index 85ed16a5..00000000 --- a/base_python/cogment_verse/agent_adapter.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import abc -import logging - -from cogment_verse.utils import LRU, get_full_class_name - -log = logging.getLogger(__name__) - - -class AgentAdapter(abc.ABC): - MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY = "_source_adapter_class_name" - VERSION_USER_DATA_MODEL_CLASS_NAME_KEY = "_model_class_name" - - def __init__(self): - """ - Create an agent adapter - """ - self._model_cache = LRU() - self._adapter_class_name = get_full_class_name(self) - - def default_get_model_registry_client(): - raise RuntimeError("`get_model_registry_client` is not defined before a call to `register_implementations`") - - self.get_model_registry_client = default_get_model_registry_client - - def _create(self, model_id, **kwargs): - """ - Create and return a model instance - Parameters: - model_id (string): unique identifier for the model - kwargs: any number of key/values paramters, forwarded from `create_and_publish_initial_version` - Returns: - model, model_user_data: a tuple containing the created model and additional user_data - """ - raise NotImplementedError - - def __create(self, model_id, **kwargs): - model, model_user_data = self._create(model_id, **kwargs) - model_user_data[self.MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY] = self._adapter_class_name - return model, model_user_data - - def _load(self, model_id, version_number, model_user_data, version_user_data, model_data_f, **kwargs): - """ - Load a serialized model instance and return it - Args: - model_id (string): unique identifier for the model - version_number (int): version number of the data - model_user_data (dict[str, str]): model user data - version_user_data (dict[str, str]): version user data - model_data_f: file object that will be used to load the version model data - kwargs: any number of key/values parameters, forwarded from `retrieve_version` - Returns: - model: the loaded model - """ - raise NotImplementedError - - def __load(self, model_id, version_number, model_user_data, version_user_data, model_data_f, **kwargs): - if ( - self.MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY in model_user_data - and model_user_data[self.MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY] != self._adapter_class_name - ): - raise RuntimeError( - f"Unable to load model '{model_id}@v{version_number}' with adapter '{self._adapter_class_name}': it was initially created by adapter '{model_user_data[self.MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY]}'" - ) - - return self._load(model_id, version_number, model_user_data, version_user_data, model_data_f, **kwargs) - - def _save(self, model, model_user_data, model_data_f, **kwargs): - """ - Serialize and save a model - Args: - model: a model, as returned by the _create method of this class - model_user_data (dict[str, str]): model user data - model_data_f: file object that will be used to save the version model data - kwargs: any number of key/values parameters, forwarded from `create_and_publish_initial_version` or `publish_version` - Returns: - version_user_data (dict[str, str]): additional version user data - """ - raise NotImplementedError - - def __save(self, model, model_user_data, model_data_f, **kwargs): - if ( - self.MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY in model_user_data - and model_user_data[self.MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY] != self._adapter_class_name - ): - raise RuntimeError( - f"Unable to save a new version of the model with adapter '{self._adapter_class_name}': it was initially created by adapter '{model_user_data[self.MODEL_USER_DATA_ADAPTER_CLASS_NAME_KEY]}'" - ) - - version_user_data = self._save(model, model_user_data, model_data_f, **kwargs) - version_user_data[self.VERSION_USER_DATA_MODEL_CLASS_NAME_KEY] = get_full_class_name(model) - - return version_user_data - - async def create_and_publish_initial_version(self, model_id, **kwargs): - """ - Create and publish to the model registry a model instance - Parameters: - model_id (string): unique identifier for the model - kwargs: any number of key/values parameters, will be forwarded to `_create` - Returns: - model, model_info, version_info: the created model, its information and the information for the initial published version - """ - - # Create the agent model locally - model, model_user_data = self.__create(model_id, **kwargs) - - # Create it in the model registry - await self.get_model_registry_client().create_model(model_id, model_user_data) - - # Publish the first version - version_info = await self.publish_version(model_id, model, **kwargs) - return model, version_info - - async def publish_version(self, model_id, model, archived=False, **kwargs): - """ - Publish to the model registry a new version of a model - Parameters: - model_id (string): unique identifier for the model - model: a model, as returned by method of this class - archive (bool - default is False): If true, the model version will be archived (i.e. stored in permanent storage) - Returns: - version_info: information for the initial published version - """ - return await self.get_model_registry_client().publish_version( - model_id=model_id, model=model, save_model=self.__save, archived=archived, **kwargs - ) - - async def retrieve_version(self, model_id, version_number=-1, **kwargs): - """ - Publish to the model registry a new version of a model - Parameters: - model_id (string): Unique id of the model - version_number (int - default is -1): The version number (-1 for the latest) - Returns: - model, model_info, version_info: A tuple containing the model, the model info and the model version info - """ - return await self.get_model_registry_client().retrieve_version( - model_id=model_id, load_model=self.__load, version_number=version_number, **kwargs - ) - - @abc.abstractmethod - def _create_actor_implementations(self): - """ - Create all the available actor implementation for this adapter - Returns: - dict[impl_name: string, (actor_impl: Callable, actor_classes: []string)]: key/value definition for the available actor implementations. - """ - return {} - - @abc.abstractmethod - def _create_run_implementations(self): - """ - Create all the available run implementation for this adapter - Returns: - dict[impl_name: string, (sample_producer_impl: Callable, run_impl: Callable, default_run_config)]: key/value definition for the available run implementations. - """ - return {} - - def register_implementations(self, context): - """ - Register all the implementations defined in this adapter - Parameters: - context: Cogment context with which the implementations are adapted - """ - for impl_name, (actor_impl, actor_classes) in self._create_actor_implementations().items(): - log.info(f"Registering actor implementation [{impl_name}]") - context.register_actor(impl=actor_impl, impl_name=impl_name, actor_classes=actor_classes) - - for impl_name, (sample_producer_impl, run_impl, default_config) in self._create_run_implementations().items(): - log.info(f"Registering run implementation [{impl_name}]") - context.register_run( - run_impl=run_impl, - run_sample_producer_impl=sample_producer_impl, - impl_name=impl_name, - default_config=default_config, - ) - - self.get_model_registry_client = context.get_model_registry_client diff --git a/base_python/cogment_verse/api/.gitkeep b/base_python/cogment_verse/api/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/base_python/cogment_verse/run/run_context.py b/base_python/cogment_verse/run/run_context.py deleted file mode 100644 index 8d44ff8a..00000000 --- a/base_python/cogment_verse/run/run_context.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import copy -import logging -import random - -import cogment -from cogment_verse.api.run_api_pb2 import DESCRIPTOR as RUN_DESCRIPTOR -from cogment_verse.api.run_api_pb2_grpc import add_RunServicer_to_server -from cogment_verse.model_registry_client import ModelRegistryClient -from cogment_verse.run.run_servicer import RunServicer -from cogment_verse.run.run_session import RunSession -from cogment_verse.trial_datastore_client import TrialDatastoreClient -from cogment_verse.utils import LRU -from grpc_reflection.v1alpha import reflection -from prometheus_client.core import REGISTRY - -log = logging.getLogger(__name__) - -# pylint: disable=too-many-arguments - - -def set_config_index(config, actor_idx): - config.actor_index = actor_idx - return config - - -# RunContext holds the context information to exectute runs -class RunContext(cogment.Context): - def __init__( - self, - user_id, - cog_settings, - services_endpoints, - asyncio_loop=None, - prometheus_registry=REGISTRY, - ): - super().__init__( - user_id, - cog_settings, - asyncio_loop=asyncio_loop, - prometheus_registry=prometheus_registry, - ) - - self._cog_settings = cog_settings - self._services_endpoints = services_endpoints - - # Pre trial hook => actor/environment config + services urls resolution - async def pre_trial_hook(pre_trial_hook_session): - log.debug(f"[pre_trial_hook] Configuring trial {pre_trial_hook_session.get_trial_id()}") - trial_parameters = pre_trial_hook_session.trial_parameters - trial_config = trial_parameters.config - - # Environment - environment_params = trial_config.environment - trial_parameters.environment_name = f"{trial_config.run_id}_environment" - trial_parameters.environment_config = environment_params.config - trial_parameters.environment_implementation = environment_params.specs.implementation - trial_parameters.environment_endpoint = self._get_service_endpoint( - trial_parameters.environment_implementation - ) - - # Datalog - trial_parameters.datalog_endpoint = services_endpoints["trial_datastore"] - - # Actors - trial_parameters.actors = [ - cogment.ActorParameters( - cog_settings=self._cog_settings, - class_name=actor.actor_class, - name=actor.name, - endpoint="cogment://client" - if actor.implementation == "client" - else self._get_service_endpoint(actor.implementation), - implementation=None if actor.implementation == "client" else actor.implementation, - config=set_config_index(getattr(actor, actor.WhichOneof("config_oneof")), actor_idx), - ) - for actor_idx, actor in enumerate(trial_config.actors) - ] - - self.register_pre_trial_hook(pre_trial_hook) - - self._run_impls = {} - - # Cache used by the model registry - self._model_registry_cache = LRU() - - def _get_service_endpoint(self, services_name): - if services_name not in self._services_endpoints: - raise Exception(f"unknown service [{services_name}]") - - desired_service_endpoints = self._services_endpoints[services_name] - - if not desired_service_endpoints: - raise Exception(f"no endpoint defined for service [{services_name}]") - - if isinstance(desired_service_endpoints, list): - return random.choice(desired_service_endpoints) - return desired_service_endpoints - - def register_run(self, run_impl, run_sample_producer_impl, impl_name, default_config): - if self._grpc_server is not None: - raise RuntimeError("Cannot register a run after the server is started") - if impl_name in self._run_impls: - raise RuntimeError(f"The run implementation name must be unique: [{impl_name}]") - self._run_impls[impl_name] = ( - run_impl, - run_sample_producer_impl, - default_config, - ) - - def _get_controller(self): - return self.get_controller(endpoint=cogment.Endpoint(self._get_service_endpoint("orchestrator"))) - - def _get_trial_datastore_client(self): - return TrialDatastoreClient(endpoint=self._get_service_endpoint("trial_datastore")) - - def get_model_registry_client(self): - return ModelRegistryClient( - endpoint=self._get_service_endpoint("model_registry"), - cache=self._model_registry_cache, - ) - - def _create_run_session(self, run_params_name, run_implementation, serialized_config, run_id=None): - if run_implementation not in self._run_impls: - raise RuntimeError(f"Unknown run implementation [{run_implementation}]") - - (run_impl, run_sample_producer_impl, default_config) = self._run_impls[run_implementation] - - merged_config = copy.deepcopy(default_config) - if serialized_config is not None: - merged_config.MergeFromString(serialized_config) - - return RunSession( - cog_settings=self._cog_settings, - controller=self._get_controller(), - trial_datastore_client=self._get_trial_datastore_client(), - config=merged_config, - run_sample_producer_impl=run_sample_producer_impl, - impl_name=run_implementation, - run_impl=run_impl, - params_name=run_params_name, - run_id=run_id, - ) - - async def exec_run(self, impl_name, config=None, run_id=None): - if impl_name not in self._run_impls: - raise RuntimeError(f"Unknown run implementation [{impl_name}]") - - (run_impl, run_sample_producer_impl, default_config) = self._run_impls[impl_name] - - merged_config = copy.deepcopy(default_config) - if config is not None: - merged_config.MergeFrom(config) - - run_session = RunSession( - cog_settings=self._cog_settings, - controller=self._get_controller(), - trial_datastore_client=self._get_trial_datastore_client(), - config=merged_config, - run_sample_producer_impl=run_sample_producer_impl, - impl_name=impl_name, - run_impl=run_impl, - params_name="manual_run", - run_id=run_id, - ) - - await run_session.exec() - - async def serve_all_registered( - self, - served_endpoint, - prometheus_port=cogment.context.DEFAULT_PROMETHEUS_PORT, - ): - serve_all_registered_task = asyncio.create_task(super().serve_all_registered(served_endpoint, prometheus_port)) - - while self._grpc_server is None: - await asyncio.sleep(0.1) - - if self._run_impls: - servicer = RunServicer(self._create_run_session) - add_RunServicer_to_server(servicer, self._grpc_server) - - reflection.enable_server_reflection( - ( - RUN_DESCRIPTOR.services_by_name["Run"].full_name, - reflection.SERVICE_NAME, - ), - self._grpc_server, - ) - - try: - await serve_all_registered_task - except Exception as error: - log.info(f"Properly canceling the server task after {error} ...") - serve_all_registered_task.cancel() - await serve_all_registered_task - raise error diff --git a/base_python/cogment_verse/run/run_sample_producer_session.py b/base_python/cogment_verse/run/run_sample_producer_session.py deleted file mode 100644 index 5e58113c..00000000 --- a/base_python/cogment_verse/run/run_sample_producer_session.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import logging - -log = logging.getLogger(__name__) - - -class TrialSample: - def __init__(self, sample_pb, actor_classes): - self._sample_pb = sample_pb - self._actor_classes = actor_classes - - def _get_payload(self, payload_idx, pb_message_class=None, default=None): - if payload_idx is None: - return default - - payload = self._sample_pb.payloads[payload_idx] - - if pb_message_class is None: - return payload - - message = pb_message_class() - message.ParseFromString(payload) - return message - - def get_trial_id(self): - return self._sample_pb.trial_id - - def get_user_id(self): - return self._sample_pb.user_id - - def get_tick_id(self): - return self._sample_pb.tick_id - - def get_timestamp(self): - return self._sample_pb.timestamp - - def get_trial_state(self): - return self._sample_pb.state - - def _get_actor(self, actor_idx): - actor = self._sample_pb.actor_samples[actor_idx] - assert actor_idx == actor.actor - - return actor - - def count_actors(self): - return len(self._sample_pb.actor_samples) - - def get_actor_observation(self, actor_idx, deserialize=True, default=None): - actor = self._get_actor(actor_idx) - return self._get_payload( - actor.observation, - pb_message_class=self._actor_classes[actor_idx].observation_space if deserialize else None, - default=default, - ) - - def get_actor_action(self, actor_idx, deserialize=True, default=None): - actor = self._get_actor(actor_idx) - return self._get_payload( - actor.action, - pb_message_class=self._actor_classes[actor_idx].action_space if deserialize else None, - default=default, - ) - - def get_actor_reward(self, actor_idx, default=None): - actor = self._get_actor(actor_idx) - reward = actor.reward - if reward is None: - return default - - return reward - - def get_actor_received_rewards(self, actor_idx): - actor = self._get_actor(actor_idx) - return [ - (reward.sender, reward.reward, reward.confidence, self._get_payload(reward.user_data)) - for reward in actor.received_rewards - ] - - def get_actor_sent_rewards(self, actor_idx): - actor = self._get_actor(actor_idx) - return [ - (reward.receiver, reward.reward, reward.confidence, self._get_payload(reward.user_data)) - for reward in actor.sent_rewards - ] - - def get_actor_received_messages(self, actor_idx): - actor = self._get_actor(actor_idx) - return [(msg.sender, self._get_payload(msg.payload)) for msg in actor.received_rewards] - - def get_actor_sent_messages(self, actor_idx): - actor = self._get_actor(actor_idx) - return [(msg.receiver, self._get_payload(msg.payload)) for msg in actor.sent_messages] - - -class RunSampleProducerSession: - def __init__( - self, - cog_settings, - run_id, - trial_id, - trial_params, - produce_training_sample, - run_config, - run_sample_producer_impl, - ): - self.run_id = run_id - self.trial_id = trial_id - self._trial_params = trial_params - self._actor_classes = [ - cog_settings.actor_classes[actor_params.actor_class] for actor_params in trial_params.actors - ] - self._trial_config_class = cog_settings.trial.config_type - self._run_sample_producer_impl = run_sample_producer_impl - self._produce_training_sample = produce_training_sample - self._current_tick_id = 0 - - self.run_config = run_config - - self._queue = asyncio.Queue(maxsize=10) - - # TODO Expose further helper functions to avoid the need to access directly _trial_params as needed - def count_actors(self): - return len(self._trial_params.actors) - - def get_trial_config(self, deserialize=True): - raw_trial_config = self._trial_params.trial_config.content - if not deserialize: - return raw_trial_config - - trial_config = self._trial_config_class() - trial_config.ParseFromString(raw_trial_config) - return trial_config - - def exec(self): - async def exec_run(): - log.debug(f"[{self.run_id}/{self.trial_id}] Starting sample producer...") - impl_task = asyncio.create_task(self._run_sample_producer_impl(self)) - try: - await impl_task - log.debug(f"[{self.run_id}/{self.trial_id}] Sample producer succeeded") - except asyncio.CancelledError: - log.debug(f"[{self.run_id}/{self.trial_id}] Terminating sample producer") - try: - await impl_task - except asyncio.CancelledError: - pass - log.debug(f"[{self.run_id}/{self.trial_id}] Sample producer terminated") - raise - except Exception as error: - log.error( - f"[{self.run_id}/{self.trial_id}] Uncaught error occured during the sample production", - exc_info=error, - ) - raise error - - self._task = asyncio.create_task(exec_run()) - return self._task - - async def on_trial_sample(self, sample): - await self._queue.put(sample) - - async def on_trial_done(self): - await self._queue.put(True) - - async def get_all_samples(self): - while True: - enqeued_item = await self._queue.get() - if enqeued_item is True: - # Trial done - return - trial_sample = TrialSample(enqeued_item, self._actor_classes) - self._current_tick_id = trial_sample.get_tick_id() - log.debug(f"[{self.run_id}] retrieving a trial sample for trial={self.trial_id}@{self._current_tick_id}") - yield trial_sample - - def produce_training_sample(self, sample): - log.debug(f"[{self.run_id}] producing a training sample for trial={self.trial_id}@{self._current_tick_id}") - return self._produce_training_sample(self.trial_id, self._current_tick_id, sample) diff --git a/base_python/cogment_verse/run/run_servicer.py b/base_python/cogment_verse/run/run_servicer.py deleted file mode 100644 index 4cf60f03..00000000 --- a/base_python/cogment_verse/run/run_servicer.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from cogment_verse.api.run_api_pb2 import ListRunsReply, RunInfo, RunStatus, StartRunReply, TerminateRunReply -from cogment_verse.api.run_api_pb2_grpc import RunServicer as AbstractRunServicer -from cogment_verse.run.run_session import RunSessionStatus -from google.protobuf.json_format import MessageToDict -from google.protobuf.timestamp_pb2 import Timestamp - -# pylint: disable=invalid-overridden-method, invalid-name, no-member - -log = logging.getLogger(__name__) - - -def run_info_from_run_session(run_session): - run_session_status = run_session.get_status() - status = RunStatus.UNKNOWN - if run_session_status is RunSessionStatus.RUNNING: - status = RunStatus.RUNNING - elif run_session_status is RunSessionStatus.SUCCESS: - status = RunStatus.SUCCESS - elif run_session_status is RunSessionStatus.TERMINATED: - status = RunStatus.TERMINATED - elif run_session_status is RunSessionStatus.ERROR: - status = RunStatus.ERROR - - start_timestamp = Timestamp() - start_timestamp.FromDatetime(run_session.start_time) - - return RunInfo( - run_id=run_session.run_id, - params_name=run_session.params_name, - implementation=run_session.impl_name, - start_timestamp=start_timestamp, - status=status, - steps_count=run_session.count_steps(), - ) - - -class RunServicer(AbstractRunServicer): - def __init__(self, create_run_session): - self._create_run_session = create_run_session - self._run_sessions = {} - - async def StartRun(self, request, _context): - run_id = request.run_id if request.run_id != "" else None - log.debug(f"StartRun(run_id=[{run_id}], run_params=[{MessageToDict(request.run_params)}])") - if run_id is not None and run_id in self._run_sessions: - raise RuntimeError(f"Run [{run_id}] already served") - - run_session = self._create_run_session( - run_params_name=request.run_params.name, - run_implementation=request.run_params.implementation, - serialized_config=request.run_params.config.content, - run_id=run_id, - ) - run_session.exec() - self._run_sessions[run_session.run_id] = run_session - - try: - return StartRunReply(run=run_info_from_run_session(run_session)) - except: - await run_session.terminate() - raise - - def ListRuns(self, _request, _context): - log.debug("ListRuns()") - - runs_reply = ListRunsReply() - - for run_info in sorted( - [run_info_from_run_session(s) for s in self._run_sessions.values()], - key=lambda info: info.start_timestamp.ToNanoseconds(), - ): - runs_reply.runs.append(run_info) - - return runs_reply - - async def TerminateRun(self, request, _context): - run_id = request.run_id - log.debug(f"TerminateRun(run_id=[{run_id}])") - if run_id not in self._run_sessions: - raise RuntimeError(f"Unknown run [{run_id}]") - - run_session = self._run_sessions[run_id] - await run_session.terminate() - - return TerminateRunReply(run=run_info_from_run_session(run_session)) diff --git a/base_python/cogment_verse/run/run_session.py b/base_python/cogment_verse/run/run_session.py deleted file mode 100644 index ff798ea2..00000000 --- a/base_python/cogment_verse/run/run_session.py +++ /dev/null @@ -1,378 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import logging -import time -from datetime import datetime -from enum import Enum, auto - -import cogment -from cogment_verse.run.run_sample_producer_session import RunSampleProducerSession -from cogment_verse.run.run_stepper import RunStepper -from names_generator import generate_name -from prometheus_client import Counter, Gauge, Summary - -log = logging.getLogger(__name__) - -# pylint: disable=too-many-arguments, too-many-instance-attributes, too-many-statements - -TRIALLAUNCHER_TRIAL_RUNNING_LEN = Gauge("triallauncher_trial_running_len", "Length of the running trials") -TRIALLAUNCHER_SAMPLE_CONSUMED_COUNTER = Counter("triallauncher_sample_consumed", "Counter of consumed samples") -TRIALLAUNCHER_SAMPLE_QUEUE_LEN = Gauge("triallauncher_sample_queue_len", "Length of the sample queue") -TRIALLAUNCHER_SAMPLE_PRODUCED_COUNTER = Counter("triallauncher_sample_produced", "Counter of produced samples") -TRIALLAUNCHER_TRIAL_TIME = Summary("triallauncher_trial_seconds", "Time spent running trials") -TRIALLAUNCHER_TRIAL_STARTED_COUNTER = Counter("triallauncher_trial_started", "Counter of started trials") -TRIALLAUNCHER_START_TRIAL_TIME = Summary("triallauncher_start_trial_seconds", "Time spent starting trials") - -PRIO_QUEUE_HIGH_PRIO = 0 -PRIO_QUEUE_LOW_PRIO = 1 -PRIO_QUEUE_END_PRIO = 2 - - -def default_on_progress(_launched_trials_count, _finished_trials_count): - pass - - -class RunSessionStatus(Enum): - CREATED = auto() - RUNNING = auto() - TERMINATING = auto() - TERMINATED = auto() - SUCCESS = auto() - ERROR = auto() - - -class RunSession: - def __init__( - self, - cog_settings, - controller, - trial_datastore_client, - config, - run_sample_producer_impl, - impl_name, - run_impl, - params_name, - run_id=None, - ): - super().__init__() - - self.run_id = run_id if run_id is not None else generate_name() - self.config = config - self.impl_name = impl_name - self.params_name = params_name - self.start_time = datetime.now() - - self._cog_settings = cog_settings - self._run_sample_producer_impl = run_sample_producer_impl - self._controller = controller - self._trial_datastore_client = trial_datastore_client - self._stepper = RunStepper() - - self._run_impl = run_impl - self._task = None - self._terminating = False - - def get_status(self): - if self._task is None: - return RunSessionStatus.CREATED - try: - self._task.result() - return RunSessionStatus.SUCCESS - except asyncio.CancelledError: - return RunSessionStatus.TERMINATED - except asyncio.InvalidStateError: - if self._terminating: - return RunSessionStatus.TERMINATING - return RunSessionStatus.RUNNING - except Exception as error: - log.error(f"[{self.run_id}] Uncaught error occured during the run", exc_info=error) - return RunSessionStatus.ERROR - - def exec(self): - if self.get_status() is not RunSessionStatus.CREATED: - raise RuntimeError(f"[{self.run_id}] already started") - - async def exec_run(): - log.info(f"[{self.run_id}] Starting run...") - impl_task = asyncio.create_task(self._run_impl(self)) - try: - await impl_task - log.info(f"[{self.run_id}] Run suceeded") - except asyncio.CancelledError: - log.info(f"[{self.run_id}] Terminating run...") - self._terminating = True - try: - await impl_task - except asyncio.CancelledError: - pass - log.info(f"[{self.run_id}] Run terminated") - raise - except Exception as error: - log.error( - f"[{self.run_id}] Uncaught error occured during the run", - exc_info=error, - ) - raise error - - self._task = asyncio.create_task(exec_run()) - return self._task - - async def terminate(self): - if self.get_status() is not RunSessionStatus.RUNNING: - raise RuntimeError(f"[{self.run_id}] not running") - - self._task.cancel() - try: - await self._task - except Exception: - # We don't want terminate to fail, exception handling is dealt with in get_status(). - pass - - return self.get_status() - - def count_steps(self): - return self._stepper.count_steps() - - @staticmethod - async def _do_enqueue_trial_configs(trial_config_queue_out, trial_configs): - for trial_config in trial_configs: - await trial_config_queue_out.put(trial_config) - - # Making sure the consumer knowns that it's finished - await trial_config_queue_out.put(None) - - async def _do_start_trials( - self, - trial_configs_queue_in, - trial_ids_chunk_prio_queue_out, - max_parallel_trials, - on_progress, - start_trial_throttle_timeout, - ): - launched_trials_count = 0 - finished_trials_count = 0 - - running_trial_ids = set() - - async def monitor_ended_trials(): - nonlocal finished_trials_count - async for ended_trial_info in self._controller.watch_trials(trial_state_filters=[cogment.TrialState.ENDED]): - if ended_trial_info.trial_id in running_trial_ids: - log.debug(f"[{self.run_id}] Trial [{ended_trial_info.trial_id}] ended") - running_trial_ids.discard(ended_trial_info.trial_id) - finished_trials_count += 1 - - try: - monitor_ended_trials_task = asyncio.create_task(monitor_ended_trials()) - - while True: - if monitor_ended_trials_task.cancelled(): - raise asyncio.CancelledError() - if monitor_ended_trials_task.done(): - # The only way `monitor_ended_trials_task` finishes is if an error occured - raise RuntimeError( - "An error occured while monitoring the trials" - ) from monitor_ended_trials_task.exception() - - trial_slots_count = max_parallel_trials - len(running_trial_ids) - on_progress(launched_trials_count, finished_trials_count) - - if trial_slots_count > 0: - to_start_trial_configs = [] - done = False - while len(to_start_trial_configs) < trial_slots_count: - trial_config = await trial_configs_queue_in.get() - if trial_config is None: - done = True - break - to_start_trial_configs.append(trial_config) - - if len(to_start_trial_configs) > 0: - log.debug(f"[{self.run_id}] Starting {len(to_start_trial_configs)} trials") - - with TRIALLAUNCHER_START_TRIAL_TIME.time(): - done_start_trial_tasks, _ = await asyncio.wait( - [ - asyncio.create_task(self._controller.start_trial(trial_config=trial_config)) - for trial_config in to_start_trial_configs - ] - ) - started_trial_ids = {r.result() for r in done_start_trial_tasks} - - log.debug(f"[{self.run_id}] Trials [{', '.join(started_trial_ids)}] started") - - launched_trials_count += len(started_trial_ids) - running_trial_ids.update(started_trial_ids) - await trial_ids_chunk_prio_queue_out.put((PRIO_QUEUE_LOW_PRIO, started_trial_ids)) - - for trial_config in to_start_trial_configs: - trial_configs_queue_in.task_done() - - if done: - # Making sure the consumer knowns that it's finished - await trial_ids_chunk_prio_queue_out.put((PRIO_QUEUE_END_PRIO, None)) - break - - else: - # at least `max_parallel_trials` currently running, let's wait a little bit before checking again - timeout_seconds = start_trial_throttle_timeout * (1 / 1000.0) - log.debug( - f"[{self.run_id}] Waiting {timeout_seconds:.1f} seconds before checking if trials need to be restarted" - ) - await asyncio.sleep(timeout_seconds) - - except asyncio.CancelledError: - # Cancelling subtasks - monitor_ended_trials_task.cancel() - await monitor_ended_trials_task - - async def _do_observe_trials(self, trial_ids_chunk_prio_queue_in, sample_queue_out, trial_datastore_timeout): - def produce_training_sample(trial_id, tick_id, sample): - step_id, step_timestamp = self._stepper.step(trial_id, tick_id) - sample_queue_out.put_nowait((step_id, step_timestamp, trial_id, tick_id, sample)) - - TRIALLAUNCHER_SAMPLE_PRODUCED_COUNTER.inc() - - return step_id, step_timestamp - - async def observe_trial_ids_chunk(trial_ids): - # Retrieve trial infos from the trial datastore - trial_infos = await self._trial_datastore_client.retrieve_trials(trial_ids, trial_datastore_timeout) - # `trial_infos` only contains info related to the trials already known to the trial datastore - known_trial_ids = {trial_info.trial_id for trial_info in trial_infos} - unknown_trials_ids = set(trial_ids).difference(known_trial_ids) - - if len(unknown_trials_ids) > 0: - log.info( - f"[{self.run_id}] Trials [{', '.join(unknown_trials_ids)}] didn't start generating data under {trial_datastore_timeout}ms, retrying" - ) - trial_ids_chunk_prio_queue_in.put_nowait((PRIO_QUEUE_HIGH_PRIO, unknown_trials_ids)) - - if len(known_trial_ids) == 0: - return - - run_sample_producer_sessions = { - trial_info.trial_id: RunSampleProducerSession( - cog_settings=self._cog_settings, - run_id=self.run_id, - trial_id=trial_info.trial_id, - trial_params=trial_info.params, - produce_training_sample=produce_training_sample, - run_config=self.config, - run_sample_producer_impl=self._run_sample_producer_impl, - ) - for trial_info in trial_infos - } - - run_sample_producer_tasks = [session.exec() for session in run_sample_producer_sessions.values()] - TRIALLAUNCHER_TRIAL_STARTED_COUNTER.inc(len(run_sample_producer_tasks)) - - trial_start_time = time.time() - sample_generator = await self._trial_datastore_client.retrieve_samples(known_trial_ids) - async for sample in sample_generator(): - await run_sample_producer_sessions[sample.trial_id].on_trial_sample(sample) - - for session in run_sample_producer_sessions.values(): - await session.on_trial_done() - - await asyncio.wait(run_sample_producer_tasks) - - trial_duration_seconds = time.time() - trial_start_time - TRIALLAUNCHER_TRIAL_TIME.observe(trial_duration_seconds) - - trial_ids_chunk_prio_queue_in.task_done() - - observe_tasks = [] - try: - while True: - done_observe_tasks = [t for t in observe_tasks if t.done()] - cancelled_observe_tasks = [t for t in done_observe_tasks if t.cancelled()] - if len(cancelled_observe_tasks) > 0: - raise asyncio.CancelledError() from cancelled_observe_tasks[0].exception() - failed_observe_tasks = [t for t in done_observe_tasks if t.exception() is not None] - if len(failed_observe_tasks) > 0: - raise RuntimeError("An error occured while observing the trials log") from failed_observe_tasks[ - 0 - ].exception() - - _, trial_ids = await trial_ids_chunk_prio_queue_in.get() - if trial_ids is None: - break - - observe_tasks.append(asyncio.create_task(observe_trial_ids_chunk(trial_ids))) - except asyncio.CancelledError: - # Cancelling subtasks - observe_tasks_gathering = asyncio.gather(*observe_tasks) - observe_tasks_gathering.cancel() - await observe_tasks_gathering - - async def start_trials_and_wait_for_termination( - self, trial_configs, max_parallel_trials=4, on_progress=default_on_progress - ): - if self.get_status() is not RunSessionStatus.RUNNING: - raise RuntimeError(f"[{self.run_id}] not running") - - trial_config_queue = asyncio.Queue() - started_trial_ids_chunk_prio_queue = asyncio.PriorityQueue() - sample_queue = asyncio.Queue() - - enqueue_trial_configs = asyncio.create_task(self._do_enqueue_trial_configs(trial_config_queue, trial_configs)) - start_trials = asyncio.create_task( - self._do_start_trials( - trial_config_queue, - started_trial_ids_chunk_prio_queue, - max_parallel_trials, - on_progress, - start_trial_throttle_timeout=500, - ) - ) - observe_trials = asyncio.create_task( - self._do_observe_trials(started_trial_ids_chunk_prio_queue, sample_queue, trial_datastore_timeout=5000) - ) - workers = asyncio.gather(enqueue_trial_configs, start_trials, observe_trials) - - try: - # We don't want the workers to be cancelled everytime a sample is retrieved - shielded_workers = asyncio.shield(workers) - - while not (sample_queue.empty() and workers.done()): - get_next_sample = asyncio.create_task(sample_queue.get()) - done, _ = await asyncio.wait({get_next_sample, shielded_workers}, return_when=asyncio.FIRST_COMPLETED) - - if get_next_sample in done: - yield get_next_sample.result() - sample_queue.task_done() - - TRIALLAUNCHER_SAMPLE_CONSUMED_COUNTER.inc() - TRIALLAUNCHER_SAMPLE_QUEUE_LEN.set(sample_queue.qsize()) - - if shielded_workers in done: - err = workers.exception() - if err is None: - # Workers have finished, trials were ran and listened to - # Let's continue as long as there's still samples to emit - continue - - if err is asyncio.CancelledError: - raise asyncio.CancelledError - - raise RuntimeError( - f"[{self.run_id}] error while running and listening for trials" - ) from workers.exception() - finally: - # Watever happens we want to cancel those workers when the function's returns - workers.cancel() - await workers diff --git a/base_python/cogment_verse/run/run_stepper.py b/base_python/cogment_verse/run/run_stepper.py deleted file mode 100644 index 0fbdc862..00000000 --- a/base_python/cogment_verse/run/run_stepper.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -from cogment_verse.utils import LRU - - -def compute_full_tick_id(trial_id, tick_id): - return f"{trial_id}-#{tick_id}" - - -class RunStepper: - def __init__(self): - self._steps_count = 0 - self._step_from_full_tick_id = LRU(100000) - - def count_steps(self): - return self._steps_count - - def get_step(self, trial_id, tick_id): - full_tick_id = compute_full_tick_id(trial_id, tick_id) - if full_tick_id not in self._step_from_full_tick_id: - # The step has not been "started" yet, or is no longer in cache - raise Exception(f"Unknown step for trial [{trial_id}] at tick [{tick_id}]") - return self._step_from_full_tick_id[full_tick_id] - - def step(self, trial_id, tick_id): - full_tick_id = compute_full_tick_id(trial_id, tick_id) - if full_tick_id in self._step_from_full_tick_id: - # The step has already been "started" - raise Exception(f"Existing step for trial [{trial_id}] at tick [{tick_id}]") - - self._steps_count += 1 - step = (self._steps_count, int(time.time() * 1000)) - self._step_from_full_tick_id[full_tick_id] = step - - return step diff --git a/base_python/cogment_verse/spaces/flatten.py b/base_python/cogment_verse/spaces/flatten.py deleted file mode 100644 index 8f0b3bdd..00000000 --- a/base_python/cogment_verse/spaces/flatten.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# no import - - -def flattened_dimensions(space): - """ - Computes the number of dimensions a flattened equivalent of the given space would have. - """ - - space_dim = 0 - for prop in space.properties: - prop_dim = 0 - if prop.WhichOneof("definition") == "discrete": - prop_dim = max(len(prop.discrete.labels), prop.discrete.num) - else: - # box - if len(prop.box.shape) > 0: - prop_dim = 1 - for dim in prop.box.shape: - prop_dim *= dim - space_dim += prop_dim - - return space_dim diff --git a/base_python/cogment_verse/trial_datastore_client.py b/base_python/cogment_verse/trial_datastore_client.py deleted file mode 100644 index 46f71ab4..00000000 --- a/base_python/cogment_verse/trial_datastore_client.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from urllib.parse import urlparse - -import grpc.aio -from cogment.api.trial_datastore_pb2 import RetrieveSamplesRequest, RetrieveTrialsRequest -from cogment.api.trial_datastore_pb2_grpc import TrialDatastoreSPStub - - -class TrialDatastoreClient: - def __init__(self, endpoint): - endpoint_components = urlparse(endpoint) - if endpoint_components.scheme != "grpc": - raise RuntimeError(f"Unsupported scheme for [{endpoint}], expected 'grpc'") - - channel = grpc.aio.insecure_channel(endpoint_components.netloc) - self._stub = TrialDatastoreSPStub(channel) - - async def retrieve_trials(self, trial_ids, timeout=30000): - req = RetrieveTrialsRequest(trial_ids=trial_ids, timeout=timeout) - - rep = await self._stub.RetrieveTrials(req) - - return rep.trial_infos - - async def retrieve_samples(self, trial_ids): - req = RetrieveSamplesRequest(trial_ids=trial_ids) - rep_stream = self._stub.RetrieveSamples(req) - - async def sample_generator(): - async for rep_msg in rep_stream: - yield rep_msg.trial_sample - - return sample_generator diff --git a/base_python/cogment_verse/utils/flatten_dict.py b/base_python/cogment_verse/utils/flatten_dict.py deleted file mode 100644 index 9025224b..00000000 --- a/base_python/cogment_verse/utils/flatten_dict.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import OrderedDict - - -def flatten_dict(nested_dict, prefix="", delimiter="/"): - flat_dict = {} - if prefix and prefix[-1] != delimiter: - prefix = prefix + delimiter - - for key, val in nested_dict.items(): - if isinstance(val, (dict, OrderedDict)): - child_dict = flatten_dict(val, f"{prefix}{key}{delimiter}") - for child_key, child_val in child_dict.items(): - flat_dict[child_key] = child_val - else: - flat_dict[f"{prefix}{key}"] = val - - return flat_dict diff --git a/base_python/pyproject.toml b/base_python/pyproject.toml deleted file mode 100644 index bed568ae..00000000 --- a/base_python/pyproject.toml +++ /dev/null @@ -1,6 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.pytest.ini_options] -addopts = "-rfs" diff --git a/base_python/requirements.txt b/base_python/requirements.txt deleted file mode 100644 index cb57295e..00000000 --- a/base_python/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -cogment[generate]==2.1.1 -names_generator~=0.1.0 -mlflow~=1.21.0 - -# For testing -pytest~=6.2.5 diff --git a/base_python/setup.cfg b/base_python/setup.cfg deleted file mode 100644 index 74b69b38..00000000 --- a/base_python/setup.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[metadata] -name = cogment-verse -version = 0.1.0 -description = cogment verse python SDK -author = AI Redefined Inc. -author_email = dev+cogment@ai-r.com -license = Apache-2.0 - -[options] -packages = find: -python_requires = >=3.7 -install_requires = - cogment[generate]==2.1.1 - names_generator~=0.1.0 - mlflow~=1.21.0 diff --git a/base_python/tests/test_run_config.py b/base_python/tests/test_run_config.py deleted file mode 100644 index acf86011..00000000 --- a/base_python/tests/test_run_config.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import yaml -from data_pb2 import RunConfig -from google.protobuf.json_format import MessageToDict, ParseDict - -# for pytest fixtures -# pylint: disable=redefined-outer-name - - -@pytest.fixture -def config_dict(): - config_str = """ -environment: - specs: - implementation: gym/CartPole-v0 - num_players: 1 - observation_space: - properties: - - box: - shape: [4] - action_space: - properties: - - discrete: - num: 2 - config: - render_width: 256 - flatten: True - framestack: 1 -epsilon_min: 0.1 -epsilon_steps: 100000 -target_net_update_schedule: 1000 -learning_rate: 1.0e-4 -lr_warmup_steps: 10000 -demonstration_count: 0 -total_trial_count: 10000 -model_publication_interval: 1000 -model_archive_interval: 4000 # Archive every 4000 training steps -batch_size: 256 -min_replay_buffer_size: 1000 -max_parallel_trials: 4 -model_kwargs: {} -max_replay_buffer_size: 100000 -aggregate_by_actor: False -replay_buffer_config: - observation_dtype: float32 - action_dtype: int8 -agent_implementation: rainbowtorch -""" - return yaml.safe_load(config_str) - - -def test_config(config_dict): - # should not raise exception - run_config = ParseDict(config_dict, RunConfig()) - dct = MessageToDict(run_config, preserving_proto_field_name=True) - assert "environment" in dct - assert "replay_buffer_config" in dct - assert "observation_dtype" in dct["replay_buffer_config"] diff --git a/client/main.py b/client/main.py deleted file mode 100644 index 138bb0ce..00000000 --- a/client/main.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import datetime -import functools -import json -import os - -import click -import yaml -from google.protobuf.json_format import ParseDict -from run_controller import RunController, RunStatus - -# from google.protobuf.json_format import MessageToDict - -# pylint: disable=too-many-arguments,import-outside-toplevel - -RUN_ENDPOINTS = json.loads(os.getenv("COGMENT_VERSE_RUN_ENDPOINTS", "{}")) - - -def import_class(class_name): - from importlib import import_module - - module_path, class_name = class_name.rsplit(".", 1) - module = import_module(module_path) - return getattr(module, class_name) - - -def load_run_params(params_path, params_name): - with open(params_path, encoding="utf-8") as f: - run_params = yaml.safe_load(f) - - if params_name not in run_params: - raise Exception(f"Undefined run '{params_name}'") - - run_implementation = run_params[params_name]["implementation"] - run_config_kwargs = run_params[params_name]["config"] - run_config_class_name = run_config_kwargs["class_name"] - del run_config_kwargs["class_name"] - run_config_class = import_class(run_config_class_name) - run_config = ParseDict(run_config_kwargs, run_config_class()) - return run_implementation, run_config - - -def make_sync(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return asyncio.run(func(*args, **kwargs)) - - return wrapper - - -def get_controller(impl_name=None): - if not impl_name: - return RunController( - endpoints={endpoint for impl_endpoints in RUN_ENDPOINTS.values() for endpoint in impl_endpoints} - ) - return RunController(endpoints=RUN_ENDPOINTS[impl_name]) - - -def pretty_print(val): - def converter(val): - if isinstance(val, datetime.datetime): - return val.__str__() - # pylint: disable=isinstance-second-argument-not-valid-type - if isinstance(val, RunStatus): - return val.name - return None - - click.echo(json.dumps(val, indent=2, default=converter)) - - -@click.group() -def cli(): - pass - - -@cli.command(name="list", help="List runs") -@make_sync -async def retrieve_list(): - ongoing_runs = await get_controller().list_runs() - pretty_print(ongoing_runs) - - -@cli.command(help="Start a run with the given params") -@click.argument("params") -@click.option("--run_id", default=None, help="Unique identifier for the run") -@click.option( - "--params_path", - default=os.getenv("COGMENT_VERSE_RUN_PARAMS_PATH", "./run_params.yaml"), - help="Path for the parameters definitions yaml file", - type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True), -) -@make_sync -async def start(run_id, params_path, params): - run_implementation, run_config = load_run_params(params_path, params) - # click.echo(f"- id: {run_id}") - # click.echo(f"- params_name: {params}") - # click.echo(f"- implementation: {run_implementation}") - # click.echo("- config:") - # pretty_print(MessageToDict(run_config)) - run = await get_controller(run_implementation).start_run( - name=params, implementation=run_implementation, config=run_config, run_id=run_id - ) - pretty_print(run) - - -@cli.command(help="Terminate a run") -@click.argument("run_id") -@make_sync -async def terminate(run_id): - run = await get_controller().terminate_run(run_id) - pretty_print(run) - - -if __name__ == "__main__": - cli() diff --git a/client/requirements.txt b/client/requirements.txt deleted file mode 100644 index f583d7ae..00000000 --- a/client/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -cogment[generate]==2.1.1 -click~=8.0.1 -PyYAML~=5.4.1 diff --git a/client/run_controller.py b/client/run_controller.py deleted file mode 100644 index 5feff565..00000000 --- a/client/run_controller.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from urllib.parse import urlparse - -import grpc.aio -from google.protobuf.json_format import MessageToDict -from run_api_pb2 import ListRunsRequest, RunParams, RunStatus, StartRunRequest, TerminateRunRequest -from run_api_pb2_grpc import RunStub - -log = logging.getLogger(__name__) - -# pylint: disable=no-member -class RunController: - def __init__(self, endpoints): - self._endpoints = [] - for endpoint_url in endpoints: - endpoint_components = urlparse(endpoint_url) - if endpoint_components.scheme != "grpc": - raise RuntimeError(f"Unsupported scheme for [{endpoint_url}], expected 'grpc'") - self._endpoints.append((endpoint_url, RunStub(grpc.aio.insecure_channel(endpoint_components.netloc)))) - - async def start_run(self, name, implementation, config, run_id=None): - log.debug( - f"start_run(run_id=[{run_id}], name=[{name}], implementation=[{implementation}], config=[{MessageToDict(config)}])" - ) - min_running_run_count = 100000 - for current_endpoint, current_stub in self._endpoints: - req = ListRunsRequest() - rep = await current_stub.ListRuns(req) - - running_run_count = len([run.status == RunStatus.RUNNING for run in rep.runs]) - if running_run_count < min_running_run_count: - min_running_run_count = running_run_count - endpoint, stub = current_endpoint, current_stub - - run_params = RunParams(name=name, implementation=implementation) - if config is not None: - run_params.config.content = config.SerializeToString() - - req = StartRunRequest(run_params=run_params, run_id=run_id) - rep = await stub.StartRun(req) - return { - **MessageToDict(rep.run), - "endpoint": endpoint, - } - - async def list_runs(self): - log.debug("list_runs()") - runs = [] - for endpoint, stub in self._endpoints: - req = ListRunsRequest() - rep = await stub.ListRuns(req) - runs.extend([{**MessageToDict(run), "endpoint": endpoint} for run in rep.runs]) - return runs - - async def terminate_run(self, run_id): - log.debug(f"terminate_run(run_id=[{run_id}])") - for endpoint, stub in self._endpoints: - req = ListRunsRequest() - rep = await stub.ListRuns(req) - - has_target_run = any(run.run_id == run_id for run in rep.runs) - if has_target_run: - req = TerminateRunRequest(run_id=run_id) - rep = await stub.TerminateRun(req) - return { - **MessageToDict(rep.run), - "endpoint": endpoint, - } - raise RuntimeError(f"Unknown run [{run_id}]") diff --git a/base_python/cogment_verse/spaces/__init__.py b/cogment_verse/__init__.py similarity index 81% rename from base_python/cogment_verse/spaces/__init__.py rename to cogment_verse/__init__.py index 49cc51d0..a5c24163 100644 --- a/base_python/cogment_verse/spaces/__init__.py +++ b/cogment_verse/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,4 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from cogment_verse.spaces.flatten import flattened_dimensions +from .app import App +from .model_registry import Model diff --git a/cogment_verse/app.py b/cogment_verse/app.py new file mode 100644 index 00000000..8b440928 --- /dev/null +++ b/cogment_verse/app.py @@ -0,0 +1,169 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from omegaconf import OmegaConf +from names_generator import generate_name + +from .processes import ( + create_actor_service, + create_environment_service, + create_model_registry_service, + create_orchestrator_service, + create_run_process, + create_trial_datastore_service, + create_web_service, +) +from .services_directory import ServiceDirectory, ServiceType +from .model_registry import ModelRegistry +from .constants import HUMAN_ACTOR_IMPL +from .utils.find_free_port import find_free_port + +log = logging.getLogger(__name__) + +SPEC_FILEPATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "specs", "cogment.yaml")) + + +def register_generate_port_resolver(): + ports = {} + + def generate_port(key): + nonlocal ports + if key in ports: + return ports[key] + port = find_free_port() + ports[key] = port + return port + + OmegaConf.register_new_resolver("generate_port", generate_port) + + +def register_generate_name_resolver(): + names = {} + + def do_generate_name(key): + nonlocal names + if key in names: + return names[key] + name = generate_name() + names[key] = name + return name + + OmegaConf.register_new_resolver("generate_name", do_generate_name) + + +register_generate_port_resolver() +register_generate_name_resolver() + + +class App: + def __init__(self, cfg, work_dir=".cogment_verse"): + self.cfg = OmegaConf.create(cfg) + OmegaConf.resolve( + cfg + ) # The configuration (or sub configurations) will be passed to other processes let's avoid surprises + self.services_directory = ServiceDirectory() + self.model_registry = ModelRegistry(self.services_directory) + self.services_process = [] + + for service_type, services_cfg in cfg.services.items(): + if service_type not in [service_type.value for service_type in ServiceType]: + raise RuntimeError(f"Unknown service type [{service_type}]") + if service_type == ServiceType.ORCHESTRATOR.value: + for orchestrator_service_cfg in services_cfg: + self.services_process.append( + create_orchestrator_service(work_dir, orchestrator_service_cfg, self.services_directory) + ) + elif service_type == ServiceType.TRIAL_DATASTORE.value: + trial_datastore_service_cfg = services_cfg + self.services_process.append( + create_trial_datastore_service(work_dir, trial_datastore_service_cfg, self.services_directory) + ) + elif service_type == ServiceType.MODEL_REGISTRY.value: + model_registry_service_cfg = services_cfg + self.services_process.append( + create_model_registry_service(work_dir, model_registry_service_cfg, self.services_directory) + ) + elif service_type == ServiceType.ENVIRONMENT.value: + environment_service_cfg = services_cfg + self.services_process.append( + create_environment_service( + work_dir, SPEC_FILEPATH, environment_service_cfg, self.services_directory + ) + ) + elif service_type == ServiceType.ACTOR.value: + for actor_service_cfg in services_cfg: + self.services_process.append( + create_actor_service( + work_dir, SPEC_FILEPATH, self.model_registry, actor_service_cfg, self.services_directory + ) + ) + elif service_type == ServiceType.WEB.value: + self.services_process.append(create_web_service(SPEC_FILEPATH, self.services_directory, services_cfg)) + self.services_directory.add( + service_type=ServiceType.ACTOR, + service_name=HUMAN_ACTOR_IMPL, + service_endpoint="cogment://client", + ) + else: + raise NotImplementedError() + + run_cfg = cfg.get("run", None) + if run_cfg is not None: + self.run_process = create_run_process( + work_dir, SPEC_FILEPATH, self.services_directory, self.model_registry, run_cfg + ) + else: + self.run_process = None + + def start(self): + log.info("Start services...") + for service_process in self.services_process: + service_process.start() + + for service_process in self.services_process: + service_process.await_ready() + log.info("Services ready") + + if self.run_process: + log.info("Run starting...") + self.run_process.start() + + def terminate(self): + if self.run_process: + self.run_process.terminate() + # TODO terminated signal ? + + log.info("Terminating services...") + for service_process in reversed(self.services_process): + service_process.terminate() + + def join(self): + try: + if self.run_process: + self.run_process.join() + log.info("Run ended") + + # Run ended, terminate the services + log.info("Terminating services...") + for service_process in reversed(self.services_process): + service_process.terminate() + + for service_process in self.services_process: + service_process.join() + log.info("Services ended") + except KeyboardInterrupt: + log.warning("interrupted by user") diff --git a/base_python/cogment_verse/constants.py b/cogment_verse/constants.py similarity index 63% rename from base_python/cogment_verse/constants.py rename to cogment_verse/constants.py index 0cf91e0c..857ff0c4 100644 --- a/base_python/cogment_verse/constants.py +++ b/cogment_verse/constants.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -HUMAN_ACTOR_NAME = "web_actor" # At the moment the web client can only join the trial using the actor name which means a unique name is required -HUMAN_ACTOR_CLASS = "teacher_agent" +COGMENT_VERSION = "v2.4.0" +WEB_ACTOR_NAME = "web_actor" # At the moment the web client can only join the trial using the actor name which means a unique name is required HUMAN_ACTOR_IMPL = "client" + +TEACHER_ACTOR_CLASS = "teacher" +PLAYER_ACTOR_CLASS = "player" +OBSERVER_ACTOR_CLASS = "observer" diff --git a/base_python/cogment_verse/mlflow_experiment_tracker.py b/cogment_verse/mlflow_experiment_tracker.py similarity index 90% rename from base_python/cogment_verse/mlflow_experiment_tracker.py rename to cogment_verse/mlflow_experiment_tracker.py index ae6c80cb..fd92bfc1 100644 --- a/base_python/cogment_verse/mlflow_experiment_tracker.py +++ b/cogment_verse/mlflow_experiment_tracker.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,11 +15,11 @@ import asyncio import logging import numbers -import os from google.protobuf.json_format import MessageToDict from google.protobuf.message import Message from prometheus_client import Counter, Summary +from omegaconf import OmegaConf from mlflow.entities import Metric, Param, RunStatus from mlflow.tracking import MlflowClient @@ -36,14 +36,14 @@ MAX_METRICS_BATCH_SIZE = 1000 # MLFlow only accepts at most 1000 metrics per batch -MLFLOW_TRACKING_URI = os.getenv("MLFLOW_TRACKING_URI") - def make_dict(ignore_non_numbers, *args, **kwargs): res = dict(kwargs) for arg in args: if isinstance(arg, Message): arg = MessageToDict(arg, preserving_proto_field_name=True, including_default_value_fields=False) + if OmegaConf.is_config(arg): + arg = OmegaConf.to_container(arg, resolve=True) if isinstance(arg, dict): for key, value in arg.items(): if ignore_non_numbers and not isinstance(value, numbers.Number): @@ -59,9 +59,10 @@ def make_dict(ignore_non_numbers, *args, **kwargs): class MlflowExperimentTracker: - def __init__(self, experiment_id, run_id, flush_frequency=5): + def __init__(self, experiment_id, run_id, mlflow_tracking_uri, flush_frequency=5): self._experiment_id = experiment_id self._run_id = run_id + self._mlflow_tracking_uri = mlflow_tracking_uri self._mlflow_exp_id = None self._mlflow_run_id = None self._metrics_buffer = [] @@ -69,8 +70,7 @@ def __init__(self, experiment_id, run_id, flush_frequency=5): self._flush_metrics_worker = None def _get_mlflow_client(self): - # This is automagically configured by the environment variable MLFLOW_TRACKING_URI - client = MlflowClient() + client = MlflowClient(tracking_uri=self._mlflow_tracking_uri) if not self._mlflow_exp_id: mlflow_experiment_name = f"/{self._experiment_id}" experiment = client.get_experiment_by_name(mlflow_experiment_name) @@ -108,10 +108,10 @@ async def worker(): except asyncio.CancelledError as cancelled_error: # Raising cancellation raise cancelled_error - except Exception as err: + except Exception as error: # pylint: disable=broad-except log.warning( - f"Error while sending metrics to mlflow server {MLFLOW_TRACKING_URI}. Will retry later in {self._flush_metrics_worker_frequency}s.", - err, + f"Error while sending metrics to mlflow server [{self._mlflow_tracking_uri}]. Will retry later in {self._flush_metrics_worker_frequency}s.", + error, ) await asyncio.sleep(self._flush_metrics_worker_frequency) diff --git a/base_python/cogment_verse/model_registry_client.py b/cogment_verse/model_registry.py similarity index 60% rename from base_python/cogment_verse/model_registry_client.py rename to cogment_verse/model_registry.py index 3d7070bb..21aa37c3 100644 --- a/base_python/cogment_verse/model_registry_client.py +++ b/cogment_verse/model_registry.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +13,8 @@ # limitations under the License. import asyncio +import abc +import copy import io import logging import time @@ -30,9 +32,11 @@ RetrieveVersionInfosRequest, ) from cogment.api.model_registry_pb2_grpc import ModelRegistrySPStub -from cogment_verse.utils import LRU from prometheus_client import Summary +from cogment_verse.services_directory import ServiceType +from cogment_verse.utils import LRU + MODEL_REGISTRY_PUBLISH_VERSION_TIME = Summary( "model_registry_publish_version_seconds", @@ -48,16 +52,57 @@ log = logging.getLogger(__name__) -class ModelRegistryClient: - def __init__(self, endpoint, cache=LRU()): +class Model(abc.ABC): + def __init__(self, model_id, version_number=0): + self.model_id = model_id + self.version_number = version_number + + @abc.abstractmethod + def get_model_user_data(self): + """ + Retrieve the user data associated with the model instance + Returns: + model_user_data (dict[str, str]): model user data that will be saved alongside the model + """ + return {} + + @classmethod + @abc.abstractmethod + def load(cls, model_id, version_number, model_user_data, version_user_data, model_data_f): + """ + Load a serialized model instance and return it + Args: + model_id (string): unique identifier for the model + model_user_data (dict[str, str]): model user data + version_user_data (dict[str, str]): version user data + model_data_f: file object that will be used to load the version model data + Returns: + model: the loaded model + """ + + @abc.abstractmethod + def save(self, model_data_f): + """ + Serialize and save the model + Args: + model_data_f: file object that will be used to save the version model data + Returns: + version_user_data (dict[str, str]): version user data that will be saved alongside the model version + """ + + +class ModelRegistry: + def __init__(self, services_directory): + self._service_directory = services_directory + self._cache = LRU() + + def _get_grpc_stub(self): + endpoint = self._service_directory.get(ServiceType.MODEL_REGISTRY) endpoint_components = urlparse(endpoint) if endpoint_components.scheme != "grpc": raise RuntimeError(f"Unsupported scheme for [{endpoint}], expected 'grpc'") - channel = grpc.aio.insecure_channel(endpoint_components.netloc) - self._stub = ModelRegistrySPStub(channel) - - self._cache = cache + return ModelRegistrySPStub(channel) @staticmethod def _build_model_version_data_cache_key(data_hash): @@ -67,25 +112,29 @@ def _build_model_version_data_cache_key(data_hash): def _build_model_info_cache_key(model_id): return f"model_info_{model_id}" - async def create_model(self, model_id, model_user_data=None): + async def publish_initial_version(self, model): """ - Create a new model in the model registry + Create a new model in the model registry and publish the initial version Parameters: - model_id (string): The model id - model_user_data (dict[str, str] - optional): model user data + model (Model): The model + Returns: + model_info, version_info (dict, dict): A tuple containing the information of the model and the information of the published initial version """ model_user_data_str = {} - if model_user_data: - for key, value in model_user_data.items(): - model_user_data_str[key] = str(value) + for key, value in model.get_model_user_data().items(): + model_user_data_str[key] = str(value) - model_info = ModelInfo(model_id=model_id, user_data=model_user_data_str) + model_info = ModelInfo(model_id=model.model_id, user_data=model_user_data_str) req = CreateOrUpdateModelRequest(model_info=model_info) - await self._stub.CreateOrUpdateModel(req) + await self._get_grpc_stub().CreateOrUpdateModel(req) - self._cache[self._build_model_info_cache_key(model_id)] = model_info + self._cache[self._build_model_info_cache_key(model.model_id)] = model_info + + version_info = await self.publish_version(model) + + return (model_info, version_info) async def retrieve_model_info(self, model_id): """ @@ -100,7 +149,7 @@ async def retrieve_model_info(self, model_id): if cache_key not in self._cache: req = RetrieveModelsRequest(model_ids=[model_id]) - rep = await self._stub.RetrieveModels(req) + rep = await self._get_grpc_stub().RetrieveModels(req) model_info = rep.model_infos[0] @@ -108,71 +157,68 @@ async def retrieve_model_info(self, model_id): return MessageToDict(self._cache[cache_key], preserving_proto_field_name=True) - async def publish_version(self, model_id, model, save_model, archived=False, **kwargs): + async def publish_version(self, model, archived=False): """ Publish a new version of the model Parameters: - model_id (string): Unique id of the model - model (ModelT): The model - save_model (f(ModelT, dict[str, str], BinaryIO, **kwargs) -> dict[str, str]): A function able to save the model, returning version_user_data + model (Model): The model archive (bool - default is False): If true, the model version will be archived (i.e. stored in permanent storage) - kwargs: any number of key/values parameters, forwarded to `save_model` Returns version_info (dict): The information of the published version """ - model_info = await self.retrieve_model_info(model_id) - model_user_data = model_info["user_data"] - def generate_chunks(): try: with io.BytesIO() as model_data_io: - version_user_data = save_model(model, model_user_data, model_data_io, **kwargs) + version_user_data = model.save(model_data_io) version_data = model_data_io.getvalue() - version_info = ModelVersionInfo(model_id=model_id, archived=archived, data_size=len(version_data)) + version_info = ModelVersionInfo(model_id=model.model_id, archived=archived, data_size=len(version_data)) for key, value in version_user_data.items(): - version_info.user_data[key] = str(value) + version_info.user_data[key] = str(value) # pylint: disable=no-member yield CreateVersionRequestChunk(header=CreateVersionRequestChunk.Header(version_info=version_info)) - chunksize = 2 * 1024 * 1024 # 2MB to keep under well under the GRPC 4MB + chunksize = 2 * 1024 * 1024 # 2MB to keep under well under the GRPC 4MB limit + sent_chunk_num = 0 while version_data: yield CreateVersionRequestChunk( body=CreateVersionRequestChunk.Body(data_chunk=version_data[:chunksize]) ) version_data = version_data[chunksize:] + sent_chunk_num += 1 + except Exception as error: log.error("Error while generating model version chunk", exc_info=error) raise error - with MODEL_REGISTRY_PUBLISH_VERSION_TIME.labels(model_id=model_id).time(): - rep = await self._stub.CreateVersion(generate_chunks()) + with MODEL_REGISTRY_PUBLISH_VERSION_TIME.labels(model_id=model.model_id).time(): + rep = await self._get_grpc_stub().CreateVersion(generate_chunks()) cache_key = self._build_model_version_data_cache_key(rep.version_info.data_hash) - self._cache[cache_key] = model + # Store a copy in the local cache + self._cache[cache_key] = copy.deepcopy(model) return MessageToDict(rep.version_info, preserving_proto_field_name=True) - async def retrieve_version(self, model_id, load_model, version_number=-1, **kwargs): + async def retrieve_version(self, model_cls, model_id, version_number=-1): """ Retrieve a version of the model Parameters: + model_cls (Model class): The class of the model model_id (string): Unique id of the model - load_model (f(string, int, dict[str, str], dict[str, str], BinaryIO)): A function able to load the model version_number (int - default is -1): The version number (-1 for the latest) - kwargs: any number of key/values parameters, forwarded to `load_model` Returns - model, model_info, version_info (ModelT, dict[str, str], dict[str, str]): A tuple containing the model version data, the model info and the model version info + model, model_info, version_info (ModelT, dict[str, str], dict[str, str]): A tuple containing the model, the model info and the version info """ start_time = time.time() # First retrieve the model info and model version info async def retrieve_version_info(model_id, version_number): req = RetrieveVersionInfosRequest(model_id=model_id, version_numbers=[version_number]) - rep = await self._stub.RetrieveVersionInfos(req) + rep = await self._get_grpc_stub().RetrieveVersionInfos(req) version_info_pb = rep.version_infos[0] version_info = MessageToDict(version_info_pb, preserving_proto_field_name=True) return version_info @@ -188,12 +234,13 @@ async def retrieve_version_info(model_id, version_number): if not cached: req = RetrieveVersionDataRequest(model_id=model_id, version_number=version_info["version_number"]) data = b"" - async for chunk in self._stub.RetrieveVersionData(req): + async for chunk in self._get_grpc_stub().RetrieveVersionData(req): data += chunk.data_chunk - model = load_model( - model_id, version_number, model_info["user_data"], version_info["user_data"], io.BytesIO(data), **kwargs + model = model_cls.load( + model_id, version_number, model_info["user_data"], version_info["user_data"], io.BytesIO(data) ) + assert model.model_id == model_id self._cache[cache_key] = model MODEL_REGISTRY_RETRIEVE_VERSION_TIME.labels(model_id=model_id, cached=cached).observe(time.time() - start_time) diff --git a/base_python/cogment_verse/utils/__init__.py b/cogment_verse/processes/__init__.py similarity index 54% rename from base_python/cogment_verse/utils/__init__.py rename to cogment_verse/processes/__init__.py index 18ba7d8e..7e910d68 100644 --- a/base_python/cogment_verse/utils/__init__.py +++ b/cogment_verse/processes/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from cogment_verse.utils.lru import LRU -from cogment_verse.utils.sizeof_fmt import sizeof_fmt -from cogment_verse.utils.throttle import throttle -from cogment_verse.utils.get_full_class_name import get_full_class_name +from .cogment_verse_process import CogmentVerseProcess +from .orchestrator import create_orchestrator_service +from .environment import create_environment_service +from .actor import create_actor_service +from .run import create_run_process +from .trial_datastore import create_trial_datastore_service +from .model_registry import create_model_registry_service +from .web import create_web_service diff --git a/cogment_verse/processes/actor.py b/cogment_verse/processes/actor.py new file mode 100644 index 00000000..36f640ea --- /dev/null +++ b/cogment_verse/processes/actor.py @@ -0,0 +1,93 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +import sys + +import cogment + +from .cogment_py_sdk_process import CogmentPySdkProcess +from ..utils.import_class import import_class +from ..utils.get_implementation_name import get_implementation_name +from ..services_directory import ServiceType + +log = logging.getLogger(__name__) + + +def actor_main( + actor_cfg, + model_registry, + name, # pylint: disable=unused-argument + on_ready, + specs_filename, # pylint: disable=unused-argument + work_dir, # pylint: disable=unused-argument +): + # Importing 'specs' only in the subprocess (i.e. where generate has been properly executed) + # pylint: disable-next=import-outside-toplevel + from cogment_verse.specs import cog_settings + + async def actor_main_async(): + actor_cls = import_class(actor_cfg.class_name) + actor = actor_cls(actor_cfg) + + actor_implementation_name = get_implementation_name(actor) + + async def impl_wrapper(actor_session): + actor_session.model_registry = model_registry + await actor.impl(actor_session) + + context = cogment.Context(cog_settings=cog_settings, user_id="cogment_verse_actor") + + context.register_actor( + impl_name=actor_implementation_name, actor_classes=actor.get_actor_classes(), impl=impl_wrapper + ) + + log.info(f"Service actor [{actor_implementation_name}] starting on port [{actor_cfg.port}]...") + + on_ready() + + await context.serve_all_registered( + cogment.ServedEndpoint(port=actor_cfg.port), prometheus_port=actor_cfg.get("prometheus_port", 0) + ) + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(actor_main_async()) + except KeyboardInterrupt: + log.warning("interrupted by user") + sys.exit(-1) + + +def create_actor_service(work_dir, specs_filename, model_registry, actor_cfg, services_directory): + process = CogmentPySdkProcess( + name="actor", + work_dir=work_dir, + specs_filename=specs_filename, + main=actor_main, + model_registry=model_registry, + actor_cfg=actor_cfg, + ) + + actor_cls = import_class(actor_cfg.class_name) + actor = actor_cls(actor_cfg) + + # Register the actor implementation + services_directory.add( + service_type=ServiceType.ACTOR, + service_name=get_implementation_name(actor), + service_endpoint=f"grpc://localhost:{actor_cfg.port}", + ) + + return process diff --git a/cogment_verse/processes/cogment_cli_process.py b/cogment_verse/processes/cogment_cli_process.py new file mode 100644 index 00000000..1181ebbd --- /dev/null +++ b/cogment_verse/processes/cogment_cli_process.py @@ -0,0 +1,78 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import re +import time + +from .popen_process import PopenProcess +from ..utils.download_cogment import download_cogment +from ..constants import COGMENT_VERSION + +log = logging.getLogger(__name__) + +LOG_LEVELS = { + "critical": logging.CRITICAL, + "error": logging.ERROR, + "warning": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG, +} + +RESERVED_MSG_KEYS = ["msg", "level", "component", "time"] + + +def on_cogment_log(name, log_line): + try: + parsed_msg = json.loads(log_line) + except json.JSONDecodeError: + log.warning(f"Unstructured log from [cogment.{name}] - {log_line}") + return + + logger_name = "cogment" + component = parsed_msg.get("component", None) + if component is not None: + logger_name = logger_name + "." + re.sub(r"[/\s]", ".", component) + logger = logging.getLogger(logger_name) + + level_str = parsed_msg.get("level", "info") + if level_str not in LOG_LEVELS: + raise RuntimeError(f"Unknown log level [{level_str}].") + level = LOG_LEVELS[level_str] + + if logger.isEnabledFor(level): + msg_str = parsed_msg.get("msg", "").strip(" \n\r") + for key, value in parsed_msg.items(): + if key not in RESERVED_MSG_KEYS: + msg_str += f" [{key}={value}]" + + logger.log(level, msg_str) + + +def on_awaiting_cogment_ready(): + time.sleep(5) # TODO replace with the status file (will require support for the trial datastore and model registry) + + +class CogmentCliProcess(PopenProcess): + def __init__(self, name, work_dir, cli_args): + cogment_path = download_cogment(output_dir=os.path.join(work_dir, "bin"), desired_version=COGMENT_VERSION) + + super().__init__( + name=name, + args=[cogment_path, *cli_args], + on_log=on_cogment_log, + on_awaiting_ready=on_awaiting_cogment_ready, + ) diff --git a/cogment_verse/processes/cogment_py_sdk_process.py b/cogment_verse/processes/cogment_py_sdk_process.py new file mode 100644 index 00000000..683d49c2 --- /dev/null +++ b/cogment_verse/processes/cogment_py_sdk_process.py @@ -0,0 +1,23 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cogment_verse_process import CogmentVerseProcess +from ..utils.generate import generate + + +class CogmentPySdkProcess(CogmentVerseProcess): + def __init__(self, name, work_dir, specs_filename, main, **kwargs): + generate(work_dir, specs_filename) + + super().__init__(name=name, target=main, work_dir=work_dir, specs_filename=specs_filename, **kwargs) diff --git a/cogment_verse/processes/cogment_verse_process.py b/cogment_verse/processes/cogment_verse_process.py new file mode 100644 index 00000000..4d6419b5 --- /dev/null +++ b/cogment_verse/processes/cogment_verse_process.py @@ -0,0 +1,56 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from multiprocessing import Process, SimpleQueue + +from hydra.core.hydra_config import HydraConfig +from hydra.core.utils import configure_log + + +def wrapped_target(target, hydra_cfg_job_logging, hydra_cfg_verbose, *args, **kwargs): + configure_log(hydra_cfg_job_logging, hydra_cfg_verbose) + return target(*args, **kwargs) + + +class SimpleSignal: + def __init__(self): + self._q = SimpleQueue() + + def trigger(self): + self._q.put(True) + + def await_trigger(self): + self._q.get() + self._q.close() + + +class CogmentVerseProcess(Process): + def __init__(self, name, target, **kwargs): + self._ready_signal = SimpleSignal() + hydra_cfg = HydraConfig.get() + super().__init__( + name=name, + target=wrapped_target, + kwargs={ + **kwargs, + "target": target, + "hydra_cfg_job_logging": hydra_cfg.job_logging, + "hydra_cfg_verbose": hydra_cfg.verbose, + "name": name, + "on_ready": self._ready_signal.trigger, + }, + ) + + def await_ready(self): + self._ready_signal.await_trigger() diff --git a/cogment_verse/processes/environment.py b/cogment_verse/processes/environment.py new file mode 100644 index 00000000..7185f1fd --- /dev/null +++ b/cogment_verse/processes/environment.py @@ -0,0 +1,101 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +import sys + +import cogment + +from .cogment_py_sdk_process import CogmentPySdkProcess +from ..utils.import_class import import_class +from ..utils.get_implementation_name import get_implementation_name +from ..services_directory import ServiceType + +log = logging.getLogger(__name__) + + +def environment_main( + environment_cfg, + name, # pylint: disable=unused-argument + on_ready, + specs_filename, # pylint: disable=unused-argument + work_dir, +): + # Importing 'specs' only in the subprocess (i.e. where generate has been properly executed) + # pylint: disable-next=import-outside-toplevel + from cogment_verse.specs import cog_settings, save_environment_specs + + environment_cls = import_class(environment_cfg.class_name) + env = environment_cls(environment_cfg) + + env_implementation_name = get_implementation_name(env) + + # Generate the environment specs if needed. + save_environment_specs(work_dir, env_implementation_name, env.get_environment_specs()) + + async def environment_main_async(): + context = cogment.Context(cog_settings=cog_settings, user_id="cogment_verse_environment") + + async def wrapped_impl(environment_session): + try: + await env.impl(environment_session) + except KeyboardInterrupt: + # Ignore this one, it's logged at a bunch of different places + pass + except Exception as error: + log.error( + f"Uncaught error occured in implementation code for environment [{env_implementation_name}]", + exc_info=error, + ) + raise + + context.register_environment(impl_name=env_implementation_name, impl=wrapped_impl) + + log.info(f"Environment [{env_implementation_name}] starting on port [{environment_cfg.port}]...") + + on_ready() + + await context.serve_all_registered( + cogment.ServedEndpoint(port=environment_cfg.port), prometheus_port=environment_cfg.get("prometheus_port", 0) + ) + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(environment_main_async()) + except KeyboardInterrupt: + log.warning("interrupted by user") + sys.exit(-1) + + +def create_environment_service(work_dir, specs_filename, environment_cfg, services_directory): + process = CogmentPySdkProcess( + name="environment", + work_dir=work_dir, + specs_filename=specs_filename, + main=environment_main, + environment_cfg=environment_cfg, + ) + + environment_cls = import_class(environment_cfg.class_name) + env = environment_cls(environment_cfg) + + # Register the environment + services_directory.add( + service_type=ServiceType.ENVIRONMENT, + service_name=get_implementation_name(env), + service_endpoint=f"grpc://localhost:{environment_cfg.port}", + ) + + return process diff --git a/cogment_verse/processes/model_registry.py b/cogment_verse/processes/model_registry.py new file mode 100644 index 00000000..99189320 --- /dev/null +++ b/cogment_verse/processes/model_registry.py @@ -0,0 +1,45 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .cogment_cli_process import CogmentCliProcess +from ..services_directory import ServiceType + + +def create_model_registry_service(work_dir, model_registry_cfg, services_directory): + model_registry_data_dir = os.path.join(work_dir, "model_registry") + os.makedirs(model_registry_data_dir, exist_ok=True) + + port = model_registry_cfg.port + + services_directory.add( + service_type=ServiceType.MODEL_REGISTRY, + service_endpoint=f"grpc://localhost:{port}", + ) + + return CogmentCliProcess( + name="model_registry", + work_dir=work_dir, + cli_args=[ + "services", + "model_registry", + "--log_format=json", + f"--log_level={model_registry_cfg.log_level}", + f"--archive_dir={model_registry_data_dir}", + f"--port={port}", + f"--sent_version_chunk_size={1024 * 1024 * 2}", # TODO make those options configurable + "--cache_max_items=100", + ], + ) diff --git a/cogment_verse/processes/orchestrator.py b/cogment_verse/processes/orchestrator.py new file mode 100644 index 00000000..768c0379 --- /dev/null +++ b/cogment_verse/processes/orchestrator.py @@ -0,0 +1,42 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cogment_cli_process import CogmentCliProcess +from ..services_directory import ServiceType + + +def create_orchestrator_service(work_dir, orchestrator_cfg, services_directory): + port = orchestrator_cfg.port + web_port = orchestrator_cfg.web_port + services_directory.add( + service_type=ServiceType.ORCHESTRATOR, + service_endpoint=f"grpc://localhost:{port}", + ) + services_directory.add( + service_type=ServiceType.ORCHESTRATOR_WEB_ENDPOINT, + service_endpoint=f"http://localhost:{web_port}", + ) + return CogmentCliProcess( + name="orchestrator", + work_dir=work_dir, + cli_args=[ + "services", + "orchestrator", + "--log_format=json", + f"--log_level={orchestrator_cfg.log_level}", + f"--actor_port={port}", + f"--lifecycle_port={port}", + f"--actor_web_port={web_port}", + ], + ) diff --git a/cogment_verse/processes/popen_process.py b/cogment_verse/processes/popen_process.py new file mode 100644 index 00000000..994567d3 --- /dev/null +++ b/cogment_verse/processes/popen_process.py @@ -0,0 +1,76 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import Thread +import logging +import platform +import signal +import subprocess + +from .cogment_verse_process import CogmentVerseProcess + +log = logging.getLogger(__name__) + +TERMINATION_TIMEOUT_SECONDS = 10 + + +def main(name, args, cwd, env, on_ready, on_log, on_awaiting_ready): + log.debug(f"Launching subprocess [{name}] with args {args}...") + with subprocess.Popen( + args=args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) as external_process: + + def killing_handler(_signum, _frame): + log.warning(f"killing [{name}]") + external_process.kill() + + def termination_handler(_signum, _frame): + log.info(f"terminating [{name}]") + external_process.terminate() + external_process.send_signal(signal.SIGINT) + try: + external_process.wait(TERMINATION_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + log.warning( + f"Subprocess [{name}] didn't terminate cleany under {TERMINATION_TIMEOUT_SECONDS}s, killing it" + ) + external_process.kill() + + if platform.system() in ["Linux"]: + signal.signal(signal.SIGKILL, killing_handler) + signal.signal(signal.SIGINT, termination_handler) + signal.signal(signal.SIGTERM, termination_handler) + + def consume_stdout(): + for log_line in external_process.stdout: + on_log(name, log_line.decode().strip(" \n\r\t")) + + log_consumer = Thread(name=f"{name}_log_consumer", target=consume_stdout) + log_consumer.start() + + on_awaiting_ready() + on_ready() + + external_process.wait() + + +def default_on_awaiting_ready(): + pass + + +class PopenProcess(CogmentVerseProcess): + def __init__(self, name, args, on_log, on_awaiting_ready=default_on_awaiting_ready, cwd=".", env=None): + super().__init__( + name=name, target=main, args=args, on_log=on_log, on_awaiting_ready=on_awaiting_ready, cwd=cwd, env=env + ) diff --git a/cogment_verse/processes/run.py b/cogment_verse/processes/run.py new file mode 100644 index 00000000..f1fcb4eb --- /dev/null +++ b/cogment_verse/processes/run.py @@ -0,0 +1,89 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +import sys + +from omegaconf import OmegaConf + +from .cogment_py_sdk_process import CogmentPySdkProcess +from ..utils.import_class import import_class +from ..services_directory import ServiceType +from ..run import RunSession + +log = logging.getLogger(__name__) + + +def run_main( + model_registry, + name, # pylint: disable=unused-argument + on_ready, + run_cfg, + services_directory, + specs_filename, # pylint: disable=unused-argument + work_dir, +): + async def run_main_async(): + # Importing 'specs' only in the subprocess (i.e. where generate has been properly executed) + # pylint: disable-next=import-outside-toplevel + from cogment_verse.specs import load_environment_specs + + _run_cfg = run_cfg + run_cls = import_class(_run_cfg.class_name) + if hasattr(run_cls, "default_cfg"): + _run_cfg = OmegaConf.merge(run_cls.default_cfg, _run_cfg) + + registered_enviroment_impl_names = services_directory.get_service_names(ServiceType.ENVIRONMENT) + enviroment_impl_name = _run_cfg.get("environment", registered_enviroment_impl_names[0]) + + try: + environment_specs = load_environment_specs(work_dir, enviroment_impl_name) + except Exception as error: + raise RuntimeError(f"Unable to start run, unknown environment: '{enviroment_impl_name}'") from error + + run_id = _run_cfg.run_id + run = run_cls(cfg=_run_cfg, environment_specs=environment_specs) + run_session = RunSession( + run_cfg=_run_cfg, run_id=run_id, services_directory=services_directory, model_registry=model_registry + ) + try: + on_ready() + log.info(f"Starting run [{run_id}] from [{_run_cfg.class_name}]") + await run.impl(run_session) + except Exception as error: + run_session.terminate_failure() + log.error(f"Error while executing run [{run_id}] from [{_run_cfg.class_name}]", exc_info=error) + raise error + + run_session.terminate_success() + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(run_main_async()) + except KeyboardInterrupt: + log.warning("interrupted by user") + sys.exit(-1) + + +def create_run_process(work_dir, specs_filename, services_directory, model_registry, run_cfg): + return CogmentPySdkProcess( + name="run", + work_dir=work_dir, + specs_filename=specs_filename, + main=run_main, + services_directory=services_directory, + model_registry=model_registry, + run_cfg=run_cfg, + ) diff --git a/cogment_verse/processes/trial_datastore.py b/cogment_verse/processes/trial_datastore.py new file mode 100644 index 00000000..9d71bd43 --- /dev/null +++ b/cogment_verse/processes/trial_datastore.py @@ -0,0 +1,39 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cogment_cli_process import CogmentCliProcess +from ..services_directory import ServiceType + + +def create_trial_datastore_service(work_dir, trial_datastore_cfg, services_directory): + # TODO support other options + + port = trial_datastore_cfg.port + + services_directory.add( + service_type=ServiceType.TRIAL_DATASTORE, + service_endpoint=f"grpc://localhost:{port}", + ) + + return CogmentCliProcess( + name="trial_datastore", + work_dir=work_dir, + cli_args=[ + "services", + "trial_datastore", + "--log_format=json", + f"--log_level={trial_datastore_cfg.log_level}", + f"--port={port}", + ], + ) diff --git a/cogment_verse/processes/web.py b/cogment_verse/processes/web.py new file mode 100644 index 00000000..f2bf7b5d --- /dev/null +++ b/cogment_verse/processes/web.py @@ -0,0 +1,80 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import time + +from ..web import server_main, generate, npm_command, create_dev_server_popen_kwargs +from .cogment_verse_process import CogmentVerseProcess +from .popen_process import PopenProcess +from ..services_directory import ServiceType + +log = logging.getLogger(__name__) + +WEB_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../web/components")) + + +def on_cra_log(_name, log_line): + log.info(log_line) + + +def on_awaiting_cra_ready(): + time.sleep(5) # TODO find a better way to do that ? + + +class NpmProcess(PopenProcess): + def __init__(self, name, specs_filename, services_directory, web_cfg): + log.info("Installing web components dependencies using `npm install`...") + npm_command(["install", "--no-audit"], WEB_DIR) + + generate(specs_filename, WEB_DIR, web_cfg.get("force_rebuild", False)) + + super().__init__( + name=name, + on_log=on_cra_log, + on_awaiting_ready=on_awaiting_cra_ready, + **create_dev_server_popen_kwargs( + port=web_cfg.port, + orchestrator_web_endpoint=services_directory.get(ServiceType.ORCHESTRATOR_WEB_ENDPOINT), + ), + ) + + +class WebProcess(CogmentVerseProcess): + def __init__(self, name, specs_filename, services_directory, web_cfg): + served_dir = os.path.abspath(os.path.join(WEB_DIR, "build")) + # TODO find a better way to detect a rebuild is needed + if not os.path.isdir(served_dir) or web_cfg.get("build", False): + log.info("Installing web components dependencies using `npm install`...") + npm_command(["install", "--no-audit"], WEB_DIR) + + generate(specs_filename, WEB_DIR, True) + + log.info("Building the web components `npm run build`...") + npm_command(["run", "build"], WEB_DIR) + + super().__init__( + name=name, + target=server_main, + port=web_cfg.port, + orchestrator_web_endpoint=services_directory.get(ServiceType.ORCHESTRATOR_WEB_ENDPOINT), + served_dir=served_dir, + ) + + +def create_web_service(specs_filename, services_directory, web_cfg): + if web_cfg.get("dev", False): + return NpmProcess("web", specs_filename, services_directory, web_cfg) + return WebProcess("web", specs_filename, services_directory, web_cfg) diff --git a/base_python/cogment_verse/run/__init__.py b/cogment_verse/run/__init__.py similarity index 83% rename from base_python/cogment_verse/run/__init__.py rename to cogment_verse/run/__init__.py index ef3a5ef5..f6abb1a9 100644 --- a/base_python/cogment_verse/run/__init__.py +++ b/cogment_verse/run/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from cogment_verse.run.run_context import RunContext +from .run_session import RunSession diff --git a/cogment_verse/run/run_session.py b/cogment_verse/run/run_session.py new file mode 100644 index 00000000..d2c37944 --- /dev/null +++ b/cogment_verse/run/run_session.py @@ -0,0 +1,85 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from multiprocessing import Queue +import time + +from ..mlflow_experiment_tracker import MlflowExperimentTracker +from .sample_producer_worker import start_sample_producer_worker +from .trial_runner_worker import start_trial_runner_worker + +log = logging.getLogger(__name__) + + +class RunSession: + def __init__(self, run_cfg, run_id, services_directory, model_registry): + self.run_id = run_id + + self._services_directory = services_directory + self._step_idx = 0 + + self._xp_tracker = MlflowExperimentTracker( + experiment_id=run_cfg.class_name, run_id=run_id, mlflow_tracking_uri=run_cfg.mlflow_tracking_uri + ) + + self.model_registry = model_registry + + def start_and_await_trials(self, trials_id_and_params, sample_producer_impl, num_parallel_trials=10): + trial_started_queue = Queue() + sample_queue = Queue() + + trial_runner_worker = start_trial_runner_worker( + trials_id_and_params=trials_id_and_params, + services_directory=self._services_directory, + trial_started_queue=trial_started_queue, + trial_ended_queue=None, + num_parallel_trials=num_parallel_trials, + ) + + sample_producer_worker = start_sample_producer_worker( + trial_started_queue=trial_started_queue, + services_directory=self._services_directory, + sample_queue=sample_queue, + impl=sample_producer_impl, + ) + + try: + while True: + sample_queue_event = sample_queue.get() + if sample_queue_event.done: + break + self._step_idx += 1 + yield (self._step_idx, sample_queue_event.trial_id, sample_queue_event.sample) + + trial_runner_worker.join() + sample_producer_worker.join() + except RuntimeError as error: + trial_runner_worker.terminate() + sample_producer_worker.terminate() + trial_runner_worker.join() + sample_producer_worker.join() + raise error + + def log_params(self, *args, **kwargs): + self._xp_tracker.log_params(*args, **kwargs) + + def log_metrics(self, *args, **kwargs): + self._xp_tracker.log_metrics(step_timestamp=int(time.time() * 1000), step_idx=self._step_idx, *args, **kwargs) + + def terminate_failure(self): + self._xp_tracker.terminate_failure() + + def terminate_success(self): + self._xp_tracker.terminate_success() diff --git a/cogment_verse/run/sample_producer_worker.py b/cogment_verse/run/sample_producer_worker.py new file mode 100644 index 00000000..c2e1bace --- /dev/null +++ b/cogment_verse/run/sample_producer_worker.py @@ -0,0 +1,125 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from multiprocessing import Process +import asyncio +import logging +import sys + +import cogment + +from ..services_directory import ServiceType + +log = logging.getLogger(__name__) + + +class SampleQueueEvent: + def __init__(self, trial_id=None, sample=None, done=False): + self.trial_id = trial_id + self.sample = sample + self.done = done + + +class SampleProducerSession: + def __init__(self, datastore, trial_info, sample_queue, impl): + self.datastore = datastore + self.trial_info = trial_info + self.sample_queue = sample_queue + self.impl = impl + + def produce_sample(self, sample): + self.sample_queue.put(SampleQueueEvent(trial_id=self.trial_info.trial_id, sample=sample)) + + def all_trial_samples(self): + return self.datastore.all_samples([self.trial_info]) + + def create_task(self): + async def wrapped_impl(): + try: + await self.impl(self) + except KeyboardInterrupt: + # This one is ignored, it's logged at a bunch of different places + pass + except Exception as error: + log.error( + f"Uncaught error occured during the sample production for trial [{self.trial_info.trial_id}]", + exc_info=error, + ) + raise + + return asyncio.create_task(wrapped_impl()) + + +async def async_sample_producer_worker(trial_started_queue, sample_queue, impl, services_directory): + # Importing 'specs' only in the subprocess (i.e. where generate has been properly executed) + # pylint: disable-next=import-outside-toplevel + from cogment_verse.specs import cog_settings + + cog_context = cogment.Context(cog_settings=cog_settings, user_id="cogment_verse_sample_producer") + datastore = cog_context.get_datastore( + endpoint=cogment.Endpoint(services_directory.get(ServiceType.TRIAL_DATASTORE)) + ) + # Define a timeout for trial info retrieval + # pylint: disable-next=protected-access + datastore._timeout = 20 + + sample_producer_tasks = [] + while True: + # Executing the retrieval from the queue as an async corouting to avoid blocking the process + trial_started_queue_event = await asyncio.get_running_loop().run_in_executor(None, trial_started_queue.get) + if trial_started_queue_event.done: + break + + trials_info = await datastore.get_trials([trial_started_queue_event.trial_id]) + if len(trials_info) == 0: + log.warning( + f"Trial [{trial_started_queue_event.trial_id}] couldn't be found in the trial datastore, retrying later" + ) + await asyncio.get_running_loop().run_in_executor(None, trial_started_queue.put, trial_started_queue_event) + continue + + [trial_info] = trials_info + + log.debug(f"[{trial_info.trial_id}] started") + + sample_producer_session = SampleProducerSession(datastore, trial_info, sample_queue, impl) + + sample_producer_tasks.append(sample_producer_session.create_task()) + + if len(sample_producer_tasks) > 0: + await asyncio.wait(sample_producer_tasks) + + sample_queue.put(SampleQueueEvent(done=True)) + + +def sample_producer_worker(trial_started_queue, sample_queue, impl, services_directory): + try: + asyncio.run(async_sample_producer_worker(trial_started_queue, sample_queue, impl, services_directory)) + except KeyboardInterrupt: + sys.exit(-1) + + +def start_sample_producer_worker(trial_started_queue, sample_queue, impl, services_directory): + worker = Process( + name="sample_producer_worker", + target=sample_producer_worker, + args=( + trial_started_queue, + sample_queue, + impl, + services_directory, + ), + ) + worker.start() + return worker diff --git a/cogment_verse/run/trial_runner_worker.py b/cogment_verse/run/trial_runner_worker.py new file mode 100644 index 00000000..51f618fe --- /dev/null +++ b/cogment_verse/run/trial_runner_worker.py @@ -0,0 +1,143 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from multiprocessing import Process +import asyncio +import logging +import sys + +import cogment + +from ..services_directory import ServiceType + +log = logging.getLogger(__name__) + + +class TrialStartedQueueEvent: + def __init__(self, trial_id=None, done=False): + self.trial_id = trial_id + self.done = done + + +class TrialEndedQueueEvent: + def __init__(self, trial_id=None, done=False): + self.trial_id = trial_id + self.done = done + + +async def async_trial_runner_worker( + trials_id_and_params, services_directory, trial_started_queue, trial_ended_queue, num_parallel_trials +): + # Importing 'specs' only in the subprocess (i.e. where generate has been properly executed) + # pylint: disable-next=import-outside-toplevel + from cogment_verse.specs import cog_settings + + context = cogment.Context(cog_settings=cog_settings, user_id="cogment_verse_trial_runner") + + controller = context.get_controller(endpoint=cogment.Endpoint(services_directory.get(ServiceType.ORCHESTRATOR))) + + num_trials = len(trials_id_and_params) + num_started_trials = 0 + num_ended_trials = 0 + running_trials = set() + + async def start_trials(): + nonlocal running_trials + nonlocal num_started_trials + num_to_start_trials = num_parallel_trials - len(running_trials) + if num_to_start_trials <= 0: + return + + trials_id_and_params_chunk = trials_id_and_params[ + num_started_trials : min(num_trials, num_started_trials + num_to_start_trials) + ] + if len(trials_id_and_params_chunk) == 0: + return + + for (trial_id, serialized_trial_params) in trials_id_and_params_chunk: + trial_params = cogment.TrialParameters(cog_settings) + trial_params.deserialize(serialized_trial_params) + + # Retrieve the endpoints from the services directory + trial_params.environment_endpoint = services_directory.get( + ServiceType.ENVIRONMENT, trial_params.environment_implementation + ) + trial_params.datalog_endpoint = services_directory.get(ServiceType.TRIAL_DATASTORE) + for actor in trial_params.actors: + actor.endpoint = services_directory.get(ServiceType.ACTOR, actor.implementation) + + actual_trial_id = await controller.start_trial(trial_id_requested=trial_id, trial_params=trial_params) + if actual_trial_id is None: + raise RuntimeError(f"Unable to start a trial with id [{trial_id}]") + running_trials.add(trial_id) + num_started_trials += 1 + log.debug(f"Trial [{trial_id}] started, {num_trials-num_started_trials} trials remaining to start.") + if trial_started_queue is not None: + trial_started_queue.put(TrialStartedQueueEvent(trial_id=trial_id)) + + async def await_trials(): + nonlocal running_trials + nonlocal num_ended_trials + async for trial_info in controller.watch_trials(trial_state_filters=[cogment.TrialState.ENDED]): + if trial_info.trial_id in running_trials: + running_trials.discard(trial_info.trial_id) + num_ended_trials += 1 + if trial_ended_queue is not None: + trial_ended_queue.put(TrialEndedQueueEvent(trial_id=trial_info.trial_id)) + log.debug(f"Trial [{trial_info.trial_id}] ended, {num_trials-num_ended_trials} trials remaining.") + if num_ended_trials == num_trials: + break + if len(running_trials) < num_parallel_trials: + await start_trials() + + try: + await_trials_start = asyncio.create_task(await_trials()) + await start_trials() + await await_trials_start + finally: + if trial_started_queue is not None: + trial_started_queue.put(TrialEndedQueueEvent(done=True)) + if trial_ended_queue is not None: + trial_ended_queue.put(TrialEndedQueueEvent(done=True)) + + +def trial_runner_worker( + trials_id_and_params, services_directory, trial_started_queue, trial_ended_queue, num_parallel_trials +): + try: + asyncio.run( + async_trial_runner_worker( + trials_id_and_params, services_directory, trial_started_queue, trial_ended_queue, num_parallel_trials + ) + ) + except KeyboardInterrupt: + sys.exit(-1) + + +def start_trial_runner_worker( + trials_id_and_params, services_directory, trial_started_queue=None, trial_ended_queue=None, num_parallel_trials=10 +): + worker = Process( + name="trial_runner_worker", + target=trial_runner_worker, + args=( + [(trial_id, trial_params.serialize()) for (trial_id, trial_params) in trials_id_and_params], + services_directory, + trial_started_queue, + trial_ended_queue, + num_parallel_trials, + ), + ) + worker.start() + return worker diff --git a/cogment_verse/services_directory.py b/cogment_verse/services_directory.py new file mode 100644 index 00000000..4ccdcf59 --- /dev/null +++ b/cogment_verse/services_directory.py @@ -0,0 +1,55 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from random import choice + + +class ServiceType(Enum): + ORCHESTRATOR = "orchestrator" + ORCHESTRATOR_WEB_ENDPOINT = "orchestrator_web_endpoint" + ENVIRONMENT = "environment" + ACTOR = "actor" + TRIAL_DATASTORE = "trial_datastore" + MODEL_REGISTRY = "model_registry" + WEB = "web" + + +class ServiceDirectory: + def __init__(self): + self._directory = {} + + def add(self, service_type, service_endpoint, service_name=None): + if service_type.value not in self._directory: + self._directory[service_type.value] = {} + + if service_name not in self._directory[service_type.value]: + self._directory[service_type.value][service_name] = [] + + self._directory[service_type.value][service_name].append(service_endpoint) + + def get(self, service_type, service_name=None): + if service_type.value not in self._directory: + raise RuntimeError(f"No service of type [{service_type.value}] registered") + + if service_name not in self._directory[service_type.value]: + raise RuntimeError(f"No service named [{service_name}] of type [{service_type.value}] registered") + + return choice(self._directory[service_type.value][service_name]) + + def get_service_names(self, service_type): + if service_type.value not in self._directory: + return [] + + return [*self._directory[service_type.value].keys()] diff --git a/cogment_verse/specs/__init__.py b/cogment_verse/specs/__init__.py new file mode 100644 index 00000000..9ca1a9d9 --- /dev/null +++ b/cogment_verse/specs/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from data_pb2 import ( # pylint: disable=import-error + AgentConfig, + EnvironmentConfig, + EnvironmentSpecs, + Observation, + PlayerAction, + Space, + SpaceValue, +) +import cog_settings # pylint: disable=import-error +from cogment_verse.constants import ( + WEB_ACTOR_NAME, + HUMAN_ACTOR_IMPL, + TEACHER_ACTOR_CLASS, + PLAYER_ACTOR_CLASS, + OBSERVER_ACTOR_CLASS, +) + +from .environment_specs import save_environment_specs, load_environment_specs +from .encode_rendered_frame import encode_rendered_frame +from .ndarray import deserialize_ndarray, serialize_ndarray +from .sample_space import sample_space +from .flatten import flattened_dimensions, flatten, unflatten diff --git a/cogment.yaml b/cogment_verse/specs/cogment.yaml similarity index 52% rename from cogment.yaml rename to cogment_verse/specs/cogment.yaml index 3e475121..dae4bda4 100644 --- a/cogment.yaml +++ b/cogment_verse/specs/cogment.yaml @@ -10,16 +10,23 @@ trial: # Static configuration actor_classes: - - name: agent + - name: player action: - space: cogment_verse.AgentAction + space: cogment_verse.PlayerAction observation: space: cogment_verse.Observation config_type: cogment_verse.AgentConfig - - name: teacher_agent + - name: teacher action: - space: cogment_verse.AgentAction + space: cogment_verse.TeacherAction observation: space: cogment_verse.Observation - config_type: cogment_verse.HumanConfig + config_type: cogment_verse.AgentConfig + + - name: observer + action: + space: cogment_verse.ObserverAction + observation: + space: cogment_verse.Observation + config_type: cogment_verse.AgentConfig diff --git a/cogment_verse/specs/data.proto b/cogment_verse/specs/data.proto new file mode 100644 index 00000000..6569b03b --- /dev/null +++ b/cogment_verse/specs/data.proto @@ -0,0 +1,126 @@ +// Copyright 2022 AI Redefined Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package cogment_verse; + +message NDArray { + string dtype = 1; + repeated uint32 shape = 2; + bytes data = 3; +} + +// Space messages are used to define environments action or observation space +// Similar to gym's "dict" space +message Space { + message Discrete { + int32 num = 1; + repeated string labels = 2; // Define labels for the discrete elements in the space, overrides `num` + } + message Bound { + optional float bound = 1; // value of the bound, not set means unbounded + } + message Box { + repeated uint32 shape = 1; + repeated Bound low = 2; // independant lower bounds for each dimensions (gets reshaped) + repeated Bound high = 3; // independant upper bounds for each dimensions (gets reshaped) + } + message Property { + string key = 1; + oneof type_oneof { + Discrete discrete = 2; + Box box = 3; + } + } + repeated Property properties = 1; +} + +// SpaceValue messages are values within a Space +// `properties` are sorted in the same way than the value's Space +message SpaceValue { + message SimpleBox { + repeated float values = 1; + } + message PropertyValue { + oneof value { + int32 discrete = 1; + NDArray box = 2; + SimpleBox simple_box = 3; + } + } + repeated PropertyValue properties = 1; +} + +message EnvironmentSpecs { + string implementation = 1; + int32 num_players = 2; + Space observation_space = 3; + Space action_space = 4; +} + +message EnvironmentConfig { + string run_id = 1; + bool render = 2; + int32 render_width = 3; + uint32 seed = 4; + bool flatten = 5; + + // uint32 framestack = 8; + // string mode = 10; +} + +message HFHubModel { + string repo_id = 1; + string filename = 2; +} + +message AgentConfig { + string run_id = 1; + EnvironmentSpecs environment_specs = 2; + string model_id = 3; + int32 model_version = 4; + int32 actor_index = 5; // Used to figure out if an agent is the current_player in the observation space + string device = 6; + uint32 threads_per_worker = 7; + HFHubModel hf_hub_model = 8; +} + +message TrialConfig { +} + +message Observation { + SpaceValue value = 1; + optional bytes rendered_frame = 2; + repeated int32 legal_moves_as_int = 3; + int32 current_player = 4; // active player for multi-agent turn-based environments + repeated string overridden_players = 5; // list of players that provided an action that was overriden during the last tick + NDArray segmentation = 6; +} + +message PlayerAction { + SpaceValue value = 1; + // NDArray policy = 2; // optional: policy from which action was drawn + // float value = 3; // optional: value of the state from which the action was taken +} + +message TeacherAction { + optional SpaceValue value = 1; + // NDArray policy = 2; // optional: policy from which action was drawn + // float value = 3; // optional: value of the state from which the action was taken +} + +message ObserverAction { + // NOTHING +} diff --git a/cogment_verse/specs/encode_rendered_frame.py b/cogment_verse/specs/encode_rendered_frame.py new file mode 100644 index 00000000..9eb9dee3 --- /dev/null +++ b/cogment_verse/specs/encode_rendered_frame.py @@ -0,0 +1,36 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cv2 + + +def encode_rendered_frame(rendered_frame, max_size=1024): + if max_size <= 0: + max_size = 1024 + # gRPC max message size hack + height, width = rendered_frame.shape[:2] + if max(height, width) > max_size: + if height > width: + new_height = max_size + new_width = int(new_height / height * width) + else: + new_width = max_size + new_height = int(height / width * new_width) + rendered_frame = cv2.resize(rendered_frame, (new_width, new_height), interpolation=cv2.INTER_AREA) + + # note rgb -> bgr for cv2 + result, encoded_frame = cv2.imencode(".jpg", rendered_frame[:, :, ::-1]) + assert result + + return encoded_frame.tobytes() diff --git a/cogment_verse/specs/environment_specs.py b/cogment_verse/specs/environment_specs.py new file mode 100644 index 00000000..f1ee0077 --- /dev/null +++ b/cogment_verse/specs/environment_specs.py @@ -0,0 +1,37 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.protobuf.json_format import MessageToDict, ParseDict +import yaml + +from data_pb2 import EnvironmentSpecs # pylint: disable=import-error + + +def save_environment_specs(work_dir, env_name, env_specs): + specs_filename = os.path.join(work_dir, "environment_specs", f"{env_name}.yaml") + os.makedirs(os.path.dirname(specs_filename), exist_ok=True) + + env_specs.implementation = env_name + + with open(specs_filename, "w", encoding="utf-8") as f: + yaml.safe_dump(MessageToDict(env_specs, preserving_proto_field_name=True), f) + + +def load_environment_specs(work_dir, env_name): + specs_filename = os.path.join(work_dir, "environment_specs", f"{env_name}.yaml") + + with open(specs_filename, "r", encoding="utf-8") as f: + return ParseDict(yaml.safe_load(f), EnvironmentSpecs()) diff --git a/cogment_verse/specs/flatten.py b/cogment_verse/specs/flatten.py new file mode 100644 index 00000000..47f74778 --- /dev/null +++ b/cogment_verse/specs/flatten.py @@ -0,0 +1,97 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +from data_pb2 import Space, SpaceValue # pylint: disable=import-error + +from .value import create_space_values +from .ndarray import deserialize_ndarray, serialize_ndarray, create_one_hot_ndarray + + +def space_prop_flattened_dimensions(prop): + """ + Computes the number of dimensions a flattened equivalent of the given space property would have. + """ + assert isinstance(prop, Space.Property) + + if prop.WhichOneof("type_oneof") == "discrete": + return max(len(prop.discrete.labels), prop.discrete.num) + # box + if len(prop.box.shape) == 0: + return 0 + prop_dim = 1 + for dim in prop.box.shape: + prop_dim *= dim + return prop_dim + + +def flattened_dimensions(space): + """ + Computes the number of dimensions a flattened equivalent of the given space would have. + """ + assert isinstance(space, Space) + + space_dim = 0 + for prop in space.properties: + space_dim += space_prop_flattened_dimensions(prop) + + return space_dim + + +def flatten_prop(space, value, prop_idx): + assert isinstance(space, Space) + assert isinstance(value, SpaceValue) + + if prop_idx >= len(value.properties): + # If a value defines less properties that its space we make the remaining zeros + return np.zeros(space_prop_flattened_dimensions(space.properties[prop_idx])) + prop_value = value.properties[prop_idx] + value_type = prop_value.WhichOneof("value") + if value_type == "discrete": + space_prop = space.properties[prop_idx] + return create_one_hot_ndarray( + prop_value.discrete, max(len(space_prop.discrete.labels), space_prop.discrete.num) + ) + if value_type == "box": + return deserialize_ndarray(prop_value.box).flatten() + # if value_type == "simple_box": + return np.array(prop_value.simple_box.values) + + +def flatten(space, value): + assert isinstance(space, Space) + assert isinstance(value, SpaceValue) + + return np.concatenate([flatten_prop(space, value, prop_idx) for prop_idx in range(len(space.properties))]) + + +def unflatten(space, flat_value): + assert isinstance(space, Space) + assert len(np.shape(flat_value)) == 1 + + [value] = create_space_values(space) + flat_value_idx = 0 + for prop_idx, prop in enumerate(space.properties): + flat_value_idx_end = flat_value_idx + space_prop_flattened_dimensions(prop) + flat_value_prop = flat_value[flat_value_idx:flat_value_idx_end] + if prop.WhichOneof("type_oneof") == "discrete": + # np.nonzero on a flat array returns a 1D tuple of the indices whose value is != 0 + value.properties[prop_idx].discrete = np.nonzero(flat_value_prop)[0][0] + else: + value.properties[prop_idx].box.CopyFrom(serialize_ndarray(flat_value_prop.reshape(prop.box.shape))) + + flat_value_idx = flat_value_idx_end + + return value diff --git a/environment/cogment_verse_environment/utils/serialization_helpers.py b/cogment_verse/specs/ndarray.py similarity index 55% rename from environment/cogment_verse_environment/utils/serialization_helpers.py rename to cogment_verse/specs/ndarray.py index d634f550..20b81e0f 100644 --- a/environment/cogment_verse_environment/utils/serialization_helpers.py +++ b/cogment_verse/specs/ndarray.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,25 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import cv2 import numpy as np -from data_pb2 import NDArray +from data_pb2 import NDArray # pylint: disable=import-error -def deserialize_np_array(nd_array): +def deserialize_ndarray(nd_array): return np.frombuffer(nd_array.data, dtype=nd_array.dtype).reshape(*nd_array.shape) -def serialize_np_array(np_array): - return NDArray(shape=np_array.shape, dtype=str(np_array.dtype), data=np_array.tobytes()) +def serialize_ndarray(nd_array): + return NDArray(shape=nd_array.shape, dtype=str(nd_array.dtype), data=nd_array.tobytes()) -def deserialize_img(img_bytes): - return cv2.imdecode(np.frombuffer(img_bytes, dtype=np.uint8), cv2.IMREAD_COLOR) - - -def serialize_img(img): - # note rgb -> bgr for cv2 - result, data = cv2.imencode(".jpg", img[:, :, ::-1]) - assert result - return data.tobytes() +def create_one_hot_ndarray(value, size): + nd_array = np.zeros(size) + nd_array[value] = 1 + return nd_array diff --git a/cogment_verse/specs/sample_space.py b/cogment_verse/specs/sample_space.py new file mode 100644 index 00000000..2a2cea72 --- /dev/null +++ b/cogment_verse/specs/sample_space.py @@ -0,0 +1,64 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +from .value import create_space_values +from .ndarray import serialize_ndarray + + +def sample_space(space, num_samples=1, seed=0): + rng = np.random.default_rng(seed) + space_values = create_space_values(space, num_samples) + + for prop_idx, prop in enumerate(space.properties): + if prop.WhichOneof("type_oneof") == "discrete": + num_action = max(len(prop.discrete.labels), prop.discrete.num) + for space_value, sampled_value in zip(space_values, rng.integers(low=0, high=num_action, size=num_samples)): + space_value.properties[prop_idx].discrete = sampled_value + else: + shape = prop.box.shape + high = np.array([bound.bound if bound.HasField("bound") else np.inf for bound in prop.box.high]).reshape( + shape + ) + low = np.array([bound.bound if bound.HasField("bound") else -np.inf for bound in prop.box.low]).reshape( + shape + ) + + sampled_values = np.zeros((num_samples, *shape)) + high = np.repeat(high[np.newaxis, :, :], num_samples, axis=0) + low = np.repeat(low[np.newaxis, :, :], num_samples, axis=0) + + unbounded_mask = (high == np.inf) & (low == -np.inf) + sampled_values[unbounded_mask] = rng.normal(size=unbounded_mask[unbounded_mask].shape) + + low_bounded_mask = (high == np.inf) & (low != -np.inf) + sampled_values[low_bounded_mask] = ( + -rng.exponential(size=low_bounded_mask[low_bounded_mask].shape) + low[low_bounded_mask] + ) + + high_bounded_mask = (high != np.inf) & (low == -np.inf) + sampled_values[high_bounded_mask] = ( + rng.exponential(size=high_bounded_mask[high_bounded_mask].shape) + high[high_bounded_mask] + ) + + bounded_mask = (high != np.inf) & (low != -np.inf) + sampled_values[bounded_mask] = rng.uniform( + low=low[bounded_mask], high=high[bounded_mask], size=bounded_mask[bounded_mask].shape + ) + + for space_value, sampled_value in zip(space_values, sampled_values): + space_value.properties[prop_idx].box.CopyFrom(serialize_ndarray(sampled_value)) + + return space_values diff --git a/cogment_verse/specs/value.py b/cogment_verse/specs/value.py new file mode 100644 index 00000000..7bf904cc --- /dev/null +++ b/cogment_verse/specs/value.py @@ -0,0 +1,22 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from data_pb2 import SpaceValue # pylint: disable=import-error + + +def create_space_values(space, num_values=1): + return [ + SpaceValue(properties=[SpaceValue.PropertyValue() for property_definition in space.properties]) + for idx_value in range(num_values) + ] diff --git a/base_python/cogment_verse/utils/get_full_class_name.py b/cogment_verse/utils/__init__.py similarity index 75% rename from base_python/cogment_verse/utils/get_full_class_name.py rename to cogment_verse/utils/__init__.py index 3df53deb..df0a641b 100644 --- a/base_python/cogment_verse/utils/get_full_class_name.py +++ b/cogment_verse/utils/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# no import - - -def get_full_class_name(instance): - return f"{type(instance).__module__}.{type(instance).__name__}" +from .lru import LRU +from .sizeof_fmt import sizeof_fmt diff --git a/cogment_verse/utils/download_cogment.py b/cogment_verse/utils/download_cogment.py new file mode 100644 index 00000000..5e082ecb --- /dev/null +++ b/cogment_verse/utils/download_cogment.py @@ -0,0 +1,141 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from tempfile import mkdtemp +import json +import logging +import os +import platform +import re +import stat +import subprocess +from urllib.request import urlretrieve, urlopen + +from cogment.errors import CogmentError + +VERSION_NUMBER_RE = re.compile(r"[0-9]+.[0-9]+.[0-9]+(?:-[a-zA-Z0-9]+)?") + +log = logging.getLogger(__name__) + + +class Arch(Enum): + AMD64 = "amd64" + + +def get_current_arch(): + py_machine = platform.machine() + if py_machine in ["x86_64", "i686", "AMD64"]: + return Arch.AMD64 + + raise CogmentError(f"Unsupported architecture [{py_machine}]") + + +class Os(Enum): + WINDOWS = "windows" + LINUX = "linux" + MACOS = "macos" + + +def get_current_os(): + py_system = platform.system() + if py_system in ["Darwin"]: + return Os.MACOS + if py_system in ["Windows"]: + return Os.WINDOWS + if py_system in ["Linux"]: + return Os.LINUX + + raise CogmentError(f"Unsupported os [{py_system}]") + + +def get_latest_release_version(): + with urlopen("https://api.github.com/repos/cogment/cogment/releases/latest") as res: + parsed_body = json.load(res) + + return parsed_body["tag_name"] + + +def download_cogment(output_dir=None, desired_version=None, desired_arch=None, desired_os=None, force_download=False): + """ + Download a version of cogment + + Parameters: + - output_dir (string, optional): the output directory, if undefined a temporary directory will be used. + - desired_version (string, optional): the desired version, + if undefined the latest released version (excluding prereleases) will be used. + - desired_arch (Arch, optional): the desired architecture, + if undefined the current architecture will be detected and used. + - os (Os, optional): the desired os, if undefined the current os will be detected and used. + + Returns: + path to the downloaded cogment + """ + if not output_dir: + output_dir = mkdtemp() + else: + output_dir = os.path.abspath(output_dir) + os.makedirs(output_dir, exist_ok=True) + + if not desired_version: + desired_version = get_latest_release_version() + + try: + desired_version = VERSION_NUMBER_RE.findall(desired_version)[0] + except Exception as error: + raise CogmentError( + f"Desired cogment version [{desired_version}] doesn't follow the expected patterns" + ) from error + + if not desired_arch: + desired_arch = get_current_arch() + + if not desired_os: + desired_os = get_current_os() + + cogment_url = ( + "https://github.com/cogment/cogment/releases/download/" + + f"v{desired_version}/cogment-{desired_os.value}-{desired_arch.value}" + ) + + cogment_filename = os.path.join(output_dir, "cogment") + if desired_os == Os.WINDOWS: + cogment_url += ".exe" + cogment_filename += ".exe" + + if not force_download: + try: + res = subprocess.run([cogment_filename, "version"], capture_output=True, check=True) + current_version = VERSION_NUMBER_RE.findall(res.stdout.decode("utf-8"))[0] + if current_version == desired_version: + # The current version of cogment matches the desired version + return cogment_filename + except FileNotFoundError: + # Unable to execute cogment let's continue to download + pass + + try: + log.info(f"Downloading Cogment [{desired_version}] from [{cogment_url}]") + cogment_filename, _ = urlretrieve(cogment_url, cogment_filename) + except Exception as error: + raise CogmentError( + f"Unable to retrieve cogment version [{desired_version}] for arch " + + f"[{desired_arch}] and os [{desired_os}] from [{cogment_url}] to [{cogment_filename}]" + ) from error + + # Make sure it is executable + cogment_stat = os.stat(cogment_filename) + os.chmod(cogment_filename, cogment_stat.st_mode | stat.S_IEXEC) + + return cogment_filename diff --git a/cogment_verse/utils/find_free_port.py b/cogment_verse/utils/find_free_port.py new file mode 100644 index 00000000..909841fe --- /dev/null +++ b/cogment_verse/utils/find_free_port.py @@ -0,0 +1,39 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket + + +def find_free_port(): + """ + from https://gist.github.com/bertjwregeer/0be94ced48383a42e70c3d9fff1f4ad0 + Finds the next free port that is available on the OS. + This is a bit of a hack, it does this by creating a new socket, and calling + bind with the 0 port. The operating system will assign a brand new port, + which we can find out using getsockname(). Once we have the new port + information we close the socket thereby returning it to the free pool. + This means it is technically possible for this function to return the same + port twice (for example if run in very quick succession), however operating + systems return a random port number in the default range (1024 - 65535), + and it is highly unlikely for two processes to get the same port number. + In other words, it is possible to flake, but incredibly unlikely. + """ + + test_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + test_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + test_socket.bind(("0.0.0.0", 0)) + portnum = test_socket.getsockname()[1] + test_socket.close() + + return portnum diff --git a/cogment_verse/utils/generate.py b/cogment_verse/utils/generate.py new file mode 100644 index 00000000..c2977073 --- /dev/null +++ b/cogment_verse/utils/generate.py @@ -0,0 +1,36 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +def generate(work_dir, specs_filename): + generate_out_dirname = os.path.join(work_dir, "generate_out") + os.makedirs(generate_out_dirname, exist_ok=True) + subprocess.run( + [ + sys.executable, + "-m", + "cogment.generate", + "--spec", + specs_filename, + "--output", + os.path.join(generate_out_dirname, "cog_settings.py"), + ], + cwd=os.path.dirname(specs_filename), + check=True, + ) + sys.path.append(generate_out_dirname) diff --git a/cogment_verse/utils/get_implementation_name.py b/cogment_verse/utils/get_implementation_name.py new file mode 100644 index 00000000..0c6f3c94 --- /dev/null +++ b/cogment_verse/utils/get_implementation_name.py @@ -0,0 +1,24 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# no import + + +def get_implementation_name(instance): + base_impl_name = f"{type(instance).__module__}.{type(instance).__name__}" + if hasattr(instance, "get_implementation_name"): + instance_impl_name = instance.get_implementation_name() + if instance_impl_name is not None and instance_impl_name != "": + return f"{base_impl_name}/{instance_impl_name}" + return base_impl_name diff --git a/cogment_verse/utils/import_class.py b/cogment_verse/utils/import_class.py new file mode 100644 index 00000000..13c6eb16 --- /dev/null +++ b/cogment_verse/utils/import_class.py @@ -0,0 +1,21 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from importlib import import_module + + +def import_class(class_name): + module_path, class_name = class_name.rsplit(".", 1) + module = import_module(module_path) + return getattr(module, class_name) diff --git a/base_python/cogment_verse/utils/lru.py b/cogment_verse/utils/lru.py similarity index 95% rename from base_python/cogment_verse/utils/lru.py rename to cogment_verse/utils/lru.py index f14e3330..f7f33a0f 100644 --- a/base_python/cogment_verse/utils/lru.py +++ b/cogment_verse/utils/lru.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/base_python/cogment_verse/utils/sizeof_fmt.py b/cogment_verse/utils/sizeof_fmt.py similarity index 93% rename from base_python/cogment_verse/utils/sizeof_fmt.py rename to cogment_verse/utils/sizeof_fmt.py index dec258fa..a375de3d 100644 --- a/base_python/cogment_verse/utils/sizeof_fmt.py +++ b/cogment_verse/utils/sizeof_fmt.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/base_python/cogment_verse/__init__.py b/cogment_verse/web/__init__.py similarity index 70% rename from base_python/cogment_verse/__init__.py rename to cogment_verse/web/__init__.py index 16785493..3ae79ad7 100644 --- a/base_python/cogment_verse/__init__.py +++ b/cogment_verse/web/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from cogment_verse.agent_adapter import AgentAdapter -from cogment_verse.mlflow_experiment_tracker import MlflowExperimentTracker -from cogment_verse.run import RunContext +from .server.server import server_main +from .server.dev_server import create_dev_server_popen_kwargs +from .utils.generate import generate +from .utils.npm import npm_command diff --git a/web_client/.eslintignore b/cogment_verse/web/components/.eslintignore similarity index 100% rename from web_client/.eslintignore rename to cogment_verse/web/components/.eslintignore diff --git a/web_client/.prettierignore b/cogment_verse/web/components/.prettierignore similarity index 100% rename from web_client/.prettierignore rename to cogment_verse/web/components/.prettierignore diff --git a/cogment_verse/web/components/build/asset-manifest.json b/cogment_verse/web/components/build/asset-manifest.json new file mode 100644 index 00000000..bd075972 --- /dev/null +++ b/cogment_verse/web/components/build/asset-manifest.json @@ -0,0 +1,15 @@ +{ + "files": { + "main.css": "/static/css/main.21cabb04.css", + "main.js": "/static/js/main.80bd9354.js", + "static/js/787.90542627.chunk.js": "/static/js/787.90542627.chunk.js", + "index.html": "/index.html", + "main.21cabb04.css.map": "/static/css/main.21cabb04.css.map", + "main.80bd9354.js.map": "/static/js/main.80bd9354.js.map", + "787.90542627.chunk.js.map": "/static/js/787.90542627.chunk.js.map" + }, + "entrypoints": [ + "static/css/main.21cabb04.css", + "static/js/main.80bd9354.js" + ] +} \ No newline at end of file diff --git a/web_client/public/assets/cogment-splash.png b/cogment_verse/web/components/build/assets/cogment-splash.png similarity index 100% rename from web_client/public/assets/cogment-splash.png rename to cogment_verse/web/components/build/assets/cogment-splash.png diff --git a/web_client/public/favicon.ico b/cogment_verse/web/components/build/favicon.ico similarity index 100% rename from web_client/public/favicon.ico rename to cogment_verse/web/components/build/favicon.ico diff --git a/cogment_verse/web/components/build/index.html b/cogment_verse/web/components/build/index.html new file mode 100644 index 00000000..e37aa772 --- /dev/null +++ b/cogment_verse/web/components/build/index.html @@ -0,0 +1 @@ +Cogment Verse
\ No newline at end of file diff --git a/web_client/public/logo192.png b/cogment_verse/web/components/build/logo192.png similarity index 100% rename from web_client/public/logo192.png rename to cogment_verse/web/components/build/logo192.png diff --git a/web_client/public/logo512.png b/cogment_verse/web/components/build/logo512.png similarity index 100% rename from web_client/public/logo512.png rename to cogment_verse/web/components/build/logo512.png diff --git a/web_client/public/manifest.json b/cogment_verse/web/components/build/manifest.json similarity index 100% rename from web_client/public/manifest.json rename to cogment_verse/web/components/build/manifest.json diff --git a/web_client/public/robots.txt b/cogment_verse/web/components/build/robots.txt similarity index 100% rename from web_client/public/robots.txt rename to cogment_verse/web/components/build/robots.txt diff --git a/cogment_verse/web/components/build/static/css/main.21cabb04.css b/cogment_verse/web/components/build/static/css/main.21cabb04.css new file mode 100644 index 00000000..35b1c5f1 --- /dev/null +++ b/cogment_verse/web/components/build/static/css/main.21cabb04.css @@ -0,0 +1,4 @@ +/* +! tailwindcss v3.0.24 | MIT License | https://tailwindcss.com +*/*,:after,:before{border:0 solid #e5e7eb;box-sizing:border-box}:after,:before{--tw-content:""}html{-webkit-text-size-adjust:100%;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;line-height:1.5;tab-size:4}body{line-height:inherit}hr{border-top-width:1px;color:inherit;height:0}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{border-collapse:collapse;border-color:inherit;text-indent:0}button,input,optgroup,select,textarea{color:inherit;font-family:inherit;font-size:100%;line-height:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}textarea{resize:vertical}input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#9ca3af;opacity:1}input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#9ca3af;opacity:1}input::placeholder,textarea::placeholder{color:#9ca3af;opacity:1}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{height:auto;max-width:100%}[hidden]{display:none}*,:after,:before{--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.static{position:static}.mx-auto{margin-left:auto;margin-right:auto}.block{display:block}.flex{display:flex}.min-h-screen{min-height:100vh}.w-full{width:100%}.w-fit{width:-webkit-fit-content;width:-moz-fit-content;width:fit-content}.max-w-screen-md{max-width:768px}.flex-1{flex:1 1}.flex-none{flex:none}.transform{-webkit-transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y));transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y))}.list-inside{list-style-position:inside}.list-disc{list-style-type:disc}.flex-row{flex-direction:row}.flex-col{flex-direction:column}.justify-center{justify-content:center}.gap-2{gap:.5rem}.gap-1{gap:.25rem}.rounded{border-radius:.25rem}.rounded-full{border-radius:9999px}.bg-indigo-600{--tw-bg-opacity:1;background-color:rgb(79 70 229/var(--tw-bg-opacity))}.bg-slate-600{--tw-bg-opacity:1;background-color:rgb(71 85 105/var(--tw-bg-opacity))}.bg-indigo-200{--tw-bg-opacity:1;background-color:rgb(199 210 254/var(--tw-bg-opacity))}.p-2{padding:.5rem}.p-5{padding:1.25rem}.py-2{padding-bottom:.5rem;padding-top:.5rem}.px-5{padding-left:1.25rem;padding-right:1.25rem}.py-3{padding-bottom:.75rem;padding-top:.75rem}.text-center{text-align:center}.text-right{text-align:right}.font-mono{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace}.text-xl{font-size:1.25rem;line-height:1.75rem}.text-base{font-size:1rem;line-height:1.5rem}.text-xs{font-size:.75rem;line-height:1rem}.text-sm{font-size:.875rem;line-height:1.25rem}.font-semibold{font-weight:600}.lowercase{text-transform:lowercase}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.ring-8{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(8px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),0 0 #0000;box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.ring-inset{--tw-ring-inset:inset}.blur{--tw-blur:blur(8px)}.blur,.filter{-webkit-filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow);filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}.duration-75{transition-duration:75ms}body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;margin:0}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}.hover\:bg-indigo-900:hover{--tw-bg-opacity:1;background-color:rgb(49 46 129/var(--tw-bg-opacity))}.RenderedScreen_container__9inx9{display:grid;position:relative}.RenderedScreen_canvas__Zmrsu,.RenderedScreen_overlay__fApA7{grid-area:1/-1;width:100%;z-index:0}.RenderedScreen_overlay__fApA7{align-items:center;display:flex;justify-content:center;z-index:1}:root{--dpad-bg-color:#fff;--dpad-bg-color-hover:#eee;--dpad-bg-color-active:#fff;--dpad-bg-color-disabled:#fff;--dpad-fg-color:#5217b8;--dpad-fg-color-hover:#5217b8;--dpad-fg-color-active:#ffb300;--dpad-fg-color-disabled:#bbb;--dpad-button-outer-radius:15%;--dpad-button-inner-radius:50%;--dpad-arrow-position:40%;--dpad-arrow-position-hover:35%;--dpad-arrow-base:19px;--dpad-arrow-height:13px}.DPad_dpad__ZZdxt{display:inline-block;height:200px;overflow:hidden;position:relative;width:200px}.DPad_down__EeuUg,.DPad_left__1AYHI,.DPad_right__AqUBc,.DPad_up__dOfw6{-webkit-tap-highlight-color:rgba(255,255,255,0);background:#fff;background:var(--dpad-bg-color);border-color:#5217b8;border-color:var(--dpad-fg-color);border-style:solid;border-width:1px;color:transparent;display:block;line-height:40%;padding:0;position:absolute;text-align:center}.DPad_down__EeuUg,.DPad_up__dOfw6{height:43%;width:33.3%}.DPad_left__1AYHI,.DPad_right__AqUBc{height:33%;width:43%}.DPad_up__dOfw6{border-radius:15% 15% 50% 50%;border-radius:var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius);top:0}.DPad_down__EeuUg,.DPad_up__dOfw6{left:50%;-webkit-transform:translate(-50%);transform:translate(-50%)}.DPad_down__EeuUg{border-radius:50% 50% 15% 15%;border-radius:var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius) var(--dpad-button-outer-radius);bottom:0}.DPad_left__1AYHI{border-radius:15% 50% 50% 15%;border-radius:var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius);left:0}.DPad_left__1AYHI,.DPad_right__AqUBc{top:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%)}.DPad_right__AqUBc{border-radius:50% 15% 15% 50%;border-radius:50% var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) 50%;right:0}.DPad_down__EeuUg:before,.DPad_left__1AYHI:before,.DPad_right__AqUBc:before,.DPad_up__dOfw6:before{border-radius:5px;border-style:solid;content:"";height:0;position:absolute;transition:all .25s;width:0}.DPad_up__dOfw6:before{border-color:transparent transparent #5217b8;border-color:transparent transparent var(--dpad-fg-color) transparent;border-width:0 13px 19px;border-width:0 var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height);left:50%;top:40%;top:var(--dpad-arrow-position);-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}.DPad_down__EeuUg:before{border-color:#5217b8 transparent transparent;border-color:var(--dpad-fg-color) transparent transparent transparent;border-width:19px 13px 0;border-width:var(--dpad-arrow-base) var(--dpad-arrow-height) 0 var(--dpad-arrow-height);bottom:40%;bottom:var(--dpad-arrow-position);left:50%;-webkit-transform:translate(-50%,50%);transform:translate(-50%,50%)}.DPad_left__1AYHI:before{border-color:transparent #5217b8 transparent transparent;border-color:transparent var(--dpad-fg-color) transparent transparent;border-width:13px 19px 13px 0;border-width:var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height) 0;left:40%;left:var(--dpad-arrow-position);top:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}.DPad_right__AqUBc:before{border-color:transparent transparent transparent #5217b8;border-color:transparent transparent transparent var(--dpad-fg-color);border-width:13px 0 13px 19px;border-width:var(--dpad-arrow-height) 0 var(--dpad-arrow-height) var(--dpad-arrow-base);right:40%;right:var(--dpad-arrow-position);top:50%;-webkit-transform:translate(50%,-50%);transform:translate(50%,-50%)}.DPad_down__EeuUg:hover,.DPad_left__1AYHI:hover,.DPad_right__AqUBc:hover,.DPad_up__dOfw6:hover{background:#eee;background:var(--dpad-bg-color-hover);border-color:#5217b8;border-color:var(--dpad-fg-color-hover)}.DPad_up__dOfw6:hover:before{border-bottom-color:#5217b8;border-bottom-color:var(--dpad-fg-color-hover);top:35%;top:var(--dpad-arrow-position-hover)}.DPad_down__EeuUg:hover:before{border-top-color:#5217b8;border-top-color:var(--dpad-fg-color-hover);bottom:35%;bottom:var(--dpad-arrow-position-hover)}.DPad_left__1AYHI:hover:before{border-right-color:#5217b8;border-right-color:var(--dpad-fg-color-hover);left:35%;left:var(--dpad-arrow-position-hover)}.DPad_right__AqUBc:hover:before{border-left-color:#5217b8;border-left-color:var(--dpad-fg-color-hover);right:35%;right:var(--dpad-arrow-position-hover)}.DPad_down__EeuUg.DPad_active__OzzXu,.DPad_down__EeuUg:active,.DPad_left__1AYHI.DPad_active__OzzXu,.DPad_left__1AYHI:active,.DPad_right__AqUBc.DPad_active__OzzXu,.DPad_right__AqUBc:active,.DPad_up__dOfw6.DPad_active__OzzXu,.DPad_up__dOfw6:active{background:#fff;background:var(--dpad-bg-color-active);border-color:#ffb300;border-color:var(--dpad-fg-color-active)}.DPad_up__dOfw6.DPad_active__OzzXu:before,.DPad_up__dOfw6:active:before{border-bottom-color:#ffb300;border-bottom-color:var(--dpad-fg-color-active)}.DPad_down__EeuUg.DPad_active__OzzXu:before,.DPad_down__EeuUg:active:before{border-top-color:#ffb300;border-top-color:var(--dpad-fg-color-active)}.DPad_left__1AYHI.DPad_active__OzzXu:before,.DPad_left__1AYHI:active:before{border-right-color:#ffb300;border-right-color:var(--dpad-fg-color-active)}.DPad_right__AqUBc.DPad_active__OzzXu:before,.DPad_right__AqUBc:active:before{border-left-color:#ffb300;border-left-color:var(--dpad-fg-color-active)}.DPad_down__EeuUg.DPad_disabled__\+IfWl,.DPad_left__1AYHI.DPad_disabled__\+IfWl,.DPad_right__AqUBc.DPad_disabled__\+IfWl,.DPad_up__dOfw6.DPad_disabled__\+IfWl{background:#fff;background:var(--dpad-bg-color-disabled);border-color:#bbb;border-color:var(--dpad-fg-color-disabled)}.DPad_up__dOfw6.DPad_disabled__\+IfWl:before{border-bottom-color:#bbb;border-bottom-color:var(--dpad-fg-color-disabled);top:40%;top:var(--dpad-arrow-position)}.DPad_down__EeuUg.DPad_disabled__\+IfWl:before{border-top-color:#bbb;border-top-color:var(--dpad-fg-color-disabled);bottom:40%;bottom:var(--dpad-arrow-position)}.DPad_left__1AYHI.DPad_disabled__\+IfWl:before{border-right-color:#bbb;border-right-color:var(--dpad-fg-color-disabled);left:40%;left:var(--dpad-arrow-position)}.DPad_right__AqUBc.DPad_disabled__\+IfWl:before{border-left-color:#bbb;border-left-color:var(--dpad-fg-color-disabled);right:40%;right:var(--dpad-arrow-position)}:root{--joystick-surface-size:200px;--joystick-stick-size:50px;--joystick-color:#5217b8;--joystick-color-active:#ffb300;--joystick-color-disabled:#bbb}.Joystick_joystick__DwFj2{align-items:center;border-color:#5217b8;border-color:var(--joystick-color);border-radius:25px;border-radius:calc(var(--joystick-stick-size)/2);border-style:solid;border-width:1px;display:flex;height:200px;height:var(--joystick-surface-size);justify-content:center;overflow:hidden;position:relative;transition:all .25s;width:200px;width:var(--joystick-surface-size)}.Joystick_joystick__DwFj2>.Joystick_stick__z5P-5{background-color:#5217b8;background-color:var(--joystick-color);border-radius:50%;height:50px;height:var(--joystick-stick-size);transition:all .25s;width:50px;width:var(--joystick-stick-size)}.Joystick_joystick__DwFj2.Joystick_active__EOXxn{border-color:#ffb300;border-color:var(--joystick-color-active)}.Joystick_joystick__DwFj2.Joystick_active__EOXxn>.Joystick_stick__z5P-5{background-color:#ffb300;background-color:var(--joystick-color-active)}.Joystick_joystick__DwFj2.Joystick_disabled__WyYwu{border-color:#bbb;border-color:var(--joystick-color-disabled)}.Joystick_joystick__DwFj2.Joystick_disabled__WyYwu>.Joystick_stick__z5P-5{background-color:#bbb;background-color:var(--joystick-color-disabled)} +/*# sourceMappingURL=main.21cabb04.css.map*/ \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/css/main.21cabb04.css.map b/cogment_verse/web/components/build/static/css/main.21cabb04.css.map new file mode 100644 index 00000000..88613926 --- /dev/null +++ b/cogment_verse/web/components/build/static/css/main.21cabb04.css.map @@ -0,0 +1 @@ +{"version":3,"file":"static/css/main.21cabb04.css","mappings":"AAAA;;CAAc,CAAd,uCAAc,CAAd,qBAAc,CAAd,8BAAc,CAAd,kCAAc,CAAd,gMAAc,CAAd,eAAc,CAAd,UAAc,CAAd,wBAAc,CAAd,uBAAc,CAAd,aAAc,CAAd,QAAc,CAAd,4DAAc,CAAd,gCAAc,CAAd,mCAAc,CAAd,mBAAc,CAAd,eAAc,CAAd,uBAAc,CAAd,2BAAc,CAAd,qHAAc,CAAd,aAAc,CAAd,mBAAc,CAAd,qBAAc,CAAd,aAAc,CAAd,iBAAc,CAAd,uBAAc,CAAd,iBAAc,CAAd,aAAc,CAAd,8BAAc,CAAd,oBAAc,CAAd,aAAc,CAAd,mDAAc,CAAd,mBAAc,CAAd,cAAc,CAAd,mBAAc,CAAd,QAAc,CAAd,SAAc,CAAd,iCAAc,CAAd,yEAAc,CAAd,4BAAc,CAAd,qBAAc,CAAd,4BAAc,CAAd,gCAAc,CAAd,gCAAc,CAAd,mEAAc,CAAd,0CAAc,CAAd,mBAAc,CAAd,mDAAc,CAAd,sDAAc,CAAd,YAAc,CAAd,yBAAc,CAAd,2DAAc,CAAd,iBAAc,CAAd,yBAAc,CAAd,0BAAc,CAAd,QAAc,CAAd,SAAc,CAAd,wBAAc,CAAd,kFAAc,CAAd,SAAc,CAAd,wEAAc,CAAd,SAAc,CAAd,sDAAc,CAAd,SAAc,CAAd,mCAAc,CAAd,wBAAc,CAAd,4DAAc,CAAd,qBAAc,CAAd,qBAAc,CAAd,cAAc,CAAd,qBAAc,CAAd,mCAAc,CAAd,kBAAc,CAAd,aAAc,CAAd,aAAc,CAAd,aAAc,CAAd,cAAc,CAAd,cAAc,CAAd,YAAc,CAAd,YAAc,CAAd,iBAAc,CAAd,qCAAc,CAAd,cAAc,CAAd,mBAAc,CAAd,qBAAc,CAAd,sBAAc,CAAd,uBAAc,CAAd,iBAAc,CAAd,0BAAc,CAAd,2BAAc,CAAd,mCAAc,CAAd,iCAAc,CAAd,0BAAc,CAAd,qBAAc,CAAd,6BAAc,CAAd,WAAc,CAAd,iBAAc,CAAd,eAAc,CAAd,gBAAc,CAAd,iBAAc,CAAd,aAAc,CAAd,eAAc,CAAd,YAAc,CAAd,kBAAc,CAAd,oBAAc,CAAd,0BAAc,CAAd,wBAAc,CAAd,yBAAc,CAAd,0BAAc,CAAd,sBAAc,CAAd,uBAAc,CAAd,wBAAc,CAAd,qBAAc,CAEd,uBAAmB,CAAnB,yBAAmB,CAAnB,iBAAmB,CAAnB,oBAAmB,CAAnB,kBAAmB,CAAnB,8BAAmB,CAAnB,kBAAmB,CAAnB,gCAAmB,CAAnB,sBAAmB,CAAnB,iBAAmB,CAAnB,gCAAmB,CAAnB,gBAAmB,CAAnB,oBAAmB,CAAnB,gNAAmB,CAAnB,6LAAmB,CAAnB,uCAAmB,CAAnB,+BAAmB,CAAnB,4BAAmB,CAAnB,+BAAmB,CAAnB,sCAAmB,CAAnB,gBAAmB,CAAnB,iBAAmB,CAAnB,6BAAmB,CAAnB,kCAAmB,CAAnB,gCAAmB,CAAnB,oDAAmB,CAAnB,+BAAmB,CAAnB,oDAAmB,CAAnB,gCAAmB,CAAnB,sDAAmB,CAAnB,kBAAmB,CAAnB,oBAAmB,CAAnB,4CAAmB,CAAnB,0BAAmB,CAAnB,qBAAmB,CAAnB,8CAAmB,CAAnB,8BAAmB,CAAnB,4BAAmB,CAAnB,8GAAmB,CAAnB,0BAAmB,CAAnB,mBAAmB,CAAnB,yBAAmB,CAAnB,kBAAmB,CAAnB,yBAAmB,CAAnB,gBAAmB,CAAnB,0BAAmB,CAAnB,mBAAmB,CAAnB,8BAAmB,CAAnB,mCAAmB,CAAnB,+BAAmB,CAAnB,6CAAmB,CAAnB,kHAAmB,CAAnB,wGAAmB,CAAnB,uEAAmB,CAAnB,wFAAmB,CAAnB,iCAAmB,CAAnB,yBAAmB,CAAnB,sMAAmB,CAAnB,gLAAmB,CAAnB,qCAAmB,CAEnB,KAIE,kCAAmC,CACnC,iCAAkC,CAHlC,mIAC4C,CAF5C,QAKF,CAEA,KACE,uEACF,CAdA,kG,CCAA,iCAEE,YAAa,CADb,iBAEF,CAEA,6DAEE,cAAe,CAGf,UAAW,CAFX,SAGF,CAEA,+BAKE,kBAAmB,CAFnB,YAAa,CACb,sBAAuB,CAHvB,SAKF,CCnBA,MACE,oBAAqB,CACrB,0BAA2B,CAC3B,2BAA4B,CAC5B,6BAA8B,CAC9B,uBAAwB,CACxB,6BAA8B,CAC9B,8BAA+B,CAC/B,6BAA8B,CAE9B,8BAA+B,CAC/B,8BAA+B,CAE/B,yBAA0B,CAC1B,+BAAgC,CAChC,sBAAuB,CACvB,wBACF,CAEA,kBAEE,oBAAqB,CAGrB,YAAa,CAEb,eAAgB,CANhB,iBAAkB,CAGlB,WAIF,CAIA,uEAME,+CAAmD,CAInD,eAAgC,CAAhC,+BAAgC,CAChC,oBAAkC,CAAlC,iCAAkC,CAClC,kBAAmB,CACnB,gBAAiB,CAEjB,iBAAkB,CAXlB,aAAc,CAId,eAAgB,CAMhB,SAAY,CATZ,iBAAkB,CAIlB,iBAOF,CAEA,kCAGE,UAAW,CADX,WAEF,CAEA,qCAGE,UAAW,CADX,SAEF,CAEA,gBAIE,6BACiC,CADjC,6IACiC,CAJjC,KAKF,CAEA,kCANE,QAAS,CACT,iCAA6B,CAA7B,yBAWF,CANA,kBAIE,6BACiC,CADjC,6IACiC,CAJjC,QAKF,CAEA,kBAIE,6BACiC,CADjC,6IACiC,CAHjC,MAIF,CAEA,qCAPE,OAAQ,CAER,kCAA6B,CAA7B,0BAUF,CALA,mBAIE,6BAAsF,CAAtF,qFAAsF,CAFtF,OAGF,CAGA,mGAQE,iBAAkB,CAClB,kBAAmB,CALnB,UAAW,CAGX,QAAS,CAFT,iBAAkB,CAKlB,mBAAqB,CAJrB,OAKF,CAEA,uBAKE,4CAAsE,CAAtE,qEAAsE,CADtE,wBAAwF,CAAxF,uFAAwF,CAFxF,QAAS,CADT,OAA+B,CAA/B,8BAA+B,CAE/B,sCAAgC,CAAhC,8BAGF,CAEA,yBAKE,4CAAsE,CAAtE,qEAAsE,CADtE,wBAA0F,CAA1F,uFAA0F,CAH1F,UAAkC,CAAlC,iCAAkC,CAClC,QAAS,CACT,qCAA+B,CAA/B,6BAGF,CAEA,yBAKE,wDAAsE,CAAtE,qEAAsE,CADtE,6BAAwF,CAAxF,uFAAwF,CAHxF,QAAgC,CAAhC,+BAAgC,CAChC,OAAQ,CACR,sCAAgC,CAAhC,8BAGF,CAEA,0BAKE,wDAAsE,CAAtE,qEAAsE,CADtE,6BAAwF,CAAxF,uFAAwF,CAHxF,SAAiC,CAAjC,gCAAiC,CACjC,OAAQ,CACR,qCAA+B,CAA/B,6BAGF,CAIA,+FAIE,eAAsC,CAAtC,qCAAsC,CACtC,oBAAwC,CAAxC,uCACF,CAEA,6BAEE,2BAA+C,CAA/C,8CAA+C,CAD/C,OAAqC,CAArC,oCAEF,CAEA,+BAEE,wBAA4C,CAA5C,2CAA4C,CAD5C,UAAwC,CAAxC,uCAEF,CAEA,+BAEE,0BAA8C,CAA9C,6CAA8C,CAD9C,QAAsC,CAAtC,qCAEF,CAEA,gCAEE,yBAA6C,CAA7C,4CAA6C,CAD7C,SAAuC,CAAvC,sCAEF,CAIA,sPAQE,eAAuC,CAAvC,sCAAuC,CACvC,oBAAyC,CAAzC,wCACF,CAEA,wEAEE,2BAAgD,CAAhD,+CACF,CAEA,4EAEE,wBAA6C,CAA7C,4CACF,CAEA,4EAEE,0BAA+C,CAA/C,8CACF,CAEA,8EAEE,yBAA8C,CAA9C,6CACF,CAIA,+JAIE,eAAyC,CAAzC,wCAAyC,CACzC,iBAA2C,CAA3C,0CACF,CAEA,6CAEE,wBAAkD,CAAlD,iDAAkD,CADlD,OAA+B,CAA/B,8BAEF,CAEA,+CAEE,qBAA+C,CAA/C,8CAA+C,CAD/C,UAAkC,CAAlC,iCAEF,CAEA,+CAEE,uBAAiD,CAAjD,gDAAiD,CADjD,QAAgC,CAAhC,+BAEF,CAEA,gDAEE,sBAAgD,CAAhD,+CAAgD,CADhD,SAAiC,CAAjC,gCAEF,CCtOA,MACE,6BAA8B,CAC9B,0BAA2B,CAE3B,wBAAyB,CACzB,+BAAgC,CAChC,8BACF,CAEA,0BAIE,kBAAmB,CAOnB,oBAAmC,CAAnC,kCAAmC,CADnC,kBAAmD,CAAnD,gDAAmD,CAGnD,kBAAmB,CADnB,gBAAiB,CAVjB,YAAa,CAMb,YAAoC,CAApC,mCAAoC,CALpC,sBAAuB,CAGvB,eAAgB,CALhB,iBAAkB,CAclB,mBAAqB,CANrB,WAAmC,CAAnC,kCAOF,CAEA,iDAKE,wBAAuC,CAAvC,sCAAuC,CADvC,iBAAkB,CAHlB,WAAkC,CAAlC,iCAAkC,CAMlC,mBAAqB,CALrB,UAAiC,CAAjC,gCAMF,CAEA,iDACE,oBAA0C,CAA1C,yCACF,CAEA,wEACE,wBAA8C,CAA9C,6CACF,CAEA,mDACE,iBAA4C,CAA5C,2CACF,CAEA,0EACE,qBAAgD,CAAhD,+CACF","sources":["index.css","components/RenderedScreen.module.css","components/DPad.module.css","components/Joystick.module.css"],"sourcesContent":["@tailwind base;\n@tailwind components;\n@tailwind utilities;\n\nbody {\n margin: 0;\n font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", \"Roboto\", \"Oxygen\", \"Ubuntu\", \"Cantarell\", \"Fira Sans\",\n \"Droid Sans\", \"Helvetica Neue\", sans-serif;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale;\n}\n\ncode {\n font-family: source-code-pro, Menlo, Monaco, Consolas, \"Courier New\", monospace;\n}\n",".container {\n position: relative;\n display: grid;\n}\n\n.canvas,\n.overlay {\n grid-area: 1/-1;\n z-index: 0;\n\n width: 100%;\n}\n\n.overlay {\n z-index: 1;\n\n display: flex;\n justify-content: center;\n align-items: center;\n}\n",":root {\n --dpad-bg-color: #fff;\n --dpad-bg-color-hover: #eee;\n --dpad-bg-color-active: #fff;\n --dpad-bg-color-disabled: #fff;\n --dpad-fg-color: #5217b8;\n --dpad-fg-color-hover: #5217b8;\n --dpad-fg-color-active: #ffb300;\n --dpad-fg-color-disabled: #bbb;\n\n --dpad-button-outer-radius: 15%;\n --dpad-button-inner-radius: 50%;\n\n --dpad-arrow-position: 40%;\n --dpad-arrow-position-hover: 35%;\n --dpad-arrow-base: 19px;\n --dpad-arrow-height: 13px;\n}\n\n.dpad {\n position: relative;\n display: inline-block;\n\n width: 200px;\n height: 200px;\n\n overflow: hidden;\n}\n\n/* Buttons background */\n\n.up,\n.right,\n.down,\n.left {\n display: block;\n position: absolute;\n -webkit-tap-highlight-color: rgba(255, 255, 255, 0);\n\n line-height: 40%;\n text-align: center;\n background: var(--dpad-bg-color);\n border-color: var(--dpad-fg-color);\n border-style: solid;\n border-width: 1px;\n padding: 0px;\n color: transparent;\n}\n\n.up,\n.down {\n width: 33.3%;\n height: 43%;\n}\n\n.left,\n.right {\n width: 43%;\n height: 33%;\n}\n\n.up {\n top: 0;\n left: 50%;\n transform: translate(-50%, 0);\n border-radius: var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) var(--dpad-button-inner-radius)\n var(--dpad-button-inner-radius);\n}\n\n.down {\n bottom: 0;\n left: 50%;\n transform: translate(-50%, 0);\n border-radius: var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius)\n var(--dpad-button-outer-radius);\n}\n\n.left {\n top: 50%;\n left: 0;\n transform: translate(0, -50%);\n border-radius: var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius)\n var(--dpad-button-outer-radius);\n}\n\n.right {\n top: 50%;\n right: 0;\n transform: translate(0, -50%);\n border-radius: 50% var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) 50%;\n}\n\n/* Buttons arrows */\n.up:before,\n.right:before,\n.down:before,\n.left:before {\n content: \"\";\n position: absolute;\n width: 0;\n height: 0;\n border-radius: 5px;\n border-style: solid;\n transition: all 0.25s;\n}\n\n.up:before {\n top: var(--dpad-arrow-position);\n left: 50%;\n transform: translate(-50%, -50%);\n border-width: 0 var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height);\n border-color: transparent transparent var(--dpad-fg-color) transparent;\n}\n\n.down:before {\n bottom: var(--dpad-arrow-position);\n left: 50%;\n transform: translate(-50%, 50%);\n border-width: var(--dpad-arrow-base) var(--dpad-arrow-height) 0px var(--dpad-arrow-height);\n border-color: var(--dpad-fg-color) transparent transparent transparent;\n}\n\n.left:before {\n left: var(--dpad-arrow-position);\n top: 50%;\n transform: translate(-50%, -50%);\n border-width: var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height) 0;\n border-color: transparent var(--dpad-fg-color) transparent transparent;\n}\n\n.right:before {\n right: var(--dpad-arrow-position);\n top: 50%;\n transform: translate(50%, -50%);\n border-width: var(--dpad-arrow-height) 0 var(--dpad-arrow-height) var(--dpad-arrow-base);\n border-color: transparent transparent transparent var(--dpad-fg-color);\n}\n\n/* Hover */\n\n.up:hover,\n.right:hover,\n.down:hover,\n.left:hover {\n background: var(--dpad-bg-color-hover);\n border-color: var(--dpad-fg-color-hover);\n}\n\n.up:hover:before {\n top: var(--dpad-arrow-position-hover);\n border-bottom-color: var(--dpad-fg-color-hover);\n}\n\n.down:hover:before {\n bottom: var(--dpad-arrow-position-hover);\n border-top-color: var(--dpad-fg-color-hover);\n}\n\n.left:hover:before {\n left: var(--dpad-arrow-position-hover);\n border-right-color: var(--dpad-fg-color-hover);\n}\n\n.right:hover:before {\n right: var(--dpad-arrow-position-hover);\n border-left-color: var(--dpad-fg-color-hover);\n}\n\n/* Active */\n\n.up:active,\n.right:active,\n.down:active,\n.left:active,\n.up.active,\n.right.active,\n.down.active,\n.left.active {\n background: var(--dpad-bg-color-active);\n border-color: var(--dpad-fg-color-active);\n}\n\n.up:active:before,\n.up.active:before {\n border-bottom-color: var(--dpad-fg-color-active);\n}\n\n.down:active:before,\n.down.active:before {\n border-top-color: var(--dpad-fg-color-active);\n}\n\n.left:active:before,\n.left.active:before {\n border-right-color: var(--dpad-fg-color-active);\n}\n\n.right:active:before,\n.right.active:before {\n border-left-color: var(--dpad-fg-color-active);\n}\n\n/* Disabled */\n\n.up.disabled,\n.right.disabled,\n.down.disabled,\n.left.disabled {\n background: var(--dpad-bg-color-disabled);\n border-color: var(--dpad-fg-color-disabled);\n}\n\n.up.disabled:before {\n top: var(--dpad-arrow-position);\n border-bottom-color: var(--dpad-fg-color-disabled);\n}\n\n.down.disabled:before {\n bottom: var(--dpad-arrow-position);\n border-top-color: var(--dpad-fg-color-disabled);\n}\n\n.left.disabled:before {\n left: var(--dpad-arrow-position);\n border-right-color: var(--dpad-fg-color-disabled);\n}\n\n.right.disabled:before {\n right: var(--dpad-arrow-position);\n border-left-color: var(--dpad-fg-color-disabled);\n}\n",":root {\n --joystick-surface-size: 200px;\n --joystick-stick-size: 50px;\n\n --joystick-color: #5217b8;\n --joystick-color-active: #ffb300;\n --joystick-color-disabled: #bbb;\n}\n\n.joystick {\n position: relative;\n display: flex;\n justify-content: center;\n align-items: center;\n\n overflow: hidden;\n\n height: var(--joystick-surface-size);\n width: var(--joystick-surface-size);\n border-radius: calc(var(--joystick-stick-size) / 2);\n border-color: var(--joystick-color);\n border-width: 1px;\n border-style: solid;\n\n transition: all 0.25s;\n}\n\n.joystick > .stick {\n height: var(--joystick-stick-size);\n width: var(--joystick-stick-size);\n\n border-radius: 50%;\n background-color: var(--joystick-color);\n\n transition: all 0.25s;\n}\n\n.joystick.active {\n border-color: var(--joystick-color-active);\n}\n\n.joystick.active > .stick {\n background-color: var(--joystick-color-active);\n}\n\n.joystick.disabled {\n border-color: var(--joystick-color-disabled);\n}\n\n.joystick.disabled > .stick {\n background-color: var(--joystick-color-disabled);\n}\n"],"names":[],"sourceRoot":""} \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/js/787.90542627.chunk.js b/cogment_verse/web/components/build/static/js/787.90542627.chunk.js new file mode 100644 index 00000000..a625b5a4 --- /dev/null +++ b/cogment_verse/web/components/build/static/js/787.90542627.chunk.js @@ -0,0 +1,2 @@ +"use strict";(self.webpackChunkcogment_verse_web=self.webpackChunkcogment_verse_web||[]).push([[787],{787:function(e,t,n){n.r(t),n.d(t,{getCLS:function(){return v},getFCP:function(){return S},getFID:function(){return k},getLCP:function(){return F},getTTFB:function(){return C}});var i,a,r,o,u=function(e,t){return{name:e,value:void 0===t?-1:t,delta:0,entries:[],id:"v1-".concat(Date.now(),"-").concat(Math.floor(8999999999999*Math.random())+1e12)}},c=function(e,t){try{if(PerformanceObserver.supportedEntryTypes.includes(e)){if("first-input"===e&&!("PerformanceEventTiming"in self))return;var n=new PerformanceObserver((function(e){return e.getEntries().map(t)}));return n.observe({type:e,buffered:!0}),n}}catch(e){}},f=function(e,t){var n=function n(i){"pagehide"!==i.type&&"hidden"!==document.visibilityState||(e(i),t&&(removeEventListener("visibilitychange",n,!0),removeEventListener("pagehide",n,!0)))};addEventListener("visibilitychange",n,!0),addEventListener("pagehide",n,!0)},s=function(e){addEventListener("pageshow",(function(t){t.persisted&&e(t)}),!0)},m="function"==typeof WeakSet?new WeakSet:new Set,p=function(e,t,n){var i;return function(){t.value>=0&&(n||m.has(t)||"hidden"===document.visibilityState)&&(t.delta=t.value-(i||0),(t.delta||void 0===i)&&(i=t.value,e(t)))}},v=function(e,t){var n,i=u("CLS",0),a=function(e){e.hadRecentInput||(i.value+=e.value,i.entries.push(e),n())},r=c("layout-shift",a);r&&(n=p(e,i,t),f((function(){r.takeRecords().map(a),n()})),s((function(){i=u("CLS",0),n=p(e,i,t)})))},d=-1,l=function(){return"hidden"===document.visibilityState?0:1/0},h=function(){f((function(e){var t=e.timeStamp;d=t}),!0)},g=function(){return d<0&&(d=l(),h(),s((function(){setTimeout((function(){d=l(),h()}),0)}))),{get timeStamp(){return d}}},S=function(e,t){var n,i=g(),a=u("FCP"),r=function(e){"first-contentful-paint"===e.name&&(f&&f.disconnect(),e.startTime=0&&a1e12?new Date:performance.now())-e.timeStamp;"pointerdown"==e.type?function(e,t){var n=function(){E(e,t),a()},i=function(){a()},a=function(){removeEventListener("pointerup",n,y),removeEventListener("pointercancel",i,y)};addEventListener("pointerup",n,y),addEventListener("pointercancel",i,y)}(t,e):E(t,e)}},b=function(e){["mousedown","keydown","touchstart","pointerdown"].forEach((function(t){return e(t,T,y)}))},k=function(e,t){var n,r=g(),v=u("FID"),d=function(e){e.startTime=0&&(n||u.has(t)||\"hidden\"===document.visibilityState)&&(t.delta=t.value-(i||0),(t.delta||void 0===i)&&(i=t.value,e(t)))}},s=function(e,t){var n,i=a(\"CLS\",0),u=function(e){e.hadRecentInput||(i.value+=e.value,i.entries.push(e),n())},s=r(\"layout-shift\",u);s&&(n=f(e,i,t),o((function(){s.takeRecords().map(u),n()})),c((function(){i=a(\"CLS\",0),n=f(e,i,t)})))},m=-1,p=function(){return\"hidden\"===document.visibilityState?0:1/0},v=function(){o((function(e){var t=e.timeStamp;m=t}),!0)},d=function(){return m<0&&(m=p(),v(),c((function(){setTimeout((function(){m=p(),v()}),0)}))),{get timeStamp(){return m}}},l=function(e,t){var n,i=d(),o=a(\"FCP\"),s=function(e){\"first-contentful-paint\"===e.name&&(p&&p.disconnect(),e.startTime=0&&t1e12?new Date:performance.now())-e.timeStamp;\"pointerdown\"==e.type?function(e,t){var n=function(){y(e,t),a()},i=function(){a()},a=function(){removeEventListener(\"pointerup\",n,h),removeEventListener(\"pointercancel\",i,h)};addEventListener(\"pointerup\",n,h),addEventListener(\"pointercancel\",i,h)}(t,e):y(t,e)}},w=function(e){[\"mousedown\",\"keydown\",\"touchstart\",\"pointerdown\"].forEach((function(t){return e(t,E,h)}))},L=function(n,s){var m,p=d(),v=a(\"FID\"),l=function(e){e.startTime>>3){case 1:i.dtype=e.string();break;case 2:if(i.shape&&i.shape.length||(i.shape=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>0}return null!=e.data&&("string"===typeof e.data?n.base64.decode(e.data,t.data=n.newBuffer(n.base64.length(e.data)),0):e.data.length>=0&&(t.data=e.data)),t},i.toObject=function(e,t){t||(t={});var r={};if((t.arrays||t.defaults)&&(r.shape=[]),t.defaults&&(r.dtype="",t.bytes===String?r.data="":(r.data=[],t.bytes!==Array&&(r.data=n.newBuffer(r.data)))),null!=e.dtype&&e.hasOwnProperty("dtype")&&(r.dtype=e.dtype),e.shape&&e.shape.length){r.shape=[];for(var o=0;o>>3===1?(i.properties&&i.properties.length||(i.properties=[]),i.properties.push(o.cogment_verse.Space.Property.decode(e,e.uint32()))):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.properties&&e.hasOwnProperty("properties")){if(!Array.isArray(e.properties))return"properties: array expected";for(var t=0;t>>3){case 1:i.num=e.int32();break;case 2:i.labels&&i.labels.length||(i.labels=[]),i.labels.push(e.string());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.num&&e.hasOwnProperty("num")&&!n.isInteger(e.num))return"num: integer expected";if(null!=e.labels&&e.hasOwnProperty("labels")){if(!Array.isArray(e.labels))return"labels: array expected";for(var t=0;t>>3===1?i.bound=e.float():e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};return null!=e.bound&&e.hasOwnProperty("bound")&&(t._bound=1,"number"!==typeof e.bound)?"bound: number expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.Space.Bound)return e;var t=new o.cogment_verse.Space.Bound;return null!=e.bound&&(t.bound=Number(e.bound)),t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.bound&&e.hasOwnProperty("bound")&&(r.bound=t.json&&!isFinite(e.bound)?String(e.bound):e.bound,t.oneofs&&(r._bound="bound")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.Space.Bound"},i}(),i.Box=function(){function i(e){if(this.shape=[],this.low=[],this.high=[],e)for(var t=Object.keys(e),r=0;r>>3){case 1:if(i.shape&&i.shape.length||(i.shape=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>0}if(e.low){if(!Array.isArray(e.low))throw TypeError(".cogment_verse.Space.Box.low: array expected");for(t.low=[],r=0;r>>3){case 1:i.key=e.string();break;case 2:i.discrete=o.cogment_verse.Space.Discrete.decode(e,e.uint32());break;case 3:i.box=o.cogment_verse.Space.Box.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.key&&e.hasOwnProperty("key")&&!n.isString(e.key))return"key: string expected";if(null!=e.discrete&&e.hasOwnProperty("discrete")&&(t.typeOneof=1,r=o.cogment_verse.Space.Discrete.verify(e.discrete)))return"discrete."+r;if(null!=e.box&&e.hasOwnProperty("box")){if(1===t.typeOneof)return"typeOneof: multiple values";var r;if(t.typeOneof=1,r=o.cogment_verse.Space.Box.verify(e.box))return"box."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.Space.Property)return e;var t=new o.cogment_verse.Space.Property;if(null!=e.key&&(t.key=String(e.key)),null!=e.discrete){if("object"!==typeof e.discrete)throw TypeError(".cogment_verse.Space.Property.discrete: object expected");t.discrete=o.cogment_verse.Space.Discrete.fromObject(e.discrete)}if(null!=e.box){if("object"!==typeof e.box)throw TypeError(".cogment_verse.Space.Property.box: object expected");t.box=o.cogment_verse.Space.Box.fromObject(e.box)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.key=""),null!=e.key&&e.hasOwnProperty("key")&&(r.key=e.key),null!=e.discrete&&e.hasOwnProperty("discrete")&&(r.discrete=o.cogment_verse.Space.Discrete.toObject(e.discrete,t),t.oneofs&&(r.typeOneof="discrete")),null!=e.box&&e.hasOwnProperty("box")&&(r.box=o.cogment_verse.Space.Box.toObject(e.box,t),t.oneofs&&(r.typeOneof="box")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.Space.Property"},i}(),i}(),i.SpaceValue=function(){function i(e){if(this.properties=[],e)for(var t=Object.keys(e),r=0;r>>3===1?(i.properties&&i.properties.length||(i.properties=[]),i.properties.push(o.cogment_verse.SpaceValue.PropertyValue.decode(e,e.uint32()))):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.properties&&e.hasOwnProperty("properties")){if(!Array.isArray(e.properties))return"properties: array expected";for(var t=0;t>>3===1)if(i.values&&i.values.length||(i.values=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>3){case 1:i.discrete=e.int32();break;case 2:i.box=o.cogment_verse.NDArray.decode(e,e.uint32());break;case 3:i.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.discrete&&e.hasOwnProperty("discrete")&&(t.value=1,!n.isInteger(e.discrete)))return"discrete: integer expected";if(null!=e.box&&e.hasOwnProperty("box")){if(1===t.value)return"value: multiple values";if(t.value=1,r=o.cogment_verse.NDArray.verify(e.box))return"box."+r}if(null!=e.simpleBox&&e.hasOwnProperty("simpleBox")){if(1===t.value)return"value: multiple values";var r;if(t.value=1,r=o.cogment_verse.SpaceValue.SimpleBox.verify(e.simpleBox))return"simpleBox."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.SpaceValue.PropertyValue)return e;var t=new o.cogment_verse.SpaceValue.PropertyValue;if(null!=e.discrete&&(t.discrete=0|e.discrete),null!=e.box){if("object"!==typeof e.box)throw TypeError(".cogment_verse.SpaceValue.PropertyValue.box: object expected");t.box=o.cogment_verse.NDArray.fromObject(e.box)}if(null!=e.simpleBox){if("object"!==typeof e.simpleBox)throw TypeError(".cogment_verse.SpaceValue.PropertyValue.simpleBox: object expected");t.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.fromObject(e.simpleBox)}return t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.discrete&&e.hasOwnProperty("discrete")&&(r.discrete=e.discrete,t.oneofs&&(r.value="discrete")),null!=e.box&&e.hasOwnProperty("box")&&(r.box=o.cogment_verse.NDArray.toObject(e.box,t),t.oneofs&&(r.value="box")),null!=e.simpleBox&&e.hasOwnProperty("simpleBox")&&(r.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.toObject(e.simpleBox,t),t.oneofs&&(r.value="simpleBox")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.SpaceValue.PropertyValue"},i}(),i}(),i.EnvironmentSpecs=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.implementation=e.string();break;case 2:i.numPlayers=e.int32();break;case 3:i.observationSpace=o.cogment_verse.Space.decode(e,e.uint32());break;case 4:i.actionSpace=o.cogment_verse.Space.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.implementation&&e.hasOwnProperty("implementation")&&!n.isString(e.implementation)?"implementation: string expected":null!=e.numPlayers&&e.hasOwnProperty("numPlayers")&&!n.isInteger(e.numPlayers)?"numPlayers: integer expected":null!=e.observationSpace&&e.hasOwnProperty("observationSpace")&&(t=o.cogment_verse.Space.verify(e.observationSpace))?"observationSpace."+t:null!=e.actionSpace&&e.hasOwnProperty("actionSpace")&&(t=o.cogment_verse.Space.verify(e.actionSpace))?"actionSpace."+t:null;var t},i.fromObject=function(e){if(e instanceof o.cogment_verse.EnvironmentSpecs)return e;var t=new o.cogment_verse.EnvironmentSpecs;if(null!=e.implementation&&(t.implementation=String(e.implementation)),null!=e.numPlayers&&(t.numPlayers=0|e.numPlayers),null!=e.observationSpace){if("object"!==typeof e.observationSpace)throw TypeError(".cogment_verse.EnvironmentSpecs.observationSpace: object expected");t.observationSpace=o.cogment_verse.Space.fromObject(e.observationSpace)}if(null!=e.actionSpace){if("object"!==typeof e.actionSpace)throw TypeError(".cogment_verse.EnvironmentSpecs.actionSpace: object expected");t.actionSpace=o.cogment_verse.Space.fromObject(e.actionSpace)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.implementation="",r.numPlayers=0,r.observationSpace=null,r.actionSpace=null),null!=e.implementation&&e.hasOwnProperty("implementation")&&(r.implementation=e.implementation),null!=e.numPlayers&&e.hasOwnProperty("numPlayers")&&(r.numPlayers=e.numPlayers),null!=e.observationSpace&&e.hasOwnProperty("observationSpace")&&(r.observationSpace=o.cogment_verse.Space.toObject(e.observationSpace,t)),null!=e.actionSpace&&e.hasOwnProperty("actionSpace")&&(r.actionSpace=o.cogment_verse.Space.toObject(e.actionSpace,t)),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.EnvironmentSpecs"},i}(),i.EnvironmentConfig=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.runId=e.string();break;case 2:i.render=e.bool();break;case 3:i.renderWidth=e.int32();break;case 4:i.seed=e.uint32();break;case 5:i.flatten=e.bool();break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.runId&&e.hasOwnProperty("runId")&&!n.isString(e.runId)?"runId: string expected":null!=e.render&&e.hasOwnProperty("render")&&"boolean"!==typeof e.render?"render: boolean expected":null!=e.renderWidth&&e.hasOwnProperty("renderWidth")&&!n.isInteger(e.renderWidth)?"renderWidth: integer expected":null!=e.seed&&e.hasOwnProperty("seed")&&!n.isInteger(e.seed)?"seed: integer expected":null!=e.flatten&&e.hasOwnProperty("flatten")&&"boolean"!==typeof e.flatten?"flatten: boolean expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.EnvironmentConfig)return e;var t=new o.cogment_verse.EnvironmentConfig;return null!=e.runId&&(t.runId=String(e.runId)),null!=e.render&&(t.render=Boolean(e.render)),null!=e.renderWidth&&(t.renderWidth=0|e.renderWidth),null!=e.seed&&(t.seed=e.seed>>>0),null!=e.flatten&&(t.flatten=Boolean(e.flatten)),t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.runId="",r.render=!1,r.renderWidth=0,r.seed=0,r.flatten=!1),null!=e.runId&&e.hasOwnProperty("runId")&&(r.runId=e.runId),null!=e.render&&e.hasOwnProperty("render")&&(r.render=e.render),null!=e.renderWidth&&e.hasOwnProperty("renderWidth")&&(r.renderWidth=e.renderWidth),null!=e.seed&&e.hasOwnProperty("seed")&&(r.seed=e.seed),null!=e.flatten&&e.hasOwnProperty("flatten")&&(r.flatten=e.flatten),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.EnvironmentConfig"},i}(),i.HFHubModel=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.repoId=e.string();break;case 2:i.filename=e.string();break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.repoId&&e.hasOwnProperty("repoId")&&!n.isString(e.repoId)?"repoId: string expected":null!=e.filename&&e.hasOwnProperty("filename")&&!n.isString(e.filename)?"filename: string expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.HFHubModel)return e;var t=new o.cogment_verse.HFHubModel;return null!=e.repoId&&(t.repoId=String(e.repoId)),null!=e.filename&&(t.filename=String(e.filename)),t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.repoId="",r.filename=""),null!=e.repoId&&e.hasOwnProperty("repoId")&&(r.repoId=e.repoId),null!=e.filename&&e.hasOwnProperty("filename")&&(r.filename=e.filename),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.HFHubModel"},i}(),i.AgentConfig=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.runId=e.string();break;case 2:i.environmentSpecs=o.cogment_verse.EnvironmentSpecs.decode(e,e.uint32());break;case 3:i.modelId=e.string();break;case 4:i.modelVersion=e.int32();break;case 5:i.actorIndex=e.int32();break;case 6:i.device=e.string();break;case 7:i.threadsPerWorker=e.uint32();break;case 8:i.hfHubModel=o.cogment_verse.HFHubModel.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.runId&&e.hasOwnProperty("runId")&&!n.isString(e.runId)?"runId: string expected":null!=e.environmentSpecs&&e.hasOwnProperty("environmentSpecs")&&(t=o.cogment_verse.EnvironmentSpecs.verify(e.environmentSpecs))?"environmentSpecs."+t:null!=e.modelId&&e.hasOwnProperty("modelId")&&!n.isString(e.modelId)?"modelId: string expected":null!=e.modelVersion&&e.hasOwnProperty("modelVersion")&&!n.isInteger(e.modelVersion)?"modelVersion: integer expected":null!=e.actorIndex&&e.hasOwnProperty("actorIndex")&&!n.isInteger(e.actorIndex)?"actorIndex: integer expected":null!=e.device&&e.hasOwnProperty("device")&&!n.isString(e.device)?"device: string expected":null!=e.threadsPerWorker&&e.hasOwnProperty("threadsPerWorker")&&!n.isInteger(e.threadsPerWorker)?"threadsPerWorker: integer expected":null!=e.hfHubModel&&e.hasOwnProperty("hfHubModel")&&(t=o.cogment_verse.HFHubModel.verify(e.hfHubModel))?"hfHubModel."+t:null;var t},i.fromObject=function(e){if(e instanceof o.cogment_verse.AgentConfig)return e;var t=new o.cogment_verse.AgentConfig;if(null!=e.runId&&(t.runId=String(e.runId)),null!=e.environmentSpecs){if("object"!==typeof e.environmentSpecs)throw TypeError(".cogment_verse.AgentConfig.environmentSpecs: object expected");t.environmentSpecs=o.cogment_verse.EnvironmentSpecs.fromObject(e.environmentSpecs)}if(null!=e.modelId&&(t.modelId=String(e.modelId)),null!=e.modelVersion&&(t.modelVersion=0|e.modelVersion),null!=e.actorIndex&&(t.actorIndex=0|e.actorIndex),null!=e.device&&(t.device=String(e.device)),null!=e.threadsPerWorker&&(t.threadsPerWorker=e.threadsPerWorker>>>0),null!=e.hfHubModel){if("object"!==typeof e.hfHubModel)throw TypeError(".cogment_verse.AgentConfig.hfHubModel: object expected");t.hfHubModel=o.cogment_verse.HFHubModel.fromObject(e.hfHubModel)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.runId="",r.environmentSpecs=null,r.modelId="",r.modelVersion=0,r.actorIndex=0,r.device="",r.threadsPerWorker=0,r.hfHubModel=null),null!=e.runId&&e.hasOwnProperty("runId")&&(r.runId=e.runId),null!=e.environmentSpecs&&e.hasOwnProperty("environmentSpecs")&&(r.environmentSpecs=o.cogment_verse.EnvironmentSpecs.toObject(e.environmentSpecs,t)),null!=e.modelId&&e.hasOwnProperty("modelId")&&(r.modelId=e.modelId),null!=e.modelVersion&&e.hasOwnProperty("modelVersion")&&(r.modelVersion=e.modelVersion),null!=e.actorIndex&&e.hasOwnProperty("actorIndex")&&(r.actorIndex=e.actorIndex),null!=e.device&&e.hasOwnProperty("device")&&(r.device=e.device),null!=e.threadsPerWorker&&e.hasOwnProperty("threadsPerWorker")&&(r.threadsPerWorker=e.threadsPerWorker),null!=e.hfHubModel&&e.hasOwnProperty("hfHubModel")&&(r.hfHubModel=o.cogment_verse.HFHubModel.toObject(e.hfHubModel,t)),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.AgentConfig"},i}(),i.TrialConfig=function(){function n(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32());break;case 2:i.renderedFrame=e.bytes();break;case 3:if(i.legalMovesAsInt&&i.legalMovesAsInt.length||(i.legalMovesAsInt=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos=0&&(t.renderedFrame=e.renderedFrame)),e.legalMovesAsInt){if(!Array.isArray(e.legalMovesAsInt))throw TypeError(".cogment_verse.Observation.legalMovesAsInt: array expected");t.legalMovesAsInt=[];for(var r=0;r>>3===1?i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32()):e.skipType(7&a)}return i},n.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},n.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.value&&e.hasOwnProperty("value")){var t=o.cogment_verse.SpaceValue.verify(e.value);if(t)return"value."+t}return null},n.fromObject=function(e){if(e instanceof o.cogment_verse.PlayerAction)return e;var t=new o.cogment_verse.PlayerAction;if(null!=e.value){if("object"!==typeof e.value)throw TypeError(".cogment_verse.PlayerAction.value: object expected");t.value=o.cogment_verse.SpaceValue.fromObject(e.value)}return t},n.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.value=null),null!=e.value&&e.hasOwnProperty("value")&&(r.value=o.cogment_verse.SpaceValue.toObject(e.value,t)),r},n.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},n.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.PlayerAction"},n}(),i.TeacherAction=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3===1?i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32()):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.value&&e.hasOwnProperty("value")){t._value=1;var r=o.cogment_verse.SpaceValue.verify(e.value);if(r)return"value."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.TeacherAction)return e;var t=new o.cogment_verse.TeacherAction;if(null!=e.value){if("object"!==typeof e.value)throw TypeError(".cogment_verse.TeacherAction.value: object expected");t.value=o.cogment_verse.SpaceValue.fromObject(e.value)}return t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.value&&e.hasOwnProperty("value")&&(r.value=o.cogment_verse.SpaceValue.toObject(e.value,t),t.oneofs&&(r._value="value")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.TeacherAction"},i}(),i.ObserverAction=function(){function n(e){if(e)for(var t=Object.keys(e),r=0;r0){var i=n.substring(0,o).trim(),a=n.substring(o+1).trim();this.append(i,a)}}},e.prototype.delete=function(e,t){var r=n.normalizeName(e);if(void 0===t)delete this.headersMap[r];else{var o=this.headersMap[r];if(o){var i=o.indexOf(t);i>=0&&o.splice(i,1),0===o.length&&delete this.headersMap[r]}}},e.prototype.append=function(e,t){var r=this,o=n.normalizeName(e);Array.isArray(this.headersMap[o])||(this.headersMap[o]=[]),Array.isArray(t)?t.forEach((function(e){r.headersMap[o].push(n.normalizeValue(e))})):this.headersMap[o].push(n.normalizeValue(t))},e.prototype.set=function(e,t){var r=n.normalizeName(e);if(Array.isArray(t)){var o=[];t.forEach((function(e){o.push(n.normalizeValue(e))})),this.headersMap[r]=o}else this.headersMap[r]=[n.normalizeValue(t)]},e.prototype.has=function(e,t){var r=this.headersMap[n.normalizeName(e)];if(!Array.isArray(r))return!1;if(void 0!==t){var o=n.normalizeValue(t);return r.indexOf(o)>=0}return!0},e.prototype.get=function(e){var t=this.headersMap[n.normalizeName(e)];return void 0!==t?t.concat():[]},e.prototype.forEach=function(e){var t=this;Object.getOwnPropertyNames(this.headersMap).forEach((function(r){e(r,t.headersMap[r])}),this)},e.prototype.toHeaders=function(){if("undefined"!=typeof Headers){var e=new Headers;return this.forEach((function(t,r){r.forEach((function(r){e.append(t,r)}))})),e}throw new Error("Headers class is not defined")},e}();t.BrowserHeaders=o},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(0);t.BrowserHeaders=n.BrowserHeaders},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.iterateHeaders=function(e,t){for(var r=e[Symbol.iterator](),n=r.next();!n.done;)t(n.value[0]),n=r.next()},t.iterateHeadersKeys=function(e,t){for(var r=e.keys(),n=r.next();!n.done;)t(n.value),n=r.next()}},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(2);t.normalizeName=function(e){if("string"!=typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.\^_`|~]/i.test(e))throw new TypeError("Invalid character in header field name");return e.toLowerCase()},t.normalizeValue=function(e){return"string"!=typeof e&&(e=String(e)),e},t.getHeaderValues=function(e,t){var r=e;if(r instanceof Headers&&r.getAll)return r.getAll(t);var n=r.get(t);return n&&"string"==typeof n?[n]:n},t.getHeaderKeys=function(e){var t=e,r={},o=[];return t.keys?n.iterateHeadersKeys(t,(function(e){r[e]||(r[e]=!0,o.push(e))})):t.forEach?t.forEach((function(e,t){r[t]||(r[t]=!0,o.push(t))})):n.iterateHeaders(t,(function(e){var t=e[0];r[t]||(r[t]=!0,o.push(t))})),o},t.splitHeaderValue=function(e){var t=[];return e.split(", ").forEach((function(e){e.split(",").forEach((function(e){t.push(e)}))})),t}}]))},617:function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.ChunkParser=t.ChunkType=t.encodeASCII=t.decodeASCII=void 0;var n,o=r(65);function i(e){return 9===(t=e)||10===t||13===t||e>=32&&e<=126;var t}function a(e){for(var t=0;t!==e.length;++t)if(!i(e[t]))throw new Error("Metadata is not valid (printable) ASCII");return String.fromCharCode.apply(String,Array.prototype.slice.call(e))}function s(e){return 128==(128&e.getUint8(0))}function l(e){return e.getUint32(1,!1)}function u(e,t,r){return e.byteLength-t>=r}function c(e,t,r){if(e.slice)return e.slice(t,r);var n=e.length;void 0!==r&&(n=r);for(var o=new Uint8Array(n-t),i=0,a=t;a=0?r:i.httpStatusToCode(t);this.props.debug&&a.debug("onHeaders.code",n);var o=e.get("grpc-message")||[];if(this.props.debug&&a.debug("onHeaders.gRPCMessage",o),this.rawOnHeaders(e),n!==i.Code.OK){var s=this.decodeGRPCStatus(o[0]);this.rawOnError(n,s,e)}}},e.prototype.onTransportChunk=function(e){var t=this;if(this.closed)this.props.debug&&a.debug("grpc.onChunk received after request was closed - ignoring");else{var r=[];try{r=this.parser.parse(e)}catch(e){return this.props.debug&&a.debug("onChunk.parsing error",e,e.message),void this.rawOnError(i.Code.Internal,"parsing error: "+e.message)}r.forEach((function(e){if(e.chunkType===o.ChunkType.MESSAGE){var r=t.methodDefinition.responseType.deserializeBinary(e.data);t.rawOnMessage(r)}else e.chunkType===o.ChunkType.TRAILERS&&(t.responseHeaders?(t.responseTrailers=new n.Metadata(e.trailers),t.props.debug&&a.debug("onChunk.trailers",t.responseTrailers)):(t.responseHeaders=new n.Metadata(e.trailers),t.rawOnHeaders(t.responseHeaders)))}))}},e.prototype.onTransportEnd=function(){if(this.props.debug&&a.debug("grpc.onEnd"),this.closed)this.props.debug&&a.debug("grpc.onEnd received after request was closed - ignoring");else if(void 0!==this.responseTrailers){var e=c(this.responseTrailers);if(null!==e){var t=this.responseTrailers.get("grpc-message"),r=this.decodeGRPCStatus(t[0]);this.rawOnEnd(e,r,this.responseTrailers)}else this.rawOnError(i.Code.Internal,"Response closed without grpc-status (Trailers provided)")}else{if(void 0===this.responseHeaders)return void this.rawOnError(i.Code.Unknown,"Response closed without headers");var n=c(this.responseHeaders),o=this.responseHeaders.get("grpc-message");if(this.props.debug&&a.debug("grpc.headers only response ",n,o),null===n)return void this.rawOnEnd(i.Code.Unknown,"Response closed without grpc-status (Headers only)",this.responseHeaders);var s=this.decodeGRPCStatus(o[0]);this.rawOnEnd(n,s,this.responseHeaders)}},e.prototype.decodeGRPCStatus=function(e){if(!e)return"";try{return decodeURIComponent(e)}catch(t){return e}},e.prototype.rawOnEnd=function(e,t,r){var n=this;this.props.debug&&a.debug("rawOnEnd",e,t,r),this.completed||(this.completed=!0,this.onEndCallbacks.forEach((function(o){if(!n.closed)try{o(e,t,r)}catch(e){setTimeout((function(){throw e}),0)}})))},e.prototype.rawOnHeaders=function(e){this.props.debug&&a.debug("rawOnHeaders",e),this.completed||this.onHeadersCallbacks.forEach((function(t){try{t(e)}catch(e){setTimeout((function(){throw e}),0)}}))},e.prototype.rawOnError=function(e,t,r){var o=this;void 0===r&&(r=new n.Metadata),this.props.debug&&a.debug("rawOnError",e,t),this.completed||(this.completed=!0,this.onEndCallbacks.forEach((function(n){if(!o.closed)try{n(e,t,r)}catch(e){setTimeout((function(){throw e}),0)}})))},e.prototype.rawOnMessage=function(e){var t=this;this.props.debug&&a.debug("rawOnMessage",e.toObject()),this.completed||this.closed||this.onMessageCallbacks.forEach((function(r){if(!t.closed)try{r(e)}catch(e){setTimeout((function(){throw e}),0)}}))},e.prototype.onHeaders=function(e){this.onHeadersCallbacks.push(e)},e.prototype.onMessage=function(e){this.onMessageCallbacks.push(e)},e.prototype.onEnd=function(e){this.onEndCallbacks.push(e)},e.prototype.start=function(e){if(this.started)throw new Error("Client already started - cannot .start()");this.started=!0;var t=new n.Metadata(e||{});t.set("content-type","application/grpc-web+proto"),t.set("x-grpc-web","1"),this.transport.start(t)},e.prototype.send=function(e){if(!this.started)throw new Error("Client not started - .start() must be called before .send()");if(this.closed)throw new Error("Client already closed - cannot .send()");if(this.finishedSending)throw new Error("Client already finished sending - cannot .send()");if(!this.methodDefinition.requestStream&&this.sentFirstMessage)throw new Error("Message already sent for non-client-streaming method - cannot .send()");this.sentFirstMessage=!0;var t=l.frameRequest(e);this.transport.sendMessage(t)},e.prototype.finishSend=function(){if(!this.started)throw new Error("Client not started - .finishSend() must be called before .close()");if(this.closed)throw new Error("Client already closed - cannot .send()");if(this.finishedSending)throw new Error("Client already finished sending - cannot .finishSend()");this.finishedSending=!0,this.transport.finishSend()},e.prototype.close=function(){if(!this.started)throw new Error("Client not started - .start() must be called before .close()");if(this.closed)throw new Error("Client already closed - cannot .close()");this.closed=!0,this.props.debug&&a.debug("request.abort aborting request"),this.transport.cancel()},e}();function c(e){var t=e.get("grpc-status")||[];if(t.length>0)try{var r=t[0];return parseInt(r,10)}catch(e){return null}return null}},346:function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.debug=void 0,t.debug=function(){for(var e=[],t=0;t=55296&&r<=56319){var n=e.charCodeAt(t+1);n>=56320&&n<=57343&&(r=65536+(r-55296<<10)+(n-56320))}return r}function g(e){for(var t=new Uint8Array(e.length),r=0,n=0;n1&&"="===e.charAt(t);)++r;return Math.ceil(3*e.length)/4-r};for(var n=new Array(64),o=new Array(123),i=0;i<64;)o[n[i]=i<26?i+65:i<52?i+71:i<62?i-4:i-59|43]=i++;r.encode=function(e,t,r){for(var o,i=null,a=[],s=0,l=0;t>2],o=(3&u)<<4,l=1;break;case 1:a[s++]=n[o|u>>4],o=(15&u)<<2,l=2;break;case 2:a[s++]=n[o|u>>6],a[s++]=n[63&u],l=0}s>8191&&((i||(i=[])).push(String.fromCharCode.apply(String,a)),s=0)}return l&&(a[s++]=n[o],a[s++]=61,1===l&&(a[s++]=61)),i?(s&&i.push(String.fromCharCode.apply(String,a.slice(0,s))),i.join("")):String.fromCharCode.apply(String,a.slice(0,s))};var a="invalid encoding";r.decode=function(e,t,r){for(var n,i=r,s=0,l=0;l1)break;if(void 0===(u=o[u]))throw Error(a);switch(s){case 0:n=u,s=1;break;case 1:t[r++]=n<<2|(48&u)>>4,n=u,s=2;break;case 2:t[r++]=(15&n)<<4|(60&u)>>2,n=u,s=3;break;case 3:t[r++]=(3&n)<<6|u,s=0}}if(1===s)throw Error(a);return r-i},r.test=function(e){return/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(e)}},"./node_modules/@protobufjs/eventemitter/index.js":function(e){"use strict";function t(){this._listeners={}}e.exports=t,t.prototype.on=function(e,t,r){return(this._listeners[e]||(this._listeners[e]=[])).push({fn:t,ctx:r||this}),this},t.prototype.off=function(e,t){if(void 0===e)this._listeners={};else if(void 0===t)this._listeners[e]=[];else for(var r=this._listeners[e],n=0;n0?0:2147483648,r,n);else if(isNaN(t))e(2143289344,r,n);else if(t>34028234663852886e22)e((o<<31|2139095040)>>>0,r,n);else if(t<11754943508222875e-54)e((o<<31|Math.round(t/1401298464324817e-60))>>>0,r,n);else{var i=Math.floor(Math.log(t)/Math.LN2);e((o<<31|i+127<<23|8388607&Math.round(t*Math.pow(2,-i)*8388608))>>>0,r,n)}}function a(e,t,r){var n=e(t,r),o=2*(n>>31)+1,i=n>>>23&255,a=8388607&n;return 255===i?a?NaN:o*(1/0):0===i?1401298464324817e-60*o*a:o*Math.pow(2,i-150)*(a+8388608)}e.writeFloatLE=t.bind(null,r),e.writeFloatBE=t.bind(null,n),e.readFloatLE=a.bind(null,o),e.readFloatBE=a.bind(null,i)}(),"undefined"!==typeof Float64Array?function(){var t=new Float64Array([-0]),r=new Uint8Array(t.buffer),n=128===r[7];function o(e,n,o){t[0]=e,n[o]=r[0],n[o+1]=r[1],n[o+2]=r[2],n[o+3]=r[3],n[o+4]=r[4],n[o+5]=r[5],n[o+6]=r[6],n[o+7]=r[7]}function i(e,n,o){t[0]=e,n[o]=r[7],n[o+1]=r[6],n[o+2]=r[5],n[o+3]=r[4],n[o+4]=r[3],n[o+5]=r[2],n[o+6]=r[1],n[o+7]=r[0]}function a(e,n){return r[0]=e[n],r[1]=e[n+1],r[2]=e[n+2],r[3]=e[n+3],r[4]=e[n+4],r[5]=e[n+5],r[6]=e[n+6],r[7]=e[n+7],t[0]}function s(e,n){return r[7]=e[n],r[6]=e[n+1],r[5]=e[n+2],r[4]=e[n+3],r[3]=e[n+4],r[2]=e[n+5],r[1]=e[n+6],r[0]=e[n+7],t[0]}e.writeDoubleLE=n?o:i,e.writeDoubleBE=n?i:o,e.readDoubleLE=n?a:s,e.readDoubleBE=n?s:a}():function(){function t(e,t,r,n,o,i){var a=n<0?1:0;if(a&&(n=-n),0===n)e(0,o,i+t),e(1/n>0?0:2147483648,o,i+r);else if(isNaN(n))e(0,o,i+t),e(2146959360,o,i+r);else if(n>17976931348623157e292)e(0,o,i+t),e((a<<31|2146435072)>>>0,o,i+r);else{var s;if(n<22250738585072014e-324)e((s=n/5e-324)>>>0,o,i+t),e((a<<31|s/4294967296)>>>0,o,i+r);else{var l=Math.floor(Math.log(n)/Math.LN2);1024===l&&(l=1023),e(4503599627370496*(s=n*Math.pow(2,-l))>>>0,o,i+t),e((a<<31|l+1023<<20|1048576*s&1048575)>>>0,o,i+r)}}}function a(e,t,r,n,o){var i=e(n,o+t),a=e(n,o+r),s=2*(a>>31)+1,l=a>>>20&2047,u=4294967296*(1048575&a)+i;return 2047===l?u?NaN:s*(1/0):0===l?5e-324*s*u:s*Math.pow(2,l-1075)*(u+4503599627370496)}e.writeDoubleLE=t.bind(null,r,0,4),e.writeDoubleBE=t.bind(null,n,4,0),e.readDoubleLE=a.bind(null,o,0,4),e.readDoubleBE=a.bind(null,i,4,0)}(),e}function r(e,t,r){t[r]=255&e,t[r+1]=e>>>8&255,t[r+2]=e>>>16&255,t[r+3]=e>>>24}function n(e,t,r){t[r]=e>>>24,t[r+1]=e>>>16&255,t[r+2]=e>>>8&255,t[r+3]=255&e}function o(e,t){return(e[t]|e[t+1]<<8|e[t+2]<<16|e[t+3]<<24)>>>0}function i(e,t){return(e[t]<<24|e[t+1]<<16|e[t+2]<<8|e[t+3])>>>0}e.exports=t(t)},"./node_modules/@protobufjs/inquire/index.js":function node_modulesProtobufjsInquireIndexJs(module){"use strict";function inquire(moduleName){try{var mod=eval("quire".replace(/^/,"re"))(moduleName);if(mod&&(mod.length||Object.keys(mod).length))return mod}catch(e){}return null}module.exports=inquire},"./node_modules/@protobufjs/pool/index.js":function(e){"use strict";e.exports=function(e,t,r){var n=r||8192,o=n>>>1,i=null,a=n;return function(r){if(r<1||r>o)return e(r);a+r>n&&(i=e(n),a=0);var s=t.call(i,a,a+=r);return 7&a&&(a=1+(7|a)),s}}},"./node_modules/@protobufjs/utf8/index.js":function(e,t){"use strict";var r=t;r.length=function(e){for(var t=0,r=0,n=0;n191&&n<224?i[a++]=(31&n)<<6|63&e[t++]:n>239&&n<365?(n=((7&n)<<18|(63&e[t++])<<12|(63&e[t++])<<6|63&e[t++])-65536,i[a++]=55296+(n>>10),i[a++]=56320+(1023&n)):i[a++]=(15&n)<<12|(63&e[t++])<<6|63&e[t++],a>8191&&((o||(o=[])).push(String.fromCharCode.apply(String,i)),a=0);return o?(a&&o.push(String.fromCharCode.apply(String,i.slice(0,a))),o.join("")):String.fromCharCode.apply(String,i.slice(0,a))},r.write=function(e,t,r){for(var n,o,i=r,a=0;a>6|192,t[r++]=63&n|128):55296===(64512&n)&&56320===(64512&(o=e.charCodeAt(a+1)))?(n=65536+((1023&n)<<10)+(1023&o),++a,t[r++]=n>>18|240,t[r++]=n>>12&63|128,t[r++]=n>>6&63|128,t[r++]=63&n|128):(t[r++]=n>>12|224,t[r++]=n>>6&63|128,t[r++]=63&n|128);return r-i}},"./node_modules/browser-headers/dist/browser-headers.umd.js":function(e){var t;t=function(){return function(e){var t={};function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}return r.m=e,r.c=t,r.i=function(e){return e},r.d=function(e,t,n){r.o(e,t)||Object.defineProperty(e,t,{configurable:!1,enumerable:!0,get:n})},r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,"a",t),t},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r.p="",r(r.s=1)}([function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(3),o=function(){function e(e,t){void 0===e&&(e={}),void 0===t&&(t={splitValues:!1});var r,o=this;this.headersMap={},e&&("undefined"!==typeof Headers&&e instanceof Headers?n.getHeaderKeys(e).forEach((function(r){n.getHeaderValues(e,r).forEach((function(e){t.splitValues?o.append(r,n.splitHeaderValue(e)):o.append(r,e)}))})):"object"===typeof(r=e)&&"object"===typeof r.headersMap&&"function"===typeof r.forEach?e.forEach((function(e,t){o.append(e,t)})):"undefined"!==typeof Map&&e instanceof Map?e.forEach((function(e,t){o.append(t,e)})):"string"===typeof e?this.appendFromString(e):"object"===typeof e&&Object.getOwnPropertyNames(e).forEach((function(t){var r=e[t];Array.isArray(r)?r.forEach((function(e){o.append(t,e)})):o.append(t,r)})))}return e.prototype.appendFromString=function(e){for(var t=e.split("\r\n"),r=0;r0){var i=n.substring(0,o).trim(),a=n.substring(o+1).trim();this.append(i,a)}}},e.prototype.delete=function(e,t){var r=n.normalizeName(e);if(void 0===t)delete this.headersMap[r];else{var o=this.headersMap[r];if(o){var i=o.indexOf(t);i>=0&&o.splice(i,1),0===o.length&&delete this.headersMap[r]}}},e.prototype.append=function(e,t){var r=this,o=n.normalizeName(e);Array.isArray(this.headersMap[o])||(this.headersMap[o]=[]),Array.isArray(t)?t.forEach((function(e){r.headersMap[o].push(n.normalizeValue(e))})):this.headersMap[o].push(n.normalizeValue(t))},e.prototype.set=function(e,t){var r=n.normalizeName(e);if(Array.isArray(t)){var o=[];t.forEach((function(e){o.push(n.normalizeValue(e))})),this.headersMap[r]=o}else this.headersMap[r]=[n.normalizeValue(t)]},e.prototype.has=function(e,t){var r=this.headersMap[n.normalizeName(e)];if(!Array.isArray(r))return!1;if(void 0!==t){var o=n.normalizeValue(t);return r.indexOf(o)>=0}return!0},e.prototype.get=function(e){var t=this.headersMap[n.normalizeName(e)];return void 0!==t?t.concat():[]},e.prototype.forEach=function(e){var t=this;Object.getOwnPropertyNames(this.headersMap).forEach((function(r){e(r,t.headersMap[r])}),this)},e.prototype.toHeaders=function(){if("undefined"!==typeof Headers){var e=new Headers;return this.forEach((function(t,r){r.forEach((function(r){e.append(t,r)}))})),e}throw new Error("Headers class is not defined")},e}();t.BrowserHeaders=o},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(0);t.BrowserHeaders=n.BrowserHeaders},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.iterateHeaders=function(e,t){for(var r=e[Symbol.iterator](),n=r.next();!n.done;)t(n.value[0]),n=r.next()},t.iterateHeadersKeys=function(e,t){for(var r=e.keys(),n=r.next();!n.done;)t(n.value),n=r.next()}},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(2);t.normalizeName=function(e){if("string"!==typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.\^_`|~]/i.test(e))throw new TypeError("Invalid character in header field name");return e.toLowerCase()},t.normalizeValue=function(e){return"string"!==typeof e&&(e=String(e)),e},t.getHeaderValues=function(e,t){var r=e;if(r instanceof Headers&&r.getAll)return r.getAll(t);var n=r.get(t);return n&&"string"===typeof n?[n]:n},t.getHeaderKeys=function(e){var t=e,r={},o=[];return t.keys?n.iterateHeadersKeys(t,(function(e){r[e]||(r[e]=!0,o.push(e))})):t.forEach?t.forEach((function(e,t){r[t]||(r[t]=!0,o.push(t))})):n.iterateHeaders(t,(function(e){var t=e[0];r[t]||(r[t]=!0,o.push(t))})),o},t.splitHeaderValue=function(e){var t=[];return e.split(", ").forEach((function(e){e.split(",").forEach((function(e){t.push(e)}))})),t}}])},e.exports=t()},"./node_modules/google-protobuf/google-protobuf.js":function node_modulesGoogleProtobufGoogleProtobufJs(__unused_webpack_module,exports,__nested_webpack_require_87310__){var $jscomp=$jscomp||{};$jscomp.scope={},$jscomp.findInternal=function(e,t,r){e instanceof String&&(e=String(e));for(var n=e.length,o=0;o=n}}),"es6","es3"),$jscomp.polyfill("Array.prototype.find",(function(e){return e||function(e,t){return $jscomp.findInternal(this,e,t).v}}),"es6","es3"),$jscomp.polyfill("String.prototype.startsWith",(function(e){return e||function(e,t){var r=$jscomp.checkStringArgs(this,e,"startsWith");e+="";var n=r.length,o=e.length;t=Math.max(0,Math.min(0|t,r.length));for(var i=0;i=o}}),"es6","es3"),$jscomp.polyfill("String.prototype.repeat",(function(e){return e||function(e){var t=$jscomp.checkStringArgs(this,null,"repeat");if(0>e||1342177279>>=1)&&(t+=t);return r}}),"es6","es3");var COMPILED=!0,goog=goog||{};goog.global=this||self,goog.isDef=function(e){return void 0!==e},goog.isString=function(e){return"string"==typeof e},goog.isBoolean=function(e){return"boolean"==typeof e},goog.isNumber=function(e){return"number"==typeof e},goog.exportPath_=function(e,t,r){e=e.split("."),r=r||goog.global,e[0]in r||"undefined"==typeof r.execScript||r.execScript("var "+e[0]);for(var n;e.length&&(n=e.shift());)!e.length&&goog.isDef(t)?r[n]=t:r=r[n]&&r[n]!==Object.prototype[n]?r[n]:r[n]={}},goog.define=function(e,t){if(!COMPILED){var r=goog.global.CLOSURE_UNCOMPILED_DEFINES,n=goog.global.CLOSURE_DEFINES;r&&void 0===r.nodeType&&Object.prototype.hasOwnProperty.call(r,e)?t=r[e]:n&&void 0===n.nodeType&&Object.prototype.hasOwnProperty.call(n,e)&&(t=n[e])}return t},goog.FEATURESET_YEAR=2012,goog.DEBUG=!0,goog.LOCALE="en",goog.TRUSTED_SITE=!0,goog.STRICT_MODE_COMPATIBLE=!1,goog.DISALLOW_TEST_ONLY_CODE=COMPILED&&!goog.DEBUG,goog.ENABLE_CHROME_APP_SAFE_SCRIPT_LOADING=!1,goog.provide=function(e){if(goog.isInModuleLoader_())throw Error("goog.provide cannot be used within a module.");if(!COMPILED&&goog.isProvided_(e))throw Error('Namespace "'+e+'" already declared.');goog.constructNamespace_(e)},goog.constructNamespace_=function(e,t){if(!COMPILED){delete goog.implicitNamespaces_[e];for(var r=e;(r=r.substring(0,r.lastIndexOf(".")))&&!goog.getObjectByName(r);)goog.implicitNamespaces_[r]=!0}goog.exportPath_(e,t)},goog.getScriptNonce=function(e){return e&&e!=goog.global?goog.getScriptNonce_(e.document):(null===goog.cspNonce_&&(goog.cspNonce_=goog.getScriptNonce_(goog.global.document)),goog.cspNonce_)},goog.NONCE_PATTERN_=/^[\w+/_-]+[=]{0,2}$/,goog.cspNonce_=null,goog.getScriptNonce_=function(e){return(e=e.querySelector&&e.querySelector("script[nonce]"))&&(e=e.nonce||e.getAttribute("nonce"))&&goog.NONCE_PATTERN_.test(e)?e:""},goog.VALID_MODULE_RE_=/^[a-zA-Z_$][a-zA-Z0-9._$]*$/,goog.module=function(e){if(!goog.isString(e)||!e||-1==e.search(goog.VALID_MODULE_RE_))throw Error("Invalid module identifier");if(!goog.isInGoogModuleLoader_())throw Error("Module "+e+" has been loaded incorrectly. Note, modules cannot be loaded as normal scripts. They require some kind of pre-processing step. You're likely trying to load a module via a script tag or as a part of a concatenated bundle without rewriting the module. For more info see: https://github.com/google/closure-library/wiki/goog.module:-an-ES6-module-like-alternative-to-goog.provide.");if(goog.moduleLoaderState_.moduleName)throw Error("goog.module may only be called once per module.");if(goog.moduleLoaderState_.moduleName=e,!COMPILED){if(goog.isProvided_(e))throw Error('Namespace "'+e+'" already declared.');delete goog.implicitNamespaces_[e]}},goog.module.get=function(e){return goog.module.getInternal_(e)},goog.module.getInternal_=function(e){if(!COMPILED){if(e in goog.loadedModules_)return goog.loadedModules_[e].exports;if(!goog.implicitNamespaces_[e])return null!=(e=goog.getObjectByName(e))?e:null}return null},goog.ModuleType={ES6:"es6",GOOG:"goog"},goog.moduleLoaderState_=null,goog.isInModuleLoader_=function(){return goog.isInGoogModuleLoader_()||goog.isInEs6ModuleLoader_()},goog.isInGoogModuleLoader_=function(){return!!goog.moduleLoaderState_&&goog.moduleLoaderState_.type==goog.ModuleType.GOOG},goog.isInEs6ModuleLoader_=function(){if(goog.moduleLoaderState_&&goog.moduleLoaderState_.type==goog.ModuleType.ES6)return!0;var e=goog.global.$jscomp;return!!e&&("function"==typeof e.getCurrentModulePath&&!!e.getCurrentModulePath())},goog.module.declareLegacyNamespace=function(){if(!COMPILED&&!goog.isInGoogModuleLoader_())throw Error("goog.module.declareLegacyNamespace must be called from within a goog.module");if(!COMPILED&&!goog.moduleLoaderState_.moduleName)throw Error("goog.module must be called prior to goog.module.declareLegacyNamespace.");goog.moduleLoaderState_.declareLegacyNamespace=!0},goog.declareModuleId=function(e){if(!COMPILED){if(!goog.isInEs6ModuleLoader_())throw Error("goog.declareModuleId may only be called from within an ES6 module");if(goog.moduleLoaderState_&&goog.moduleLoaderState_.moduleName)throw Error("goog.declareModuleId may only be called once per module.");if(e in goog.loadedModules_)throw Error('Module with namespace "'+e+'" already exists.')}if(goog.moduleLoaderState_)goog.moduleLoaderState_.moduleName=e;else{var t=goog.global.$jscomp;if(!t||"function"!=typeof t.getCurrentModulePath)throw Error('Module with namespace "'+e+'" has been loaded incorrectly.');t=t.require(t.getCurrentModulePath()),goog.loadedModules_[e]={exports:t,type:goog.ModuleType.ES6,moduleId:e}}},goog.setTestOnly=function(e){if(goog.DISALLOW_TEST_ONLY_CODE)throw e=e||"",Error("Importing test-only code into non-debug environment"+(e?": "+e:"."))},goog.forwardDeclare=function(e){},COMPILED||(goog.isProvided_=function(e){return e in goog.loadedModules_||!goog.implicitNamespaces_[e]&&goog.isDefAndNotNull(goog.getObjectByName(e))},goog.implicitNamespaces_={"goog.module":!0}),goog.getObjectByName=function(e,t){e=e.split("."),t=t||goog.global;for(var r=0;r>>0),goog.uidCounter_=0,goog.getHashCode=goog.getUid,goog.removeHashCode=goog.removeUid,goog.cloneObject=function(e){var t=goog.typeOf(e);if("object"==t||"array"==t){if("function"===typeof e.clone)return e.clone();for(var r in t="array"==t?[]:{},e)t[r]=goog.cloneObject(e[r]);return t}return e},goog.bindNative_=function(e,t,r){return e.call.apply(e.bind,arguments)},goog.bindJs_=function(e,t,r){if(!e)throw Error();if(2{"use strict";class X{constructor(){if(new.target!=String)throw 1;this.x=42}}let q=Reflect.construct(X,[],String);if(q.x!=42||!(q instanceof String))throw 1;for(const a of[2,3]){if(a==2)continue;function f(z={a}){let a=0;return z.a}{function f(){return 0;}}return f()==3}})()')})),a("es7",(function(){return b("2 ** 2 == 4")})),a("es8",(function(){return b("async () => 1, true")})),a("es9",(function(){return b("({...rest} = {}), true")})),a("es_next",(function(){return!1})),{target:c,map:d}},goog.Transpiler.prototype.needsTranspile=function(e,t){if("always"==goog.TRANSPILE)return!0;if("never"==goog.TRANSPILE)return!1;if(!this.requiresTranspilation_){var r=this.createRequiresTranspilation_();this.requiresTranspilation_=r.map,this.transpilationTarget_=this.transpilationTarget_||r.target}if(e in this.requiresTranspilation_)return!!this.requiresTranspilation_[e]||!(!goog.inHtmlDocument_()||"es6"!=t||"noModule"in goog.global.document.createElement("script"));throw Error("Unknown language mode: "+e)},goog.Transpiler.prototype.transpile=function(e,t){return goog.transpile_(e,t,this.transpilationTarget_)},goog.transpiler_=new goog.Transpiler,goog.protectScriptTag_=function(e){return e.replace(/<\/(SCRIPT)/gi,"\\x3c/$1")},goog.DebugLoader_=function(){this.dependencies_={},this.idToPath_={},this.written_={},this.loadingDeps_=[],this.depsToLoad_=[],this.paused_=!1,this.factory_=new goog.DependencyFactory(goog.transpiler_),this.deferredCallbacks_={},this.deferredQueue_=[]},goog.DebugLoader_.prototype.bootstrap=function(e,t){function r(){n&&(goog.global.setTimeout(n,0),n=null)}var n=t;if(e.length){t=[];for(var o=0;o<\/script>",t.write(goog.TRUSTED_TYPES_POLICY_?goog.TRUSTED_TYPES_POLICY_.createHTML(n):n)}else{var o=t.createElement("script");o.defer=goog.Dependency.defer_,o.async=!1,o.type="text/javascript",(n=goog.getScriptNonce())&&o.setAttribute("nonce",n),goog.DebugLoader_.IS_OLD_IE_?(e.pause(),o.onreadystatechange=function(){"loaded"!=o.readyState&&"complete"!=o.readyState||(e.loaded(),e.resume())}):o.onload=function(){o.onload=null,e.loaded()},o.src=goog.TRUSTED_TYPES_POLICY_?goog.TRUSTED_TYPES_POLICY_.createScriptURL(this.path):this.path,t.head.appendChild(o)}}else goog.logToConsole_("Cannot use default debug loader outside of HTML documents."),"deps.js"==this.relativePath?(goog.logToConsole_("Consider setting CLOSURE_IMPORT_SCRIPT before loading base.js, or setting CLOSURE_NO_DEPS to true."),e.loaded()):e.pause()},goog.Es6ModuleDependency=function(e,t,r,n,o){goog.Dependency.call(this,e,t,r,n,o)},goog.inherits(goog.Es6ModuleDependency,goog.Dependency),goog.Es6ModuleDependency.prototype.load=function(e){if(goog.global.CLOSURE_IMPORT_SCRIPT)goog.global.CLOSURE_IMPORT_SCRIPT(this.path)?e.loaded():e.pause();else if(goog.inHtmlDocument_()){var t=goog.global.document,r=this;if(goog.isDocumentLoading_()){var n=function(e,r){e=r?' diff --git a/cogment_verse/web/components/public/logo192.png b/cogment_verse/web/components/public/logo192.png new file mode 100644 index 0000000000000000000000000000000000000000..fc44b0a3796c0e0a64c3d858ca038bd4570465d9 GIT binary patch literal 5347 zcmZWtbyO6NvR-oO24RV%BvuJ&=?+<7=`LvyB&A_#M7mSDYw1v6DJkiYl9XjT!%$dLEBTQ8R9|wd3008in6lFF3GV-6mLi?MoP_y~}QUnaDCHI#t z7w^m$@6DI)|C8_jrT?q=f8D?0AM?L)Z}xAo^e^W>t$*Y0KlT5=@bBjT9kxb%-KNdk zeOS1tKO#ChhG7%{ApNBzE2ZVNcxbrin#E1TiAw#BlUhXllzhN$qWez5l;h+t^q#Eav8PhR2|T}y5kkflaK`ba-eoE+Z2q@o6P$)=&` z+(8}+-McnNO>e#$Rr{32ngsZIAX>GH??tqgwUuUz6kjns|LjsB37zUEWd|(&O!)DY zQLrq%Y>)Y8G`yYbYCx&aVHi@-vZ3|ebG!f$sTQqMgi0hWRJ^Wc+Ibv!udh_r%2|U) zPi|E^PK?UE!>_4`f`1k4hqqj_$+d!EB_#IYt;f9)fBOumGNyglU(ofY`yHq4Y?B%- zp&G!MRY<~ajTgIHErMe(Z8JG*;D-PJhd@RX@QatggM7+G(Lz8eZ;73)72Hfx5KDOE zkT(m}i2;@X2AT5fW?qVp?@WgN$aT+f_6eo?IsLh;jscNRp|8H}Z9p_UBO^SJXpZew zEK8fz|0Th%(Wr|KZBGTM4yxkA5CFdAj8=QSrT$fKW#tweUFqr0TZ9D~a5lF{)%-tTGMK^2tz(y2v$i%V8XAxIywrZCp=)83p(zIk6@S5AWl|Oa2hF`~~^W zI;KeOSkw1O#TiQ8;U7OPXjZM|KrnN}9arP)m0v$c|L)lF`j_rpG(zW1Qjv$=^|p*f z>)Na{D&>n`jOWMwB^TM}slgTEcjxTlUby89j1)|6ydRfWERn3|7Zd2&e7?!K&5G$x z`5U3uFtn4~SZq|LjFVrz$3iln-+ucY4q$BC{CSm7Xe5c1J<=%Oagztj{ifpaZk_bQ z9Sb-LaQMKp-qJA*bP6DzgE3`}*i1o3GKmo2pn@dj0;He}F=BgINo};6gQF8!n0ULZ zL>kC0nPSFzlcB7p41doao2F7%6IUTi_+!L`MM4o*#Y#0v~WiO8uSeAUNp=vA2KaR&=jNR2iVwG>7t%sG2x_~yXzY)7K& zk3p+O0AFZ1eu^T3s};B%6TpJ6h-Y%B^*zT&SN7C=N;g|#dGIVMSOru3iv^SvO>h4M=t-N1GSLLDqVTcgurco6)3&XpU!FP6Hlrmj}f$ zp95;b)>M~`kxuZF3r~a!rMf4|&1=uMG$;h^g=Kl;H&Np-(pFT9FF@++MMEx3RBsK?AU0fPk-#mdR)Wdkj)`>ZMl#^<80kM87VvsI3r_c@_vX=fdQ`_9-d(xiI z4K;1y1TiPj_RPh*SpDI7U~^QQ?%0&!$Sh#?x_@;ag)P}ZkAik{_WPB4rHyW#%>|Gs zdbhyt=qQPA7`?h2_8T;-E6HI#im9K>au*(j4;kzwMSLgo6u*}-K`$_Gzgu&XE)udQ zmQ72^eZd|vzI)~!20JV-v-T|<4@7ruqrj|o4=JJPlybwMg;M$Ud7>h6g()CT@wXm` zbq=A(t;RJ^{Xxi*Ff~!|3!-l_PS{AyNAU~t{h;(N(PXMEf^R(B+ZVX3 z8y0;0A8hJYp@g+c*`>eTA|3Tgv9U8#BDTO9@a@gVMDxr(fVaEqL1tl?md{v^j8aUv zm&%PX4^|rX|?E4^CkplWWNv*OKM>DxPa z!RJ)U^0-WJMi)Ksc!^ixOtw^egoAZZ2Cg;X7(5xZG7yL_;UJ#yp*ZD-;I^Z9qkP`} zwCTs0*%rIVF1sgLervtnUo&brwz?6?PXRuOCS*JI-WL6GKy7-~yi0giTEMmDs_-UX zo=+nFrW_EfTg>oY72_4Z0*uG>MnXP=c0VpT&*|rvv1iStW;*^={rP1y?Hv+6R6bxFMkxpWkJ>m7Ba{>zc_q zEefC3jsXdyS5??Mz7IET$Kft|EMNJIv7Ny8ZOcKnzf`K5Cd)&`-fTY#W&jnV0l2vt z?Gqhic}l}mCv1yUEy$%DP}4AN;36$=7aNI^*AzV(eYGeJ(Px-j<^gSDp5dBAv2#?; zcMXv#aj>%;MiG^q^$0MSg-(uTl!xm49dH!{X0){Ew7ThWV~Gtj7h%ZD zVN-R-^7Cf0VH!8O)uUHPL2mO2tmE*cecwQv_5CzWeh)ykX8r5Hi`ehYo)d{Jnh&3p z9ndXT$OW51#H5cFKa76c<%nNkP~FU93b5h-|Cb}ScHs@4Q#|}byWg;KDMJ#|l zE=MKD*F@HDBcX@~QJH%56eh~jfPO-uKm}~t7VkHxHT;)4sd+?Wc4* z>CyR*{w@4(gnYRdFq=^(#-ytb^5ESD?x<0Skhb%Pt?npNW1m+Nv`tr9+qN<3H1f<% zZvNEqyK5FgPsQ`QIu9P0x_}wJR~^CotL|n zk?dn;tLRw9jJTur4uWoX6iMm914f0AJfB@C74a;_qRrAP4E7l890P&{v<}>_&GLrW z)klculcg`?zJO~4;BBAa=POU%aN|pmZJn2{hA!d!*lwO%YSIzv8bTJ}=nhC^n}g(ld^rn#kq9Z3)z`k9lvV>y#!F4e{5c$tnr9M{V)0m(Z< z#88vX6-AW7T2UUwW`g<;8I$Jb!R%z@rCcGT)-2k7&x9kZZT66}Ztid~6t0jKb&9mm zpa}LCb`bz`{MzpZR#E*QuBiZXI#<`5qxx=&LMr-UUf~@dRk}YI2hbMsAMWOmDzYtm zjof16D=mc`^B$+_bCG$$@R0t;e?~UkF?7<(vkb70*EQB1rfUWXh$j)R2)+dNAH5%R zEBs^?N;UMdy}V};59Gu#0$q53$}|+q7CIGg_w_WlvE}AdqoS<7DY1LWS9?TrfmcvT zaypmplwn=P4;a8-%l^e?f`OpGb}%(_mFsL&GywhyN(-VROj`4~V~9bGv%UhcA|YW% zs{;nh@aDX11y^HOFXB$a7#Sr3cEtNd4eLm@Y#fc&j)TGvbbMwze zXtekX_wJqxe4NhuW$r}cNy|L{V=t#$%SuWEW)YZTH|!iT79k#?632OFse{+BT_gau zJwQcbH{b}dzKO?^dV&3nTILYlGw{27UJ72ZN){BILd_HV_s$WfI2DC<9LIHFmtyw? zQ;?MuK7g%Ym+4e^W#5}WDLpko%jPOC=aN)3!=8)s#Rnercak&b3ESRX3z{xfKBF8L z5%CGkFmGO@x?_mPGlpEej!3!AMddChabyf~nJNZxx!D&{@xEb!TDyvqSj%Y5@A{}9 zRzoBn0?x}=krh{ok3Nn%e)#~uh;6jpezhA)ySb^b#E>73e*frBFu6IZ^D7Ii&rsiU z%jzygxT-n*joJpY4o&8UXr2s%j^Q{?e-voloX`4DQyEK+DmrZh8A$)iWL#NO9+Y@!sO2f@rI!@jN@>HOA< z?q2l{^%mY*PNx2FoX+A7X3N}(RV$B`g&N=e0uvAvEN1W^{*W?zT1i#fxuw10%~))J zjx#gxoVlXREWZf4hRkgdHx5V_S*;p-y%JtGgQ4}lnA~MBz-AFdxUxU1RIT$`sal|X zPB6sEVRjGbXIP0U+?rT|y5+ev&OMX*5C$n2SBPZr`jqzrmpVrNciR0e*Wm?fK6DY& zl(XQZ60yWXV-|Ps!A{EF;=_z(YAF=T(-MkJXUoX zI{UMQDAV2}Ya?EisdEW;@pE6dt;j0fg5oT2dxCi{wqWJ<)|SR6fxX~5CzblPGr8cb zUBVJ2CQd~3L?7yfTpLNbt)He1D>*KXI^GK%<`bq^cUq$Q@uJifG>p3LU(!H=C)aEL zenk7pVg}0{dKU}&l)Y2Y2eFMdS(JS0}oZUuVaf2+K*YFNGHB`^YGcIpnBlMhO7d4@vV zv(@N}(k#REdul8~fP+^F@ky*wt@~&|(&&meNO>rKDEnB{ykAZ}k>e@lad7to>Ao$B zz<1(L=#J*u4_LB=8w+*{KFK^u00NAmeNN7pr+Pf+N*Zl^dO{LM-hMHyP6N!~`24jd zXYP|Ze;dRXKdF2iJG$U{k=S86l@pytLx}$JFFs8e)*Vi?aVBtGJ3JZUj!~c{(rw5>vuRF$`^p!P8w1B=O!skwkO5yd4_XuG^QVF z`-r5K7(IPSiKQ2|U9+`@Js!g6sfJwAHVd|s?|mnC*q zp|B|z)(8+mxXyxQ{8Pg3F4|tdpgZZSoU4P&9I8)nHo1@)9_9u&NcT^FI)6|hsAZFk zZ+arl&@*>RXBf-OZxhZerOr&dN5LW9@gV=oGFbK*J+m#R-|e6(Loz(;g@T^*oO)0R zN`N=X46b{7yk5FZGr#5&n1!-@j@g02g|X>MOpF3#IjZ_4wg{dX+G9eqS+Es9@6nC7 zD9$NuVJI}6ZlwtUm5cCAiYv0(Yi{%eH+}t)!E^>^KxB5^L~a`4%1~5q6h>d;paC9c zTj0wTCKrhWf+F#5>EgX`sl%POl?oyCq0(w0xoL?L%)|Q7d|Hl92rUYAU#lc**I&^6p=4lNQPa0 znQ|A~i0ip@`B=FW-Q;zh?-wF;Wl5!+q3GXDu-x&}$gUO)NoO7^$BeEIrd~1Dh{Tr` z8s<(Bn@gZ(mkIGnmYh_ehXnq78QL$pNDi)|QcT*|GtS%nz1uKE+E{7jdEBp%h0}%r zD2|KmYGiPa4;md-t_m5YDz#c*oV_FqXd85d@eub?9N61QuYcb3CnVWpM(D-^|CmkL z(F}L&N7qhL2PCq)fRh}XO@U`Yn<?TNGR4L(mF7#4u29{i~@k;pLsgl({YW5`Mo+p=zZn3L*4{JU;++dG9 X@eDJUQo;Ye2mwlRs?y0|+_a0zY+Zo%Dkae}+MySoIppb75o?vUW_?)>@g{U2`ERQIXV zeY$JrWnMZ$QC<=ii4X|@0H8`si75jB(ElJb00HAB%>SlLR{!zO|C9P3zxw_U8?1d8uRZ=({Ga4shyN}3 zAK}WA(ds|``G4jA)9}Bt2Hy0+f3rV1E6b|@?hpGA=PI&r8)ah|)I2s(P5Ic*Ndhn^ z*T&j@gbCTv7+8rpYbR^Ty}1AY)YH;p!m948r#%7x^Z@_-w{pDl|1S4`EM3n_PaXvK z1JF)E3qy$qTj5Xs{jU9k=y%SQ0>8E$;x?p9ayU0bZZeo{5Z@&FKX>}s!0+^>C^D#z z>xsCPvxD3Z=dP}TTOSJhNTPyVt14VCQ9MQFN`rn!c&_p?&4<5_PGm4a;WS&1(!qKE z_H$;dDdiPQ!F_gsN`2>`X}$I=B;={R8%L~`>RyKcS$72ai$!2>d(YkciA^J0@X%G4 z4cu!%Ps~2JuJ8ex`&;Fa0NQOq_nDZ&X;^A=oc1&f#3P1(!5il>6?uK4QpEG8z0Rhu zvBJ+A9RV?z%v?!$=(vcH?*;vRs*+PPbOQ3cdPr5=tOcLqmfx@#hOqX0iN)wTTO21jH<>jpmwRIAGw7`a|sl?9y9zRBh>(_%| zF?h|P7}~RKj?HR+q|4U`CjRmV-$mLW>MScKnNXiv{vD3&2@*u)-6P@h0A`eeZ7}71 zK(w%@R<4lLt`O7fs1E)$5iGb~fPfJ?WxhY7c3Q>T-w#wT&zW522pH-B%r5v#5y^CF zcC30Se|`D2mY$hAlIULL%-PNXgbbpRHgn<&X3N9W!@BUk@9g*P5mz-YnZBb*-$zMM z7Qq}ic0mR8n{^L|=+diODdV}Q!gwr?y+2m=3HWwMq4z)DqYVg0J~^}-%7rMR@S1;9 z7GFj6K}i32X;3*$SmzB&HW{PJ55kT+EI#SsZf}bD7nW^Haf}_gXciYKX{QBxIPSx2Ma? zHQqgzZq!_{&zg{yxqv3xq8YV+`S}F6A>Gtl39_m;K4dA{pP$BW0oIXJ>jEQ!2V3A2 zdpoTxG&V=(?^q?ZTj2ZUpDUdMb)T?E$}CI>r@}PFPWD9@*%V6;4Ag>D#h>!s)=$0R zRXvdkZ%|c}ubej`jl?cS$onl9Tw52rBKT)kgyw~Xy%z62Lr%V6Y=f?2)J|bZJ5(Wx zmji`O;_B+*X@qe-#~`HFP<{8$w@z4@&`q^Q-Zk8JG3>WalhnW1cvnoVw>*R@c&|o8 zZ%w!{Z+MHeZ*OE4v*otkZqz11*s!#s^Gq>+o`8Z5 z^i-qzJLJh9!W-;SmFkR8HEZJWiXk$40i6)7 zZpr=k2lp}SasbM*Nbn3j$sn0;rUI;%EDbi7T1ZI4qL6PNNM2Y%6{LMIKW+FY_yF3) zSKQ2QSujzNMSL2r&bYs`|i2Dnn z=>}c0>a}>|uT!IiMOA~pVT~R@bGlm}Edf}Kq0?*Af6#mW9f9!}RjW7om0c9Qlp;yK z)=XQs(|6GCadQbWIhYF=rf{Y)sj%^Id-ARO0=O^Ad;Ph+ z0?$eE1xhH?{T$QI>0JP75`r)U_$#%K1^BQ8z#uciKf(C701&RyLQWBUp*Q7eyn76} z6JHpC9}R$J#(R0cDCkXoFSp;j6{x{b&0yE@P7{;pCEpKjS(+1RQy38`=&Yxo%F=3y zCPeefABp34U-s?WmU#JJw23dcC{sPPFc2#J$ZgEN%zod}J~8dLm*fx9f6SpO zn^Ww3bt9-r0XaT2a@Wpw;C23XM}7_14#%QpubrIw5aZtP+CqIFmsG4`Cm6rfxl9n5 z7=r2C-+lM2AB9X0T_`?EW&Byv&K?HS4QLoylJ|OAF z`8atBNTzJ&AQ!>sOo$?^0xj~D(;kS$`9zbEGd>f6r`NC3X`tX)sWgWUUOQ7w=$TO&*j;=u%25ay-%>3@81tGe^_z*C7pb9y*Ed^H3t$BIKH2o+olp#$q;)_ zfpjCb_^VFg5fU~K)nf*d*r@BCC>UZ!0&b?AGk_jTPXaSnCuW110wjHPPe^9R^;jo3 zwvzTl)C`Zl5}O2}3lec=hZ*$JnkW#7enKKc)(pM${_$9Hc=Sr_A9Biwe*Y=T?~1CK z6eZ9uPICjy-sMGbZl$yQmpB&`ouS8v{58__t0$JP%i3R&%QR3ianbZqDs<2#5FdN@n5bCn^ZtH992~5k(eA|8|@G9u`wdn7bnpg|@{m z^d6Y`*$Zf2Xr&|g%sai#5}Syvv(>Jnx&EM7-|Jr7!M~zdAyjt*xl;OLhvW-a%H1m0 z*x5*nb=R5u><7lyVpNAR?q@1U59 zO+)QWwL8t zyip?u_nI+K$uh{y)~}qj?(w0&=SE^8`_WMM zTybjG=999h38Yes7}-4*LJ7H)UE8{mE(6;8voE+TYY%33A>S6`G_95^5QHNTo_;Ao ztIQIZ_}49%{8|=O;isBZ?=7kfdF8_@azfoTd+hEJKWE!)$)N%HIe2cplaK`ry#=pV z0q{9w-`i0h@!R8K3GC{ivt{70IWG`EP|(1g7i_Q<>aEAT{5(yD z=!O?kq61VegV+st@XCw475j6vS)_z@efuqQgHQR1T4;|-#OLZNQJPV4k$AX1Uk8Lm z{N*b*ia=I+MB}kWpupJ~>!C@xEN#Wa7V+7{m4j8c?)ChV=D?o~sjT?0C_AQ7B-vxqX30s0I_`2$in86#`mAsT-w?j{&AL@B3$;P z31G4(lV|b}uSDCIrjk+M1R!X7s4Aabn<)zpgT}#gE|mIvV38^ODy@<&yflpCwS#fRf9ZX3lPV_?8@C5)A;T zqmouFLFk;qIs4rA=hh=GL~sCFsXHsqO6_y~*AFt939UYVBSx1s(=Kb&5;j7cSowdE;7()CC2|-i9Zz+_BIw8#ll~-tyH?F3{%`QCsYa*b#s*9iCc`1P1oC26?`g<9))EJ3%xz+O!B3 zZ7$j~To)C@PquR>a1+Dh>-a%IvH_Y7^ys|4o?E%3`I&ADXfC8++hAdZfzIT#%C+Jz z1lU~K_vAm0m8Qk}K$F>|>RPK%<1SI0(G+8q~H zAsjezyP+u!Se4q3GW)`h`NPSRlMoBjCzNPesWJwVTY!o@G8=(6I%4XHGaSiS3MEBK zhgGFv6Jc>L$4jVE!I?TQuwvz_%CyO!bLh94nqK11C2W$*aa2ueGopG8DnBICVUORP zgytv#)49fVXDaR$SukloYC3u7#5H)}1K21=?DKj^U)8G;MS)&Op)g^zR2($<>C*zW z;X7`hLxiIO#J`ANdyAOJle4V%ppa*(+0i3w;8i*BA_;u8gOO6)MY`ueq7stBMJTB; z-a0R>hT*}>z|Gg}@^zDL1MrH+2hsR8 zHc}*9IvuQC^Ju)^#Y{fOr(96rQNPNhxc;mH@W*m206>Lo<*SaaH?~8zg&f&%YiOEG zGiz?*CP>Bci}!WiS=zj#K5I}>DtpregpP_tfZtPa(N<%vo^#WCQ5BTv0vr%Z{)0q+ z)RbfHktUm|lg&U3YM%lMUM(fu}i#kjX9h>GYctkx9Mt_8{@s%!K_EI zScgwy6%_fR?CGJQtmgNAj^h9B#zmaMDWgH55pGuY1Gv7D z;8Psm(vEPiwn#MgJYu4Ty9D|h!?Rj0ddE|&L3S{IP%H4^N!m`60ZwZw^;eg4sk6K{ ziA^`Sbl_4~f&Oo%n;8Ye(tiAdlZKI!Z=|j$5hS|D$bDJ}p{gh$KN&JZYLUjv4h{NY zBJ>X9z!xfDGY z+oh_Z&_e#Q(-}>ssZfm=j$D&4W4FNy&-kAO1~#3Im;F)Nwe{(*75(p=P^VI?X0GFakfh+X-px4a%Uw@fSbmp9hM1_~R>?Z8+ ziy|e9>8V*`OP}4x5JjdWp}7eX;lVxp5qS}0YZek;SNmm7tEeSF*-dI)6U-A%m6YvCgM(}_=k#a6o^%-K4{`B1+}O4x zztDT%hVb;v#?j`lTvlFQ3aV#zkX=7;YFLS$uIzb0E3lozs5`Xy zi~vF+%{z9uLjKvKPhP%x5f~7-Gj+%5N`%^=yk*Qn{`> z;xj&ROY6g`iy2a@{O)V(jk&8#hHACVDXey5a+KDod_Z&}kHM}xt7}Md@pil{2x7E~ zL$k^d2@Ec2XskjrN+IILw;#7((abu;OJii&v3?60x>d_Ma(onIPtcVnX@ELF0aL?T zSmWiL3(dOFkt!x=1O!_0n(cAzZW+3nHJ{2S>tgSK?~cFha^y(l@-Mr2W$%MN{#af8J;V*>hdq!gx=d0h$T7l}>91Wh07)9CTX zh2_ZdQCyFOQ)l(}gft0UZG`Sh2`x-w`5vC2UD}lZs*5 zG76$akzn}Xi))L3oGJ75#pcN=cX3!=57$Ha=hQ2^lwdyU#a}4JJOz6ddR%zae%#4& za)bFj)z=YQela(F#Y|Q#dp}PJghITwXouVaMq$BM?K%cXn9^Y@g43$=O)F&ZlOUom zJiad#dea;-eywBA@e&D6Pdso1?2^(pXiN91?jvcaUyYoKUmvl5G9e$W!okWe*@a<^ z8cQQ6cNSf+UPDx%?_G4aIiybZHHagF{;IcD(dPO!#=u zWfqLcPc^+7Uu#l(Bpxft{*4lv#*u7X9AOzDO z1D9?^jIo}?%iz(_dwLa{ex#T}76ZfN_Z-hwpus9y+4xaUu9cX}&P{XrZVWE{1^0yw zO;YhLEW!pJcbCt3L8~a7>jsaN{V3>tz6_7`&pi%GxZ=V3?3K^U+*ryLSb)8^IblJ0 zSRLNDvIxt)S}g30?s_3NX>F?NKIGrG_zB9@Z>uSW3k2es_H2kU;Rnn%j5qP)!XHKE zPB2mHP~tLCg4K_vH$xv`HbRsJwbZMUV(t=ez;Ec(vyHH)FbfLg`c61I$W_uBB>i^r z&{_P;369-&>23R%qNIULe=1~T$(DA`ev*EWZ6j(B$(te}x1WvmIll21zvygkS%vwG zzkR6Z#RKA2!z!C%M!O>!=Gr0(J0FP=-MN=5t-Ir)of50y10W}j`GtRCsXBakrKtG& zazmITDJMA0C51&BnLY)SY9r)NVTMs);1<=oosS9g31l{4ztjD3#+2H7u_|66b|_*O z;Qk6nalpqdHOjx|K&vUS_6ITgGll;TdaN*ta=M_YtyC)I9Tmr~VaPrH2qb6sd~=AcIxV+%z{E&0@y=DPArw zdV7z(G1hBx7hd{>(cr43^WF%4Y@PXZ?wPpj{OQ#tvc$pABJbvPGvdR`cAtHn)cSEV zrpu}1tJwQ3y!mSmH*uz*x0o|CS<^w%&KJzsj~DU0cLQUxk5B!hWE>aBkjJle8z~;s z-!A=($+}Jq_BTK5^B!`R>!MulZN)F=iXXeUd0w5lUsE5VP*H*oCy(;?S$p*TVvTxwAeWFB$jHyb0593)$zqalVlDX=GcCN1gU0 zlgU)I$LcXZ8Oyc2TZYTPu@-;7<4YYB-``Qa;IDcvydIA$%kHhJKV^m*-zxcvU4viy&Kr5GVM{IT>WRywKQ9;>SEiQD*NqplK-KK4YR`p0@JW)n_{TU3bt0 zim%;(m1=#v2}zTps=?fU5w^(*y)xT%1vtQH&}50ZF!9YxW=&7*W($2kgKyz1mUgfs zfV<*XVVIFnohW=|j+@Kfo!#liQR^x>2yQdrG;2o8WZR+XzU_nG=Ed2rK?ntA;K5B{ z>M8+*A4!Jm^Bg}aW?R?6;@QG@uQ8&oJ{hFixcfEnJ4QH?A4>P=q29oDGW;L;= z9-a0;g%c`C+Ai!UmK$NC*4#;Jp<1=TioL=t^YM)<<%u#hnnfSS`nq63QKGO1L8RzX z@MFDqs1z ztYmxDl@LU)5acvHk)~Z`RW7=aJ_nGD!mOSYD>5Odjn@TK#LY{jf?+piB5AM-CAoT_ z?S-*q7}wyLJzK>N%eMPuFgN)Q_otKP;aqy=D5f!7<=n(lNkYRXVpkB{TAYLYg{|(jtRqYmg$xH zjmq?B(RE4 zQx^~Pt}gxC2~l=K$$-sYy_r$CO(d=+b3H1MB*y_5g6WLaWTXn+TKQ|hNY^>Mp6k*$ zwkovomhu776vQATqT4blf~g;TY(MWCrf^^yfWJvSAB$p5l;jm@o#=!lqw+Lqfq>X= z$6~kxfm7`3q4zUEB;u4qa#BdJxO!;xGm)wwuisj{0y2x{R(IGMrsIzDY9LW>m!Y`= z04sx3IjnYvL<4JqxQ8f7qYd0s2Ig%`ytYPEMKI)s(LD}D@EY>x`VFtqvnADNBdeao zC96X+MxnwKmjpg{U&gP3HE}1=s!lv&D{6(g_lzyF3A`7Jn*&d_kL<;dAFx!UZ>hB8 z5A*%LsAn;VLp>3${0>M?PSQ)9s3}|h2e?TG4_F{}{Cs>#3Q*t$(CUc}M)I}8cPF6% z=+h(Kh^8)}gj(0}#e7O^FQ6`~fd1#8#!}LMuo3A0bN`o}PYsm!Y}sdOz$+Tegc=qT z8x`PH$7lvnhJp{kHWb22l;@7B7|4yL4UOOVM0MP_>P%S1Lnid)+k9{+3D+JFa#Pyf zhVc#&df87APl4W9X)F3pGS>@etfl=_E5tBcVoOfrD4hmVeTY-cj((pkn%n@EgN{0f zwb_^Rk0I#iZuHK!l*lN`ceJn(sI{$Fq6nN& zE<-=0_2WN}m+*ivmIOxB@#~Q-cZ>l136w{#TIJe478`KE7@=a{>SzPHsKLzYAyBQO zAtuuF$-JSDy_S@6GW0MOE~R)b;+0f%_NMrW(+V#c_d&U8Z9+ec4=HmOHw?gdjF(Lu zzra83M_BoO-1b3;9`%&DHfuUY)6YDV21P$C!Rc?mv&{lx#f8oc6?0?x zK08{WP65?#>(vPfA-c=MCY|%*1_<3D4NX zeVTi-JGl2uP_2@0F{G({pxQOXt_d{g_CV6b?jNpfUG9;8yle-^4KHRvZs-_2siata zt+d_T@U$&t*xaD22(fH(W1r$Mo?3dc%Tncm=C6{V9y{v&VT#^1L04vDrLM9qBoZ4@ z6DBN#m57hX7$C(=#$Y5$bJmwA$T8jKD8+6A!-IJwA{WOfs%s}yxUw^?MRZjF$n_KN z6`_bGXcmE#5e4Ym)aQJ)xg3Pg0@k`iGuHe?f(5LtuzSq=nS^5z>vqU0EuZ&75V%Z{ zYyhRLN^)$c6Ds{f7*FBpE;n5iglx5PkHfWrj3`x^j^t z7ntuV`g!9Xg#^3!x)l*}IW=(Tz3>Y5l4uGaB&lz{GDjm2D5S$CExLT`I1#n^lBH7Y zDgpMag@`iETKAI=p<5E#LTkwzVR@=yY|uBVI1HG|8h+d;G-qfuj}-ZR6fN>EfCCW z9~wRQoAPEa#aO?3h?x{YvV*d+NtPkf&4V0k4|L=uj!U{L+oLa(z#&iuhJr3-PjO3R z5s?=nn_5^*^Rawr>>Nr@K(jwkB#JK-=+HqwfdO<+P5byeim)wvqGlP-P|~Nse8=XF zz`?RYB|D6SwS}C+YQv+;}k6$-%D(@+t14BL@vM z2q%q?f6D-A5s$_WY3{^G0F131bbh|g!}#BKw=HQ7mx;Dzg4Z*bTLQSfo{ed{4}NZW zfrRm^Ca$rlE{Ue~uYv>R9{3smwATcdM_6+yWIO z*ZRH~uXE@#p$XTbCt5j7j2=86e{9>HIB6xDzV+vAo&B?KUiMP|ttOElepnl%|DPqL b{|{}U^kRn2wo}j7|0ATu<;8xA7zX}7|B6mN literal 0 HcmV?d00001 diff --git a/cogment_verse/web/components/public/manifest.json b/cogment_verse/web/components/public/manifest.json new file mode 100644 index 00000000..cb88ef90 --- /dev/null +++ b/cogment_verse/web/components/public/manifest.json @@ -0,0 +1,25 @@ +{ + "short_name": "CogVerse", + "name": "CogVerse Web Client", + "icons": [ + { + "src": "favicon.ico", + "sizes": "64x64 32x32 24x24 16x16", + "type": "image/x-icon" + }, + { + "src": "logo192.png", + "type": "image/png", + "sizes": "192x192" + }, + { + "src": "logo512.png", + "type": "image/png", + "sizes": "512x512" + } + ], + "start_url": ".", + "display": "standalone", + "theme_color": "#000000", + "background_color": "#ffffff" +} diff --git a/cogment_verse/web/components/public/robots.txt b/cogment_verse/web/components/public/robots.txt new file mode 100644 index 00000000..e9e57dc4 --- /dev/null +++ b/cogment_verse/web/components/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/web_client/src/App.jsx b/cogment_verse/web/components/src/App.jsx similarity index 86% rename from web_client/src/App.jsx rename to cogment_verse/web/components/src/App.jsx index 4f05751c..8e1d91a0 100644 --- a/web_client/src/App.jsx +++ b/cogment_verse/web/components/src/App.jsx @@ -19,20 +19,17 @@ import { useActions } from "./hooks/useActions"; import { RenderedScreen } from "./components/RenderedScreen"; import { Button } from "./components/Button"; import { Controls } from "./controls/Controls"; +import { ORCHESTRATOR_WEB_ENDPOINT } from "./utils/constants"; function App() { const [trialStatus, setTrialStatus] = useState("no trial started"); const [countdown, setCountdown] = useState(false); - const [{ trialId, runId, environment, role }, setTrialInfo] = useState({}); + const [{ trialId, runId, environment }, setTrialInfo] = useState({}); - const grpcURL = process.env.REACT_APP_ORCHESTRATOR_HTTP_ENDPOINT || "http://localhost:8081"; - - const [event, joinAnyTrial, _sendAction, trialJoined, actorConfig] = useActions( + const [event, joinAnyTrial, _sendAction, trialJoined, actorClass, actorConfig] = useActions( cogSettings, - "web_actor", // actor name - "teacher_agent", // actor class - grpcURL + ORCHESTRATOR_WEB_ENDPOINT ); const actionLock = useRef(false); @@ -58,7 +55,6 @@ function App() { ...trialInfo, environment: actorConfig?.environmentSpecs?.implementation || undefined, runId: actorConfig?.runId || undefined, - role: actorConfig?.role, })); }, [actorConfig]); const joinTrial = useCallback(() => { @@ -101,7 +97,7 @@ function App() { } />
- {trialJoined ? : null} + {trialJoined ? : null}
Status: {trialStatus}
diff --git a/web_client/src/components/Button.jsx b/cogment_verse/web/components/src/components/Button.jsx similarity index 100% rename from web_client/src/components/Button.jsx rename to cogment_verse/web/components/src/components/Button.jsx diff --git a/web_client/src/components/Countdown.jsx b/cogment_verse/web/components/src/components/Countdown.jsx similarity index 100% rename from web_client/src/components/Countdown.jsx rename to cogment_verse/web/components/src/components/Countdown.jsx diff --git a/web_client/src/components/DPad.jsx b/cogment_verse/web/components/src/components/DPad.jsx similarity index 100% rename from web_client/src/components/DPad.jsx rename to cogment_verse/web/components/src/components/DPad.jsx diff --git a/web_client/src/components/DPad.module.css b/cogment_verse/web/components/src/components/DPad.module.css similarity index 100% rename from web_client/src/components/DPad.module.css rename to cogment_verse/web/components/src/components/DPad.module.css diff --git a/web_client/src/components/FpsCounter.jsx b/cogment_verse/web/components/src/components/FpsCounter.jsx similarity index 100% rename from web_client/src/components/FpsCounter.jsx rename to cogment_verse/web/components/src/components/FpsCounter.jsx diff --git a/web_client/src/components/Joystick.jsx b/cogment_verse/web/components/src/components/Joystick.jsx similarity index 100% rename from web_client/src/components/Joystick.jsx rename to cogment_verse/web/components/src/components/Joystick.jsx diff --git a/web_client/src/components/Joystick.module.css b/cogment_verse/web/components/src/components/Joystick.module.css similarity index 100% rename from web_client/src/components/Joystick.module.css rename to cogment_verse/web/components/src/components/Joystick.module.css diff --git a/web_client/src/components/KeyboardControlList.jsx b/cogment_verse/web/components/src/components/KeyboardControlList.jsx similarity index 93% rename from web_client/src/components/KeyboardControlList.jsx rename to cogment_verse/web/components/src/components/KeyboardControlList.jsx index fc44b1c1..8d180759 100644 --- a/web_client/src/components/KeyboardControlList.jsx +++ b/cogment_verse/web/components/src/components/KeyboardControlList.jsx @@ -19,8 +19,8 @@ export const KeyboardControlList = ({ items, className, ...props }) => {
    {items .filter((item) => !!item) - .map(([label, description]) => ( -
  • + .map(([label, description], index) => ( +
  • {`${label}:`} {` ${description}`}
    • diff --git a/web_client/src/components/RenderedScreen.jsx b/cogment_verse/web/components/src/components/RenderedScreen.jsx similarity index 79% rename from web_client/src/components/RenderedScreen.jsx rename to cogment_verse/web/components/src/components/RenderedScreen.jsx index 39749e57..3e1a5be2 100644 --- a/web_client/src/components/RenderedScreen.jsx +++ b/cogment_verse/web/components/src/components/RenderedScreen.jsx @@ -29,18 +29,19 @@ const DEFAULT_SCREEN_SRC = `${process.env.PUBLIC_URL}/assets/cogment-splash.png` export const RenderedScreen = ({ observation, overlay, className, ...props }) => { const canvasRef = useRef(); + const teacherOverride = observation?.overriddenPlayers != null && observation.overriddenPlayers.length > 0; useEffect(() => { const canvas = canvasRef?.current; if (!canvas) { return; } - const pixelData = observation?.pixelData; - if (!pixelData) { + const renderedFrame = observation?.renderedFrame; + if (!renderedFrame) { return; } - canvas.src = "data:image/png;base64," + bufferToBase64(pixelData); + canvas.src = "data:image/png;base64," + bufferToBase64(renderedFrame); }, [canvasRef, observation]); return ( @@ -52,6 +53,9 @@ export const RenderedScreen = ({ observation, overlay, className, ...props }) => alt="current observation rendered pixels" /> {overlay ?
    {overlay}
    : null} + {teacherOverride ? ( +
    + ) : null}
    ); }; diff --git a/web_client/src/components/RenderedScreen.module.css b/cogment_verse/web/components/src/components/RenderedScreen.module.css similarity index 100% rename from web_client/src/components/RenderedScreen.module.css rename to cogment_verse/web/components/src/components/RenderedScreen.module.css diff --git a/web_client/src/controls/AtariPitfallControls.jsx b/cogment_verse/web/components/src/controls/AtariPitfallControls.jsx similarity index 52% rename from web_client/src/controls/AtariPitfallControls.jsx rename to cogment_verse/web/components/src/controls/AtariPitfallControls.jsx index 40c7654d..bcb2534c 100644 --- a/web_client/src/controls/AtariPitfallControls.jsx +++ b/cogment_verse/web/components/src/controls/AtariPitfallControls.jsx @@ -17,34 +17,49 @@ import { cogment_verse } from "../data_pb"; import { useDocumentKeypressListener, usePressedKeys } from "../hooks/usePressedKeys"; import { useRealTimeUpdate } from "../hooks/useRealTimeUpdate"; import { createLookup } from "../utils/controlLookup"; -import { TEACHER_NOOP_ACTION } from "../utils/constants"; +import { TEACHER_ACTOR_CLASS, TEACHER_NOOP_ACTION } from "../utils/constants"; import { Button } from "../components/Button"; import { FpsCounter } from "../components/FpsCounter"; import { KeyboardControlList } from "../components/KeyboardControlList"; // cf. https://www.gymlibrary.ml/environments/atari/#action-space const ATARI_LOOKUP = createLookup(); -ATARI_LOOKUP.setAction([], new cogment_verse.AgentAction({ discreteAction: 0 })); -ATARI_LOOKUP.setAction(["FIRE"], new cogment_verse.AgentAction({ discreteAction: 1 })); -ATARI_LOOKUP.setAction(["UP"], new cogment_verse.AgentAction({ discreteAction: 2 })); -ATARI_LOOKUP.setAction(["RIGHT"], new cogment_verse.AgentAction({ discreteAction: 3 })); -ATARI_LOOKUP.setAction(["LEFT"], new cogment_verse.AgentAction({ discreteAction: 4 })); -ATARI_LOOKUP.setAction(["DOWN"], new cogment_verse.AgentAction({ discreteAction: 5 })); -ATARI_LOOKUP.setAction(["UP", "RIGHT"], new cogment_verse.AgentAction({ discreteAction: 6 })); -ATARI_LOOKUP.setAction(["UP", "LEFT"], new cogment_verse.AgentAction({ discreteAction: 7 })); -ATARI_LOOKUP.setAction(["DOWN", "RIGHT"], new cogment_verse.AgentAction({ discreteAction: 8 })); -ATARI_LOOKUP.setAction(["DOWN", "LEFT"], new cogment_verse.AgentAction({ discreteAction: 9 })); -ATARI_LOOKUP.setAction(["UP", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 10 })); -ATARI_LOOKUP.setAction(["RIGHT", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 11 })); -ATARI_LOOKUP.setAction(["LEFT", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 12 })); -ATARI_LOOKUP.setAction(["DOWN", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 13 })); -ATARI_LOOKUP.setAction(["UP", "RIGHT", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 14 })); -ATARI_LOOKUP.setAction(["UP", "LEFT", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 15 })); -ATARI_LOOKUP.setAction(["DOWN", "RIGHT", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 16 })); -ATARI_LOOKUP.setAction(["DOWN", "LEFT", "FIRE"], new cogment_verse.AgentAction({ discreteAction: 17 })); +ATARI_LOOKUP.setAction([], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 0 }] } })); +ATARI_LOOKUP.setAction(["FIRE"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 1 }] } })); +ATARI_LOOKUP.setAction(["UP"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 2 }] } })); +ATARI_LOOKUP.setAction(["RIGHT"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 3 }] } })); +ATARI_LOOKUP.setAction(["LEFT"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 4 }] } })); +ATARI_LOOKUP.setAction(["DOWN"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 5 }] } })); +ATARI_LOOKUP.setAction(["UP", "RIGHT"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 6 }] } })); +ATARI_LOOKUP.setAction(["UP", "LEFT"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 7 }] } })); +ATARI_LOOKUP.setAction(["DOWN", "RIGHT"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 8 }] } })); +ATARI_LOOKUP.setAction(["DOWN", "LEFT"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 9 }] } })); +ATARI_LOOKUP.setAction(["UP", "FIRE"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 10 }] } })); +ATARI_LOOKUP.setAction( + ["RIGHT", "FIRE"], + new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 11 }] } }) +); +ATARI_LOOKUP.setAction(["LEFT", "FIRE"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 12 }] } })); +ATARI_LOOKUP.setAction(["DOWN", "FIRE"], new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 13 }] } })); +ATARI_LOOKUP.setAction( + ["UP", "RIGHT", "FIRE"], + new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 14 }] } }) +); +ATARI_LOOKUP.setAction( + ["UP", "LEFT", "FIRE"], + new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 15 }] } }) +); +ATARI_LOOKUP.setAction( + ["DOWN", "RIGHT", "FIRE"], + new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 16 }] } }) +); +ATARI_LOOKUP.setAction( + ["DOWN", "LEFT", "FIRE"], + new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 17 }] } }) +); export const AtariPitfallEnvironments = ["atari/Pitfall"]; -export const AtariPitfallControls = ({ sendAction, fps = 30, role, ...props }) => { +export const AtariPitfallControls = ({ sendAction, fps = 30, actorClass, ...props }) => { const [paused, setPaused] = useState(false); const togglePause = useCallback(() => setPaused((paused) => !paused), [setPaused]); useDocumentKeypressListener("p", togglePause); @@ -53,7 +68,7 @@ export const AtariPitfallControls = ({ sendAction, fps = 30, role, ...props }) = const computeAndSendAction = useCallback( (dt) => { - if (pressedKeys.size === 0 && role === cogment_verse.HumanRole.TEACHER) { + if (pressedKeys.size === 0 && actorClass === TEACHER_ACTOR_CLASS) { sendAction(TEACHER_NOOP_ACTION); return; } @@ -76,7 +91,7 @@ export const AtariPitfallControls = ({ sendAction, fps = 30, role, ...props }) = const action = ATARI_LOOKUP.getAction(controls); sendAction(action); }, - [pressedKeys, sendAction, role] + [pressedKeys, sendAction, actorClass] ); const { currentFps } = useRealTimeUpdate(computeAndSendAction, fps, paused); diff --git a/web_client/src/controls/Controls.jsx b/cogment_verse/web/components/src/controls/Controls.jsx similarity index 80% rename from web_client/src/controls/Controls.jsx rename to cogment_verse/web/components/src/controls/Controls.jsx index c296d15c..94404fc7 100644 --- a/web_client/src/controls/Controls.jsx +++ b/cogment_verse/web/components/src/controls/Controls.jsx @@ -13,7 +13,6 @@ // limitations under the License. import { useMemo } from "react"; -import * as data_pb from "../data_pb"; import { ObserverControls } from "./ObserverControls"; import { GymLunarLanderControls, GymLunarLanderEnvironments } from "./GymLunarLanderControls"; import { @@ -24,6 +23,7 @@ import { GymCartPoleEnvironments, GymCartPoleControls } from "./GymCartPoleContr import { GymMountainCarEnvironments, GymMountainCarControls } from "./GymMountainCarControls"; import { AtariPitfallEnvironments, AtariPitfallControls } from "./AtariPitfallControls"; import { TetrisEnvironments, TetrisControls } from "./TetrisControls"; +import { TEACHER_ACTOR_CLASS, PLAYER_ACTOR_CLASS, OBSERVER_ACTOR_CLASS } from "../utils/constants"; const CONTROLS = [ { environments: GymLunarLanderEnvironments, component: GymLunarLanderControls }, @@ -34,20 +34,20 @@ const CONTROLS = [ { environments: TetrisEnvironments, component: TetrisControls }, ]; -export const Controls = ({ environment, role, sendAction, fps }) => { +export const Controls = ({ environment, actorClass, sendAction, fps }) => { const ControlsComponent = useMemo(() => { - if (data_pb.cogment_verse.HumanRole.OBSERVER === role) { + if (OBSERVER_ACTOR_CLASS === actorClass) { return ObserverControls; } - if ([data_pb.cogment_verse.HumanRole.PLAYER, data_pb.cogment_verse.HumanRole.TEACHER].includes(role)) { + if ([PLAYER_ACTOR_CLASS, TEACHER_ACTOR_CLASS].includes(actorClass)) { const control = CONTROLS.find(({ environments }) => environments.includes(environment)); if (control == null) { return () =>
    {environment} is not playable
    ; } return control.component; } - return () =>
    Unknown role "{role}"
    ; - }, [environment, role]); + return () =>
    Unknown actor class "{actorClass}"
    ; + }, [environment, actorClass]); - return ; + return ; }; diff --git a/web_client/src/controls/GymCartPoleControls.jsx b/cogment_verse/web/components/src/controls/GymCartPoleControls.jsx similarity index 75% rename from web_client/src/controls/GymCartPoleControls.jsx rename to cogment_verse/web/components/src/controls/GymCartPoleControls.jsx index 95d14f34..f6f1478d 100644 --- a/web_client/src/controls/GymCartPoleControls.jsx +++ b/cogment_verse/web/components/src/controls/GymCartPoleControls.jsx @@ -16,13 +16,13 @@ import { useCallback, useState } from "react"; import { cogment_verse } from "../data_pb"; import { useDocumentKeypressListener, usePressedKeys } from "../hooks/usePressedKeys"; import { useRealTimeUpdate } from "../hooks/useRealTimeUpdate"; -import { TEACHER_NOOP_ACTION } from "../utils/constants"; +import { TEACHER_ACTOR_CLASS, TEACHER_NOOP_ACTION } from "../utils/constants"; import { Button } from "../components/Button"; import { FpsCounter } from "../components/FpsCounter"; import { KeyboardControlList } from "../components/KeyboardControlList"; -export const GymCartPoleEnvironments = ["gym/CartPole-v0"]; -export const GymCartPoleControls = ({ sendAction, fps = 30, role, ...props }) => { +export const GymCartPoleEnvironments = ["environments.gym_adapter.Environment/CartPole-v1"]; +export const GymCartPoleControls = ({ sendAction, fps = 30, actorClass, ...props }) => { const [paused, setPaused] = useState(true); const togglePause = useCallback(() => setPaused((paused) => !paused), [setPaused]); useDocumentKeypressListener("p", togglePause); @@ -31,24 +31,30 @@ export const GymCartPoleControls = ({ sendAction, fps = 30, role, ...props }) => const computeAndSendAction = useCallback( (dt) => { - if (pressedKeys.size === 0 && role === cogment_verse.HumanRole.TEACHER) { + if (pressedKeys.size === 0 && actorClass === TEACHER_ACTOR_CLASS) { sendAction(TEACHER_NOOP_ACTION); return; } const action_params = { - discreteAction: 0, + value: { + properties: [ + { + discrete: 0, + }, + ], + }, }; if (pressedKeys.has("ArrowLeft")) { - action_params.discreteAction = 0; + action_params.value.properties[0].discrete = 0; } else if (pressedKeys.has("ArrowRight")) { - action_params.discreteAction = 1; + action_params.value.properties[0].discrete = 1; } - sendAction(new cogment_verse.AgentAction(action_params)); + sendAction(new cogment_verse.PlayerAction(action_params)); }, - [pressedKeys, sendAction, role] + [pressedKeys, sendAction, actorClass] ); const { currentFps } = useRealTimeUpdate(computeAndSendAction, fps, paused); diff --git a/web_client/src/controls/GymLunarLanderContinuousControls.jsx b/cogment_verse/web/components/src/controls/GymLunarLanderContinuousControls.jsx similarity index 83% rename from web_client/src/controls/GymLunarLanderContinuousControls.jsx rename to cogment_verse/web/components/src/controls/GymLunarLanderContinuousControls.jsx index 71c7a223..8bb1841d 100644 --- a/web_client/src/controls/GymLunarLanderContinuousControls.jsx +++ b/cogment_verse/web/components/src/controls/GymLunarLanderContinuousControls.jsx @@ -16,15 +16,15 @@ import { useCallback, useState } from "react"; import { cogment_verse } from "../data_pb"; import { useDocumentKeypressListener, usePressedKeys } from "../hooks/usePressedKeys"; import { useRealTimeUpdate } from "../hooks/useRealTimeUpdate"; -import { TEACHER_NOOP_ACTION } from "../utils/constants"; +import { TEACHER_ACTOR_CLASS, TEACHER_NOOP_ACTION } from "../utils/constants"; import { Button } from "../components/Button"; import { useJoystickState, Joystick } from "../components/Joystick"; import { FpsCounter } from "../components/FpsCounter"; import { KeyboardControlList } from "../components/KeyboardControlList"; -export const GymLunarLanderContinuousEnvironments = ["gym/LunarLanderContinuous-v2"]; -export const GymLunarLanderContinuousControls = ({ sendAction, fps = 20, role, ...props }) => { - const isTeacher = role === cogment_verse.HumanRole.TEACHER; +export const GymLunarLanderContinuousEnvironments = ["environments.gym_adapter.Environment/LunarLanderContinuous-v2"]; +export const GymLunarLanderContinuousControls = ({ sendAction, fps = 20, actorClass, ...props }) => { + const isTeacher = actorClass === TEACHER_ACTOR_CLASS; const [paused, setPaused] = useState(false); const togglePause = useCallback(() => setPaused((paused) => !paused), [setPaused]); useDocumentKeypressListener("p", togglePause); @@ -38,10 +38,8 @@ export const GymLunarLanderContinuousControls = ({ sendAction, fps = 20, role, . (dt) => { if (isJoystickActive) { sendAction( - new cogment_verse.AgentAction({ - continuousAction: { - data: [joystickPosition[1], -joystickPosition[0]], - }, + new cogment_verse.PlayerAction({ + value: { properties: [{ simpleBox: { values: [joystickPosition[1], -joystickPosition[0]] } }] }, }) ); setKeyboardActive(false); @@ -65,11 +63,7 @@ export const GymLunarLanderContinuousControls = ({ sendAction, fps = 20, role, . if (keyboardAction != null) { sendAction( - new cogment_verse.AgentAction({ - continuousAction: { - data: keyboardAction, - }, - }) + new cogment_verse.PlayerAction({ value: { properties: [{ simpleBox: { values: keyboardAction } }] } }) ); setKeyboardActive(true); setJoystickState([-keyboardAction[1], keyboardAction[0]], false); @@ -79,13 +73,7 @@ export const GymLunarLanderContinuousControls = ({ sendAction, fps = 20, role, . if (isTeacher) { sendAction(TEACHER_NOOP_ACTION); } else { - sendAction( - new cogment_verse.AgentAction({ - continuousAction: { - data: [0, 0], - }, - }) - ); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ simpleBox: { values: [0, 0] } }] } })); } setKeyboardActive(false); setJoystickState([0, 0], false); diff --git a/web_client/src/controls/GymLunarLanderControls.jsx b/cogment_verse/web/components/src/controls/GymLunarLanderControls.jsx similarity index 80% rename from web_client/src/controls/GymLunarLanderControls.jsx rename to cogment_verse/web/components/src/controls/GymLunarLanderControls.jsx index 4c1a0aff..d7a78175 100644 --- a/web_client/src/controls/GymLunarLanderControls.jsx +++ b/cogment_verse/web/components/src/controls/GymLunarLanderControls.jsx @@ -16,7 +16,7 @@ import { useCallback, useState } from "react"; import { cogment_verse } from "../data_pb"; import { useDocumentKeypressListener, usePressedKeys } from "../hooks/usePressedKeys"; import { useRealTimeUpdate } from "../hooks/useRealTimeUpdate"; -import { TEACHER_NOOP_ACTION } from "../utils/constants"; +import { TEACHER_ACTOR_CLASS, TEACHER_NOOP_ACTION } from "../utils/constants"; import { DPad, usePressedButtons, DPAD_BUTTONS } from "../components/DPad"; import { Button } from "../components/Button"; import { FpsCounter } from "../components/FpsCounter"; @@ -25,10 +25,10 @@ import { KeyboardControlList } from "../components/KeyboardControlList"; const DEACTIVATED_BUTTONS_TEACHER = []; const DEACTIVATED_BUTTONS_PLAYER = [DPAD_BUTTONS.UP]; -export const GymLunarLanderEnvironments = ["gym/LunarLander-v2"]; +export const GymLunarLanderEnvironments = ["environments.gym_adapter.Environment/LunarLander-v2"]; -export const GymLunarLanderControls = ({ sendAction, fps = 20, role, ...props }) => { - const isTeacher = role === cogment_verse.HumanRole.TEACHER; +export const GymLunarLanderControls = ({ sendAction, fps = 20, actorClass, ...props }) => { + const isTeacher = actorClass === TEACHER_ACTOR_CLASS; const [paused, setPaused] = useState(false); const togglePause = useCallback(() => setPaused((paused) => !paused), [setPaused]); useDocumentKeypressListener("p", togglePause); @@ -41,36 +41,20 @@ export const GymLunarLanderControls = ({ sendAction, fps = 20, role, ...props }) (dt) => { if (pressedKeys.has("ArrowRight") || isButtonPressed(DPAD_BUTTONS.RIGHT)) { setActiveButtons([DPAD_BUTTONS.RIGHT]); - sendAction( - new cogment_verse.AgentAction({ - discreteAction: 1, - }) - ); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 1 }] } })); return; } else if (pressedKeys.has("ArrowDown") || isButtonPressed(DPAD_BUTTONS.DOWN)) { setActiveButtons([DPAD_BUTTONS.DOWN]); - sendAction( - new cogment_verse.AgentAction({ - discreteAction: 2, - }) - ); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 2 }] } })); return; } else if (pressedKeys.has("ArrowLeft") || isButtonPressed(DPAD_BUTTONS.LEFT)) { setActiveButtons([DPAD_BUTTONS.LEFT]); - sendAction( - new cogment_verse.AgentAction({ - discreteAction: 3, - }) - ); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 3 }] } })); return; } else if (isTeacher) { if (pressedKeys.has("ArrowUp") || isButtonPressed(DPAD_BUTTONS.UP)) { setActiveButtons([DPAD_BUTTONS.UP]); - sendAction( - new cogment_verse.AgentAction({ - discreteAction: 0, - }) - ); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 0 }] } })); return; } setActiveButtons([]); @@ -78,11 +62,7 @@ export const GymLunarLanderControls = ({ sendAction, fps = 20, role, ...props }) return; } setActiveButtons([]); - sendAction( - new cogment_verse.AgentAction({ - discreteAction: 0, - }) - ); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 0 }] } })); }, [isButtonPressed, pressedKeys, sendAction, setActiveButtons, isTeacher] ); diff --git a/cogment_verse/web/components/src/controls/GymMountainCarControls.jsx b/cogment_verse/web/components/src/controls/GymMountainCarControls.jsx new file mode 100644 index 00000000..fcbc68d1 --- /dev/null +++ b/cogment_verse/web/components/src/controls/GymMountainCarControls.jsx @@ -0,0 +1,90 @@ +// Copyright 2021 AI Redefined Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { useCallback, useState } from "react"; +import { cogment_verse } from "../data_pb"; +import { useDocumentKeypressListener, usePressedKeys } from "../hooks/usePressedKeys"; +import { useRealTimeUpdate } from "../hooks/useRealTimeUpdate"; +import { TEACHER_ACTOR_CLASS, TEACHER_NOOP_ACTION } from "../utils/constants"; +import { DPad, usePressedButtons, DPAD_BUTTONS } from "../components/DPad"; +import { Button } from "../components/Button"; +import { FpsCounter } from "../components/FpsCounter"; +import { KeyboardControlList } from "../components/KeyboardControlList"; + +const DEACTIVATED_BUTTONS_TEACHER = [DPAD_BUTTONS.UP]; +const DEACTIVATED_BUTTONS_PLAYER = [DPAD_BUTTONS.UP, DPAD_BUTTONS.DOWN]; + +export const GymMountainCarEnvironments = ["environments.gym_adapter.Environment/MountainCar-v0"]; +export const GymMountainCarControls = ({ sendAction, fps = 30, actorClass, ...props }) => { + const isTeacher = actorClass === TEACHER_ACTOR_CLASS; + const [paused, setPaused] = useState(false); + const togglePause = useCallback(() => setPaused((paused) => !paused), [setPaused]); + useDocumentKeypressListener("p", togglePause); + + const pressedKeys = usePressedKeys(); + const { pressedButtons, isButtonPressed, setPressedButtons } = usePressedButtons(); + const [activeButtons, setActiveButtons] = useState([]); + + const computeAndSendAction = useCallback( + (dt) => { + if (pressedKeys.has("ArrowLeft") || isButtonPressed(DPAD_BUTTONS.LEFT)) { + setActiveButtons([DPAD_BUTTONS.LEFT]); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 0 }] } })); + } else if (pressedKeys.has("ArrowRight") || isButtonPressed(DPAD_BUTTONS.RIGHT)) { + setActiveButtons([DPAD_BUTTONS.RIGHT]); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 2 }] } })); + } else if (isTeacher) { + if (pressedKeys.has("ArrowDown") || isButtonPressed(DPAD_BUTTONS.DOWN)) { + setActiveButtons([DPAD_BUTTONS.DOWN]); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 1 }] } })); + } else { + setActiveButtons([]); + sendAction(TEACHER_NOOP_ACTION); + } + } else { + setActiveButtons([]); + sendAction(new cogment_verse.PlayerAction({ value: { properties: [{ discrete: 1 }] } })); + } + }, + [isButtonPressed, isTeacher, pressedKeys, sendAction] + ); + + const { currentFps } = useRealTimeUpdate(computeAndSendAction, fps, paused); + + return ( +
    +
    + +
    +
    + + +
    + +
    + ); +}; diff --git a/web_client/src/controls/ObserverControls.jsx b/cogment_verse/web/components/src/controls/ObserverControls.jsx similarity index 100% rename from web_client/src/controls/ObserverControls.jsx rename to cogment_verse/web/components/src/controls/ObserverControls.jsx diff --git a/web_client/src/controls/TetrisControls.jsx b/cogment_verse/web/components/src/controls/TetrisControls.jsx similarity index 71% rename from web_client/src/controls/TetrisControls.jsx rename to cogment_verse/web/components/src/controls/TetrisControls.jsx index 1e3844f7..e8b7b37e 100644 --- a/web_client/src/controls/TetrisControls.jsx +++ b/cogment_verse/web/components/src/controls/TetrisControls.jsx @@ -17,27 +17,27 @@ import { cogment_verse } from "../data_pb"; import { useDocumentKeypressListener, usePressedKeys } from "../hooks/usePressedKeys"; import { useRealTimeUpdate } from "../hooks/useRealTimeUpdate"; import { createLookup } from "../utils/controlLookup"; -import { TEACHER_NOOP_ACTION } from "../utils/constants"; +import { TEACHER_ACTOR_CLASS, TEACHER_NOOP_ACTION } from "../utils/constants"; import { Button } from "../components/Button"; import { FpsCounter } from "../components/FpsCounter"; import { KeyboardControlList } from "../components/KeyboardControlList"; const TETRIS_LOOKUP = createLookup(); -TETRIS_LOOKUP.setAction([], new cogment_verse.AgentAction({ discreteAction: 0 })); -TETRIS_LOOKUP.setAction(["TURN_CW"], new cogment_verse.AgentAction({ discreteAction: 1 })); -TETRIS_LOOKUP.setAction(["TURN_CCW"], new cogment_verse.AgentAction({ discreteAction: 2 })); -TETRIS_LOOKUP.setAction(["RIGHT"], new cogment_verse.AgentAction({ discreteAction: 3 })); -TETRIS_LOOKUP.setAction(["RIGHT", "TURN_CW"], new cogment_verse.AgentAction({ discreteAction: 4 })); -TETRIS_LOOKUP.setAction(["RIGHT", "TURN_CCW"], new cogment_verse.AgentAction({ discreteAction: 5 })); -TETRIS_LOOKUP.setAction(["LEFT"], new cogment_verse.AgentAction({ discreteAction: 6 })); -TETRIS_LOOKUP.setAction(["LEFT", "TURN_CW"], new cogment_verse.AgentAction({ discreteAction: 7 })); -TETRIS_LOOKUP.setAction(["LEFT", "TURN_CCW"], new cogment_verse.AgentAction({ discreteAction: 8 })); -TETRIS_LOOKUP.setAction(["DOWN"], new cogment_verse.AgentAction({ discreteAction: 9 })); -TETRIS_LOOKUP.setAction(["DOWN", "TURN_CW"], new cogment_verse.AgentAction({ discreteAction: 10 })); -TETRIS_LOOKUP.setAction(["DOWN", "TURN_CCW"], new cogment_verse.AgentAction({ discreteAction: 11 })); +TETRIS_LOOKUP.setAction([], new cogment_verse.PlayerAction({ discreteAction: 0 })); +TETRIS_LOOKUP.setAction(["TURN_CW"], new cogment_verse.PlayerAction({ discreteAction: 1 })); +TETRIS_LOOKUP.setAction(["TURN_CCW"], new cogment_verse.PlayerAction({ discreteAction: 2 })); +TETRIS_LOOKUP.setAction(["RIGHT"], new cogment_verse.PlayerAction({ discreteAction: 3 })); +TETRIS_LOOKUP.setAction(["RIGHT", "TURN_CW"], new cogment_verse.PlayerAction({ discreteAction: 4 })); +TETRIS_LOOKUP.setAction(["RIGHT", "TURN_CCW"], new cogment_verse.PlayerAction({ discreteAction: 5 })); +TETRIS_LOOKUP.setAction(["LEFT"], new cogment_verse.PlayerAction({ discreteAction: 6 })); +TETRIS_LOOKUP.setAction(["LEFT", "TURN_CW"], new cogment_verse.PlayerAction({ discreteAction: 7 })); +TETRIS_LOOKUP.setAction(["LEFT", "TURN_CCW"], new cogment_verse.PlayerAction({ discreteAction: 8 })); +TETRIS_LOOKUP.setAction(["DOWN"], new cogment_verse.PlayerAction({ discreteAction: 9 })); +TETRIS_LOOKUP.setAction(["DOWN", "TURN_CW"], new cogment_verse.PlayerAction({ discreteAction: 10 })); +TETRIS_LOOKUP.setAction(["DOWN", "TURN_CCW"], new cogment_verse.PlayerAction({ discreteAction: 11 })); export const TetrisEnvironments = ["tetris/TetrisA-v0"]; -export const TetrisControls = ({ sendAction, fps = 30, role, ...props }) => { +export const TetrisControls = ({ sendAction, fps = 30, actorClass, ...props }) => { const [paused, setPaused] = useState(false); const togglePause = useCallback(() => setPaused((paused) => !paused), [setPaused]); useDocumentKeypressListener("p", togglePause); @@ -46,7 +46,7 @@ export const TetrisControls = ({ sendAction, fps = 30, role, ...props }) => { const computeAndSendAction = useCallback( (dt) => { - if (pressedKeys.size === 0 && role === cogment_verse.HumanRole.TEACHER) { + if (pressedKeys.size === 0 && actorClass === TEACHER_ACTOR_CLASS) { sendAction(TEACHER_NOOP_ACTION); return; } @@ -67,7 +67,7 @@ export const TetrisControls = ({ sendAction, fps = 30, role, ...props }) => { const action = TETRIS_LOOKUP.getAction(controls); sendAction(action); }, - [pressedKeys, sendAction, role] + [pressedKeys, sendAction, actorClass] ); const { currentFps } = useRealTimeUpdate(computeAndSendAction, fps, paused); diff --git a/web_client/src/hooks/useActions.ts b/cogment_verse/web/components/src/hooks/useActions.ts similarity index 65% rename from web_client/src/hooks/useActions.ts rename to cogment_verse/web/components/src/hooks/useActions.ts index 6033b391..599eb4a8 100644 --- a/web_client/src/hooks/useActions.ts +++ b/cogment_verse/web/components/src/hooks/useActions.ts @@ -1,4 +1,4 @@ -// Copyright 2021 AI Redefined Inc. +// Copyright 2022 AI Redefined Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { CogSettings, Context, MessageBase, Reward } from "@cogment/cogment-js-sdk"; +import { ActorImplementation, CogSettings, Context, MessageBase, Reward } from "@cogment/cogment-js-sdk"; import { CogMessage } from "@cogment/cogment-js-sdk/dist/cogment/types/CogMessage"; import { useCallback, useEffect, useState } from "react"; +import { WEB_ACTOR_NAME, PLAYER_ACTOR_CLASS } from "../utils/constants"; + export type SendAction = (action: ActionT) => void; type JoinTrial = (trialId: string) => false | Promise; @@ -38,22 +40,19 @@ export type Policy = (event: Event) => Acti export type WatchTrials = () => void; export type UseActions = ( _cogSettings: CogSettings, - actorName: string, - actorClass: string, - grpcURL: string + cogmentOrchestratorWebEndpoint: string ) => [ event: Event, joinAnyTrial: JoinAnyTrial, sendAction: SendAction | undefined, trialJoined: boolean, + actorClassName: string | undefined, actorConfig: ActorConfigT | undefined ]; export const useActions: UseActions = ( cogSettings: CogSettings, - actorName: string, - actorClass: string, - grpcURL: string + cogmentOrchestratorWebEndpoint: string ) => { type EventT = Event; @@ -73,60 +72,61 @@ export const useActions: UseActions = (); - const [actorConfig, setActorConfig] = useState(); + const [actorConfig, setActorConfig] = useState(); + const [actorClassName, setActorClassName] = useState(); //Set up the connection and register the actor only once, regardless of re-rendering useEffect(() => { - const actor = { name: actorName, actorClass: actorClass }; - - const context = new Context(cogSettings, actorName); + const context = new Context(cogSettings, "cogment_verse_web"); - context.registerActor( - async (actorSession) => { - let tickId = 0; + const actorImplementation: ActorImplementation = async (actorSession) => { + let tickId = 0; - actorSession.start(); + actorSession.start(); - // todo: figure out why this cast is necessary (wrong template argument somewhere?) - setActorConfig(actorSession.config as ActorConfigT); + setActorConfig(actorSession.config); + setActorClassName(actorSession.className); - //Double arrow function here beause react will turn a single one into a lazy loaded function - setSendAction(() => (action: ActionT) => { - actorSession.doAction(action); - }); + //Double arrow function here beause react will turn a single one into a lazy loaded function + setSendAction(() => (action: ActionT) => { + actorSession.doAction(action); + }); - for await (const { observation, messages, rewards, type } of actorSession.eventLoop()) { - //Parse the observation into a regular JS object - //TODO: this will eventually be part of the API + for await (const { observation, messages, rewards, type } of actorSession.eventLoop()) { + //Parse the observation into a regular JS object + //TODO: this will eventually be part of the API - let observationOBJ = observation && (observation as ObservationT | undefined); + let observationOBJ = observation && (observation as ObservationT | undefined); - let next_event = { - observation: observationOBJ, - message: messages[0], - reward: rewards[0], - last: type === 3, - tickId: tickId++, - }; + let next_event = { + observation: observationOBJ, + message: messages[0], + reward: rewards[0], + last: type === 3, + tickId: tickId++, + }; - setEvent(next_event); + setEvent(next_event); - if (next_event.last) { - break; - } + if (next_event.last) { + break; } - }, - actor.name, - actor.actorClass + } + } + + context.registerActor( + actorImplementation, + WEB_ACTOR_NAME, + PLAYER_ACTOR_CLASS // actually what we should do is [TEACHER_ACTOR_CLASS, PLAYER_ACTOR_CLASS, OBSERVER_ACTOR_CLASS] ); //Creating the trial controller must happen after actors are registered - const trialController = context.getController(grpcURL); + const trialController = context.getController(cogmentOrchestratorWebEndpoint); setJoinTrial(() => (trialId: string) => { try { setTrialJoined(true); - const joinTrialPromise = context.joinTrial(trialId, grpcURL, actor.name).then(() => setTrialJoined(false)); + const joinTrialPromise = context.joinTrial(trialId, cogmentOrchestratorWebEndpoint, WEB_ACTOR_NAME).then(() => setTrialJoined(false)); return joinTrialPromise; } catch (error) { console.log(`failed to start trial: ${error}`); @@ -152,7 +152,7 @@ export const useActions: UseActions = { if (!watchTrials) return; @@ -168,5 +168,5 @@ export const useActions: UseActions = +// Copyright 2022 AI Redefined Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/web_client/src/hooks/usePressedKeys.jsx b/cogment_verse/web/components/src/hooks/usePressedKeys.jsx similarity index 100% rename from web_client/src/hooks/usePressedKeys.jsx rename to cogment_verse/web/components/src/hooks/usePressedKeys.jsx diff --git a/web_client/src/hooks/useRealTimeUpdate.jsx b/cogment_verse/web/components/src/hooks/useRealTimeUpdate.jsx similarity index 100% rename from web_client/src/hooks/useRealTimeUpdate.jsx rename to cogment_verse/web/components/src/hooks/useRealTimeUpdate.jsx diff --git a/web_client/src/index.css b/cogment_verse/web/components/src/index.css similarity index 100% rename from web_client/src/index.css rename to cogment_verse/web/components/src/index.css diff --git a/web_client/src/index.tsx b/cogment_verse/web/components/src/index.tsx similarity index 100% rename from web_client/src/index.tsx rename to cogment_verse/web/components/src/index.tsx diff --git a/web_client/src/logo.svg b/cogment_verse/web/components/src/logo.svg similarity index 100% rename from web_client/src/logo.svg rename to cogment_verse/web/components/src/logo.svg diff --git a/web_client/src/react-app-env.d.ts b/cogment_verse/web/components/src/react-app-env.d.ts similarity index 91% rename from web_client/src/react-app-env.d.ts rename to cogment_verse/web/components/src/react-app-env.d.ts index 407e86e1..11f7d617 100644 --- a/web_client/src/react-app-env.d.ts +++ b/cogment_verse/web/components/src/react-app-env.d.ts @@ -1,4 +1,4 @@ -// Copyright 2021 AI Redefined Inc. +// Copyright 2022 AI Redefined Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/web_client/src/reportWebVitals.js b/cogment_verse/web/components/src/reportWebVitals.js similarity index 93% rename from web_client/src/reportWebVitals.js rename to cogment_verse/web/components/src/reportWebVitals.js index ed84e248..306c662a 100644 --- a/web_client/src/reportWebVitals.js +++ b/cogment_verse/web/components/src/reportWebVitals.js @@ -1,4 +1,4 @@ -// Copyright 2021 AI Redefined Inc. +// Copyright 2022 AI Redefined Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/web_client/src/utils/constants.js b/cogment_verse/web/components/src/utils/constants.js similarity index 54% rename from web_client/src/utils/constants.js rename to cogment_verse/web/components/src/utils/constants.js index ee0b90ac..5aa0cf8d 100644 --- a/web_client/src/utils/constants.js +++ b/cogment_verse/web/components/src/utils/constants.js @@ -1,4 +1,4 @@ -// Copyright 2021 AI Redefined Inc. +// Copyright 2022 AI Redefined Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,4 +14,15 @@ import { cogment_verse } from "../data_pb"; -export const TEACHER_NOOP_ACTION = new cogment_verse.AgentAction({ discreteAction: -1 }); +export const TEACHER_NOOP_ACTION = new cogment_verse.TeacherAction({ value: null }); + +export const WEB_ACTOR_NAME = "web_actor"; + +export const TEACHER_ACTOR_CLASS = "teacher"; +export const PLAYER_ACTOR_CLASS = "player"; +export const OBSERVER_ACTOR_CLASS = "observer"; + +export const ORCHESTRATOR_WEB_ENDPOINT = + window.ORCHESTRATOR_WEB_ENDPOINT !== "" + ? window.ORCHESTRATOR_WEB_ENDPOINT + : process.env.REACT_APP_ORCHESTRATOR_WEB_ENDPOINT; diff --git a/web_client/src/utils/controlLookup.js b/cogment_verse/web/components/src/utils/controlLookup.js similarity index 95% rename from web_client/src/utils/controlLookup.js rename to cogment_verse/web/components/src/utils/controlLookup.js index cf8dda43..4aef3197 100644 --- a/web_client/src/utils/controlLookup.js +++ b/cogment_verse/web/components/src/utils/controlLookup.js @@ -1,4 +1,4 @@ -// Copyright 2021 AI Redefined Inc. +// Copyright 2022 AI Redefined Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/web_client/tailwind.config.js b/cogment_verse/web/components/tailwind.config.js similarity index 91% rename from web_client/tailwind.config.js rename to cogment_verse/web/components/tailwind.config.js index a9b6e6e9..b7aa3de0 100644 --- a/web_client/tailwind.config.js +++ b/cogment_verse/web/components/tailwind.config.js @@ -1,4 +1,4 @@ -// Copyright 2021 AI Redefined Inc. +// Copyright 2022 AI Redefined Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/web_client/tsconfig.json b/cogment_verse/web/components/tsconfig.json similarity index 100% rename from web_client/tsconfig.json rename to cogment_verse/web/components/tsconfig.json diff --git a/cogment_verse/web/package-lock.json b/cogment_verse/web/package-lock.json new file mode 100644 index 00000000..48e341a0 --- /dev/null +++ b/cogment_verse/web/package-lock.json @@ -0,0 +1,3 @@ +{ + "lockfileVersion": 1 +} diff --git a/cogment_verse/web/server/dev_server.py b/cogment_verse/web/server/dev_server.py new file mode 100644 index 00000000..59516d57 --- /dev/null +++ b/cogment_verse/web/server/dev_server.py @@ -0,0 +1,33 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from ..utils.npm import NPM_BIN + + +def create_dev_server_popen_kwargs(port, orchestrator_web_endpoint, **kwargs): + extended_env = os.environ.copy() + extended_env |= { + "PORT": f"{port}", + "BROWSER": "none", # Don't automatically open a browser + "REACT_APP_ORCHESTRATOR_WEB_ENDPOINT": orchestrator_web_endpoint, + } + + return { + **kwargs, + "args": [NPM_BIN, "run", "start"], + "env": extended_env, + "cwd": os.path.abspath(os.path.join(os.path.dirname(__file__), "../components")), + } diff --git a/cogment_verse/web/server/server.py b/cogment_verse/web/server/server.py new file mode 100644 index 00000000..8f68b56a --- /dev/null +++ b/cogment_verse/web/server/server.py @@ -0,0 +1,72 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os + +from starlette.applications import Starlette +from starlette.responses import HTMLResponse +from starlette.routing import Mount, Route +from starlette.staticfiles import StaticFiles +import uvicorn + +log = logging.getLogger(__name__) + +WEB_CFG_ENV_VAR = "COGMENT_VERSE_ORCHESTRATOR_WEB_CFG" + + +def create_app(): + web_app_cfg = json.loads(os.getenv(WEB_CFG_ENV_VAR)) + + with open(os.path.join(web_app_cfg["served_dir"], "index.html"), encoding="utf-8") as homepage_file: + homepage_content = homepage_file.read() + + orchestrator_web_endpoint = web_app_cfg["orchestrator_web_endpoint"] + homepage_content = homepage_content.replace( + 'ORCHESTRATOR_WEB_ENDPOINT=""', f'ORCHESTRATOR_WEB_ENDPOINT="{orchestrator_web_endpoint}"' + ) + + async def homepage(_request): + return HTMLResponse(homepage_content) + + return Starlette( + routes=[ + Route("/", endpoint=homepage), + Mount("/", app=StaticFiles(directory=web_app_cfg["served_dir"], html=True)), + ] + ) + + +def server_main( + name, # pylint: disable=unused-argument + on_ready, + orchestrator_web_endpoint, + port, + served_dir, +): + # Writing the configuration for the web app in an environment variable + web_app_cfg = {"orchestrator_web_endpoint": orchestrator_web_endpoint, "served_dir": served_dir} + os.environ[WEB_CFG_ENV_VAR] = json.dumps(web_app_cfg) + + on_ready() + + log.info(f"Starting production web server on port [{port}]") + uvicorn.run( + "cogment_verse.web.server.server:create_app", + factory=True, + host="0.0.0.0", + port=port, + log_level="error", + ) diff --git a/cogment_verse/web/utils/generate.py b/cogment_verse/web/utils/generate.py new file mode 100644 index 00000000..4f4950b2 --- /dev/null +++ b/cogment_verse/web/utils/generate.py @@ -0,0 +1,49 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import shutil + + +from .npm import npm_command + +log = logging.getLogger(__name__) + + +def generate(specs_filename, web_dir, force=False): + cog_generate_sample_out_file = os.path.join(web_dir, "src/CogSettings.d.ts") + cog_generate_source_files = [ + specs_filename, + os.path.join(os.path.dirname(specs_filename), "data.proto"), + ] # TODO make that more generic + + do_generate = force + + if not do_generate and not os.path.isfile(cog_generate_sample_out_file): + do_generate = True + + if not do_generate: + cog_generate_output_mtime = os.stat(cog_generate_sample_out_file).st_mtime + do_generate = any( + os.stat(source_file).st_mtime > cog_generate_output_mtime for source_file in cog_generate_source_files + ) + + if do_generate: + for source_file in cog_generate_source_files: + shutil.copy(source_file, web_dir) + log.info("Running code generation for the web components using `npm run cogment_generate`...") + npm_command(["run", "cogment_generate"], web_dir) + else: + log.info("Nothing to do for the web components code generation") diff --git a/cogment_verse/web/utils/npm.py b/cogment_verse/web/utils/npm.py new file mode 100644 index 00000000..dc9f7430 --- /dev/null +++ b/cogment_verse/web/utils/npm.py @@ -0,0 +1,31 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import shutil +import subprocess + +NPM_BIN = shutil.which("npm") + +log = logging.getLogger(__name__) + + +def npm_command(args, cwd): + try: + subprocess.run([NPM_BIN, *args], cwd=cwd, capture_output=True, check=True) + except subprocess.CalledProcessError as err: + log.error( + f"Error while running [{args}] in [{cwd}]\n---STDOUT---\n{err.stdout.decode('utf-8')}\n---STDERR---\n{err.stderr.decode('utf-8')}" + ) + raise RuntimeError(f"Error while running [{args}] in [{cwd}]") from err diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 00000000..71878888 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,14 @@ +defaults: + - services: local_base_services + - services/environment: mountain_car + - services/actor: random + - run: play + - _self_ + +hydra: + job: + name: ${run.class_name} + id: ${run.run_id} +run: + run_id: ${generate_name:run_id}_${hydra:job.num} + mlflow_tracking_uri: http://localhost:3000 diff --git a/config/experiment/simple_a2c/cartpole.yaml b/config/experiment/simple_a2c/cartpole.yaml new file mode 100644 index 00000000..4f627c7e --- /dev/null +++ b/config/experiment/simple_a2c/cartpole.yaml @@ -0,0 +1,18 @@ +# @package _global_ +defaults: + - override /services/actor: simple_a2c + - override /run: simple_a2c + - override /services/environment: cartpole + +run: + actor_network: + hidden_size: 32 + critic_network: + hidden_size: 32 + training: + discount_factor: 0.95 + entropy_loss_coef: 0.05 + value_loss_coef: 1.0 + action_loss_coef: 0.1 + environment_config: + seed: 15 diff --git a/config/run/observe.yaml b/config/run/observe.yaml new file mode 100644 index 00000000..af59754f --- /dev/null +++ b/config/run/observe.yaml @@ -0,0 +1,6 @@ +class_name: runs.play.PlayRun +observer: True +trial_count: 10 +players: + - name: player_1 + implementation: actors.random_actor.RandomActor diff --git a/config/run/play.yaml b/config/run/play.yaml new file mode 100644 index 00000000..1e8117b3 --- /dev/null +++ b/config/run/play.yaml @@ -0,0 +1,5 @@ +class_name: runs.play.PlayRun +trial_count: 1 +players: + - name: player_1 + implementation: client diff --git a/config/run/simple_a2c.yaml b/config/run/simple_a2c.yaml new file mode 100644 index 00000000..02e36ae6 --- /dev/null +++ b/config/run/simple_a2c.yaml @@ -0,0 +1,16 @@ +class_name: actors.simple_a2c.SimpleA2CTraining +environment_config: + seed: 618 +training: + epoch_count: 500 + epoch_trial_count: 10 + num_parallel_trials: 8 + discount_factor: 0.95 + entropy_loss_coef: 0.05 + value_loss_coef: 0.5 + action_loss_coef: 1.0 + learning_rate: 0.01 +actor_network: + hidden_size: 64 +critic_network: + hidden_size": 64 diff --git a/config/run/simple_bc.yaml b/config/run/simple_bc.yaml new file mode 100644 index 00000000..5cbcbb20 --- /dev/null +++ b/config/run/simple_bc.yaml @@ -0,0 +1,5 @@ +class_name: actors.tutorial.tutorial_4.SimpleBCTraining +environment_config: + seed: 618 +training: + num_trials: 15 diff --git a/config/services/actor/random.yaml b/config/services/actor/random.yaml new file mode 100644 index 00000000..a2367c1d --- /dev/null +++ b/config/services/actor/random.yaml @@ -0,0 +1,2 @@ +- class_name: actors.random_actor.RandomActor + port: ${generate_port:actors.random_actor.RandomActor} diff --git a/config/services/actor/simple_a2c.yaml b/config/services/actor/simple_a2c.yaml new file mode 100644 index 00000000..95d31cec --- /dev/null +++ b/config/services/actor/simple_a2c.yaml @@ -0,0 +1,2 @@ +- class_name: actors.simple_a2c.SimpleA2CActor + port: ${generate_port:actors.simple_a2c.SimpleA2CActor} diff --git a/config/services/actor/simple_bc.yaml b/config/services/actor/simple_bc.yaml new file mode 100644 index 00000000..ed7e02e3 --- /dev/null +++ b/config/services/actor/simple_bc.yaml @@ -0,0 +1,2 @@ +- class_name: actors.tutorial.tutorial_4.SimpleBCActor + port: ${generate_port:actor_2} diff --git a/config/services/environment/cartpole.yaml b/config/services/environment/cartpole.yaml new file mode 100644 index 00000000..337fadd5 --- /dev/null +++ b/config/services/environment/cartpole.yaml @@ -0,0 +1,3 @@ +class_name: environments.gym_adapter.Environment +env_name: CartPole-v1 +port: ${generate_port:environments.gym_adapter.Environment} diff --git a/config/services/environment/lunar_lander.yaml b/config/services/environment/lunar_lander.yaml new file mode 100644 index 00000000..dc458efc --- /dev/null +++ b/config/services/environment/lunar_lander.yaml @@ -0,0 +1,3 @@ +class_name: environments.gym_adapter.Environment +env_name: LunarLander-v2 +port: ${generate_port:environments.gym_adapter.Environment} diff --git a/config/services/environment/lunar_lander_continuous.yaml b/config/services/environment/lunar_lander_continuous.yaml new file mode 100644 index 00000000..e2f08259 --- /dev/null +++ b/config/services/environment/lunar_lander_continuous.yaml @@ -0,0 +1,3 @@ +class_name: environments.gym_adapter.Environment +env_name: LunarLanderContinuous-v2 +port: ${generate_port:environments.gym_adapter.Environment} diff --git a/config/services/environment/mountain_car.yaml b/config/services/environment/mountain_car.yaml new file mode 100644 index 00000000..49569751 --- /dev/null +++ b/config/services/environment/mountain_car.yaml @@ -0,0 +1,3 @@ +class_name: environments.gym_adapter.Environment +env_name: MountainCar-v0 +port: ${generate_port:environments.gym_adapter.Environment} diff --git a/config/services/local_base_services.yaml b/config/services/local_base_services.yaml new file mode 100644 index 00000000..2b0c701b --- /dev/null +++ b/config/services/local_base_services.yaml @@ -0,0 +1,14 @@ +orchestrator: + - port: ${generate_port:orchestrator} + web_port: ${generate_port:orchestrator_web} + log_level: warning +trial_datastore: + port: ${generate_port:trial_datastore} + log_level: warning +model_registry: + port: ${generate_port:model_registry} + log_level: warning +web: + port: 8080 + build: False + dev: False diff --git a/data.proto b/data.proto deleted file mode 100644 index 12039367..00000000 --- a/data.proto +++ /dev/null @@ -1,369 +0,0 @@ -// Copyright 2021 AI Redefined Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package cogment_verse; - -message NDArray { - string dtype = 1; - repeated uint32 shape = 2; - bytes data = 3; -} - -message EnvironmentConfig { - string run_id = 1; - bool render = 5; - bool flatten = 6; - int32 render_width = 7; - uint32 framestack = 8; - uint32 seed = 9; - string mode = 10; -} - -// Space messages are used to define environments action or observation space -// Similar to gym's "dict" space -message Space { - message Discrete { - int32 num = 1; - repeated string labels = 2; // Define labels for the discrete elements in the space, overrides `num` - } - message Box { - repeated uint32 shape = 1; - optional float low = 2; // lower bound on all dimensions, not set means unbounded - repeated float lows = 3; // independant lower bounds for each dimensions (gets reshaped), overrides `low` - optional float high = 4; // higher bound on all dimensions, not set means unbounded - repeated float highs = 5; // independant upper bounds for each dimensions (gets reshaped), overrides `high` - } - message Property { - string key = 1; - oneof definition { - Discrete discrete = 2; - Box box = 3; - } - } - repeated Property properties = 1; -} - -// SpaceValue messages are values within a Space -// `properties` are sorted in the same way than the value's Space -message SpaceValue { - message SimpleBox { - repeated float values = 1; - } - message PropertyValue { - oneof value { - int32 discrete = 1; - NDArray box = 2; - SimpleBox simple_box = 3; - } - } - repeated PropertyValue properties = 1; -} - -message EnvironmentSpecs { - string implementation = 1; - int32 num_players = 2; - Space observation_space = 3; - Space action_space = 4; -} - -message EnvironmentParams { - EnvironmentSpecs specs = 2; // Keeping that here for the moment, could probably be defined somewhere else with only the implementation name defined in the params - EnvironmentConfig config = 3; -} - -message HFHubModel { - string repo_id = 1; - string filename = 2; -} - -message AgentConfig { - string run_id = 1; - EnvironmentSpecs environment_specs = 2; - string model_id = 3; - int32 model_version = 4; - int32 actor_index = 5; // Used to figure out if an agent is the current_player in the observation space - string device = 6; - uint32 threads_per_worker = 7; - HFHubModel hf_hub_model = 8; -} - -enum HumanRole { - TEACHER = 0; - OBSERVER = 1; - PLAYER = 2; -} - -message HumanConfig { - string run_id = 1; - EnvironmentSpecs environment_specs = 2; - int32 actor_index = 3; // Used to figure out if an agent is the current_player in the observation space - HumanRole role = 4; -} - -message ActorParams { - string name = 1; - string actor_class = 2; - string implementation = 3; - oneof config_oneof { - AgentConfig agent_config = 4; - HumanConfig human_config = 5; - } -} - -message TrialConfig { - string run_id = 1; - EnvironmentParams environment = 3; - repeated ActorParams actors = 4; - int32 distinguished_actor = 6; -} - -message Observation { - NDArray vectorized = 1; - bytes pixel_data = 2; - repeated int32 legal_moves_as_int = 3; - int32 current_player = 4; // active player for multi-agent turn-based environments - int32 player_override = 5; // player that _actually_ acted (in case of override/intervention) - NDArray segmentation = 6; -} - -message ContinuousAction { - repeated float data = 1; -} - -message AgentAction { - oneof action { - ContinuousAction continuous_action = 1; - int32 discrete_action = 2; - }; - NDArray policy = 3; // optional: policy from which action was drawn - float value = 4; // optional: value of the state from which the action was taken -} - -message ModelArgs { - float v_min = 1; - float v_max = 2; - uint32 start_timesteps = 4; - repeated float high_action = 5; - repeated float low_action = 6; - float expl_noise = 7; - bool target_net_soft_update = 8; - uint32 screensize = 9; -} - -message ReplayBufferConfig { - string action_dtype = 1; - string observation_dtype = 2; -} - -message RunConfig { - EnvironmentParams environment = 1; - string agent_implementation = 2; - float epsilon_min = 3; - uint32 epsilon_steps = 4; - uint32 target_net_update_schedule = 5; - float learning_rate = 6; - uint32 lr_warmup_steps = 7; - uint32 demonstration_count = 8; - uint32 total_trial_count = 9; - uint32 model_publication_interval = 10; - uint32 model_archive_interval = 11; - uint32 batch_size = 12; - uint32 min_replay_buffer_size = 13; - uint32 max_parallel_trials = 14; - ModelArgs model_kwargs = 15; - uint32 max_replay_buffer_size = 16; - bool aggregate_by_actor = 17; - ReplayBufferConfig replay_buffer_config = 18; - float discount_factor = 19; -} - -message MLPNetworkConfig { - uint32 input_size = 1; - uint32 hidden_size = 2; - uint32 num_hidden_layers = 3; - uint32 output_size = 4; -} - -message SimpleA2CTrainingConfig { - uint32 epoch_count = 1; - uint32 epoch_trial_count = 2; - uint32 max_parallel_trials = 3; - float discount_factor = 4; - float entropy_coef = 5; - float value_loss_coef = 6; - float action_loss_coef = 7; - float learning_rate = 8; -} - -message SimpleA2CTrainingRunConfig { - EnvironmentParams environment = 1; - SimpleA2CTrainingConfig training = 2; - MLPNetworkConfig actor_network = 3; - MLPNetworkConfig critic_network = 4; -} - -message DistributionConfig { - float min_value = 1; - float max_value = 2; - uint32 num_bins = 3; -} - -message MCTSConfig { - uint32 max_depth = 1; - uint32 num_samples = 2; - float temperature = 3; - float ucb_c1 = 4; - float ucb_c2 = 5; - float exploration_alpha = 6; - float exploration_epsilon = 7; - uint32 rollout_length = 8; - float epsilon_min = 9; - float epsilon_decay_steps = 10; - float min_temperature = 11; - uint32 temperature_decay_steps = 12; -} - -message OptimizerConfig { - float learning_rate = 1; - float weight_decay = 2; - float min_learning_rate = 3; - uint32 lr_warmup_steps = 4; - uint32 lr_decay_steps = 5; - float max_norm = 6; -} - -message MuZeroTrainingConfig { - uint32 model_publication_interval = 1; - float discount_rate = 2; - OptimizerConfig optimizer = 3; - uint32 bootstrap_steps = 4; - uint32 batch_size = 5; - uint32 max_replay_buffer_size = 6; - uint32 min_replay_buffer_size = 7; - uint32 log_interval = 8; - float similarity_weight = 9; - float value_weight = 10; -} - -message MuZeroRunConfig { - EnvironmentParams environment = 1; - AgentConfig actor = 2; - MuZeroTrainingConfig training = 3; - MCTSConfig mcts = 4; - MLPNetworkConfig representation_network = 5; - MLPNetworkConfig projector_network = 6; - MLPNetworkConfig dynamics_network = 7; - MLPNetworkConfig policy_network = 8; - MLPNetworkConfig value_network = 9; - DistributionConfig reward_distribution = 10; - DistributionConfig value_distribution = 11; - uint32 trial_count = 12; - uint32 max_parallel_trials = 13; - uint32 demonstration_trials = 14; - string train_device = 15; - string actor_device = 16; - string reanalyze_device = 17; - uint32 reanalyze_workers = 18; - uint32 threads_per_worker = 19; -} - -message SimpleBCTrainingConfig { - uint32 trial_count = 2; - uint32 max_parallel_trials = 3; - float discount_factor = 4; - float learning_rate = 8; - uint32 batch_size = 5; -} - -message SimpleBCTrainingRunConfig { - EnvironmentParams environment = 1; - SimpleBCTrainingConfig training = 2; - MLPNetworkConfig policy_network = 3; -} - -message PlayRunConfig { - EnvironmentParams environment = 1; - repeated ActorParams actors = 2; - bool observer = 3; - uint32 trial_count = 4; -} - -message SelfPlayActorConfig{ - uint32 num_input = 1; - uint32 num_input_2 = 2; - uint32 num_action = 3; - MLPNetworkConfig actor_network = 4; - MLPNetworkConfig critic_network = 5; - ModelArgs model_kwargs = 6; - string run_id = 7; - string model_id = 8; - int32 model_version = 9; - string environment_implementation = 10; - repeated float action_scale = 11; - repeated float action_bias = 12; - uint32 max_action = 13; - repeated uint32 alice_grid_shape = 14; - repeated uint32 bob_grid_shape = 15; -} - -message SelfPlayRolloutConfig{ - uint32 epoch_count = 1; - uint32 epoch_train_trial_count = 2; - uint32 epoch_test_trial_count = 3; - uint32 max_parallel_trials = 4; - uint32 model_publication_interval = 5; - uint32 number_turns_per_trial = 6; - uint32 test_freq = 7; -} - -message SelfPlayReplaybufferConfig{ - uint32 min_replay_buffer_size = 1; - uint32 max_replay_buffer_size = 2; - ReplayBufferConfig replay_buffer_config = 3; -} - -message SelfPlayTrainingConfig{ - float discount_factor = 1; - float tau = 2; - float policy_noise = 3; - float noise_clip = 4; - float learning_rate = 5; - uint32 policy_freq = 6; - uint32 batch_size = 7; - float SIGMA = 8; - uint32 num_training_steps = 9; - float beta = 10; - float alice_reward = 11; - float bob_reward = 12; - float alice_penalty = 13; - float bob_penalty = 14; -} - - -message SelfPlayActorParams{ - string name = 1; - string actor_class = 2; - string implementation = 3; - SelfPlayActorConfig config = 4; -} - -message SelfPlayTD3TrainingRunConfig { - EnvironmentParams environment = 1; - SelfPlayActorParams actor = 2; - SelfPlayRolloutConfig rollout = 3; - SelfPlayTrainingConfig training = 4; - SelfPlayReplaybufferConfig replaybuffer = 5; -} diff --git a/data/mlflow/.gitkeep b/data/mlflow/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/data/model-registry/.gitkeep b/data/model-registry/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/development_setup.md b/docs/development_setup.md index 77b4e464..48f2aa34 100644 --- a/docs/development_setup.md +++ b/docs/development_setup.md @@ -2,4 +2,44 @@ This is a practical guide for developers wanting to develop within cogment verse. +## Dependencies + +To support development, additional dependencies are required. + +- [Node.JS v14](https://nodejs.org/) or above is required to develop the web components of cogment verse. + > 🚧 _in construction_ 🚧 + +## Linting + +### Python code formatting + +Check the code style using `black` by running the following in the virtual environment in the root directory: + +```console +$ black --check --diff . +``` + +Fix the code style by running: + +```console +$ black . +``` + +### Python code quality + +Check the code quality using `pylint` by running the following in the virtual environment in the root directory: + +```console +$ pylint --recursive=y . +``` + +## Testing + +### Python test suite + +Run the test suite on the python codebase using `pytest` by running the following in the virtual environment in the root directory: + +```console +$ python -m pytest +``` diff --git a/docs/results/a2c.md b/docs/results/a2c.md index 9ee1bcec..514ffe8d 100644 --- a/docs/results/a2c.md +++ b/docs/results/a2c.md @@ -39,7 +39,7 @@ simple_a2c_cartpole: epoch_trial_count: 15 max_parallel_trials: 8 discount_factor: 0.95 - entropy_coef: 0.01 + entropy_loss_coef: 0.01 value_loss_coef: 0.5 action_loss_coef: 1.0 learning_rate: 0.01 diff --git a/environment/cogment_verse_environment/base_agent_adapter.py b/environment/cogment_verse_environment/base_agent_adapter.py deleted file mode 100644 index da3dc292..00000000 --- a/environment/cogment_verse_environment/base_agent_adapter.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import copy - -import numpy as np -import cogment - -from cogment.api.common_pb2 import TrialState -from cogment_verse import AgentAdapter, MlflowExperimentTracker -from cogment_verse.spaces import flattened_dimensions -from cogment_verse.constants import HUMAN_ACTOR_NAME, HUMAN_ACTOR_CLASS, HUMAN_ACTOR_IMPL -from data_pb2 import ( - ActorParams, - AgentAction, - AgentConfig, - EnvironmentConfig, - EnvironmentParams, - HumanConfig, - HumanRole, - PlayRunConfig, - TrialConfig, -) - - -log = logging.getLogger(__name__) - - -def extend_actor_config(actor_config_template, run_id, environment_specs): - config = AgentConfig() - config.CopyFrom(actor_config_template) - config.run_id = run_id - # pylint: disable=no-member - config.environment_specs.CopyFrom(environment_specs) - return config - - -# pylint: disable=arguments-differ -class BaseAgentAdapter(AgentAdapter): - def _create_actor_implementations(self): - async def random_impl(actor_session): - actor_session.start() - - config = actor_session.config - - num_action = flattened_dimensions(config.environment_specs.action_space) - - async for event in actor_session.all_events(): - if event.observation and event.type == cogment.EventType.ACTIVE: - action = np.random.default_rng().integers(0, num_action) - actor_session.do_action(AgentAction(discrete_action=action)) - - return { - "random": (random_impl, ["agent"]), - } - - def _create_run_implementations(self): - async def total_rewards_producer_impl(run_sample_producer_session): - actors_total_rewards = [0.0 for actor_idx in range(run_sample_producer_session.count_actors())] - num_ticks = 0 - async for sample in run_sample_producer_session.get_all_samples(): - if sample.get_trial_state() == TrialState.ENDED: - break - - num_ticks += 1 - for actor_idx in range(run_sample_producer_session.count_actors()): - actors_total_rewards[actor_idx] += sample.get_actor_reward(actor_idx, 0.0) - - run_sample_producer_session.produce_training_sample((actors_total_rewards, num_ticks)) - - async def play_impl(run_session): - xp_tracker = MlflowExperimentTracker(run_session.params_name, run_session.run_id) - - config = run_session.config - # We ignore additional actor configs - if config.environment.specs.num_players > len(config.actors): - raise RuntimeError( - f"Expecting at least {config.environment.specs.num_players} configured actors, got {len(config.actors)}" - ) - - actors_params = [] - has_human_actor = False - for actor_params in config.actors[: config.environment.specs.num_players]: - if actor_params.implementation == HUMAN_ACTOR_IMPL: - if has_human_actor: - raise RuntimeError("Can't have more than one human involved in the trial") - # Human actor - actors_params.append( - ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, - implementation=HUMAN_ACTOR_IMPL, - human_config=HumanConfig( - run_id=run_session.run_id, - environment_specs=config.environment.specs, - role=HumanRole.PLAYER, - ), - ) - ) - has_human_actor = True - else: - actors_params.append( - ActorParams( - name=actor_params.name, - actor_class=actor_params.actor_class, - implementation=actor_params.implementation, - agent_config=extend_actor_config( - actor_config_template=actor_params.agent_config, - run_id=run_session.run_id, - environment_specs=config.environment.specs, - ), - ) - ) - - xp_tracker.log_params( - config.environment.config, - **{ - f"actor_{actor_idx}_implementation": actor_params.implementation - for actor_idx, actor_params in enumerate(actors_params) - }, - **{ - f"actor_{actor_idx}_model_id": actor_params.agent_config.model_id - for actor_idx, actor_params in enumerate(actors_params) - }, - **{ - f"actor_{actor_idx}_model_version": actor_params.agent_config.model_version - for actor_idx, actor_params in enumerate(actors_params) - }, - environment=config.environment.specs.implementation, - ) - - if config.observer: - if has_human_actor: - raise RuntimeError("Can't have more than one human involved in the trial") - # Add an observer agent - actors_params.append( - ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, - implementation=HUMAN_ACTOR_IMPL, - human_config=HumanConfig( - run_id=run_session.run_id, - environment_specs=config.environment.specs, - role=HumanRole.OBSERVER, - ), - ) - ) - - # Helper function to create a trial configuration - def create_trial_config(trial_idx): - env_params = copy.deepcopy(config.environment) - env_params.config.seed = env_params.config.seed + trial_idx - if has_human_actor: - env_params.config.render = True - - return TrialConfig( - run_id=run_session.run_id, - environment=env_params, - actors=actors_params, - ) - - # Rollout a bunch of trials - async for ( - step_idx, - step_timestamp, - _trial_id, - _tick_id, - sample, - ) in run_session.start_trials_and_wait_for_termination( - trial_configs=[create_trial_config(trial_idx) for trial_idx in range(config.trial_count)], - max_parallel_trials=1, - ): - (actors_total_rewards, num_ticks) = sample - xp_tracker.log_metrics( - step_timestamp, - step_idx, - **{ - f"actor_{actor_idx}_reward": actors_total_rewards[actor_idx] - for actor_idx in range(config.environment.specs.num_players) - }, - total_reward=sum(actors_total_rewards), - num_ticks=num_ticks, - ) - - return { - "play": ( - total_rewards_producer_impl, - play_impl, - PlayRunConfig( - environment=EnvironmentParams( - specs=None, # Needs to be specified - config=EnvironmentConfig(seed=12, framestack=1, render=True, render_width=256), - ), - actors=[], - trial_count=5, - ), - ) - } diff --git a/environments/gym_adapter.py b/environments/gym_adapter.py new file mode 100644 index 00000000..960365f4 --- /dev/null +++ b/environments/gym_adapter.py @@ -0,0 +1,186 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import cogment +import gym +import numpy as np + +from cogment_verse.specs import ( + encode_rendered_frame, + deserialize_ndarray, + serialize_ndarray, + EnvironmentSpecs, + Observation, + Space, + SpaceValue, +) +from cogment_verse.constants import PLAYER_ACTOR_CLASS, TEACHER_ACTOR_CLASS + +# configure pygame to use a dummy video server to be able to render headlessly +os.environ["SDL_VIDEODRIVER"] = "dummy" + +SPACES_BOUND_MAX = float.fromhex("0x1.fffffep+127") +SPACES_BOUND_MIN = -SPACES_BOUND_MAX + + +def space_from_gym_space(gym_space): + if isinstance(gym_space, gym.spaces.Box): + return Space( + properties=[ + Space.Property( + box=Space.Box( + shape=list(gym_space.shape), + low=[ + Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() + for v in gym_space.low.flat + ], + high=[ + Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() + for v in gym_space.high.flat + ], + ) + ) + ] + ) + if isinstance(gym_space, gym.spaces.Discrete): + return Space(properties=[Space.Property(discrete=Space.Discrete(num=gym_space.n))]) + raise RuntimeError(f"[{type(gym_space)}] is not a supported gym space type") + + +def gym_action_from_action(space, action): + if len(action.properties) == 1: + prop_value = action.properties[0] + value_type = prop_value.WhichOneof("value") + if value_type == "discrete": + return prop_value.discrete + if value_type == "box": + return deserialize_ndarray(prop_value.box) + # value_type == "simple_box" + return np.array(prop_value.simple_box.values).reshape(space.properties[0].box.shape) + raise RuntimeError(f"Not supporting spaces not having one property, got {len(action.properties)}") + + +def observation_from_gym_observation(gym_space, gym_observation): + if isinstance(gym_space, gym.spaces.Box): + return SpaceValue(properties=[SpaceValue.PropertyValue(box=serialize_ndarray(gym_observation))]) + if isinstance(gym_space, gym.spaces.Discrete): + return SpaceValue(properties=[SpaceValue.PropertyValue(discrete=gym_observation.item())]) + raise RuntimeError(f"[{type(gym_space)}] is not a supported gym space type") + + +class Environment: + def __init__(self, cfg): + self.gym_env_name = cfg.env_name + + gym_env = gym.make(self.gym_env_name) + self.env_specs = EnvironmentSpecs( + num_players=1, + observation_space=space_from_gym_space(gym_env.observation_space), + action_space=space_from_gym_space(gym_env.action_space), + ) + + def get_implementation_name(self): + return self.gym_env_name + + def get_environment_specs(self): + return self.env_specs + + async def impl(self, environment_session): + actors = environment_session.get_active_actors() + player_actors = [ + (actor_idx, actor.actor_name) + for (actor_idx, actor) in enumerate(actors) + if actor.actor_class_name == PLAYER_ACTOR_CLASS + ] + assert len(player_actors) == 1 + [(player_actor_idx, player_actor_name)] = player_actors + + teacher_actors = [ + (actor_idx, actor.actor_name) + for (actor_idx, actor) in enumerate(actors) + if actor.actor_class_name == TEACHER_ACTOR_CLASS + ] + assert len(teacher_actors) <= 1 + has_teacher = len(teacher_actors) == 1 + if has_teacher: + [(teacher_actor_idx, _teacher_actor_name)] = teacher_actors + + session_cfg = environment_session.config + + gym_env = gym.make(self.gym_env_name) + + gym_observation, _info = gym_env.reset(seed=session_cfg.seed, return_info=True) + observation_value = observation_from_gym_observation(gym_env.observation_space, gym_observation) + + rendered_frame = None + if session_cfg.render: + rendered_frame = encode_rendered_frame(gym_env.render(mode="rgb_array"), session_cfg.render_width) + + environment_session.start([("*", Observation(value=observation_value, rendered_frame=rendered_frame))]) + + async for event in environment_session.all_events(): + if event.actions: + player_action_value = event.actions[player_actor_idx].action.value + action_value = player_action_value + overridden_players = [] + if has_teacher and event.actions[teacher_actor_idx].action.HasField("value"): + teacher_action_value = event.actions[teacher_actor_idx].action.value + action_value = teacher_action_value + overridden_players = [player_actor_name] + + gym_action = gym_action_from_action( + self.env_specs.action_space, action_value + ) # pylint: disable=no-member + + gym_observation, reward, done, _info = gym_env.step(gym_action) + observation_value = observation_from_gym_observation(gym_env.observation_space, gym_observation) + + rendered_frame = None + if session_cfg.render: + if session_cfg.render: + rendered_frame = encode_rendered_frame( + gym_env.render(mode="rgb_array"), session_cfg.render_width + ) + + observations = [ + ( + "*", + Observation( + value=observation_value, + rendered_frame=rendered_frame, + overridden_players=overridden_players, + ), + ) + ] + + if reward is not None: + environment_session.add_reward( + value=reward, + confidence=1.0, + to=[player_actor_name], + ) + + if done: + # The trial ended + environment_session.end(observations) + elif event.type != cogment.EventType.ACTIVE: + # The trial termination has been requested + environment_session.end(observations) + else: + # The trial is active + environment_session.produce_observations(observations) + + gym_env.close() diff --git a/main.py b/main.py new file mode 100644 index 00000000..41a3ad85 --- /dev/null +++ b/main.py @@ -0,0 +1,35 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import hydra + +import cogment_verse + +log = logging.getLogger(__name__) + + +@hydra.main(version_base=None, config_path="config", config_name="config") +def main(cfg): + work_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".cogment_verse")) + app = cogment_verse.App(cfg, work_dir=work_dir) + + app.start() + app.join() + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index dfbb863e..2b71d6d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,37 +1,45 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + [tool.black] line-length = 120 target-version = ["py39"] -extend-exclude = ''' -/( - .*/third_party/.* -)/ -''' color = true [tool.pylint.MASTER] disable=[ - "C0330", "C0326", "C0199", "C0412", # Disable checks in conflict with black - "line-too-long", # Dealt with by black - "import-error", "no-name-in-module", "no-member", # Import checks don't work given the way the projects are setup - "missing-module-docstring", "missing-function-docstring", "missing-class-docstring", # Don't require docstrings for this demo code + "ungrouped-imports", "line-too-long", # Disable checks in conflict with black + "missing-module-docstring", "missing-function-docstring", "missing-class-docstring", # Don't require docstrings (yet) "logging-fstring-interpolation", - "broad-except", + "fixme", # Don't consider TODO as errors + "no-value-for-parameter", # seems to be buggy + # The following are basically stupid tests + "duplicate-code", "too-few-public-methods", "too-many-arguments", - "too-many-locals", - "too-many-statements", "too-many-branches", "too-many-function-args", "too-many-instance-attributes", - "attribute-defined-outside-init", # seems to be buggy - "fixme", # Don't consider TODO as errors - "duplicate-code", # This is a stupid test + "too-many-locals", + "too-many-statements", ] ignore-patterns=[".*_pb2.py",".*_pb2_grpc.py","cog_settings.py"] -ignore-paths=[".*/third_party/.*", ".*/node_modules/.*"] +ignore-paths=[ + ".*/node_modules/.*", + ".*/_old/.*", + ".*/.venv/.*" +] jobs=0 + [tool.pylint.LOGGING] logging-format-style="new" [tool.pylint.FORMAT] good-names=["i","j","k","c","h","w","x","id","f","to"] + +[tool.pylint.Typecheck] +generated-members=["cv2", "torch"] + +[tool.pytest.ini_options] +addopts = "-rfs --ignore=_old" diff --git a/requirements.txt b/requirements.txt index 71f1fc26..440b1d39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,21 @@ +cogment[generate]==2.1.1 +grpcio==1.43.0 +hydra-core==1.2.0 mlflow==1.26.1 +names_generator~=0.1.0 +opencv-python~=4.5.5 +PyYAML~=5.4.1 +starlette==0.20.0 +uvicorn==0.17.6 + +# environments +gym[atari,box2d]==0.23.1 + +# actors +torch==1.11.0 +numpy>=1.21.5,<1.22 # For testing -black -pylint~=2.9.6 +black~=22.3.0 +pylint~=2.14 +pytest~=6.2.5 diff --git a/runs/play.py b/runs/play.py new file mode 100644 index 00000000..0fd0e13f --- /dev/null +++ b/runs/play.py @@ -0,0 +1,173 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import cogment +from cogment.api.common_pb2 import TrialState + +from cogment_verse.specs import ( + AgentConfig, + cog_settings, + EnvironmentConfig, + HUMAN_ACTOR_IMPL, + OBSERVER_ACTOR_CLASS, + PLAYER_ACTOR_CLASS, + WEB_ACTOR_NAME, +) + +log = logging.getLogger(__name__) + + +def extend_actor_config(actor_config_template, run_id, environment_specs): + config = AgentConfig() + if actor_config_template is not None: + config.CopyFrom(actor_config_template) + config.run_id = run_id + # pylint: disable=no-member + config.environment_specs.CopyFrom(environment_specs) + return config + + +class PlayRun: + default_cfg = {"trial_count": 1, "observer": False, "players": []} + + def __init__(self, environment_specs, cfg): + super().__init__() + self._environment_specs = environment_specs + self._cfg = cfg + + async def total_rewards_producer_impl(self, sample_producer_session): + actors_total_rewards = { + actor_parameters.name: 0.0 for actor_parameters in sample_producer_session.trial_info.parameters.actors + } + num_ticks = 0 + async for sample in sample_producer_session.all_trial_samples(): + if sample.trial_state == TrialState.ENDED: + break + + num_ticks += 1 + for actor_name, actor_sample in sample.actors_data.items(): + actors_total_rewards[actor_name] += actor_sample.reward if actor_sample.reward is not None else 0.0 + + sample = ( + actors_total_rewards, + num_ticks, + ) + sample_producer_session.produce_sample(sample) + + async def impl(self, run_session): + # We ignore additional actor configs + if self._environment_specs.num_players > len(self._cfg.players): + raise RuntimeError( + f"Expecting at least {self._environment_specs.num_players} configured actors, got {len(self._cfg.players)}" + ) + + actors_params = [] + has_human_actor = False + for actor_params in self._cfg.players[: self._environment_specs.num_players]: + if actor_params.implementation == HUMAN_ACTOR_IMPL: + if has_human_actor: + raise RuntimeError("Can't have more than one human involved in the trial") + # Human actor + actors_params.append( + cogment.ActorParameters( + cog_settings, + name=WEB_ACTOR_NAME, + class_name=PLAYER_ACTOR_CLASS, + implementation=HUMAN_ACTOR_IMPL, + config=extend_actor_config( + actor_config_template=actor_params.get("agent_config", None), + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + ) + has_human_actor = True + else: + actors_params.append( + cogment.ActorParameters( + cog_settings, + name=actor_params.name, + class_name=PLAYER_ACTOR_CLASS, + implementation=actor_params.implementation, + config=extend_actor_config( + actor_config_template=actor_params.get("agent_config", None), + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + ) + + if self._cfg.observer: + if has_human_actor: + raise RuntimeError("Can't have more than one human involved in the trial") + # Add an observer agent + actors_params.append( + cogment.ActorParameters( + cog_settings, + name=WEB_ACTOR_NAME, + class_name=OBSERVER_ACTOR_CLASS, + implementation=HUMAN_ACTOR_IMPL, + config=AgentConfig( + run_id=run_session.run_id, + environment_specs=self._environment_specs, + ), + ) + ) + has_human_actor = True + + run_session.log_params( + **{ + f"actor_{actor_idx}_implementation": actor_params.implementation + for actor_idx, actor_params in enumerate(actors_params) + }, + **{ + f"actor_{actor_idx}_model_id": actor_params.config.model_id + for actor_idx, actor_params in enumerate(actors_params) + }, + **{ + f"actor_{actor_idx}_model_version": actor_params.config.model_version + for actor_idx, actor_params in enumerate(actors_params) + }, + environment=self._environment_specs.implementation, + ) + + # Helper function to create a trial configuration + trial_params = cogment.TrialParameters( + cog_settings, + environment_name="env", + environment_implementation=self._environment_specs.implementation, + environment_config=EnvironmentConfig(run_id=run_session.run_id, render=has_human_actor, seed=50), + actors=actors_params, + ) + + # Rollout a bunch of trials + for (_step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + trials_id_and_params=[ + (f"{run_session.run_id}_{trial_idx}", trial_params) for trial_idx in range(self._cfg.trial_count) + ], + sample_producer_impl=self.total_rewards_producer_impl, + num_parallel_trials=1, + ): + (actors_total_rewards, num_ticks) = sample + run_session.log_metrics( + **{ + f"actor_{actor_name}_reward": actor_total_reward + for actor_name, actor_total_reward in actors_total_rewards.items() + }, + total_reward=sum(actors_total_rewards.values()), + num_ticks=num_ticks, + ) + log.info("play ending") diff --git a/simple_mlflow.py b/simple_mlflow.py new file mode 100644 index 00000000..579d73fd --- /dev/null +++ b/simple_mlflow.py @@ -0,0 +1,44 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from subprocess import check_call + + +def main(): + parser = argparse.ArgumentParser(description="Start a simple mlflow server") + parser.add_argument("port", type=int, default=3000, nargs="?", help="TCP port (optional, default is 3000)") + args = parser.parse_args() + + mlflow_data_dir = os.path.join(os.path.dirname(__file__), ".cogment_verse/mlflow") + os.makedirs(mlflow_data_dir, exist_ok=True) + check_call( + args=[ + "mlflow", + "server", + "--host", + "0.0.0.0", + "--port", + f"{args.port}", + "--backend-store-uri", + f"sqlite:///{mlflow_data_dir}/mlflow.db", + "--default-artifact-root", + f"{mlflow_data_dir}", + ] + ) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..8bcf1275 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,24 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from cogment_verse.utils.generate import generate + +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + +generate( + work_dir=os.path.join(ROOT_DIR, ".cogment_verse"), + specs_filename=os.path.join(ROOT_DIR, "cogment_verse/specs/cogment.yaml"), +) diff --git a/base_python/tests/test_flatten.py b/tests/test_flatten.py similarity index 64% rename from base_python/tests/test_flatten.py rename to tests/test_flatten.py index 171bfe21..993b6241 100644 --- a/base_python/tests/test_flatten.py +++ b/tests/test_flatten.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from data_pb2 import Space -from cogment_verse.spaces import flattened_dimensions +import numpy as np + +from cogment_verse.specs import Space, sample_space, flattened_dimensions, flatten, unflatten def test_flattened_dimensions_discrete(): @@ -92,3 +93,43 @@ def test_flattened_dimensions_mixed(): ) == 37 ) + + +def test_flatten(): + space = Space( + properties=[ + Space.Property(key="a", discrete=Space.Discrete(labels=["brake", "accelerate", "do nothing"])), + Space.Property( + key="b", + box=Space.Box( + shape=[2, 3], + low=[ + Space.Bound(), + Space.Bound(), + Space.Bound(), + Space.Bound(), + Space.Bound(), + Space.Bound(), + ], + high=[ + Space.Bound(), + Space.Bound(), + Space.Bound(), + Space.Bound(), + Space.Bound(), + Space.Bound(), + ], + ), + ), + ] + ) + [value] = sample_space(space) + flat_value = flatten(space, value) + assert flat_value.shape == (9,) + unflattened_value = unflatten(space, flat_value) + assert unflattened_value == value + + unflattened_random_value = unflatten(space, np.random.default_rng(64).normal(size=(9,))) + assert len(unflattened_random_value.properties) == 2 + assert unflattened_random_value.properties[0].discrete in [0, 1, 2] + assert unflattened_random_value.properties[0].box is not None diff --git a/tests/test_sample.py b/tests/test_sample.py new file mode 100644 index 00000000..fe26b60c --- /dev/null +++ b/tests/test_sample.py @@ -0,0 +1,91 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cogment_verse.specs import ( + Space, + sample_space, + deserialize_ndarray, +) + + +def test_sample_discrete(): + space = Space(properties=[Space.Property(discrete=Space.Discrete(num=2))]) + + values = sample_space(space, 4) + assert len(values) == 4 + + for value in values: + assert len(value.properties) == 1 + assert value.properties[0].discrete in range(2) + + +def test_sample_discrete_named(): + space = Space(properties=[Space.Property(discrete=Space.Discrete(labels=["brake", "accelerate", "do nothing"]))]) + + values = sample_space(space, 12) + assert len(values) == 12 + + for value in values: + assert len(value.properties) == 1 + assert value.properties[0].discrete in range(3) + + +def test_sample_discrete_named_and_more(): + space = Space( + properties=[Space.Property(discrete=Space.Discrete(labels=["brake", "accelerate", "do nothing"], num=12))] + ) + + values = sample_space(space, 12) + assert len(values) == 12 + + for value in values: + assert len(value.properties) == 1 + assert value.properties[0].discrete in range(12) + + +def test_sample_box(): + space = Space( + properties=[ + Space.Property( + box=Space.Box( + shape=[2, 3], + low=[ + Space.Bound(bound=-1), + Space.Bound(bound=-2), + Space.Bound(), + Space.Bound(bound=-4), + Space.Bound(), + Space.Bound(bound=-6), + ], + high=[ + Space.Bound(bound=1), + Space.Bound(bound=2), + Space.Bound(), + Space.Bound(), + Space.Bound(bound=5), + Space.Bound(bound=6), + ], + ) + ) + ] + ) + + values = sample_space(space, 4) + assert len(values) == 4 + + for value in values: + assert len(value.properties) == 1 + assert value.properties[0].box.shape == [2, 3] + ndarray = deserialize_ndarray(value.properties[0].box) + assert ndarray.shape == (2, 3) diff --git a/torch_agents/cogment_verse_torch_agents/simple_a2c/simple_a2c_agent.py b/torch_agents/cogment_verse_torch_agents/simple_a2c/simple_a2c_agent.py deleted file mode 100644 index 8fcca83d..00000000 --- a/torch_agents/cogment_verse_torch_agents/simple_a2c/simple_a2c_agent.py +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from collections import namedtuple - -import cogment -import torch -from cogment.api.common_pb2 import TrialState -from cogment_verse import AgentAdapter, MlflowExperimentTracker -from cogment_verse.spaces import flattened_dimensions -from cogment_verse_torch_agents.utils.tensors import cog_action_from_tensor, tensor_from_cog_action, tensor_from_cog_obs -from data_pb2 import ( - AgentConfig, - ActorParams, - EnvironmentConfig, - EnvironmentParams, - MLPNetworkConfig, - SimpleA2CTrainingConfig, - SimpleA2CTrainingRunConfig, - TrialConfig, -) - -log = logging.getLogger(__name__) - -SimpleA2CModel = namedtuple("SimpleA2CModel", ["model_id", "version_number", "actor_network", "critic_network"]) - -# pylint: disable=arguments-differ - - -class SimpleA2CAgentAdapter(AgentAdapter): - def __init__(self): - super().__init__() - self._dtype = torch.float - - def _create( - self, - model_id, - environment_specs, - actor_network_hidden_size=64, - critic_network_hidden_size=64, - **kwargs, - ): - num_input = flattened_dimensions(environment_specs.observation_space) - num_output = flattened_dimensions(environment_specs.action_space) - model = SimpleA2CModel( - model_id=model_id, - version_number=1, - actor_network=torch.nn.Sequential( - torch.nn.Linear(num_input, actor_network_hidden_size), - torch.nn.Tanh(), - torch.nn.Linear(actor_network_hidden_size, actor_network_hidden_size), - torch.nn.Tanh(), - torch.nn.Linear(actor_network_hidden_size, num_output), - ).to(self._dtype), - critic_network=torch.nn.Sequential( - torch.nn.Linear(num_input, critic_network_hidden_size), - torch.nn.Tanh(), - torch.nn.Linear(critic_network_hidden_size, critic_network_hidden_size), - torch.nn.Tanh(), - torch.nn.Linear(critic_network_hidden_size, 1), - ).to(self._dtype), - ) - - model_user_data = { - "environment_implementation": environment_specs.implementation, - "num_input": num_input, - "num_output": num_output, - } - - return model, model_user_data - - def _load( - self, - model_id, - version_number, - model_user_data, - version_user_data, - model_data_f, - environment_specs, - **kwargs, - ): - (actor_network, critic_network) = torch.load(model_data_f) - assert model_user_data["environment_implementation"] == environment_specs.implementation - assert isinstance(actor_network, torch.nn.Sequential) - assert isinstance(critic_network, torch.nn.Sequential) - return SimpleA2CModel( - model_id=model_id, version_number=version_number, actor_network=actor_network, critic_network=critic_network - ) - - def _save(self, model, model_user_data, model_data_f, environment_specs, epoch_idx=-1, total_samples=0, **kwargs): - assert model_user_data["environment_implementation"] == environment_specs.implementation - assert isinstance(model, SimpleA2CModel) - torch.save((model.actor_network, model.critic_network), model_data_f) - return {"epoch_idx": epoch_idx, "total_samples": total_samples} - - def _create_actor_implementations(self): - async def impl(actor_session): - actor_session.start() - - config = actor_session.config - - model, _, _ = await self.retrieve_version( - config.model_id, config.model_version, environment_specs=config.environment_specs - ) - - async for event in actor_session.all_events(): - if event.observation and event.type == cogment.EventType.ACTIVE: - obs = tensor_from_cog_obs(event.observation.snapshot, dtype=self._dtype) - scores = model.actor_network(obs) - probs = torch.softmax(scores, dim=-1) - action = torch.distributions.Categorical(probs).sample() - actor_session.do_action(cog_action_from_tensor(action)) - - return { - "simple_a2c": (impl, ["agent"]), - } - - def _create_run_implementations(self): - async def sample_producer_impl(run_sample_producer_session): - assert run_sample_producer_session.count_actors() == 1 - observation = [] - action = [] - reward = [] - done = [] - async for sample in run_sample_producer_session.get_all_samples(): - if sample.get_trial_state() == TrialState.ENDED: - # This sample includes the last observation and no action - # The last sample was the last useful one - done[-1] = torch.ones(1, dtype=self._dtype) - break - observation.append(tensor_from_cog_obs(sample.get_actor_observation(0), dtype=self._dtype)) - action.append(tensor_from_cog_action(sample.get_actor_action(0))) - reward.append(torch.tensor(sample.get_actor_reward(0), dtype=self._dtype)) - done.append(torch.zeros(1, dtype=self._dtype)) - - # Keeping the samples grouped by trial by emitting only one grouped sample at the end of the trial - run_sample_producer_session.produce_training_sample((observation, action, reward, done)) - - async def run_impl(run_session): - xp_tracker = MlflowExperimentTracker(run_session.params_name, run_session.run_id) - - # Initializing a model - model_id = f"{run_session.run_id}_model" - - config = run_session.config - assert config.environment.specs.num_players == 1 - - model, _ = await self.create_and_publish_initial_version( - model_id, - environment_specs=config.environment.specs, - actor_network_hidden_size=config.actor_network.hidden_size, - critic_network_hidden_size=config.critic_network.hidden_size, - ) - model_version_number = 1 - - xp_tracker.log_params( - config.training, - config.environment.config, - environment_implementation=config.environment.specs.implementation, - actor_network_hidden_size=config.actor_network.hidden_size, - critic_network_hidden_size=config.critic_network.hidden_size, - ) - - # Configure the optimizer over the two models - optimizer = torch.optim.Adam( - torch.nn.Sequential(model.actor_network, model.critic_network).parameters(), - lr=config.training.learning_rate, - ) - - total_samples = 0 - for epoch_idx in range(config.training.epoch_count): - # Rollout a bunch of trials - observation = [] - action = [] - reward = [] - done = [] - epoch_last_step_idx = None - epoch_last_step_timestamp = None - async for ( - step_idx, - step_timestamp, - _trial_id, - _tick_id, - sample, - ) in run_session.start_trials_and_wait_for_termination( - trial_configs=[ - TrialConfig( - run_id=run_session.run_id, - environment=config.environment, - actors=[ - ActorParams( - name="agent_1", - actor_class="agent", - implementation="simple_a2c", - agent_config=AgentConfig( - run_id=run_session.run_id, - model_id=model_id, - model_version=model_version_number, - environment_specs=config.environment.specs, - ), - ) - ], - ) - for trial_idx in range(config.training.epoch_trial_count) - ], - max_parallel_trials=config.training.max_parallel_trials, - ): - (trial_observation, trial_action, trial_reward, trial_done) = sample - observation.extend(trial_observation) - action.extend(trial_action) - reward.extend(trial_reward) - done.extend(trial_done) - epoch_last_step_idx = step_idx - epoch_last_step_timestamp = step_timestamp - - xp_tracker.log_metrics(step_timestamp, step_idx, total_reward=sum([r.item() for r in trial_reward])) - - if len(observation) == 0: - log.warning( - f"[{run_session.params_name}/{run_session.run_id}] epoch #{epoch_idx + 1}/{config.training.epoch_count} finished without generating any sample (every trial ended at the first tick), skipping training." - ) - continue - - total_samples += len(observation) - - # Convert the accumulated observation/action/reward over the epoch to tensors - observation = torch.vstack(observation) - action = torch.vstack(action) - reward = torch.vstack(reward) - done = torch.vstack(done) - - # Compute the action probability and the critic value over the epoch - action_probs = torch.softmax(model.actor_network(observation), dim=-1) - critic = model.critic_network(observation).squeeze(-1) - - # Compute the estimated advantage over the epoch - advantage = ( - reward[1:] + config.training.discount_factor * critic[1:].detach() * (1 - done[1:]) - critic[:-1] - ) - - # Compute critic loss - value_loss = advantage.pow(2).mean() - - # Compute entropy loss - entropy_loss = torch.distributions.Categorical(action_probs).entropy().mean() - - # Compute A2C loss - action_log_probs = torch.gather(action_probs, -1, action.long()).log() - action_loss = -(action_log_probs[:-1] * advantage.detach()).mean() - - # Compute the complete loss - loss = ( - -config.training.entropy_coef * entropy_loss - + config.training.value_loss_coef * value_loss - + config.training.action_loss_coef * action_loss - ) - - # Backprop! - optimizer.zero_grad() - loss.backward() - optimizer.step() - - # Publish the newly trained version - last_epoch = epoch_idx + 1 == config.training.epoch_count - version_info = await self.publish_version( - model_id, - model, - archived=last_epoch, - epoch_idx=epoch_idx, - total_samples=total_samples, - environment_specs=config.environment.specs, - ) - model_version_number = version_info["version_number"] - xp_tracker.log_metrics( - epoch_last_step_timestamp, - epoch_last_step_idx, - model_version_number=model_version_number, - epoch_idx=epoch_idx, - entropy_loss=entropy_loss.item(), - value_loss=value_loss.item(), - action_loss=action_loss.item(), - loss=loss.item(), - total_samples=total_samples, - ) - log.info( - f"[{run_session.params_name}/{run_session.run_id}] epoch #{epoch_idx + 1}/{config.training.epoch_count} finished ({total_samples} samples seen)" - ) - - xp_tracker.terminate_success() - - return { - "simple_a2c_training": ( - sample_producer_impl, - run_impl, - SimpleA2CTrainingRunConfig( - environment=EnvironmentParams( - specs=None, # Needs to be specified - config=EnvironmentConfig(seed=12, framestack=1), - ), - training=SimpleA2CTrainingConfig( - epoch_count=100, - epoch_trial_count=15, - max_parallel_trials=8, - discount_factor=0.95, - entropy_coef=0.01, - value_loss_coef=0.5, - action_loss_coef=1.0, - learning_rate=0.01, - ), - actor_network=MLPNetworkConfig(hidden_size=64), - critic_network=MLPNetworkConfig(hidden_size=64), - ), - ) - } diff --git a/torch_agents/cogment_verse_torch_agents/simple_bc/__init__.py b/torch_agents/cogment_verse_torch_agents/simple_bc/__init__.py deleted file mode 100644 index c08c2100..00000000 --- a/torch_agents/cogment_verse_torch_agents/simple_bc/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cogment_verse_torch_agents.simple_bc.tutorial_1 import SimpleBCAgentAdapterTutorialStep1 -from cogment_verse_torch_agents.simple_bc.tutorial_2 import SimpleBCAgentAdapterTutorialStep2 -from cogment_verse_torch_agents.simple_bc.tutorial_3 import SimpleBCAgentAdapterTutorialStep3 -from cogment_verse_torch_agents.simple_bc.tutorial_4 import SimpleBCAgentAdapterTutorialStep4 - -SimpleBCAgentAdapter = SimpleBCAgentAdapterTutorialStep4 diff --git a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_1.py b/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_1.py deleted file mode 100644 index 61767f44..00000000 --- a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_1.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import copy -import logging - -import cogment -import numpy as np -import torch -from cogment.api.common_pb2 import TrialState -from cogment_verse import AgentAdapter, MlflowExperimentTracker -from cogment_verse.constants import HUMAN_ACTOR_NAME, HUMAN_ACTOR_CLASS, HUMAN_ACTOR_IMPL -from data_pb2 import ( - ActorParams, - AgentAction, - AgentConfig, - EnvironmentConfig, - EnvironmentParams, - HumanConfig, - HumanRole, - SimpleBCTrainingRunConfig, - TrialConfig, -) - -log = logging.getLogger(__name__) - -# pylint: disable=arguments-differ - - -class SimpleBCAgentAdapterTutorialStep1(AgentAdapter): - def __init__(self): - super().__init__() - self._dtype = torch.float - - @staticmethod - async def run_async(func, *args): - """Run a given function asynchronously in the default thread pool""" - all_events = asyncio.get_running_loop() - return await all_events.run_in_executor(None, func, *args) - - def _create_actor_implementations(self): - async def impl(actor_session): - actor_session.start() - - config = actor_session.config - - async for event in actor_session.all_events(): - if event.observation and event.type == cogment.EventType.ACTIVE: - action = np.random.default_rng().integers(0, config.environment_specs.num_action) - actor_session.do_action(AgentAction(discrete_action=action)) - - return { - "simple_bc": (impl, ["agent"]), - } - - def _create_run_implementations(self): - async def sample_producer_impl(run_sample_producer_session): - assert run_sample_producer_session.count_actors() == 2 - - async for sample in run_sample_producer_session.get_all_samples(): - if sample.get_trial_state() == TrialState.ENDED: - break - - log.info("Got raw sample") - - async def run_impl(run_session): - xp_tracker = MlflowExperimentTracker(run_session.params_name, run_session.run_id) - - config = run_session.config - assert config.environment.specs.num_players == 1 - - xp_tracker.log_params( - config.training, - config.environment.config, - environment=config.environment.specs.implementation, - policy_network_hidden_size=config.policy_network.hidden_size, - ) - - # Helper function to create a trial configuration - def create_trial_config(trial_idx): - env_params = copy.deepcopy(config.environment) - env_params.config.seed = env_params.config.seed + trial_idx - agent_actor_params = ActorParams( - name="agent_1", - actor_class="agent", - implementation="simple_bc", - agent_config=AgentConfig( - run_id=run_session.run_id, - environment_specs=env_params.specs, - ), - ) - - teacher_actor_params = ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, - implementation=HUMAN_ACTOR_IMPL, - human_config=HumanConfig( - environment_specs=env_params.specs, - role=HumanRole.TEACHER, - ), - ) - - return TrialConfig( - run_id=run_session.run_id, - environment=env_params, - actors=[agent_actor_params, teacher_actor_params], - ) - - # Rollout a bunch of trials - async for ( - _step_idx, - _step_timestamp, - _trial_id, - _tick_id, - sample, - ) in run_session.start_trials_and_wait_for_termination( - trial_configs=[create_trial_config(trial_idx) for trial_idx in range(config.training.trial_count)], - max_parallel_trials=config.training.max_parallel_trials, - ): - log.info(f"Got sample {sample}") - - return { - "simple_bc_training": ( - sample_producer_impl, - run_impl, - SimpleBCTrainingRunConfig( - environment=EnvironmentParams( - specs=None, # Needs to be specified - config=EnvironmentConfig(seed=12, framestack=1, render=True, render_width=256), - ) - ), - ) - } diff --git a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_2.py b/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_2.py deleted file mode 100644 index da9ed2a9..00000000 --- a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_2.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import copy -import logging - -import cogment -import numpy as np -import torch -from cogment.api.common_pb2 import TrialState -from cogment_verse import AgentAdapter, MlflowExperimentTracker -from cogment_verse.constants import HUMAN_ACTOR_NAME, HUMAN_ACTOR_CLASS, HUMAN_ACTOR_IMPL - - -############ TUTORIAL STEP 2 ############ -from cogment_verse_torch_agents.utils.tensors import tensor_from_cog_action, tensor_from_cog_obs -from data_pb2 import ( - ActorParams, - AgentAction, - AgentConfig, - EnvironmentConfig, - EnvironmentParams, - HumanConfig, - HumanRole, - SimpleBCTrainingRunConfig, - TrialConfig, -) - -########################################## - - -log = logging.getLogger(__name__) - -# pylint: disable=arguments-differ - - -class SimpleBCAgentAdapterTutorialStep2(AgentAdapter): - def __init__(self): - super().__init__() - self._dtype = torch.float - - @staticmethod - async def run_async(func, *args): - """Run a given function asynchronously in the default thread pool""" - all_events = asyncio.get_running_loop() - return await all_events.run_in_executor(None, func, *args) - - def _create_actor_implementations(self): - async def impl(actor_session): - actor_session.start() - - config = actor_session.config - - async for event in actor_session.all_events(): - if event.observation and event.type == cogment.EventType.ACTIVE: - action = np.random.default_rng().integers(0, config.environment_specs.num_action) - actor_session.do_action(AgentAction(discrete_action=action)) - - return { - "simple_bc": (impl, ["agent"]), - } - - def _create_run_implementations(self): - async def sample_producer_impl(run_sample_producer_session): - assert run_sample_producer_session.count_actors() == 2 - - async for sample in run_sample_producer_session.get_all_samples(): - if sample.get_trial_state() == TrialState.ENDED: - break - - ############ TUTORIAL STEP 2 ############ - observation = tensor_from_cog_obs(sample.get_actor_observation(0), dtype=self._dtype) - - # The actor order matches the order in the trial configuration - agent_action = sample.get_actor_action(0) - teacher_action = sample.get_actor_action(1) - - # Check for teacher override. - # Teacher action -1 corresponds to teacher approval, - # i.e. the teacher considers the action taken by the agent to be correct - if teacher_action.discrete_action != -1: - action = tensor_from_cog_action(teacher_action) - run_sample_producer_session.produce_training_sample((True, observation, action)) - else: - action = tensor_from_cog_action(agent_action) - run_sample_producer_session.produce_training_sample((False, observation, action)) - ########################################## - - async def run_impl(run_session): - xp_tracker = MlflowExperimentTracker(run_session.params_name, run_session.run_id) - - config = run_session.config - assert config.environment.specs.num_players == 1 - - xp_tracker.log_params( - config.training, - config.environment.config, - environment=config.environment.specs.implementation, - policy_network_hidden_size=config.policy_network.hidden_size, - ) - - # Helper function to create a trial configuration - def create_trial_config(trial_idx): - env_params = copy.deepcopy(config.environment) - env_params.config.seed = env_params.config.seed + trial_idx - agent_actor_params = ActorParams( - name="agent_1", - actor_class="agent", - implementation="simple_bc", - agent_config=AgentConfig( - run_id=run_session.run_id, - environment_specs=env_params.specs, - ), - ) - - teacher_actor_params = ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, - implementation=HUMAN_ACTOR_IMPL, - human_config=HumanConfig( - environment_specs=env_params.specs, - role=HumanRole.TEACHER, - ), - ) - - return TrialConfig( - run_id=run_session.run_id, - environment=env_params, - actors=[agent_actor_params, teacher_actor_params], - ) - - # Rollout a bunch of trials - async for ( - _step_idx, - _step_timestamp, - _trial_id, - _tick_id, - sample, - ) in run_session.start_trials_and_wait_for_termination( - trial_configs=[create_trial_config(trial_idx) for trial_idx in range(config.training.trial_count)], - max_parallel_trials=config.training.max_parallel_trials, - ): - log.info(f"Got sample {sample}") - - return { - "simple_bc_training": ( - sample_producer_impl, - run_impl, - SimpleBCTrainingRunConfig( - environment=EnvironmentParams( - specs=None, # Needs to be specified - config=EnvironmentConfig(seed=12, framestack=1, render=True, render_width=256), - ) - ), - ) - } diff --git a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_3.py b/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_3.py deleted file mode 100644 index bbd34a74..00000000 --- a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_3.py +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import copy -import logging - -############ TUTORIAL STEP 3 ############ -from collections import namedtuple - -########################################## - -import cogment -import torch -from cogment.api.common_pb2 import TrialState -from cogment_verse import AgentAdapter, MlflowExperimentTracker -from cogment_verse.constants import HUMAN_ACTOR_NAME, HUMAN_ACTOR_CLASS, HUMAN_ACTOR_IMPL - -############ TUTORIAL STEP 3 ############ -from cogment_verse_torch_agents.utils.tensors import cog_action_from_tensor, tensor_from_cog_action, tensor_from_cog_obs -from cogment_verse.spaces import flattened_dimensions -from data_pb2 import ( - ActorParams, - AgentConfig, - EnvironmentConfig, - EnvironmentParams, - HumanConfig, - HumanRole, - MLPNetworkConfig, - SimpleBCTrainingRunConfig, - TrialConfig, -) - -########################################## - -############ TUTORIAL STEP 3 ############ -SimpleBCModel = namedtuple("SimpleBCModel", ["model_id", "version_number", "policy_network"]) -########################################## - -log = logging.getLogger(__name__) - -# pylint: disable=arguments-differ -class SimpleBCAgentAdapterTutorialStep3(AgentAdapter): - def __init__(self): - super().__init__() - self._dtype = torch.float - - @staticmethod - async def run_async(func, *args): - """Run a given function asynchronously in the default thread pool""" - all_events = asyncio.get_running_loop() - return await all_events.run_in_executor(None, func, *args) - - ############ TUTORIAL STEP 3 ############ - def _create( - self, - model_id, - environment_specs, - policy_network_hidden_size=64, - **kwargs, - ): - num_input = flattened_dimensions(environment_specs.observation_space) - num_output = flattened_dimensions(environment_specs.action_space) - - model = SimpleBCModel( - model_id=model_id, - version_number=1, - policy_network=torch.nn.Sequential( - torch.nn.Linear(num_input, policy_network_hidden_size), - torch.nn.BatchNorm1d(policy_network_hidden_size), - torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, policy_network_hidden_size), - torch.nn.BatchNorm1d(policy_network_hidden_size), - torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, num_output), - ).to(self._dtype), - ) - - model_user_data = { - "environment_implementation": environment_specs.implementation, - "num_input": num_input, - "num_output": num_output, - } - - return model, model_user_data - - def _load(self, model_id, version_number, model_user_data, version_user_data, model_data_f, **kwargs): - policy_network = torch.load(model_data_f) - assert isinstance(policy_network, torch.nn.Sequential) - return SimpleBCModel(model_id=model_id, version_number=version_number, policy_network=policy_network) - - def _save(self, model, model_user_data, model_data_f, **kwargs): - assert isinstance(model, SimpleBCModel) - torch.save(model.policy_network, model_data_f) - return {} - - ########################################## - - def _create_actor_implementations(self): - async def impl(actor_session): - actor_session.start() - - config = actor_session.config - - ############ TUTORIAL STEP 3 ############ - model, _model_info, version_info = await self.retrieve_version(config.model_id, config.model_version) - model_version_number = version_info["version_number"] - log.info(f"Starting trial with model v{model_version_number}") - - # Retrieve the policy network and set it to "eval" mode - policy_network = copy.deepcopy(model.policy_network) - policy_network.eval() - - @torch.no_grad() - def compute_action(event): - obs = tensor_from_cog_obs(event.observation.snapshot, dtype=self._dtype) - scores = policy_network(obs.view(1, -1)) - probs = torch.softmax(scores, dim=-1) - action = torch.distributions.Categorical(probs).sample() - return action - - async for event in actor_session.all_events(): - if event.observation and event.type == cogment.EventType.ACTIVE: - action = await self.run_async(compute_action, event) - actor_session.do_action(cog_action_from_tensor(action)) - ########################################## - - return { - "simple_bc": (impl, ["agent"]), - } - - def _create_run_implementations(self): - async def sample_producer_impl(run_sample_producer_session): - assert run_sample_producer_session.count_actors() == 2 - - async for sample in run_sample_producer_session.get_all_samples(): - if sample.get_trial_state() == TrialState.ENDED: - break - - observation = tensor_from_cog_obs(sample.get_actor_observation(0), dtype=self._dtype) - - agent_action = sample.get_actor_action(0) - teacher_action = sample.get_actor_action(1) - - # Check for teacher override. - # Teacher action -1 corresponds to teacher approval, - # i.e. the teacher considers the action taken by the agent to be correct - if teacher_action.discrete_action != -1: - action = tensor_from_cog_action(teacher_action) - run_sample_producer_session.produce_training_sample((True, observation, action)) - else: - action = tensor_from_cog_action(agent_action) - run_sample_producer_session.produce_training_sample((False, observation, action)) - - async def run_impl(run_session): - xp_tracker = MlflowExperimentTracker(run_session.params_name, run_session.run_id) - - config = run_session.config - assert config.environment.specs.num_players == 1 - - xp_tracker.log_params( - config.training, - config.environment.config, - environment=config.environment.specs.implementation, - policy_network_hidden_size=config.policy_network.hidden_size, - ) - - ############ TUTORIAL STEP 3 ############ - model_id = f"{run_session.run_id}_model" - - # Initializing a model - _model, _version_info = await self.create_and_publish_initial_version( - model_id, - environment_specs=config.environment.specs, - policy_network_hidden_size=config.policy_network.hidden_size, - ) - ########################################## - - # Helper function to create a trial configuration - def create_trial_config(trial_idx): - env_params = copy.deepcopy(config.environment) - env_params.config.seed = env_params.config.seed + trial_idx - agent_actor_params = ActorParams( - name="agent_1", - actor_class="agent", - implementation="simple_bc", - agent_config=AgentConfig( - run_id=run_session.run_id, - ############ TUTORIAL STEP 3 ############ - model_id=model_id, - model_version=-1, - ########################################## - environment_specs=env_params.specs, - ), - ) - - teacher_actor_params = ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, - implementation=HUMAN_ACTOR_IMPL, - human_config=HumanConfig( - environment_specs=env_params.specs, - role=HumanRole.TEACHER, - ), - ) - - return TrialConfig( - run_id=run_session.run_id, - environment=env_params, - actors=[agent_actor_params, teacher_actor_params], - ) - - # Rollout a bunch of trials - async for ( - _step_idx, - _step_timestamp, - _trial_id, - _tick_id, - sample, - ) in run_session.start_trials_and_wait_for_termination( - trial_configs=[create_trial_config(trial_idx) for trial_idx in range(config.training.trial_count)], - max_parallel_trials=config.training.max_parallel_trials, - ): - log.info(f"Got sample {sample}") - - return { - "simple_bc_training": ( - sample_producer_impl, - run_impl, - SimpleBCTrainingRunConfig( - environment=EnvironmentParams( - specs=None, # Needs to be specified - config=EnvironmentConfig(seed=12, framestack=1, render=True, render_width=256), - ), - ############ TUTORIAL STEP 3 ############ - policy_network=MLPNetworkConfig(hidden_size=64), - ########################################## - ), - ) - } diff --git a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_4.py b/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_4.py deleted file mode 100644 index 6a2f7bea..00000000 --- a/torch_agents/cogment_verse_torch_agents/simple_bc/tutorial_4.py +++ /dev/null @@ -1,305 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import copy -import logging -from collections import namedtuple - -import cogment - -############ TUTORIAL STEP 4 ############ -import numpy as np - -########################################## -import torch -from cogment.api.common_pb2 import TrialState -from cogment_verse import AgentAdapter, MlflowExperimentTracker -from cogment_verse.constants import HUMAN_ACTOR_NAME, HUMAN_ACTOR_CLASS, HUMAN_ACTOR_IMPL -from cogment_verse_torch_agents.utils.tensors import cog_action_from_tensor, tensor_from_cog_action, tensor_from_cog_obs -from cogment_verse.spaces import flattened_dimensions -from data_pb2 import ( - ActorParams, - AgentConfig, - EnvironmentConfig, - EnvironmentParams, - HumanConfig, - HumanRole, - MLPNetworkConfig, - ############ TUTORIAL STEP 4 ############ - SimpleBCTrainingConfig, - ########################################## - SimpleBCTrainingRunConfig, - TrialConfig, -) - -SimpleBCModel = namedtuple("SimpleBCModel", ["model_id", "version_number", "policy_network"]) - -log = logging.getLogger(__name__) - -# pylint: disable=arguments-differ -class SimpleBCAgentAdapterTutorialStep4(AgentAdapter): - def __init__(self): - super().__init__() - self._dtype = torch.float - - @staticmethod - async def run_async(func, *args): - """Run a given function asynchronously in the default thread pool""" - all_events = asyncio.get_running_loop() - return await all_events.run_in_executor(None, func, *args) - - def _create( - self, - model_id, - environment_specs, - policy_network_hidden_size=64, - **kwargs, - ): - num_input = flattened_dimensions(environment_specs.observation_space) - num_output = flattened_dimensions(environment_specs.action_space) - - model = SimpleBCModel( - model_id=model_id, - version_number=1, - policy_network=torch.nn.Sequential( - torch.nn.Linear(num_input, policy_network_hidden_size), - torch.nn.BatchNorm1d(policy_network_hidden_size), - torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, policy_network_hidden_size), - torch.nn.BatchNorm1d(policy_network_hidden_size), - torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, num_output), - ).to(self._dtype), - ) - - model_user_data = { - "environment_implementation": environment_specs.implementation, - "num_input": num_input, - "num_output": num_output, - } - - return model, model_user_data - - def _load(self, model_id, version_number, model_user_data, version_user_data, model_data_f, **kwargs): - policy_network = torch.load(model_data_f) - assert isinstance(policy_network, torch.nn.Sequential) - return SimpleBCModel(model_id=model_id, version_number=version_number, policy_network=policy_network) - - def _save(self, model, model_user_data, model_data_f, **kwargs): - assert isinstance(model, SimpleBCModel) - torch.save(model.policy_network, model_data_f) - return {} - - def _create_actor_implementations(self): - async def impl(actor_session): - actor_session.start() - - config = actor_session.config - - model, _model_info, version_info = await self.retrieve_version(config.model_id, config.model_version) - model_version_number = version_info["version_number"] - log.info(f"Starting trial with model v{model_version_number}") - - # Retrieve the policy network and set it to "eval" mode - policy_network = copy.deepcopy(model.policy_network) - policy_network.eval() - - @torch.no_grad() - def compute_action(event): - obs = tensor_from_cog_obs(event.observation.snapshot, dtype=self._dtype) - scores = policy_network(obs.view(1, -1)) - probs = torch.softmax(scores, dim=-1) - action = torch.distributions.Categorical(probs).sample() - return action - - async for event in actor_session.all_events(): - if event.observation and event.type == cogment.EventType.ACTIVE: - action = await self.run_async(compute_action, event) - actor_session.do_action(cog_action_from_tensor(action)) - - return { - "simple_bc": (impl, ["agent"]), - } - - def _create_run_implementations(self): - async def sample_producer_impl(run_sample_producer_session): - assert run_sample_producer_session.count_actors() == 2 - - async for sample in run_sample_producer_session.get_all_samples(): - if sample.get_trial_state() == TrialState.ENDED: - break - - observation = tensor_from_cog_obs(sample.get_actor_observation(0), dtype=self._dtype) - - agent_action = sample.get_actor_action(0) - teacher_action = sample.get_actor_action(1) - - # Check for teacher override. - # Teacher action -1 corresponds to teacher approval, - # i.e. the teacher considers the action taken by the agent to be correct - if teacher_action.discrete_action != -1: - action = tensor_from_cog_action(teacher_action) - run_sample_producer_session.produce_training_sample((True, observation, action)) - else: - action = tensor_from_cog_action(agent_action) - run_sample_producer_session.produce_training_sample((False, observation, action)) - - async def run_impl(run_session): - xp_tracker = MlflowExperimentTracker(run_session.params_name, run_session.run_id) - - config = run_session.config - assert config.environment.specs.num_players == 1 - - xp_tracker.log_params( - config.training, - config.environment.config, - environment=config.environment.specs.implementation, - policy_network_hidden_size=config.policy_network.hidden_size, - ) - - model_id = f"{run_session.run_id}_model" - - # Initializing a model - model, _version_info = await self.create_and_publish_initial_version( - model_id, - environment_specs=config.environment.specs, - policy_network_hidden_size=config.policy_network.hidden_size, - ) - - # Helper function to create a trial configuration - def create_trial_config(trial_idx): - env_params = copy.deepcopy(config.environment) - env_params.config.seed = env_params.config.seed + trial_idx - agent_actor_params = ActorParams( - name="agent_1", - actor_class="agent", - implementation="simple_bc", - agent_config=AgentConfig( - run_id=run_session.run_id, - model_id=model_id, - model_version=-1, - environment_specs=env_params.specs, - ), - ) - - teacher_actor_params = ActorParams( - name=HUMAN_ACTOR_NAME, - actor_class=HUMAN_ACTOR_CLASS, - implementation=HUMAN_ACTOR_IMPL, - human_config=HumanConfig( - environment_specs=env_params.specs, - role=HumanRole.TEACHER, - ), - ) - - return TrialConfig( - run_id=run_session.run_id, - environment=env_params, - actors=[agent_actor_params, teacher_actor_params], - ) - - ############ TUTORIAL STEP 4 ############ - # Configure the optimizer - optimizer = torch.optim.Adam( - model.policy_network.parameters(), - lr=config.training.learning_rate, - ) - - # Keep accumulated observations/actions around - observations = [] - actions = [] - - loss_fn = torch.nn.CrossEntropyLoss() - - def train_step(): - # Sample a batch of observations/actions - batch_indices = np.random.default_rng().integers(0, len(observations), config.training.batch_size) - batch_obs = torch.vstack([observations[i] for i in batch_indices]) - batch_act = torch.vstack([actions[i] for i in batch_indices]).view(-1) - - model.policy_network.train() - pred_policy = model.policy_network(batch_obs) - loss = loss_fn(pred_policy, batch_act) - - # Backprop! - optimizer.zero_grad() - loss.backward() - optimizer.step() - - return loss.item() - - ########################################## - - # Rollout a bunch of trials - async for ( - ############ TUTORIAL STEP 4 ############ - step_idx, - step_timestamp, - ########################################## - _trial_id, - _tick_id, - sample, - ) in run_session.start_trials_and_wait_for_termination( - trial_configs=[create_trial_config(trial_idx) for trial_idx in range(config.training.trial_count)], - max_parallel_trials=config.training.max_parallel_trials, - ): - ############ TUTORIAL STEP 4 ############ - (_demonstration, observation, action) = sample - # Can be uncommented to only use samples coming from the teacher - # (demonstration, observation, action) = sample - # if not demonstration: - # continue - observations.append(observation) - actions.append(action) - - if len(observations) < config.training.batch_size: - continue - - loss = await self.run_async(train_step) - - # Publish the newly trained version every 100 steps - if step_idx % 100 == 0: - version_info = await self.publish_version(model_id, model) - - xp_tracker.log_metrics( - step_timestamp, - step_idx, - model_version_number=version_info["version_number"], - loss=loss, - total_samples=len(observations), - ) - ########################################## - - return { - "simple_bc_training": ( - sample_producer_impl, - run_impl, - SimpleBCTrainingRunConfig( - environment=EnvironmentParams( - specs=None, # Needs to be specified, - config=EnvironmentConfig(seed=12, framestack=1, render=True, render_width=256), - ), - ############ TUTORIAL STEP 4 ############ - training=SimpleBCTrainingConfig( - trial_count=100, - max_parallel_trials=1, - discount_factor=0.95, - learning_rate=0.01, - ), - ########################################## - policy_network=MLPNetworkConfig(hidden_size=64), - ), - ) - } diff --git a/torch_agents/cogment_verse_torch_agents/utils/throttle.py b/torch_agents/cogment_verse_torch_agents/utils/throttle.py deleted file mode 100644 index 0718e665..00000000 --- a/torch_agents/cogment_verse_torch_agents/utils/throttle.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime, timedelta -from functools import wraps - - -# pylint: disable=invalid-name -class throttle: - """ - Decorator that prevents a function from being called more than once every - time period. - To create a function that cannot be called more than once a minute: - @throttle(seconds=60) - def my_fun(): - pass - """ - - def __init__(self, seconds=0): - self.throttle_period = timedelta(seconds=seconds) - self.time_of_last_call = datetime.min - - def __call__(self, function): - @wraps(function) - def wrapper(*args, **kwargs): - now = datetime.now() - time_since_last_call = now - self.time_of_last_call - - if time_since_last_call > self.throttle_period: - self.time_of_last_call = now - return function(*args, **kwargs) - - return None - - return wrapper diff --git a/web_client/.gitignore b/web_client/.gitignore deleted file mode 100644 index 7b9f4143..00000000 --- a/web_client/.gitignore +++ /dev/null @@ -1,29 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.js - -# testing -/coverage - -# production -/build - -# misc -.DS_Store -.env.local -.env.development.local -.env.test.local -.env.production.local - -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -src/CogSettings.* -src/data_pb.* -src/data.* -src/data_pb_service.* -src/CogTypes.d.ts diff --git a/web_client/src/controls/GymMountainCarControls.jsx b/web_client/src/controls/GymMountainCarControls.jsx deleted file mode 100644 index e56456d3..00000000 --- a/web_client/src/controls/GymMountainCarControls.jsx +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2021 AI Redefined Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { useCallback, useState } from "react"; -import { cogment_verse } from "../data_pb"; -import { useDocumentKeypressListener, usePressedKeys } from "../hooks/usePressedKeys"; -import { useRealTimeUpdate } from "../hooks/useRealTimeUpdate"; -import { TEACHER_NOOP_ACTION } from "../utils/constants"; -import { Button } from "../components/Button"; -import { FpsCounter } from "../components/FpsCounter"; -import { KeyboardControlList } from "../components/KeyboardControlList"; - -export const GymMountainCarEnvironments = ["gym/MountainCar-v0"]; -export const GymMountainCarControls = ({ sendAction, fps = 30, role, ...props }) => { - const [paused, setPaused] = useState(false); - const togglePause = useCallback(() => setPaused((paused) => !paused), [setPaused]); - useDocumentKeypressListener("p", togglePause); - - const pressedKeys = usePressedKeys(); - - const computeAndSendAction = useCallback( - (dt) => { - if (pressedKeys.size === 0 && role === cogment_verse.HumanRole.TEACHER) { - sendAction(TEACHER_NOOP_ACTION); - return; - } - - const action_params = { - discreteAction: 1, - }; - - if (pressedKeys.has("ArrowLeft")) { - action_params.discreteAction = 0; - } else if (pressedKeys.has("ArrowRight")) { - action_params.discreteAction = 2; - } - - const action = new cogment_verse.AgentAction(action_params); - sendAction(action); - }, - [pressedKeys, sendAction, role] - ); - - const { currentFps } = useRealTimeUpdate(computeAndSendAction, fps, paused); - - return ( -
    -
    - - -
    - -
    - ); -}; From 68953eb5e32c66d37b7c181d9451e6966e914cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clod=C3=A9ric=20Mars?= Date: Fri, 17 Jun 2022 12:59:50 -0400 Subject: [PATCH 02/64] Introducing DQN --- README.md | 4 +- .../reinforce/replaybuffer.py | 68 --- actors/random_actor.py | 3 +- actors/simple_a2c.py | 97 +++-- actors/simple_dqn.py | 396 ++++++++++++++++++ actors/tutorial/tutorial_1.py | 15 +- actors/tutorial/tutorial_2.py | 15 +- actors/tutorial/tutorial_3.py | 42 +- actors/tutorial/tutorial_4.py | 59 ++- cogment_verse/__init__.py | 1 + cogment_verse/mlflow_experiment_tracker.py | 3 + cogment_verse/replay_buffers/__init__.py | 15 + .../replay_buffers/torch_replay_buffer.py | 87 ++++ cogment_verse/run/run_session.py | 7 +- cogment_verse/run/sample_producer_worker.py | 14 +- cogment_verse/run/trial_runner_worker.py | 18 +- cogment_verse/specs/__init__.py | 6 + cogment_verse/specs/data.proto | 14 +- cogment_verse/specs/gym_spaces_adapter.py | 86 ++++ cogment_verse/specs/sample_space.py | 6 +- config/config.yaml | 2 +- config/experiment/simple_a2c/cartpole.yaml | 21 +- config/experiment/simple_bc/mountain_car.yaml | 9 + config/experiment/simple_dqn/cartpole.yaml | 22 + config/run/observe.yaml | 2 +- config/run/play.yaml | 2 +- config/run/simple_a2c.yaml | 16 - config/run/simple_bc.yaml | 5 - config/services/actor/simple_dqn.yaml | 2 + docs/results/a2c.md | 6 +- environments/gym_adapter.py | 60 +-- pyproject.toml | 2 +- requirements.txt | 2 +- runs/play.py | 6 +- tests/test_torch_replay_buffer.py | 48 +++ 35 files changed, 850 insertions(+), 311 deletions(-) delete mode 100644 _old/tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py create mode 100644 actors/simple_dqn.py create mode 100644 cogment_verse/replay_buffers/__init__.py create mode 100644 cogment_verse/replay_buffers/torch_replay_buffer.py create mode 100644 cogment_verse/specs/gym_spaces_adapter.py create mode 100644 config/experiment/simple_bc/mountain_car.yaml create mode 100644 config/experiment/simple_dqn/cartpole.yaml delete mode 100644 config/run/simple_a2c.yaml delete mode 100644 config/run/simple_bc.yaml create mode 100644 config/services/actor/simple_dqn.yaml create mode 100644 tests/test_torch_replay_buffer.py diff --git a/README.md b/README.md index b13ea473..268b4466 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,11 @@ Here are a few examples: - Launch a Simple Behavior Cloning run with the [Mountain Car Gym environment](https://www.gymlibrary.ml/environments/classic_control/mountain_car/) (which is the default environment) ```console - $ python -m main services/actor=simple_bc run=simple_bc + $ python -m main +experiment=simple_bc/mountain_car ``` - Launch a Simple Behavior Cloning run with the [Lunar Lander Gym environment](https://www.gymlibrary.ml/environments/box2d/lunar_lander/) ```console - $ python -m main services/actor=simple_bc services/environment=lunar_lander run=simple_bc + $ python -m main +experiment=simple_bc/mountain_car services/environment=lunar_lander ``` - Launch and play a single trial of the Lunar Lander Gym environment with continuous controls ```console diff --git a/_old/tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py b/_old/tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py deleted file mode 100644 index 7d7c8d3f..00000000 --- a/_old/tf_agents/cogment_verse_tf_agents/reinforce/replaybuffer.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2021 AI Redefined Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np - - -class Memory: - def __init__(self, number_features, number_actions, buffer_size): - """Initialize buffer - Params - ====== - number_features (int): number of features in player's observation - number_actions (int): number of actions of a player - buffer_size (int): maximum size of the buffer - """ - - self.buffer_size = buffer_size - self.number_features = number_features - self.number_actions = number_actions - - # initialize state, action, reward, next_state, done arrays - self._data = {} - self._data["observations"] = np.zeros((self.buffer_size, self.number_features)) - self._data["actions"] = np.zeros((self.buffer_size,)) - self._data["rewards"] = np.zeros((self.buffer_size,)) - self._data["next_observations"] = np.zeros((self.buffer_size, self.number_features)) - self._data["dones"] = np.zeros((self.buffer_size,)) - - self._ptr = 0 - - def add(self, data): - """Add an experience to buffer - Params - ====== - data (tuple): (observation, action, reward, next_observation, done) - """ - for idx, key in enumerate(self._data): - self._data[key][self._ptr] = np.asarray(data[idx]) - - self._ptr = (self._ptr + 1) % self.buffer_size - - def sample(self): - """Sample previous trajectory data from the buffer""" - - rval = {} - for key, _ in self._data.items(): - rval[key] = self._data[key][0 : self._ptr] - - return rval - - def reset_replay_buffer(self): - """ - Resets the pointer of the buffer to its initial position, - thereby making previous trajectory data unavailable to the - Reinforce agent - """ - self._ptr = 0 diff --git a/actors/random_actor.py b/actors/random_actor.py index c4851650..464d54a9 100644 --- a/actors/random_actor.py +++ b/actors/random_actor.py @@ -35,8 +35,7 @@ async def impl(self, actor_session): action_space = config.environment_specs.action_space - # TODO this is something that could be configured - random_seed = 0 + random_seed = config.seed if config.seed is not None else 0 async for event in actor_session.all_events(): if event.observation and event.type == cogment.EventType.ACTIVE: diff --git a/actors/simple_a2c.py b/actors/simple_a2c.py index ca97b977..dff25531 100644 --- a/actors/simple_a2c.py +++ b/actors/simple_a2c.py @@ -40,8 +40,8 @@ def __init__( environment_implementation, num_input, num_output, - actor_network_hidden_size=64, - critic_network_hidden_size=64, + actor_network_num_hidden_nodes=64, + critic_network_num_hidden_nodes=64, dtype=torch.float, version_number=0, ): @@ -50,23 +50,27 @@ def __init__( self._environment_implementation = environment_implementation self._num_input = num_input self._num_output = num_output - self._actor_network_hidden_size = actor_network_hidden_size - self._critic_network_hidden_size = critic_network_hidden_size + self._actor_network_num_hidden_nodes = actor_network_num_hidden_nodes + self._critic_network_num_hidden_nodes = critic_network_num_hidden_nodes self.actor_network = torch.nn.Sequential( - torch.nn.Linear(self._num_input, self._actor_network_hidden_size, dtype=self._dtype), + torch.nn.Linear(self._num_input, self._actor_network_num_hidden_nodes, dtype=self._dtype), torch.nn.Tanh(), - torch.nn.Linear(self._actor_network_hidden_size, self._actor_network_hidden_size, dtype=self._dtype), + torch.nn.Linear( + self._actor_network_num_hidden_nodes, self._actor_network_num_hidden_nodes, dtype=self._dtype + ), torch.nn.Tanh(), - torch.nn.Linear(self._actor_network_hidden_size, self._num_output, dtype=self._dtype), + torch.nn.Linear(self._actor_network_num_hidden_nodes, self._num_output, dtype=self._dtype), ) self.critic_network = torch.nn.Sequential( - torch.nn.Linear(self._num_input, self._critic_network_hidden_size, dtype=self._dtype), + torch.nn.Linear(self._num_input, self._critic_network_num_hidden_nodes, dtype=self._dtype), torch.nn.Tanh(), - torch.nn.Linear(self._critic_network_hidden_size, self._critic_network_hidden_size, dtype=self._dtype), + torch.nn.Linear( + self._critic_network_num_hidden_nodes, self._critic_network_num_hidden_nodes, dtype=self._dtype + ), torch.nn.Tanh(), - torch.nn.Linear(self._critic_network_hidden_size, 1, dtype=self._dtype), + torch.nn.Linear(self._critic_network_num_hidden_nodes, 1, dtype=self._dtype), ) # version user data @@ -78,8 +82,8 @@ def get_model_user_data(self): "environment_implementation": self._environment_implementation, "num_input": self._num_input, "num_output": self._num_output, - "actor_network_hidden_size": self._actor_network_hidden_size, - "critic_network_hidden_size": self._critic_network_hidden_size, + "actor_network_num_hidden_nodes": self._actor_network_num_hidden_nodes, + "critic_network_num_hidden_nodes": self._critic_network_num_hidden_nodes, } def save(self, model_data_f): @@ -96,8 +100,8 @@ def load(cls, model_id, version_number, model_user_data, version_user_data, mode environment_implementation=model_user_data["environment_implementation"], num_input=int(model_user_data["num_input"]), num_output=int(model_user_data["num_output"]), - actor_network_hidden_size=int(model_user_data["actor_network_hidden_size"]), - critic_network_hidden_size=int(model_user_data["critic_network_hidden_size"]), + actor_network_num_hidden_nodes=int(model_user_data["actor_network_num_hidden_nodes"]), + critic_network_num_hidden_nodes=int(model_user_data["critic_network_num_hidden_nodes"]), ) # Load the saved states @@ -148,19 +152,17 @@ async def impl(self, actor_session): class SimpleA2CTraining: default_cfg = { - "environment_config": {"seed": 10}, - "training": { - "epoch_count": 500, - "epoch_trial_count": 10, - "num_parallel_trials": 8, - "discount_factor": 0.99, - "entropy_loss_coef": 0.05, - "value_loss_coef": 0.5, - "action_loss_coef": 1.0, - "learning_rate": 0.01, - }, - "actor_network": {"hidden_size": 64}, - "critic_network": {"hidden_size": 64}, + "seed": 10, + "num_epochs": 500, + "epoch_num_trials": 10, + "num_parallel_trials": 8, + "discount_factor": 0.99, + "entropy_loss_coef": 0.05, + "value_loss_coef": 0.5, + "action_loss_coef": 1.0, + "learning_rate": 0.01, + "actor_network": {"num_hidden_nodes": 64}, + "critic_network": {"num_hidden_nodes": 64}, } def __init__(self, environment_specs, cfg): @@ -218,34 +220,33 @@ async def impl(self, run_session): environment_implementation=self._environment_specs.implementation, num_input=flattened_dimensions(self._environment_specs.observation_space), num_output=flattened_dimensions(self._environment_specs.action_space), - actor_network_hidden_size=self._cfg.actor_network.hidden_size, - critic_network_hidden_size=self._cfg.critic_network.hidden_size, + actor_network_num_hidden_nodes=self._cfg.actor_network.num_hidden_nodes, + critic_network_num_hidden_nodes=self._cfg.critic_network.num_hidden_nodes, dtype=self._dtype, ) _model_info, version_info = await run_session.model_registry.publish_initial_version(model) run_session.log_params( - self._cfg.training, - self._cfg.environment_config, + self._cfg, environment_implementation=self._environment_specs.implementation, - actor_network_hidden_size=self._cfg.actor_network.hidden_size, - critic_network_hidden_size=self._cfg.critic_network.hidden_size, + actor_network_num_hidden_nodes=self._cfg.actor_network.num_hidden_nodes, + critic_network_num_hidden_nodes=self._cfg.critic_network.num_hidden_nodes, ) # Configure the optimizer over the two models optimizer = torch.optim.Adam( torch.nn.Sequential(model.actor_network, model.critic_network).parameters(), - lr=self._cfg.training.learning_rate, + lr=self._cfg.learning_rate, ) total_samples = 0 - for epoch_idx in range(self._cfg.training.epoch_count): + for epoch_idx in range(self._cfg.num_epochs): # Rollout a bunch of trials observation = [] action = [] reward = [] done = [] - for (_step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + for (_step_idx, _trial_id, _trial_idx, sample,) in run_session.start_and_await_trials( trials_id_and_params=[ ( f"{run_session.run_id}_{epoch_idx}_{trial_idx}", @@ -256,9 +257,7 @@ async def impl(self, run_session): environment_config=EnvironmentConfig( run_id=run_session.run_id, render=False, - seed=self._cfg.environment_config.seed - + trial_idx - + epoch_idx * self._cfg.training.epoch_trial_count, + seed=self._cfg.seed + trial_idx + epoch_idx * self._cfg.epoch_num_trials, ), actors=[ cogment.ActorParameters( @@ -276,10 +275,10 @@ async def impl(self, run_session): ], ), ) - for trial_idx in range(self._cfg.training.epoch_trial_count) + for trial_idx in range(self._cfg.epoch_num_trials) ], sample_producer_impl=self.trial_sample_sequences_producer_impl, - num_parallel_trials=self._cfg.training.num_parallel_trials, + num_parallel_trials=self._cfg.num_parallel_trials, ): (trial_observation, trial_action, trial_reward, trial_done) = sample observation.extend(trial_observation) @@ -291,7 +290,7 @@ async def impl(self, run_session): if len(observation) == 0: log.warning( - f"[SimpleA2CTraining/{run_session.run_id}] epoch #{epoch_idx + 1}/{self._cfg.training.epoch_count} finished without generating any sample (every trial ended at the first tick), skipping training." + f"[SimpleA2CTraining/{run_session.run_id}] epoch #{epoch_idx + 1}/{self._cfg.num_epochs} finished without generating any sample (every trial ended at the first tick), skipping training." ) continue @@ -309,9 +308,7 @@ async def impl(self, run_session): # Compute the estimated advantage over the epoch advantage = ( - reward[1:] - + self._cfg.training.discount_factor * critic[1:].detach() * (1.0 - done[1:].float()) - - critic[:-1] + reward[1:] + self._cfg.discount_factor * critic[1:].detach() * (1.0 - done[1:].float()) - critic[:-1] ) # Compute critic loss @@ -326,9 +323,9 @@ async def impl(self, run_session): # Compute the complete loss loss = ( - -self._cfg.training.entropy_loss_coef * entropy_loss - + self._cfg.training.value_loss_coef * value_loss - + self._cfg.training.action_loss_coef * action_loss + -self._cfg.entropy_loss_coef * entropy_loss + + self._cfg.value_loss_coef * value_loss + + self._cfg.action_loss_coef * action_loss ) # Backprop! @@ -337,7 +334,7 @@ async def impl(self, run_session): optimizer.step() # Publish the newly trained version - last_epoch = (epoch_idx + 1) == self._cfg.training.epoch_count + last_epoch = (epoch_idx + 1) == self._cfg.num_epochs model.epoch_idx = epoch_idx model.total_samples = total_samples @@ -353,5 +350,5 @@ async def impl(self, run_session): total_samples=total_samples, ) log.info( - f"[SimpleA2CTraining/{run_session.run_id}] epoch #{epoch_idx + 1}/{self._cfg.training.epoch_count} finished ({total_samples} samples seen)" + f"[SimpleA2CTraining/{run_session.run_id}] epoch #{epoch_idx + 1}/{self._cfg.num_epochs} finished ({total_samples} samples seen)" ) diff --git a/actors/simple_dqn.py b/actors/simple_dqn.py new file mode 100644 index 00000000..95df25dc --- /dev/null +++ b/actors/simple_dqn.py @@ -0,0 +1,396 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import copy +import time +import json + +import cogment +import torch +import numpy as np + +from cogment_verse.specs import ( + AgentConfig, + cog_settings, + EnvironmentConfig, + flatten, + flattened_dimensions, + PLAYER_ACTOR_CLASS, + PlayerAction, + SpaceValue, + sample_space, +) + +from cogment_verse import Model, TorchReplayBuffer + + +log = logging.getLogger(__name__) + + +def create_linear_schedule(start, end, duration): + slope = (end - start) / duration + + def compute_value(t): + return max(slope * t + start, end) + + return compute_value + + +class SimpleDQNModel(Model): + def __init__( + self, + model_id, + environment_implementation, + num_input, + num_output, + num_hidden_nodes, + epsilon, + dtype=torch.float, + version_number=0, + ): + super().__init__(model_id, version_number) + self._dtype = dtype + self._environment_implementation = environment_implementation + self._num_input = num_input + self._num_output = num_output + self._num_hidden_nodes = list(num_hidden_nodes) + + self.epsilon = epsilon + self.network = torch.nn.Sequential( + torch.nn.Linear(self._num_input, self._num_hidden_nodes[0]), + torch.nn.ReLU(), + *[ + layer + for hidden_node_idx in range(len(self._num_hidden_nodes) - 1) + for layer in [ + torch.nn.Linear(self._num_hidden_nodes[hidden_node_idx], self._num_hidden_nodes[-1]), + torch.nn.ReLU(), + ] + ], + torch.nn.Linear(self._num_hidden_nodes[-1], self._num_output), + ) + + # version user data + self.num_samples_seen = 0 + + def get_model_user_data(self): + return { + "environment_implementation": self._environment_implementation, + "num_input": self._num_input, + "num_output": self._num_output, + "num_hidden_nodes": json.dumps(self._num_hidden_nodes), + } + + def save(self, model_data_f): + torch.save((self.network.state_dict(), self.epsilon), model_data_f) + + return {"num_samples_seen": 0} + + @classmethod + def load(cls, model_id, version_number, model_user_data, version_user_data, model_data_f): + # Create the model instance + model = SimpleDQNModel( + model_id=model_id, + version_number=version_number, + environment_implementation=model_user_data["environment_implementation"], + num_input=int(model_user_data["num_input"]), + num_output=int(model_user_data["num_output"]), + num_hidden_nodes=json.loads(model_user_data["num_hidden_nodes"]), + epsilon=0, + ) + + # Load the saved states + (network_state_dict, epsilon) = torch.load(model_data_f) + model.network.load_state_dict(network_state_dict) + model.epsilon = epsilon + + # Load version data + model.num_samples_seen = int(version_user_data["num_samples_seen"]) + + return model + + +class SimpleDQNActor: + def __init__(self, _cfg): + self._dtype = torch.float + + def get_actor_classes(self): + return [PLAYER_ACTOR_CLASS] + + async def impl(self, actor_session): + actor_session.start() + + config = actor_session.config + + assert config.environment_specs.num_players == 1 + assert len(config.environment_specs.action_space.properties) == 1 + assert config.environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + + observation_space = config.environment_specs.observation_space + action_space = config.environment_specs.action_space + + rng = np.random.default_rng(config.seed if config.seed is not None else 0) + + model, _, _ = await actor_session.model_registry.retrieve_version( + SimpleDQNModel, config.model_id, config.model_version + ) + model.network.eval() + + async for event in actor_session.all_events(): + if event.observation and event.type == cogment.EventType.ACTIVE: + if config.model_version == -1 and actor_session.get_tick_id() % config.model_update_frequency == 0: + model, _, _ = await actor_session.model_registry.retrieve_version( + SimpleDQNModel, config.model_id, config.model_version + ) + model.network.eval() + if rng.random() < model.epsilon: + [action_value] = sample_space(action_space, rng=rng) + else: + obs_tensor = torch.tensor( + flatten(observation_space, event.observation.observation.value), dtype=self._dtype + ) + action_probs = model.network(obs_tensor) + discrete_action_tensor = torch.argmax(action_probs) + action_value = SpaceValue( + properties=[SpaceValue.PropertyValue(discrete=discrete_action_tensor.item())] + ) + + actor_session.do_action(PlayerAction(value=action_value)) + + +class SimpleDQNTraining: + default_cfg = { + "seed": 10, + "num_trials": 5000, + "num_parallel_trials": 10, + "learning_rate": 0.00025, + "buffer_size": 10000, + "discount_factor": 0.99, + "target_network_frequency": 500, + "batch_size": 128, + "epsilon_schedule_start": 1, + "epsilon_schedule_end": 0.05, + "epsilon_schedule_duration_ratio": 0.75, + "learning_starts": 10000, + "train_frequency": 10, + "model_update_frequency": 10, + "value_network": {"num_hidden_nodes": [128, 64]}, + } + + def __init__(self, environment_specs, cfg): + super().__init__() + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._dtype = torch.float + self._environment_specs = environment_specs + self._cfg = cfg + + async def sample_producer_impl(self, sample_producer_session): + player_actor_params = sample_producer_session.trial_info.parameters.actors[0] + + player_actor_name = player_actor_params.name + player_observation_space = player_actor_params.config.environment_specs.observation_space + + observation = None + action = None + reward = None + + total_reward = 0 + + async for sample in sample_producer_session.all_trial_samples(): + actor_sample = sample.actors_data[player_actor_name] + if actor_sample.observation is None: + # This can happen when there is several "end-of-trial" samples + continue + + next_observation = torch.tensor( + flatten(player_observation_space, actor_sample.observation.value), dtype=self._dtype + ) + + if observation is not None: + # It's not the first sample, let's check if it is the last + done = sample.trial_state == cogment.TrialState.ENDED + sample_producer_session.produce_sample( + ( + observation, + next_observation, + action, + reward, + torch.ones(1, dtype=torch.int8) if done else torch.zeros(1, dtype=torch.int8), + total_reward, + ) + ) + if done: + break + + observation = next_observation + action_value = actor_sample.action.value + action = torch.tensor( + action_value.properties[0].discrete if len(action_value.properties) > 0 else 0, dtype=torch.int64 + ) + reward = torch.tensor(actor_sample.reward if actor_sample.reward is not None else 0, dtype=self._dtype) + total_reward += reward.item() + + async def impl(self, run_session): + # Initializing a model + model_id = f"{run_session.run_id}_model" + + assert self._environment_specs.num_players == 1 + assert len(self._environment_specs.action_space.properties) == 1 + assert self._environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + + epsilon_schedule = create_linear_schedule( + self._cfg.epsilon_schedule_start, + self._cfg.epsilon_schedule_end, + self._cfg.epsilon_schedule_duration_ratio * self._cfg.num_trials, + ) + + model = SimpleDQNModel( + model_id, + environment_implementation=self._environment_specs.implementation, + num_input=flattened_dimensions(self._environment_specs.observation_space), + num_output=flattened_dimensions(self._environment_specs.action_space), + num_hidden_nodes=self._cfg.value_network.num_hidden_nodes, + epsilon=epsilon_schedule(0), + dtype=self._dtype, + ) + _model_info, version_info = await run_session.model_registry.publish_initial_version(model) + + run_session.log_params( + self._cfg, + environment_implementation=self._environment_specs.implementation, + ) + + # Configure the optimizer + optimizer = torch.optim.Adam( + model.network.parameters(), + lr=self._cfg.learning_rate, + ) + + # Initialize the target model + target_network = copy.deepcopy(model.network) + + replay_buffer = TorchReplayBuffer( + capacity=self._cfg.buffer_size, + observation_shape=(flattened_dimensions(self._environment_specs.observation_space),), + observation_dtype=self._dtype, + action_shape=(1,), + action_dtype=torch.int64, + reward_dtype=self._dtype, + seed=self._cfg.seed, + ) + + start_time = time.time() + total_reward_cum = 0 + + for (step_idx, _trial_id, trial_idx, sample,) in run_session.start_and_await_trials( + trials_id_and_params=[ + ( + f"{run_session.run_id}_{trial_idx}", + cogment.TrialParameters( + cog_settings, + environment_name="env", + environment_implementation=self._environment_specs.implementation, + environment_config=EnvironmentConfig( + run_id=run_session.run_id, + render=False, + seed=self._cfg.seed + trial_idx, + ), + actors=[ + cogment.ActorParameters( + cog_settings, + name="player", + class_name=PLAYER_ACTOR_CLASS, + implementation="actors.simple_dqn.SimpleDQNActor", + config=AgentConfig( + run_id=run_session.run_id, + seed=self._cfg.seed + trial_idx, + model_id=model_id, + model_version=-1, + model_update_frequency=self._cfg.model_update_frequency, + environment_specs=self._environment_specs, + ), + ) + ], + ), + ) + for trial_idx in range(self._cfg.num_trials) + ], + sample_producer_impl=self.sample_producer_impl, + num_parallel_trials=self._cfg.num_parallel_trials, + ): + (observation, next_observation, action, reward, done, total_reward) = sample + replay_buffer.add( + observation=observation, next_observation=next_observation, action=action, reward=reward, done=done + ) + + trial_done = done.item() == 1 + + if trial_done: + run_session.log_metrics(trial_idx=trial_idx, total_reward=total_reward) + total_reward_cum += total_reward + if (trial_idx + 1) % 100 == 0: + total_reward_avg = total_reward_cum / 100 + run_session.log_metrics(total_reward_avg=total_reward_avg) + total_reward_cum = 0 + log.info( + f"[SimpleDQN/{run_session.run_id}] trial #{trial_idx + 1}/{self._cfg.num_trials} done (average total reward = {total_reward_avg})." + ) + + if ( + step_idx > self._cfg.learning_starts + and replay_buffer.size() > self._cfg.batch_size + and step_idx % self._cfg.train_frequency == 0 + ): + data = replay_buffer.sample(self._cfg.batch_size) + + with torch.no_grad(): + target_values, _ = target_network(data.next_observation).max(dim=1) + td_target = data.reward.flatten() + self._cfg.discount_factor * target_values * ( + 1 - data.done.flatten() + ) + + action_values = model.network(data.observation).gather(1, data.action).squeeze() + loss = torch.nn.functional.mse_loss(td_target, action_values) + + # optimize the model + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # Update the epsilon + model.epsilon = epsilon_schedule(trial_idx) + + # Update the version info + model.num_samples_seen = replay_buffer.num_total + + if step_idx % self._cfg.target_network_frequency == 0: + target_network.load_state_dict(model.network.state_dict()) + + version_info = await run_session.model_registry.publish_version(model) + + if step_idx % 100 == 0: + end_time = time.time() + steps_per_seconds = 100 / (end_time - start_time) + start_time = end_time + run_session.log_metrics( + model_version_number=version_info["version_number"], + loss=loss.item(), + q_values=action_values.mean().item(), + batch_avg_reward=data.reward.mean().item(), + epsilon=model.epsilon, + steps_per_seconds=steps_per_seconds, + ) + + version_info = await run_session.model_registry.publish_version(model, archived=True) diff --git a/actors/tutorial/tutorial_1.py b/actors/tutorial/tutorial_1.py index 0f282eed..0b98a2f9 100644 --- a/actors/tutorial/tutorial_1.py +++ b/actors/tutorial/tutorial_1.py @@ -51,10 +51,8 @@ async def impl(self, actor_session): class SimpleBCTraining: default_cfg = { - "environment_config": {"seed": 12}, - "training": { - "num_trials": 10, - }, + "seed": 12, + "num_trials": 10, } def __init__(self, environment_specs, cfg): @@ -92,8 +90,7 @@ async def impl(self, run_session): assert self._environment_specs.num_players == 1 run_session.log_params( - self._cfg.training, - self._cfg.environment_config, + self._cfg, environment_implementation=self._environment_specs.implementation, ) @@ -126,16 +123,16 @@ def create_trial_params(trial_idx): environment_name="env", environment_implementation=self._environment_specs.implementation, environment_config=EnvironmentConfig( - run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + run_id=run_session.run_id, render=True, seed=self._cfg.seed + trial_idx ), actors=[agent_actor_params, teacher_actor_params], ) # Rollout a bunch of trials - for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + for (step_idx, _trial_id, _trial_idx, sample,) in run_session.start_and_await_trials( trials_id_and_params=[ (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) - for trial_idx in range(self._cfg.training.num_trials) + for trial_idx in range(self._cfg.num_trials) ], sample_producer_impl=self.sample_producer, num_parallel_trials=1, diff --git a/actors/tutorial/tutorial_2.py b/actors/tutorial/tutorial_2.py index ba767236..ef69cde6 100644 --- a/actors/tutorial/tutorial_2.py +++ b/actors/tutorial/tutorial_2.py @@ -59,10 +59,8 @@ async def impl(self, actor_session): class SimpleBCTraining: default_cfg = { - "environment_config": {"seed": 12}, - "training": { - "num_trials": 10, - }, + "seed": 12, + "num_trials": 10, } def __init__(self, environment_specs, cfg): @@ -123,8 +121,7 @@ async def impl(self, run_session): assert self._environment_specs.num_players == 1 run_session.log_params( - self._cfg.training, - self._cfg.environment_config, + self._cfg, environment_implementation=self._environment_specs.implementation, ) @@ -159,16 +156,16 @@ def create_trial_params(trial_idx): environment_name="env", environment_implementation=self._environment_specs.implementation, environment_config=EnvironmentConfig( - run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + run_id=run_session.run_id, render=True, seed=self._cfg.seed + trial_idx ), actors=[agent_actor_params, teacher_actor_params], ) # Rollout a bunch of trials - for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + for (step_idx, _trial_id, _trial_idx, sample,) in run_session.start_and_await_trials( trials_id_and_params=[ (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) - for trial_idx in range(self._cfg.training.num_trials) + for trial_idx in range(self._cfg.num_trials) ], sample_producer_impl=self.sample_producer, num_parallel_trials=1, diff --git a/actors/tutorial/tutorial_3.py b/actors/tutorial/tutorial_3.py index a1fb4e22..4bb48919 100644 --- a/actors/tutorial/tutorial_3.py +++ b/actors/tutorial/tutorial_3.py @@ -49,7 +49,7 @@ def __init__( environment_implementation, num_input, num_output, - policy_network_hidden_size=64, + policy_network_num_hidden_nodes=64, version_number=0, ): super().__init__(model_id, version_number) @@ -58,16 +58,16 @@ def __init__( self._environment_implementation = environment_implementation self._num_input = num_input self._num_output = num_output - self._policy_network_hidden_size = policy_network_hidden_size + self._policy_network_num_hidden_nodes = policy_network_num_hidden_nodes self.policy_network = torch.nn.Sequential( - torch.nn.Linear(num_input, policy_network_hidden_size, dtype=self._dtype), - torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.Linear(num_input, policy_network_num_hidden_nodes, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_num_hidden_nodes, dtype=self._dtype), torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, policy_network_hidden_size, dtype=self._dtype), - torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.Linear(policy_network_num_hidden_nodes, policy_network_num_hidden_nodes, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_num_hidden_nodes, dtype=self._dtype), torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, num_output, dtype=self._dtype), + torch.nn.Linear(policy_network_num_hidden_nodes, num_output, dtype=self._dtype), ) self.total_samples = 0 @@ -77,7 +77,7 @@ def get_model_user_data(self): "environment_implementation": self._environment_implementation, "num_input": self._num_input, "num_output": self._num_output, - "policy_network_hidden_size": self._policy_network_hidden_size, + "policy_network_num_hidden_nodes": self._policy_network_num_hidden_nodes, } def save(self, model_data_f): @@ -94,7 +94,7 @@ def load(cls, model_id, version_number, model_user_data, version_user_data, mode environment_implementation=model_user_data["environment_implementation"], num_input=int(model_user_data["num_input"]), num_output=int(model_user_data["num_output"]), - policy_network_hidden_size=int(model_user_data["policy_network_hidden_size"]), + policy_network_num_hidden_nodes=int(model_user_data["policy_network_num_hidden_nodes"]), ) # Load the saved states @@ -152,12 +152,10 @@ async def impl(self, actor_session): class SimpleBCTraining: default_cfg = { - "environment_config": {"seed": 12}, - "training": { - "num_trials": 10, - }, + "seed": 12, + "num_trials": 10, ############ TUTORIAL STEP 3 ############ - "policy_network": {"hidden_size": 64}, + "policy_network": {"num_hidden_nodes": 64}, ########################################## } @@ -206,9 +204,6 @@ async def sample_producer(self, sample_producer_session): ) sample_producer_session.produce_sample((demonstration, observation_tensor, action_tensor)) - if sample.trial_state == cogment.TrialState.ENDED: - break - async def impl(self, run_session): assert self._environment_specs.num_players == 1 @@ -221,17 +216,16 @@ async def impl(self, run_session): environment_implementation=self._environment_specs.implementation, num_input=flattened_dimensions(self._environment_specs.observation_space), num_output=flattened_dimensions(self._environment_specs.action_space), - policy_network_hidden_size=self._cfg.policy_network.hidden_size, + policy_network_num_hidden_nodes=self._cfg.policy_network.num_hidden_nodes, ) _model_info, _version_info = await run_session.model_registry.publish_initial_version(model) ########################################## run_session.log_params( - self._cfg.training, - self._cfg.environment_config, + self._cfg, environment_implementation=self._environment_specs.implementation, ############ TUTORIAL STEP 3 ############ - policy_network_hidden_size=self._cfg.policy_network.hidden_size, + policy_network_num_hidden_nodes=self._cfg.policy_network.num_hidden_nodes, ######################################### ) @@ -270,16 +264,16 @@ def create_trial_params(trial_idx): environment_name="env", environment_implementation=self._environment_specs.implementation, environment_config=EnvironmentConfig( - run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + run_id=run_session.run_id, render=True, seed=self._cfg.seed + trial_idx ), actors=[agent_actor_params, teacher_actor_params], ) # Rollout a bunch of trials - for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + for (step_idx, _trial_id, _trial_idx, sample,) in run_session.start_and_await_trials( trials_id_and_params=[ (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) - for trial_idx in range(self._cfg.training.num_trials) + for trial_idx in range(self._cfg.num_trials) ], sample_producer_impl=self.sample_producer, num_parallel_trials=1, diff --git a/actors/tutorial/tutorial_4.py b/actors/tutorial/tutorial_4.py index cef0e43c..715afd69 100644 --- a/actors/tutorial/tutorial_4.py +++ b/actors/tutorial/tutorial_4.py @@ -47,7 +47,7 @@ def __init__( environment_implementation, num_input, num_output, - policy_network_hidden_size=64, + policy_network_num_hidden_nodes=64, version_number=0, ): super().__init__(model_id, version_number) @@ -56,16 +56,16 @@ def __init__( self._environment_implementation = environment_implementation self._num_input = num_input self._num_output = num_output - self._policy_network_hidden_size = policy_network_hidden_size + self._policy_network_num_hidden_nodes = policy_network_num_hidden_nodes self.policy_network = torch.nn.Sequential( - torch.nn.Linear(num_input, policy_network_hidden_size, dtype=self._dtype), - torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.Linear(num_input, policy_network_num_hidden_nodes, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_num_hidden_nodes, dtype=self._dtype), torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, policy_network_hidden_size, dtype=self._dtype), - torch.nn.BatchNorm1d(policy_network_hidden_size, dtype=self._dtype), + torch.nn.Linear(policy_network_num_hidden_nodes, policy_network_num_hidden_nodes, dtype=self._dtype), + torch.nn.BatchNorm1d(policy_network_num_hidden_nodes, dtype=self._dtype), torch.nn.ReLU(), - torch.nn.Linear(policy_network_hidden_size, num_output, dtype=self._dtype), + torch.nn.Linear(policy_network_num_hidden_nodes, num_output, dtype=self._dtype), ) self.total_samples = 0 @@ -75,7 +75,7 @@ def get_model_user_data(self): "environment_implementation": self._environment_implementation, "num_input": self._num_input, "num_output": self._num_output, - "policy_network_hidden_size": self._policy_network_hidden_size, + "policy_network_num_hidden_nodes": self._policy_network_num_hidden_nodes, } def save(self, model_data_f): @@ -92,7 +92,7 @@ def load(cls, model_id, version_number, model_user_data, version_user_data, mode environment_implementation=model_user_data["environment_implementation"], num_input=int(model_user_data["num_input"]), num_output=int(model_user_data["num_output"]), - policy_network_hidden_size=int(model_user_data["policy_network_hidden_size"]), + policy_network_num_hidden_nodes=int(model_user_data["policy_network_num_hidden_nodes"]), ) # Load the saved states @@ -141,17 +141,15 @@ async def impl(self, actor_session): class SimpleBCTraining: default_cfg = { - "environment_config": {"seed": 12}, - "training": { - "num_trials": 10, - ############ TUTORIAL STEP 4 ############ - "discount_factor": 0.95, - "learning_rate": 0.01, - "batch_size": 32, - "train_only_from_demonstration": False - ######################################### - }, - "policy_network": {"hidden_size": 64}, + "seed": 12, + "num_trials": 10, + ############ TUTORIAL STEP 4 ############ + "discount_factor": 0.95, + "learning_rate": 0.01, + "batch_size": 32, + "train_only_from_demonstration": False, + ######################################### + "policy_network": {"num_hidden_nodes": 64}, } def __init__(self, environment_specs, cfg): @@ -210,15 +208,14 @@ async def impl(self, run_session): environment_implementation=self._environment_specs.implementation, num_input=flattened_dimensions(self._environment_specs.observation_space), num_output=flattened_dimensions(self._environment_specs.action_space), - policy_network_hidden_size=self._cfg.policy_network.hidden_size, + policy_network_num_hidden_nodes=self._cfg.policy_network.num_hidden_nodes, ) _model_info, _version_info = await run_session.model_registry.publish_initial_version(model) run_session.log_params( - self._cfg.training, - self._cfg.environment_config, + self._cfg, environment_implementation=self._environment_specs.implementation, - policy_network_hidden_size=self._cfg.policy_network.hidden_size, + policy_network_num_hidden_nodes=self._cfg.policy_network.num_hidden_nodes, ) # Helper function to create a trial configuration @@ -254,7 +251,7 @@ def create_trial_params(trial_idx): environment_name="env", environment_implementation=self._environment_specs.implementation, environment_config=EnvironmentConfig( - run_id=run_session.run_id, render=True, seed=self._cfg.environment_config.seed + trial_idx + run_id=run_session.run_id, render=True, seed=self._cfg.seed + trial_idx ), actors=[agent_actor_params, teacher_actor_params], ) @@ -263,7 +260,7 @@ def create_trial_params(trial_idx): # Configure the optimizer optimizer = torch.optim.Adam( model.policy_network.parameters(), - lr=self._cfg.training.learning_rate, + lr=self._cfg.learning_rate, ) # Keep accumulated observations/actions around @@ -274,27 +271,27 @@ def create_trial_params(trial_idx): ########################################## # Rollout a bunch of trials - for (step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + for (step_idx, _trial_id, _trial_idx, sample,) in run_session.start_and_await_trials( trials_id_and_params=[ (f"{run_session.run_id}_{trial_idx}", create_trial_params(trial_idx)) - for trial_idx in range(self._cfg.training.num_trials) + for trial_idx in range(self._cfg.num_trials) ], sample_producer_impl=self.sample_producer, num_parallel_trials=1, ): ############ TUTORIAL STEP 4 ############ (demonstration, observation, action) = sample - if self._cfg.training.train_only_from_demonstration and not demonstration: + if self._cfg.train_only_from_demonstration and not demonstration: continue observations.append(observation) actions.append(action) - if len(observations) < self._cfg.training.batch_size: + if len(observations) < self._cfg.batch_size: continue # Sample a batch of observations/actions - batch_indices = np.random.default_rng().integers(0, len(observations), self._cfg.training.batch_size) + batch_indices = np.random.default_rng().integers(0, len(observations), self._cfg.batch_size) batch_obs = torch.vstack([observations[i] for i in batch_indices]) batch_act = torch.vstack([actions[i] for i in batch_indices]) diff --git a/cogment_verse/__init__.py b/cogment_verse/__init__.py index a5c24163..f76dff68 100644 --- a/cogment_verse/__init__.py +++ b/cogment_verse/__init__.py @@ -14,3 +14,4 @@ from .app import App from .model_registry import Model +from .replay_buffers import TorchReplayBuffer diff --git a/cogment_verse/mlflow_experiment_tracker.py b/cogment_verse/mlflow_experiment_tracker.py index fd92bfc1..7661a383 100644 --- a/cogment_verse/mlflow_experiment_tracker.py +++ b/cogment_verse/mlflow_experiment_tracker.py @@ -69,6 +69,9 @@ def __init__(self, experiment_id, run_id, mlflow_tracking_uri, flush_frequency=5 self._flush_metrics_worker_frequency = flush_frequency self._flush_metrics_worker = None + def __del__(self): + self._stop_flush_metrics_worker() + def _get_mlflow_client(self): client = MlflowClient(tracking_uri=self._mlflow_tracking_uri) if not self._mlflow_exp_id: diff --git a/cogment_verse/replay_buffers/__init__.py b/cogment_verse/replay_buffers/__init__.py new file mode 100644 index 00000000..1ba0dd0b --- /dev/null +++ b/cogment_verse/replay_buffers/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .torch_replay_buffer import TorchReplayBuffer diff --git a/cogment_verse/replay_buffers/torch_replay_buffer.py b/cogment_verse/replay_buffers/torch_replay_buffer.py new file mode 100644 index 00000000..3cb8f961 --- /dev/null +++ b/cogment_verse/replay_buffers/torch_replay_buffer.py @@ -0,0 +1,87 @@ +# Copyright 2021 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch + + +class TorchReplayBufferSample: + def __init__(self, observation, next_observation, action, reward, done): + self.observation = observation + self.next_observation = next_observation + self.action = action + self.reward = reward + self.done = done + + def size(self): + return self.reward.size(dim=0) + + +class TorchReplayBuffer: + def __init__( + self, + capacity, + observation_shape, + action_shape, + seed=0, + observation_dtype=torch.float, + action_dtype=torch.float, + reward_dtype=torch.float, + ): + self.capacity = capacity + self.observation_shape = observation_shape + self.action_shape = action_shape + + self.observation = torch.zeros((self.capacity, *self.observation_shape), dtype=observation_dtype) + self.next_observation = torch.zeros((self.capacity, *self.observation_shape), dtype=observation_dtype) + self.action = torch.zeros((self.capacity, *self.action_shape), dtype=action_dtype) + self.reward = torch.zeros((self.capacity,), dtype=reward_dtype) + self.done = torch.zeros((self.capacity,), dtype=torch.int8) + + self._ptr = 0 + self.num_total = 0 + + self._rng = np.random.default_rng(seed) + + def add(self, observation, next_observation, action, reward, done): + self.observation[self._ptr] = ( + observation.clone().detach() if torch.is_tensor(observation) else torch.tensor(observation) + ) + self.next_observation[self._ptr] = ( + next_observation.clone().detach() if torch.is_tensor(next_observation) else torch.tensor(next_observation) + ) + self.action[self._ptr] = action.clone().detach() if torch.is_tensor(action) else torch.tensor(action) + self.reward[self._ptr] = reward.clone().detach() if torch.is_tensor(reward) else torch.tensor(reward) + self.done[self._ptr] = done.clone().detach() if torch.is_tensor(done) else torch.tensor(done) + + self._ptr = (self._ptr + 1) % self.capacity + self.num_total += 1 + + def sample(self, num): + size = self.size() + if size < num: + indices = range(size) + else: + indices = self._rng.choice(self.size(), size=num, replace=False) + + return TorchReplayBufferSample( + observation=self.observation[indices], + next_observation=self.next_observation[indices], + action=self.action[indices], + reward=self.reward[indices], + done=self.done[indices], + ) + + def size(self): + return self.num_total if self.num_total < self.capacity else self.capacity diff --git a/cogment_verse/run/run_session.py b/cogment_verse/run/run_session.py index d2c37944..659899fc 100644 --- a/cogment_verse/run/run_session.py +++ b/cogment_verse/run/run_session.py @@ -61,7 +61,12 @@ def start_and_await_trials(self, trials_id_and_params, sample_producer_impl, num if sample_queue_event.done: break self._step_idx += 1 - yield (self._step_idx, sample_queue_event.trial_id, sample_queue_event.sample) + yield ( + self._step_idx, + sample_queue_event.trial_id, + sample_queue_event.trial_idx, + sample_queue_event.sample, + ) trial_runner_worker.join() sample_producer_worker.join() diff --git a/cogment_verse/run/sample_producer_worker.py b/cogment_verse/run/sample_producer_worker.py index c2e1bace..f7f49cb9 100644 --- a/cogment_verse/run/sample_producer_worker.py +++ b/cogment_verse/run/sample_producer_worker.py @@ -25,21 +25,25 @@ class SampleQueueEvent: - def __init__(self, trial_id=None, sample=None, done=False): + def __init__(self, trial_id=None, trial_idx=None, sample=None, done=False): self.trial_id = trial_id + self.trial_idx = trial_idx self.sample = sample self.done = done class SampleProducerSession: - def __init__(self, datastore, trial_info, sample_queue, impl): + def __init__(self, datastore, trial_idx, trial_info, sample_queue, impl): + self.trial_idx = trial_idx self.datastore = datastore self.trial_info = trial_info self.sample_queue = sample_queue self.impl = impl def produce_sample(self, sample): - self.sample_queue.put(SampleQueueEvent(trial_id=self.trial_info.trial_id, sample=sample)) + self.sample_queue.put( + SampleQueueEvent(trial_id=self.trial_info.trial_id, trial_idx=self.trial_idx, sample=sample) + ) def all_trial_samples(self): return self.datastore.all_samples([self.trial_info]) @@ -93,7 +97,9 @@ async def async_sample_producer_worker(trial_started_queue, sample_queue, impl, log.debug(f"[{trial_info.trial_id}] started") - sample_producer_session = SampleProducerSession(datastore, trial_info, sample_queue, impl) + sample_producer_session = SampleProducerSession( + datastore, trial_started_queue_event.trial_idx, trial_info, sample_queue, impl + ) sample_producer_tasks.append(sample_producer_session.create_task()) diff --git a/cogment_verse/run/trial_runner_worker.py b/cogment_verse/run/trial_runner_worker.py index 51f618fe..b228c60c 100644 --- a/cogment_verse/run/trial_runner_worker.py +++ b/cogment_verse/run/trial_runner_worker.py @@ -25,13 +25,15 @@ class TrialStartedQueueEvent: - def __init__(self, trial_id=None, done=False): + def __init__(self, trial_id=None, trial_idx=None, done=False): + self.trial_idx = trial_idx self.trial_id = trial_id self.done = done class TrialEndedQueueEvent: - def __init__(self, trial_id=None, done=False): + def __init__(self, trial_id=None, trial_idx=None, done=False): + self.trial_idx = trial_idx self.trial_id = trial_id self.done = done @@ -50,7 +52,7 @@ async def async_trial_runner_worker( num_trials = len(trials_id_and_params) num_started_trials = 0 num_ended_trials = 0 - running_trials = set() + running_trials = {} async def start_trials(): nonlocal running_trials @@ -80,21 +82,23 @@ async def start_trials(): actual_trial_id = await controller.start_trial(trial_id_requested=trial_id, trial_params=trial_params) if actual_trial_id is None: raise RuntimeError(f"Unable to start a trial with id [{trial_id}]") - running_trials.add(trial_id) + trial_idx = num_started_trials + running_trials[trial_id] = trial_idx num_started_trials += 1 log.debug(f"Trial [{trial_id}] started, {num_trials-num_started_trials} trials remaining to start.") if trial_started_queue is not None: - trial_started_queue.put(TrialStartedQueueEvent(trial_id=trial_id)) + trial_started_queue.put(TrialStartedQueueEvent(trial_id=trial_id, trial_idx=trial_idx)) async def await_trials(): nonlocal running_trials nonlocal num_ended_trials async for trial_info in controller.watch_trials(trial_state_filters=[cogment.TrialState.ENDED]): if trial_info.trial_id in running_trials: - running_trials.discard(trial_info.trial_id) + trial_idx = running_trials[trial_info.trial_id] + del running_trials[trial_info.trial_id] num_ended_trials += 1 if trial_ended_queue is not None: - trial_ended_queue.put(TrialEndedQueueEvent(trial_id=trial_info.trial_id)) + trial_ended_queue.put(TrialEndedQueueEvent(trial_id=trial_info.trial_id, trial_idx=trial_idx)) log.debug(f"Trial [{trial_info.trial_id}] ended, {num_trials-num_ended_trials} trials remaining.") if num_ended_trials == num_trials: break diff --git a/cogment_verse/specs/__init__.py b/cogment_verse/specs/__init__.py index 9ca1a9d9..86bb4030 100644 --- a/cogment_verse/specs/__init__.py +++ b/cogment_verse/specs/__init__.py @@ -35,3 +35,9 @@ from .ndarray import deserialize_ndarray, serialize_ndarray from .sample_space import sample_space from .flatten import flattened_dimensions, flatten, unflatten +from .gym_spaces_adapter import ( + gym_action_from_action, + gym_space_from_space, + observation_from_gym_observation, + space_from_gym_space, +) diff --git a/cogment_verse/specs/data.proto b/cogment_verse/specs/data.proto index 6569b03b..7714b185 100644 --- a/cogment_verse/specs/data.proto +++ b/cogment_verse/specs/data.proto @@ -89,12 +89,14 @@ message HFHubModel { message AgentConfig { string run_id = 1; EnvironmentSpecs environment_specs = 2; - string model_id = 3; - int32 model_version = 4; - int32 actor_index = 5; // Used to figure out if an agent is the current_player in the observation space - string device = 6; - uint32 threads_per_worker = 7; - HFHubModel hf_hub_model = 8; + uint32 seed = 3; + string model_id = 4; + int32 model_version = 5; + int32 model_update_frequency = 6; + // int32 actor_index = 6; // Used to figure out if an agent is the current_player in the observation space + // string device = 7; + // uint32 threads_per_worker = 8; + // HFHubModel hf_hub_model = 9; } message TrialConfig { diff --git a/cogment_verse/specs/gym_spaces_adapter.py b/cogment_verse/specs/gym_spaces_adapter.py new file mode 100644 index 00000000..f70fb94f --- /dev/null +++ b/cogment_verse/specs/gym_spaces_adapter.py @@ -0,0 +1,86 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gym +import numpy as np + +from data_pb2 import Space, SpaceValue # pylint: disable=import-error + +from .ndarray import deserialize_ndarray, serialize_ndarray + +SPACES_BOUND_MAX = float.fromhex("0x1.fffffep+127") +SPACES_BOUND_MIN = -SPACES_BOUND_MAX + + +def gym_space_from_space(space): + gym_spaces_dict = {} + for prop in space.properties: + prop_type = prop.WhichOneof("type_oneof") + if prop_type == "discrete": + num_action = max(len(prop.discrete.labels), prop.discrete.num) + gym_spaces_dict[prop.key] = gym.spaces.Discrete(num_action) + if prop_type == "box": + gym_spaces_dict[prop.key] = gym.spaces.Box( + low=np.array([bound.bound if bound.bound is not None else -np.inf for bound in prop.box.low]), + high=np.array([bound.bound if bound.bound is not None else np.inf for bound in prop.box.high]), + shape=prop.box.shape, + ) + if len(gym_spaces_dict) == 1: + return next(iter(gym_spaces_dict.values())) + return gym.spaces.Dict(gym_spaces_dict) + + +def space_from_gym_space(gym_space): + if isinstance(gym_space, gym.spaces.Box): + return Space( + properties=[ + Space.Property( + box=Space.Box( + shape=list(gym_space.shape), + low=[ + Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() + for v in gym_space.low.flat + ], + high=[ + Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() + for v in gym_space.high.flat + ], + ) + ) + ] + ) + if isinstance(gym_space, gym.spaces.Discrete): + return Space(properties=[Space.Property(discrete=Space.Discrete(num=gym_space.n))]) + raise RuntimeError(f"[{type(gym_space)}] is not a supported gym space type") + + +def gym_action_from_action(space, action): + if len(action.properties) == 1: + prop_value = action.properties[0] + value_type = prop_value.WhichOneof("value") + if value_type == "discrete": + return prop_value.discrete + if value_type == "box": + return deserialize_ndarray(prop_value.box) + # value_type == "simple_box" + return np.array(prop_value.simple_box.values).reshape(space.properties[0].box.shape) + raise RuntimeError(f"Not supporting spaces not having one property, got {len(action.properties)}") + + +def observation_from_gym_observation(gym_space, gym_observation): + if isinstance(gym_space, gym.spaces.Box): + return SpaceValue(properties=[SpaceValue.PropertyValue(box=serialize_ndarray(gym_observation))]) + if isinstance(gym_space, gym.spaces.Discrete): + return SpaceValue(properties=[SpaceValue.PropertyValue(discrete=gym_observation.item())]) + raise RuntimeError(f"[{type(gym_space)}] is not a supported gym space type") diff --git a/cogment_verse/specs/sample_space.py b/cogment_verse/specs/sample_space.py index 2a2cea72..2249cf2f 100644 --- a/cogment_verse/specs/sample_space.py +++ b/cogment_verse/specs/sample_space.py @@ -18,8 +18,10 @@ from .ndarray import serialize_ndarray -def sample_space(space, num_samples=1, seed=0): - rng = np.random.default_rng(seed) +def sample_space(space, num_samples=1, seed=0, rng=None): + if rng is None: + rng = np.random.default_rng(seed) + space_values = create_space_values(space, num_samples) for prop_idx, prop in enumerate(space.properties): diff --git a/config/config.yaml b/config/config.yaml index 71878888..723ac1b2 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,6 +1,6 @@ defaults: - services: local_base_services - - services/environment: mountain_car + - services/environment: lunar_lander - services/actor: random - run: play - _self_ diff --git a/config/experiment/simple_a2c/cartpole.yaml b/config/experiment/simple_a2c/cartpole.yaml index 4f627c7e..11e526ba 100644 --- a/config/experiment/simple_a2c/cartpole.yaml +++ b/config/experiment/simple_a2c/cartpole.yaml @@ -5,14 +5,17 @@ defaults: - override /services/environment: cartpole run: + class_name: actors.simple_a2c.SimpleA2CTraining + seed: 618 + num_epochs: 500 + epoch_num_trials: 10 + num_parallel_trials: 8 + learning_rate: 0.01 actor_network: - hidden_size: 32 + num_hidden_nodes: 32 critic_network: - hidden_size: 32 - training: - discount_factor: 0.95 - entropy_loss_coef: 0.05 - value_loss_coef: 1.0 - action_loss_coef: 0.1 - environment_config: - seed: 15 + num_hidden_nodes: 32 + discount_factor: 0.95 + entropy_loss_coef: 0.05 + value_loss_coef: 1.0 + action_loss_coef: 0.1 diff --git a/config/experiment/simple_bc/mountain_car.yaml b/config/experiment/simple_bc/mountain_car.yaml new file mode 100644 index 00000000..9e3e15e8 --- /dev/null +++ b/config/experiment/simple_bc/mountain_car.yaml @@ -0,0 +1,9 @@ +# @package _global_ +defaults: + - override /services/actor: simple_bc + - override /services/environment: mountain_car + - override /run: simple_dqn +run: + class_name: actors.tutorial.tutorial_4.SimpleBCTraining + seed: 618 + num_trials: 15 diff --git a/config/experiment/simple_dqn/cartpole.yaml b/config/experiment/simple_dqn/cartpole.yaml new file mode 100644 index 00000000..9c7fb768 --- /dev/null +++ b/config/experiment/simple_dqn/cartpole.yaml @@ -0,0 +1,22 @@ +# @package _global_ +defaults: + - override /services/actor: simple_dqn + - override /services/environment: cartpole +run: + class_name: actors.simple_dqn.SimpleDQNTraining + seed: 618 + num_trials: 10000 + num_parallel_trials: 10 + learning_starts: ${run.batch_size} + learning_rate: 0.000125 + discount_factor: 0.95 + target_network_frequency: 2000 + batch_size: 64 + buffer_size: 10000 + epsilon_schedule_start: 1 + epsilon_schedule_end: 0.05 + epsilon_schedule_duration_ratio: 0.75 + train_frequency: 10 + model_update_frequency: 10 + value_network: + num_hidden_nodes: [128, 64] diff --git a/config/run/observe.yaml b/config/run/observe.yaml index af59754f..4fe01ea2 100644 --- a/config/run/observe.yaml +++ b/config/run/observe.yaml @@ -1,6 +1,6 @@ class_name: runs.play.PlayRun observer: True -trial_count: 10 +num_trials: 10 players: - name: player_1 implementation: actors.random_actor.RandomActor diff --git a/config/run/play.yaml b/config/run/play.yaml index 1e8117b3..52ea3ca9 100644 --- a/config/run/play.yaml +++ b/config/run/play.yaml @@ -1,5 +1,5 @@ class_name: runs.play.PlayRun -trial_count: 1 +num_trials: 1 players: - name: player_1 implementation: client diff --git a/config/run/simple_a2c.yaml b/config/run/simple_a2c.yaml deleted file mode 100644 index 02e36ae6..00000000 --- a/config/run/simple_a2c.yaml +++ /dev/null @@ -1,16 +0,0 @@ -class_name: actors.simple_a2c.SimpleA2CTraining -environment_config: - seed: 618 -training: - epoch_count: 500 - epoch_trial_count: 10 - num_parallel_trials: 8 - discount_factor: 0.95 - entropy_loss_coef: 0.05 - value_loss_coef: 0.5 - action_loss_coef: 1.0 - learning_rate: 0.01 -actor_network: - hidden_size: 64 -critic_network: - hidden_size": 64 diff --git a/config/run/simple_bc.yaml b/config/run/simple_bc.yaml deleted file mode 100644 index 5cbcbb20..00000000 --- a/config/run/simple_bc.yaml +++ /dev/null @@ -1,5 +0,0 @@ -class_name: actors.tutorial.tutorial_4.SimpleBCTraining -environment_config: - seed: 618 -training: - num_trials: 15 diff --git a/config/services/actor/simple_dqn.yaml b/config/services/actor/simple_dqn.yaml new file mode 100644 index 00000000..27cca925 --- /dev/null +++ b/config/services/actor/simple_dqn.yaml @@ -0,0 +1,2 @@ +- class_name: actors.simple_dqn.SimpleDQNActor + port: ${generate_port:actors.simple_dqn.SimpleDQNActor} diff --git a/docs/results/a2c.md b/docs/results/a2c.md index 514ffe8d..53fcb7aa 100644 --- a/docs/results/a2c.md +++ b/docs/results/a2c.md @@ -35,7 +35,7 @@ simple_a2c_cartpole: num_input: 4 num_action: 2 training: - epoch_count: 100 + num_epochs: 100 epoch_trial_count: 15 max_parallel_trials: 8 discount_factor: 0.95 @@ -44,9 +44,9 @@ simple_a2c_cartpole: action_loss_coef: 1.0 learning_rate: 0.01 actor_network: - hidden_size: 64 + num_hidden_nodes: 64 critic_network: - hidden_size: 64 + num_hidden_nodes: 64 ``` This is a plot of the total trial reward against the number of trials with a exponential moving average over 60 trials. diff --git a/environments/gym_adapter.py b/environments/gym_adapter.py index 960365f4..8901d1cb 100644 --- a/environments/gym_adapter.py +++ b/environments/gym_adapter.py @@ -16,70 +16,20 @@ import cogment import gym -import numpy as np from cogment_verse.specs import ( encode_rendered_frame, - deserialize_ndarray, - serialize_ndarray, EnvironmentSpecs, Observation, - Space, - SpaceValue, + space_from_gym_space, + gym_action_from_action, + observation_from_gym_observation, ) from cogment_verse.constants import PLAYER_ACTOR_CLASS, TEACHER_ACTOR_CLASS # configure pygame to use a dummy video server to be able to render headlessly os.environ["SDL_VIDEODRIVER"] = "dummy" -SPACES_BOUND_MAX = float.fromhex("0x1.fffffep+127") -SPACES_BOUND_MIN = -SPACES_BOUND_MAX - - -def space_from_gym_space(gym_space): - if isinstance(gym_space, gym.spaces.Box): - return Space( - properties=[ - Space.Property( - box=Space.Box( - shape=list(gym_space.shape), - low=[ - Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() - for v in gym_space.low.flat - ], - high=[ - Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() - for v in gym_space.high.flat - ], - ) - ) - ] - ) - if isinstance(gym_space, gym.spaces.Discrete): - return Space(properties=[Space.Property(discrete=Space.Discrete(num=gym_space.n))]) - raise RuntimeError(f"[{type(gym_space)}] is not a supported gym space type") - - -def gym_action_from_action(space, action): - if len(action.properties) == 1: - prop_value = action.properties[0] - value_type = prop_value.WhichOneof("value") - if value_type == "discrete": - return prop_value.discrete - if value_type == "box": - return deserialize_ndarray(prop_value.box) - # value_type == "simple_box" - return np.array(prop_value.simple_box.values).reshape(space.properties[0].box.shape) - raise RuntimeError(f"Not supporting spaces not having one property, got {len(action.properties)}") - - -def observation_from_gym_observation(gym_space, gym_observation): - if isinstance(gym_space, gym.spaces.Box): - return SpaceValue(properties=[SpaceValue.PropertyValue(box=serialize_ndarray(gym_observation))]) - if isinstance(gym_space, gym.spaces.Discrete): - return SpaceValue(properties=[SpaceValue.PropertyValue(discrete=gym_observation.item())]) - raise RuntimeError(f"[{type(gym_space)}] is not a supported gym space type") - class Environment: def __init__(self, cfg): @@ -142,8 +92,8 @@ async def impl(self, environment_session): overridden_players = [player_actor_name] gym_action = gym_action_from_action( - self.env_specs.action_space, action_value - ) # pylint: disable=no-member + self.env_specs.action_space, action_value # pylint: disable=no-member + ) gym_observation, reward, done, _info = gym_env.step(gym_action) observation_value = observation_from_gym_observation(gym_env.observation_space, gym_observation) diff --git a/pyproject.toml b/pyproject.toml index 2b71d6d0..b0af633e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ jobs=0 logging-format-style="new" [tool.pylint.FORMAT] -good-names=["i","j","k","c","h","w","x","id","f","to"] +good-names=["i","j","k","c","h","t","w","x","id","f","to"] [tool.pylint.Typecheck] generated-members=["cv2", "torch"] diff --git a/requirements.txt b/requirements.txt index 440b1d39..299184b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ starlette==0.20.0 uvicorn==0.17.6 # environments -gym[atari,box2d]==0.23.1 +gym[atari,box2d,classic_control]>=0.24,<0.25 # actors torch==1.11.0 diff --git a/runs/play.py b/runs/play.py index 0fd0e13f..626c127d 100644 --- a/runs/play.py +++ b/runs/play.py @@ -41,7 +41,7 @@ def extend_actor_config(actor_config_template, run_id, environment_specs): class PlayRun: - default_cfg = {"trial_count": 1, "observer": False, "players": []} + default_cfg = {"num_trials": 1, "observer": False, "players": []} def __init__(self, environment_specs, cfg): super().__init__() @@ -154,9 +154,9 @@ async def impl(self, run_session): ) # Rollout a bunch of trials - for (_step_idx, _trial_id, sample,) in run_session.start_and_await_trials( + for (_step_idx, _trial_id, _trial_idx, sample,) in run_session.start_and_await_trials( trials_id_and_params=[ - (f"{run_session.run_id}_{trial_idx}", trial_params) for trial_idx in range(self._cfg.trial_count) + (f"{run_session.run_id}_{trial_idx}", trial_params) for trial_idx in range(self._cfg.num_trials) ], sample_producer_impl=self.total_rewards_producer_impl, num_parallel_trials=1, diff --git a/tests/test_torch_replay_buffer.py b/tests/test_torch_replay_buffer.py new file mode 100644 index 00000000..87ddf1de --- /dev/null +++ b/tests/test_torch_replay_buffer.py @@ -0,0 +1,48 @@ +# Copyright 2022 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cogment_verse import TorchReplayBuffer + + +def test_torch_replay_buffer(): + replay_buffer = TorchReplayBuffer(capacity=10, observation_shape=(2,), action_shape=(2, 2)) + assert replay_buffer.size() == 0 + assert replay_buffer.capacity == 10 + assert replay_buffer.num_total == 0 + + replay_buffer.add(observation=[0, 1], next_observation=[2, 3], action=[[4, 5], [6, 7]], reward=12, done=1) + + assert replay_buffer.size() == 1 + assert replay_buffer.capacity == 10 + assert replay_buffer.num_total == 1 + + sample = replay_buffer.sample(2) + assert sample.size() == 1 + + for i in range(100): + replay_buffer.add( + observation=[i, i + 1], + next_observation=[i + 2, i + 3], + action=[[i + 4, i + 5], [i + 6, i + 7]], + reward=i + 8, + done=0, + ) + + sample = replay_buffer.sample(10) + assert sample.size() == 10 + assert sample.observation.shape == (10, 2) + assert sample.next_observation.shape == (10, 2) + assert sample.action.shape == (10, 2, 2) + assert sample.reward.shape == (10,) + assert sample.done.shape == (10,) From 4212224544e933af6e3eefdda78370a22280cda2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clod=C3=A9ric=20Mars?= Date: Sun, 19 Jun 2022 23:01:22 -0400 Subject: [PATCH 03/64] Add support for petting zoo classic environment - action mask - turn based play - connect four UI --- .gitlab-ci.yml | 2 +- actors/random_actor.py | 12 +- actors/simple_a2c.py | 4 +- actors/simple_dqn.py | 18 +- actors/tutorial/tutorial_1.py | 2 +- actors/tutorial/tutorial_2.py | 2 +- cogment_verse/replay_buffers/__init__.py | 2 +- .../replay_buffers/torch_replay_buffer.py | 2 +- cogment_verse/specs/__init__.py | 3 +- cogment_verse/specs/data.proto | 26 +- cogment_verse/specs/flatten.py | 34 ++- cogment_verse/specs/gym_spaces_adapter.py | 49 ++-- cogment_verse/specs/ndarray.py | 4 +- cogment_verse/specs/sample_space.py | 12 +- cogment_verse/utils/__init__.py | 1 + .../web/components/build/asset-manifest.json | 12 +- cogment_verse/web/components/build/index.html | 2 +- .../build/static/css/main.21cabb04.css | 4 - .../build/static/css/main.21cabb04.css.map | 1 - .../build/static/css/main.b6c8fc15.css | 4 + .../build/static/css/main.b6c8fc15.css.map | 1 + .../build/static/js/main.80bd9354.js | 3 - .../build/static/js/main.80bd9354.js.map | 1 - .../build/static/js/main.b399feaf.js | 3 + ...CENSE.txt => main.b399feaf.js.LICENSE.txt} | 0 .../build/static/js/main.b399feaf.js.map | 1 + cogment_verse/web/components/data.proto | 40 +-- .../web/components/package-lock.json | 12 +- cogment_verse/web/components/package.json | 2 +- cogment_verse/web/components/src/App.jsx | 13 +- .../web/components/src/components/Button.jsx | 2 + .../src/controls/ConnectFourControls.jsx | 101 ++++++++ .../web/components/src/controls/Controls.jsx | 24 +- ...trols.jsx => RealTimeObserverControls.jsx} | 2 +- .../controls/TurnBasedObserverControls.jsx | 75 ++++++ config/run/headless_play.yaml | 8 + config/run/observe.yaml | 3 + config/run/play.yaml | 3 + config/services/environment/connect_four.yaml | 3 + config/services/environment/tictactoe.yaml | 3 + environments/gym_adapter.py | 6 +- environments/pettingzoo_adapter.py | 227 ++++++++++++++++++ requirements.txt | 1 + runs/play.py | 138 ++++++----- tests/test_flatten.py | 36 ++- tests/test_sample.py | 31 +++ 46 files changed, 770 insertions(+), 165 deletions(-) delete mode 100644 cogment_verse/web/components/build/static/css/main.21cabb04.css delete mode 100644 cogment_verse/web/components/build/static/css/main.21cabb04.css.map create mode 100644 cogment_verse/web/components/build/static/css/main.b6c8fc15.css create mode 100644 cogment_verse/web/components/build/static/css/main.b6c8fc15.css.map delete mode 100644 cogment_verse/web/components/build/static/js/main.80bd9354.js delete mode 100644 cogment_verse/web/components/build/static/js/main.80bd9354.js.map create mode 100644 cogment_verse/web/components/build/static/js/main.b399feaf.js rename cogment_verse/web/components/build/static/js/{main.80bd9354.js.LICENSE.txt => main.b399feaf.js.LICENSE.txt} (100%) create mode 100644 cogment_verse/web/components/build/static/js/main.b399feaf.js.map create mode 100644 cogment_verse/web/components/src/controls/ConnectFourControls.jsx rename cogment_verse/web/components/src/controls/{ObserverControls.jsx => RealTimeObserverControls.jsx} (95%) create mode 100644 cogment_verse/web/components/src/controls/TurnBasedObserverControls.jsx create mode 100644 config/run/headless_play.yaml create mode 100644 config/services/environment/connect_four.yaml create mode 100644 config/services/environment/tictactoe.yaml create mode 100644 environments/pettingzoo_adapter.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 674b4f1e..c5a2e505 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ stages: before_script: - mkdir -p ${PIP_CACHE_DIR} - apt-get update - - apt-get install -y python3-opencv + - apt-get install -y swig python3-opencv - python -m venv .venv - source .venv/bin/activate - pip install -r requirements.txt diff --git a/actors/random_actor.py b/actors/random_actor.py index 464d54a9..56dfaefa 100644 --- a/actors/random_actor.py +++ b/actors/random_actor.py @@ -13,6 +13,7 @@ # limitations under the License. import cogment +import numpy as np from cogment_verse.specs import ( PLAYER_ACTOR_CLASS, @@ -35,9 +36,16 @@ async def impl(self, actor_session): action_space = config.environment_specs.action_space - random_seed = config.seed if config.seed is not None else 0 + rng = np.random.default_rng(config.seed if config.seed is not None else 0) async for event in actor_session.all_events(): if event.observation and event.type == cogment.EventType.ACTIVE: - [action_value] = sample_space(action_space, seed=random_seed + actor_session.get_tick_id()) + if ( + event.observation.observation.HasField("current_player") + and event.observation.observation.current_player != actor_session.name + ): + # Not the turn of the agent + actor_session.do_action(PlayerAction()) + continue + [action_value] = sample_space(action_space, rng=rng, mask=event.observation.observation.action_mask) actor_session.do_action(PlayerAction(value=action_value)) diff --git a/actors/simple_a2c.py b/actors/simple_a2c.py index dff25531..ca917261 100644 --- a/actors/simple_a2c.py +++ b/actors/simple_a2c.py @@ -129,7 +129,7 @@ async def impl(self, actor_session): assert config.environment_specs.num_players == 1 assert len(config.environment_specs.action_space.properties) == 1 - assert config.environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + assert config.environment_specs.action_space.properties[0].WhichOneof("type") == "discrete" observation_space = config.environment_specs.observation_space @@ -213,7 +213,7 @@ async def impl(self, run_session): assert self._environment_specs.num_players == 1 assert len(self._environment_specs.action_space.properties) == 1 - assert self._environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + assert self._environment_specs.action_space.properties[0].WhichOneof("type") == "discrete" model = SimpleA2CModel( model_id, diff --git a/actors/simple_dqn.py b/actors/simple_dqn.py index 95df25dc..f2c19eb3 100644 --- a/actors/simple_dqn.py +++ b/actors/simple_dqn.py @@ -27,6 +27,7 @@ EnvironmentConfig, flatten, flattened_dimensions, + flatten_mask, PLAYER_ACTOR_CLASS, PlayerAction, SpaceValue, @@ -136,7 +137,7 @@ async def impl(self, actor_session): assert config.environment_specs.num_players == 1 assert len(config.environment_specs.action_space.properties) == 1 - assert config.environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + assert config.environment_specs.action_space.properties[0].WhichOneof("type") == "discrete" observation_space = config.environment_specs.observation_space action_space = config.environment_specs.action_space @@ -150,18 +151,29 @@ async def impl(self, actor_session): async for event in actor_session.all_events(): if event.observation and event.type == cogment.EventType.ACTIVE: + if ( + event.observation.observation.HasField("current_player") + and event.observation.observation.current_player != actor_session.name + ): + # Not the turn of the agent + actor_session.do_action(PlayerAction()) + continue + if config.model_version == -1 and actor_session.get_tick_id() % config.model_update_frequency == 0: model, _, _ = await actor_session.model_registry.retrieve_version( SimpleDQNModel, config.model_id, config.model_version ) model.network.eval() if rng.random() < model.epsilon: - [action_value] = sample_space(action_space, rng=rng) + [action_value] = sample_space(action_space, rng=rng, mask=event.observation.observation.action_mask) else: obs_tensor = torch.tensor( flatten(observation_space, event.observation.observation.value), dtype=self._dtype ) action_probs = model.network(obs_tensor) + if event.observation.observation.HasField("action_mask"): + action_mask = flatten_mask(event.observation.observation.action_mask) + action_probs = action_probs * action_mask discrete_action_tensor = torch.argmax(action_probs) action_value = SpaceValue( properties=[SpaceValue.PropertyValue(discrete=discrete_action_tensor.item())] @@ -248,7 +260,7 @@ async def impl(self, run_session): assert self._environment_specs.num_players == 1 assert len(self._environment_specs.action_space.properties) == 1 - assert self._environment_specs.action_space.properties[0].WhichOneof("type_oneof") == "discrete" + assert self._environment_specs.action_space.properties[0].WhichOneof("type") == "discrete" epsilon_schedule = create_linear_schedule( self._cfg.epsilon_schedule_start, diff --git a/actors/tutorial/tutorial_1.py b/actors/tutorial/tutorial_1.py index 0b98a2f9..549b01d8 100644 --- a/actors/tutorial/tutorial_1.py +++ b/actors/tutorial/tutorial_1.py @@ -45,7 +45,7 @@ async def impl(self, actor_session): async for event in actor_session.all_events(): if event.observation and event.type == cogment.EventType.ACTIVE: - [action_value] = sample_space(config.environment_specs.action_space, seed=actor_session.get_tick_id()) + [action_value] = sample_space(config.environment_specs.action_space) actor_session.do_action(PlayerAction(value=action_value)) diff --git a/actors/tutorial/tutorial_2.py b/actors/tutorial/tutorial_2.py index ef69cde6..91247d56 100644 --- a/actors/tutorial/tutorial_2.py +++ b/actors/tutorial/tutorial_2.py @@ -53,7 +53,7 @@ async def impl(self, actor_session): async for event in actor_session.all_events(): if event.observation and event.type == cogment.EventType.ACTIVE: - [action_value] = sample_space(config.environment_specs.action_space, seed=actor_session.get_tick_id()) + [action_value] = sample_space(config.environment_specs.action_space) actor_session.do_action(PlayerAction(value=action_value)) diff --git a/cogment_verse/replay_buffers/__init__.py b/cogment_verse/replay_buffers/__init__.py index 1ba0dd0b..abcedc75 100644 --- a/cogment_verse/replay_buffers/__init__.py +++ b/cogment_verse/replay_buffers/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cogment_verse/replay_buffers/torch_replay_buffer.py b/cogment_verse/replay_buffers/torch_replay_buffer.py index 3cb8f961..2d879e2d 100644 --- a/cogment_verse/replay_buffers/torch_replay_buffer.py +++ b/cogment_verse/replay_buffers/torch_replay_buffer.py @@ -1,4 +1,4 @@ -# Copyright 2021 AI Redefined Inc. +# Copyright 2022 AI Redefined Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cogment_verse/specs/__init__.py b/cogment_verse/specs/__init__.py index 86bb4030..ce48b720 100644 --- a/cogment_verse/specs/__init__.py +++ b/cogment_verse/specs/__init__.py @@ -19,6 +19,7 @@ Observation, PlayerAction, Space, + SpaceMask, SpaceValue, ) import cog_settings # pylint: disable=import-error @@ -34,7 +35,7 @@ from .encode_rendered_frame import encode_rendered_frame from .ndarray import deserialize_ndarray, serialize_ndarray from .sample_space import sample_space -from .flatten import flattened_dimensions, flatten, unflatten +from .flatten import flattened_dimensions, flatten, unflatten, flatten_mask from .gym_spaces_adapter import ( gym_action_from_action, gym_space_from_space, diff --git a/cogment_verse/specs/data.proto b/cogment_verse/specs/data.proto index 7714b185..30d78b1a 100644 --- a/cogment_verse/specs/data.proto +++ b/cogment_verse/specs/data.proto @@ -39,7 +39,7 @@ message Space { } message Property { string key = 1; - oneof type_oneof { + oneof type { Discrete discrete = 2; Box box = 3; } @@ -63,11 +63,21 @@ message SpaceValue { repeated PropertyValue properties = 1; } +// SpaceMask messages are value masks within a space +// `properties` are sorted in the same way than the value's Space +message SpaceMask { + message PropertyMask { + repeated int32 discrete = 1; // The discrete actions that are valid + } + repeated PropertyMask properties = 1; +} + message EnvironmentSpecs { string implementation = 1; - int32 num_players = 2; - Space observation_space = 3; - Space action_space = 4; + bool turn_based = 2; + int32 num_players = 3; + Space observation_space = 4; + Space action_space = 5; } message EnvironmentConfig { @@ -104,11 +114,11 @@ message TrialConfig { message Observation { SpaceValue value = 1; - optional bytes rendered_frame = 2; - repeated int32 legal_moves_as_int = 3; - int32 current_player = 4; // active player for multi-agent turn-based environments + optional string current_player = 2; // active player for multi-agent turn-based environments + optional SpaceMask action_mask = 3; + optional bytes rendered_frame = 4; repeated string overridden_players = 5; // list of players that provided an action that was overriden during the last tick - NDArray segmentation = 6; + //NDArray segmentation = 6; } message PlayerAction { diff --git a/cogment_verse/specs/flatten.py b/cogment_verse/specs/flatten.py index 47f74778..f9895acc 100644 --- a/cogment_verse/specs/flatten.py +++ b/cogment_verse/specs/flatten.py @@ -14,7 +14,7 @@ import numpy as np -from data_pb2 import Space, SpaceValue # pylint: disable=import-error +from data_pb2 import Space, SpaceMask, SpaceValue # pylint: disable=import-error from .value import create_space_values from .ndarray import deserialize_ndarray, serialize_ndarray, create_one_hot_ndarray @@ -26,7 +26,7 @@ def space_prop_flattened_dimensions(prop): """ assert isinstance(prop, Space.Property) - if prop.WhichOneof("type_oneof") == "discrete": + if prop.WhichOneof("type") == "discrete": return max(len(prop.discrete.labels), prop.discrete.num) # box if len(prop.box.shape) == 0: @@ -55,14 +55,14 @@ def flatten_prop(space, value, prop_idx): assert isinstance(value, SpaceValue) if prop_idx >= len(value.properties): - # If a value defines less properties that its space we make the remaining zeros + # If a value defines less properties than its space we make the remaining zeros return np.zeros(space_prop_flattened_dimensions(space.properties[prop_idx])) prop_value = value.properties[prop_idx] value_type = prop_value.WhichOneof("value") if value_type == "discrete": space_prop = space.properties[prop_idx] return create_one_hot_ndarray( - prop_value.discrete, max(len(space_prop.discrete.labels), space_prop.discrete.num) + [prop_value.discrete], max(len(space_prop.discrete.labels), space_prop.discrete.num) ) if value_type == "box": return deserialize_ndarray(prop_value.box).flatten() @@ -86,7 +86,7 @@ def unflatten(space, flat_value): for prop_idx, prop in enumerate(space.properties): flat_value_idx_end = flat_value_idx + space_prop_flattened_dimensions(prop) flat_value_prop = flat_value[flat_value_idx:flat_value_idx_end] - if prop.WhichOneof("type_oneof") == "discrete": + if prop.WhichOneof("type") == "discrete": # np.nonzero on a flat array returns a 1D tuple of the indices whose value is != 0 value.properties[prop_idx].discrete = np.nonzero(flat_value_prop)[0][0] else: @@ -95,3 +95,27 @@ def unflatten(space, flat_value): flat_value_idx = flat_value_idx_end return value + + +def flatten_mask_prop(space, mask, prop_idx): + assert isinstance(space, Space) + assert isinstance(mask, SpaceMask) + + space_prop = space.properties[prop_idx] + if prop_idx >= len(mask.properties): + # If a mask defines less properties than its space we make the remaining ones + return np.ones(space_prop_flattened_dimensions(space_prop)) + + value_type = space_prop.WhichOneof("type") + prop_mask = mask.properties[prop_idx] + if value_type == "discrete": + return create_one_hot_ndarray(prop_mask.discrete, max(len(space_prop.discrete.labels), space_prop.discrete.num)) + # if value_type == "box": + return np.ones(space_prop_flattened_dimensions(space_prop)) + + +def flatten_mask(space, mask): + assert isinstance(space, Space) + assert isinstance(mask, SpaceMask) + + return np.concatenate([flatten_mask_prop(space, mask, prop_idx) for prop_idx in range(len(space.properties))]) diff --git a/cogment_verse/specs/gym_spaces_adapter.py b/cogment_verse/specs/gym_spaces_adapter.py index f70fb94f..70b02d32 100644 --- a/cogment_verse/specs/gym_spaces_adapter.py +++ b/cogment_verse/specs/gym_spaces_adapter.py @@ -26,7 +26,7 @@ def gym_space_from_space(space): gym_spaces_dict = {} for prop in space.properties: - prop_type = prop.WhichOneof("type_oneof") + prop_type = prop.WhichOneof("type") if prop_type == "discrete": num_action = max(len(prop.discrete.labels), prop.discrete.num) gym_spaces_dict[prop.key] = gym.spaces.Discrete(num_action) @@ -41,30 +41,41 @@ def gym_space_from_space(space): return gym.spaces.Dict(gym_spaces_dict) -def space_from_gym_space(gym_space): +def space_properties_from_gym_space(gym_space): if isinstance(gym_space, gym.spaces.Box): - return Space( - properties=[ - Space.Property( - box=Space.Box( - shape=list(gym_space.shape), - low=[ - Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() - for v in gym_space.low.flat - ], - high=[ - Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() - for v in gym_space.high.flat - ], - ) + return [ + Space.Property( + box=Space.Box( + shape=list(gym_space.shape), + low=[ + Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() + for v in gym_space.low.flat + ], + high=[ + Space.Bound(bound=v) if SPACES_BOUND_MIN < v < SPACES_BOUND_MAX else Space.Bound() + for v in gym_space.high.flat + ], ) - ] - ) + ) + ] if isinstance(gym_space, gym.spaces.Discrete): - return Space(properties=[Space.Property(discrete=Space.Discrete(num=gym_space.n))]) + return [Space.Property(discrete=Space.Discrete(num=gym_space.n))] + if isinstance(gym_space, gym.spaces.Dict): + properties = [] + for prop_key, gym_sub_space in gym_space.properties: + for sub_prop in space_properties_from_gym_space(gym_sub_space): + sub_prop.key = ( + f"{prop_key}.{sub_prop.key}" if sub_prop.key is not None else prop_key # pylint: disable=no-member + ) + properties.append(sub_prop) + return properties raise RuntimeError(f"[{type(gym_space)}] is not a supported gym space type") +def space_from_gym_space(gym_space): + return Space(properties=space_properties_from_gym_space(gym_space)) + + def gym_action_from_action(space, action): if len(action.properties) == 1: prop_value = action.properties[0] diff --git a/cogment_verse/specs/ndarray.py b/cogment_verse/specs/ndarray.py index 20b81e0f..d662a3cd 100644 --- a/cogment_verse/specs/ndarray.py +++ b/cogment_verse/specs/ndarray.py @@ -24,7 +24,7 @@ def serialize_ndarray(nd_array): return NDArray(shape=nd_array.shape, dtype=str(nd_array.dtype), data=nd_array.tobytes()) -def create_one_hot_ndarray(value, size): +def create_one_hot_ndarray(values, size): nd_array = np.zeros(size) - nd_array[value] = 1 + nd_array[values] = 1 return nd_array diff --git a/cogment_verse/specs/sample_space.py b/cogment_verse/specs/sample_space.py index 2249cf2f..e33da851 100644 --- a/cogment_verse/specs/sample_space.py +++ b/cogment_verse/specs/sample_space.py @@ -18,14 +18,20 @@ from .ndarray import serialize_ndarray -def sample_space(space, num_samples=1, seed=0, rng=None): +def sample_space(space, num_samples=1, rng=None, mask=None): if rng is None: - rng = np.random.default_rng(seed) + rng = np.random.default_rng() space_values = create_space_values(space, num_samples) for prop_idx, prop in enumerate(space.properties): - if prop.WhichOneof("type_oneof") == "discrete": + if prop.WhichOneof("type") == "discrete": + if mask is not None and prop_idx < len(mask.properties) and len(mask.properties[prop_idx].discrete) > 0: + for space_value, sampled_value in zip( + space_values, rng.choice(a=mask.properties[prop_idx].discrete, size=num_samples) + ): + space_value.properties[prop_idx].discrete = sampled_value + continue num_action = max(len(prop.discrete.labels), prop.discrete.num) for space_value, sampled_value in zip(space_values, rng.integers(low=0, high=num_action, size=num_samples)): space_value.properties[prop_idx].discrete = sampled_value diff --git a/cogment_verse/utils/__init__.py b/cogment_verse/utils/__init__.py index df0a641b..76f07fa9 100644 --- a/cogment_verse/utils/__init__.py +++ b/cogment_verse/utils/__init__.py @@ -14,3 +14,4 @@ from .lru import LRU from .sizeof_fmt import sizeof_fmt +from .import_class import import_class diff --git a/cogment_verse/web/components/build/asset-manifest.json b/cogment_verse/web/components/build/asset-manifest.json index bd075972..2a50d235 100644 --- a/cogment_verse/web/components/build/asset-manifest.json +++ b/cogment_verse/web/components/build/asset-manifest.json @@ -1,15 +1,15 @@ { "files": { - "main.css": "/static/css/main.21cabb04.css", - "main.js": "/static/js/main.80bd9354.js", + "main.css": "/static/css/main.b6c8fc15.css", + "main.js": "/static/js/main.b399feaf.js", "static/js/787.90542627.chunk.js": "/static/js/787.90542627.chunk.js", "index.html": "/index.html", - "main.21cabb04.css.map": "/static/css/main.21cabb04.css.map", - "main.80bd9354.js.map": "/static/js/main.80bd9354.js.map", + "main.b6c8fc15.css.map": "/static/css/main.b6c8fc15.css.map", + "main.b399feaf.js.map": "/static/js/main.b399feaf.js.map", "787.90542627.chunk.js.map": "/static/js/787.90542627.chunk.js.map" }, "entrypoints": [ - "static/css/main.21cabb04.css", - "static/js/main.80bd9354.js" + "static/css/main.b6c8fc15.css", + "static/js/main.b399feaf.js" ] } \ No newline at end of file diff --git a/cogment_verse/web/components/build/index.html b/cogment_verse/web/components/build/index.html index e37aa772..6cde68bc 100644 --- a/cogment_verse/web/components/build/index.html +++ b/cogment_verse/web/components/build/index.html @@ -1 +1 @@ -Cogment Verse
    \ No newline at end of file +Cogment Verse
    \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/css/main.21cabb04.css b/cogment_verse/web/components/build/static/css/main.21cabb04.css deleted file mode 100644 index 35b1c5f1..00000000 --- a/cogment_verse/web/components/build/static/css/main.21cabb04.css +++ /dev/null @@ -1,4 +0,0 @@ -/* -! tailwindcss v3.0.24 | MIT License | https://tailwindcss.com -*/*,:after,:before{border:0 solid #e5e7eb;box-sizing:border-box}:after,:before{--tw-content:""}html{-webkit-text-size-adjust:100%;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;line-height:1.5;tab-size:4}body{line-height:inherit}hr{border-top-width:1px;color:inherit;height:0}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{border-collapse:collapse;border-color:inherit;text-indent:0}button,input,optgroup,select,textarea{color:inherit;font-family:inherit;font-size:100%;line-height:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}textarea{resize:vertical}input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#9ca3af;opacity:1}input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#9ca3af;opacity:1}input::placeholder,textarea::placeholder{color:#9ca3af;opacity:1}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{height:auto;max-width:100%}[hidden]{display:none}*,:after,:before{--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.static{position:static}.mx-auto{margin-left:auto;margin-right:auto}.block{display:block}.flex{display:flex}.min-h-screen{min-height:100vh}.w-full{width:100%}.w-fit{width:-webkit-fit-content;width:-moz-fit-content;width:fit-content}.max-w-screen-md{max-width:768px}.flex-1{flex:1 1}.flex-none{flex:none}.transform{-webkit-transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y));transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y))}.list-inside{list-style-position:inside}.list-disc{list-style-type:disc}.flex-row{flex-direction:row}.flex-col{flex-direction:column}.justify-center{justify-content:center}.gap-2{gap:.5rem}.gap-1{gap:.25rem}.rounded{border-radius:.25rem}.rounded-full{border-radius:9999px}.bg-indigo-600{--tw-bg-opacity:1;background-color:rgb(79 70 229/var(--tw-bg-opacity))}.bg-slate-600{--tw-bg-opacity:1;background-color:rgb(71 85 105/var(--tw-bg-opacity))}.bg-indigo-200{--tw-bg-opacity:1;background-color:rgb(199 210 254/var(--tw-bg-opacity))}.p-2{padding:.5rem}.p-5{padding:1.25rem}.py-2{padding-bottom:.5rem;padding-top:.5rem}.px-5{padding-left:1.25rem;padding-right:1.25rem}.py-3{padding-bottom:.75rem;padding-top:.75rem}.text-center{text-align:center}.text-right{text-align:right}.font-mono{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace}.text-xl{font-size:1.25rem;line-height:1.75rem}.text-base{font-size:1rem;line-height:1.5rem}.text-xs{font-size:.75rem;line-height:1rem}.text-sm{font-size:.875rem;line-height:1.25rem}.font-semibold{font-weight:600}.lowercase{text-transform:lowercase}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.ring-8{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(8px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),0 0 #0000;box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.ring-inset{--tw-ring-inset:inset}.blur{--tw-blur:blur(8px)}.blur,.filter{-webkit-filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow);filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}.duration-75{transition-duration:75ms}body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;margin:0}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}.hover\:bg-indigo-900:hover{--tw-bg-opacity:1;background-color:rgb(49 46 129/var(--tw-bg-opacity))}.RenderedScreen_container__9inx9{display:grid;position:relative}.RenderedScreen_canvas__Zmrsu,.RenderedScreen_overlay__fApA7{grid-area:1/-1;width:100%;z-index:0}.RenderedScreen_overlay__fApA7{align-items:center;display:flex;justify-content:center;z-index:1}:root{--dpad-bg-color:#fff;--dpad-bg-color-hover:#eee;--dpad-bg-color-active:#fff;--dpad-bg-color-disabled:#fff;--dpad-fg-color:#5217b8;--dpad-fg-color-hover:#5217b8;--dpad-fg-color-active:#ffb300;--dpad-fg-color-disabled:#bbb;--dpad-button-outer-radius:15%;--dpad-button-inner-radius:50%;--dpad-arrow-position:40%;--dpad-arrow-position-hover:35%;--dpad-arrow-base:19px;--dpad-arrow-height:13px}.DPad_dpad__ZZdxt{display:inline-block;height:200px;overflow:hidden;position:relative;width:200px}.DPad_down__EeuUg,.DPad_left__1AYHI,.DPad_right__AqUBc,.DPad_up__dOfw6{-webkit-tap-highlight-color:rgba(255,255,255,0);background:#fff;background:var(--dpad-bg-color);border-color:#5217b8;border-color:var(--dpad-fg-color);border-style:solid;border-width:1px;color:transparent;display:block;line-height:40%;padding:0;position:absolute;text-align:center}.DPad_down__EeuUg,.DPad_up__dOfw6{height:43%;width:33.3%}.DPad_left__1AYHI,.DPad_right__AqUBc{height:33%;width:43%}.DPad_up__dOfw6{border-radius:15% 15% 50% 50%;border-radius:var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius);top:0}.DPad_down__EeuUg,.DPad_up__dOfw6{left:50%;-webkit-transform:translate(-50%);transform:translate(-50%)}.DPad_down__EeuUg{border-radius:50% 50% 15% 15%;border-radius:var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius) var(--dpad-button-outer-radius);bottom:0}.DPad_left__1AYHI{border-radius:15% 50% 50% 15%;border-radius:var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius);left:0}.DPad_left__1AYHI,.DPad_right__AqUBc{top:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%)}.DPad_right__AqUBc{border-radius:50% 15% 15% 50%;border-radius:50% var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) 50%;right:0}.DPad_down__EeuUg:before,.DPad_left__1AYHI:before,.DPad_right__AqUBc:before,.DPad_up__dOfw6:before{border-radius:5px;border-style:solid;content:"";height:0;position:absolute;transition:all .25s;width:0}.DPad_up__dOfw6:before{border-color:transparent transparent #5217b8;border-color:transparent transparent var(--dpad-fg-color) transparent;border-width:0 13px 19px;border-width:0 var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height);left:50%;top:40%;top:var(--dpad-arrow-position);-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}.DPad_down__EeuUg:before{border-color:#5217b8 transparent transparent;border-color:var(--dpad-fg-color) transparent transparent transparent;border-width:19px 13px 0;border-width:var(--dpad-arrow-base) var(--dpad-arrow-height) 0 var(--dpad-arrow-height);bottom:40%;bottom:var(--dpad-arrow-position);left:50%;-webkit-transform:translate(-50%,50%);transform:translate(-50%,50%)}.DPad_left__1AYHI:before{border-color:transparent #5217b8 transparent transparent;border-color:transparent var(--dpad-fg-color) transparent transparent;border-width:13px 19px 13px 0;border-width:var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height) 0;left:40%;left:var(--dpad-arrow-position);top:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}.DPad_right__AqUBc:before{border-color:transparent transparent transparent #5217b8;border-color:transparent transparent transparent var(--dpad-fg-color);border-width:13px 0 13px 19px;border-width:var(--dpad-arrow-height) 0 var(--dpad-arrow-height) var(--dpad-arrow-base);right:40%;right:var(--dpad-arrow-position);top:50%;-webkit-transform:translate(50%,-50%);transform:translate(50%,-50%)}.DPad_down__EeuUg:hover,.DPad_left__1AYHI:hover,.DPad_right__AqUBc:hover,.DPad_up__dOfw6:hover{background:#eee;background:var(--dpad-bg-color-hover);border-color:#5217b8;border-color:var(--dpad-fg-color-hover)}.DPad_up__dOfw6:hover:before{border-bottom-color:#5217b8;border-bottom-color:var(--dpad-fg-color-hover);top:35%;top:var(--dpad-arrow-position-hover)}.DPad_down__EeuUg:hover:before{border-top-color:#5217b8;border-top-color:var(--dpad-fg-color-hover);bottom:35%;bottom:var(--dpad-arrow-position-hover)}.DPad_left__1AYHI:hover:before{border-right-color:#5217b8;border-right-color:var(--dpad-fg-color-hover);left:35%;left:var(--dpad-arrow-position-hover)}.DPad_right__AqUBc:hover:before{border-left-color:#5217b8;border-left-color:var(--dpad-fg-color-hover);right:35%;right:var(--dpad-arrow-position-hover)}.DPad_down__EeuUg.DPad_active__OzzXu,.DPad_down__EeuUg:active,.DPad_left__1AYHI.DPad_active__OzzXu,.DPad_left__1AYHI:active,.DPad_right__AqUBc.DPad_active__OzzXu,.DPad_right__AqUBc:active,.DPad_up__dOfw6.DPad_active__OzzXu,.DPad_up__dOfw6:active{background:#fff;background:var(--dpad-bg-color-active);border-color:#ffb300;border-color:var(--dpad-fg-color-active)}.DPad_up__dOfw6.DPad_active__OzzXu:before,.DPad_up__dOfw6:active:before{border-bottom-color:#ffb300;border-bottom-color:var(--dpad-fg-color-active)}.DPad_down__EeuUg.DPad_active__OzzXu:before,.DPad_down__EeuUg:active:before{border-top-color:#ffb300;border-top-color:var(--dpad-fg-color-active)}.DPad_left__1AYHI.DPad_active__OzzXu:before,.DPad_left__1AYHI:active:before{border-right-color:#ffb300;border-right-color:var(--dpad-fg-color-active)}.DPad_right__AqUBc.DPad_active__OzzXu:before,.DPad_right__AqUBc:active:before{border-left-color:#ffb300;border-left-color:var(--dpad-fg-color-active)}.DPad_down__EeuUg.DPad_disabled__\+IfWl,.DPad_left__1AYHI.DPad_disabled__\+IfWl,.DPad_right__AqUBc.DPad_disabled__\+IfWl,.DPad_up__dOfw6.DPad_disabled__\+IfWl{background:#fff;background:var(--dpad-bg-color-disabled);border-color:#bbb;border-color:var(--dpad-fg-color-disabled)}.DPad_up__dOfw6.DPad_disabled__\+IfWl:before{border-bottom-color:#bbb;border-bottom-color:var(--dpad-fg-color-disabled);top:40%;top:var(--dpad-arrow-position)}.DPad_down__EeuUg.DPad_disabled__\+IfWl:before{border-top-color:#bbb;border-top-color:var(--dpad-fg-color-disabled);bottom:40%;bottom:var(--dpad-arrow-position)}.DPad_left__1AYHI.DPad_disabled__\+IfWl:before{border-right-color:#bbb;border-right-color:var(--dpad-fg-color-disabled);left:40%;left:var(--dpad-arrow-position)}.DPad_right__AqUBc.DPad_disabled__\+IfWl:before{border-left-color:#bbb;border-left-color:var(--dpad-fg-color-disabled);right:40%;right:var(--dpad-arrow-position)}:root{--joystick-surface-size:200px;--joystick-stick-size:50px;--joystick-color:#5217b8;--joystick-color-active:#ffb300;--joystick-color-disabled:#bbb}.Joystick_joystick__DwFj2{align-items:center;border-color:#5217b8;border-color:var(--joystick-color);border-radius:25px;border-radius:calc(var(--joystick-stick-size)/2);border-style:solid;border-width:1px;display:flex;height:200px;height:var(--joystick-surface-size);justify-content:center;overflow:hidden;position:relative;transition:all .25s;width:200px;width:var(--joystick-surface-size)}.Joystick_joystick__DwFj2>.Joystick_stick__z5P-5{background-color:#5217b8;background-color:var(--joystick-color);border-radius:50%;height:50px;height:var(--joystick-stick-size);transition:all .25s;width:50px;width:var(--joystick-stick-size)}.Joystick_joystick__DwFj2.Joystick_active__EOXxn{border-color:#ffb300;border-color:var(--joystick-color-active)}.Joystick_joystick__DwFj2.Joystick_active__EOXxn>.Joystick_stick__z5P-5{background-color:#ffb300;background-color:var(--joystick-color-active)}.Joystick_joystick__DwFj2.Joystick_disabled__WyYwu{border-color:#bbb;border-color:var(--joystick-color-disabled)}.Joystick_joystick__DwFj2.Joystick_disabled__WyYwu>.Joystick_stick__z5P-5{background-color:#bbb;background-color:var(--joystick-color-disabled)} -/*# sourceMappingURL=main.21cabb04.css.map*/ \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/css/main.21cabb04.css.map b/cogment_verse/web/components/build/static/css/main.21cabb04.css.map deleted file mode 100644 index 88613926..00000000 --- a/cogment_verse/web/components/build/static/css/main.21cabb04.css.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"static/css/main.21cabb04.css","mappings":"AAAA;;CAAc,CAAd,uCAAc,CAAd,qBAAc,CAAd,8BAAc,CAAd,kCAAc,CAAd,gMAAc,CAAd,eAAc,CAAd,UAAc,CAAd,wBAAc,CAAd,uBAAc,CAAd,aAAc,CAAd,QAAc,CAAd,4DAAc,CAAd,gCAAc,CAAd,mCAAc,CAAd,mBAAc,CAAd,eAAc,CAAd,uBAAc,CAAd,2BAAc,CAAd,qHAAc,CAAd,aAAc,CAAd,mBAAc,CAAd,qBAAc,CAAd,aAAc,CAAd,iBAAc,CAAd,uBAAc,CAAd,iBAAc,CAAd,aAAc,CAAd,8BAAc,CAAd,oBAAc,CAAd,aAAc,CAAd,mDAAc,CAAd,mBAAc,CAAd,cAAc,CAAd,mBAAc,CAAd,QAAc,CAAd,SAAc,CAAd,iCAAc,CAAd,yEAAc,CAAd,4BAAc,CAAd,qBAAc,CAAd,4BAAc,CAAd,gCAAc,CAAd,gCAAc,CAAd,mEAAc,CAAd,0CAAc,CAAd,mBAAc,CAAd,mDAAc,CAAd,sDAAc,CAAd,YAAc,CAAd,yBAAc,CAAd,2DAAc,CAAd,iBAAc,CAAd,yBAAc,CAAd,0BAAc,CAAd,QAAc,CAAd,SAAc,CAAd,wBAAc,CAAd,kFAAc,CAAd,SAAc,CAAd,wEAAc,CAAd,SAAc,CAAd,sDAAc,CAAd,SAAc,CAAd,mCAAc,CAAd,wBAAc,CAAd,4DAAc,CAAd,qBAAc,CAAd,qBAAc,CAAd,cAAc,CAAd,qBAAc,CAAd,mCAAc,CAAd,kBAAc,CAAd,aAAc,CAAd,aAAc,CAAd,aAAc,CAAd,cAAc,CAAd,cAAc,CAAd,YAAc,CAAd,YAAc,CAAd,iBAAc,CAAd,qCAAc,CAAd,cAAc,CAAd,mBAAc,CAAd,qBAAc,CAAd,sBAAc,CAAd,uBAAc,CAAd,iBAAc,CAAd,0BAAc,CAAd,2BAAc,CAAd,mCAAc,CAAd,iCAAc,CAAd,0BAAc,CAAd,qBAAc,CAAd,6BAAc,CAAd,WAAc,CAAd,iBAAc,CAAd,eAAc,CAAd,gBAAc,CAAd,iBAAc,CAAd,aAAc,CAAd,eAAc,CAAd,YAAc,CAAd,kBAAc,CAAd,oBAAc,CAAd,0BAAc,CAAd,wBAAc,CAAd,yBAAc,CAAd,0BAAc,CAAd,sBAAc,CAAd,uBAAc,CAAd,wBAAc,CAAd,qBAAc,CAEd,uBAAmB,CAAnB,yBAAmB,CAAnB,iBAAmB,CAAnB,oBAAmB,CAAnB,kBAAmB,CAAnB,8BAAmB,CAAnB,kBAAmB,CAAnB,gCAAmB,CAAnB,sBAAmB,CAAnB,iBAAmB,CAAnB,gCAAmB,CAAnB,gBAAmB,CAAnB,oBAAmB,CAAnB,gNAAmB,CAAnB,6LAAmB,CAAnB,uCAAmB,CAAnB,+BAAmB,CAAnB,4BAAmB,CAAnB,+BAAmB,CAAnB,sCAAmB,CAAnB,gBAAmB,CAAnB,iBAAmB,CAAnB,6BAAmB,CAAnB,kCAAmB,CAAnB,gCAAmB,CAAnB,oDAAmB,CAAnB,+BAAmB,CAAnB,oDAAmB,CAAnB,gCAAmB,CAAnB,sDAAmB,CAAnB,kBAAmB,CAAnB,oBAAmB,CAAnB,4CAAmB,CAAnB,0BAAmB,CAAnB,qBAAmB,CAAnB,8CAAmB,CAAnB,8BAAmB,CAAnB,4BAAmB,CAAnB,8GAAmB,CAAnB,0BAAmB,CAAnB,mBAAmB,CAAnB,yBAAmB,CAAnB,kBAAmB,CAAnB,yBAAmB,CAAnB,gBAAmB,CAAnB,0BAAmB,CAAnB,mBAAmB,CAAnB,8BAAmB,CAAnB,mCAAmB,CAAnB,+BAAmB,CAAnB,6CAAmB,CAAnB,kHAAmB,CAAnB,wGAAmB,CAAnB,uEAAmB,CAAnB,wFAAmB,CAAnB,iCAAmB,CAAnB,yBAAmB,CAAnB,sMAAmB,CAAnB,gLAAmB,CAAnB,qCAAmB,CAEnB,KAIE,kCAAmC,CACnC,iCAAkC,CAHlC,mIAC4C,CAF5C,QAKF,CAEA,KACE,uEACF,CAdA,kG,CCAA,iCAEE,YAAa,CADb,iBAEF,CAEA,6DAEE,cAAe,CAGf,UAAW,CAFX,SAGF,CAEA,+BAKE,kBAAmB,CAFnB,YAAa,CACb,sBAAuB,CAHvB,SAKF,CCnBA,MACE,oBAAqB,CACrB,0BAA2B,CAC3B,2BAA4B,CAC5B,6BAA8B,CAC9B,uBAAwB,CACxB,6BAA8B,CAC9B,8BAA+B,CAC/B,6BAA8B,CAE9B,8BAA+B,CAC/B,8BAA+B,CAE/B,yBAA0B,CAC1B,+BAAgC,CAChC,sBAAuB,CACvB,wBACF,CAEA,kBAEE,oBAAqB,CAGrB,YAAa,CAEb,eAAgB,CANhB,iBAAkB,CAGlB,WAIF,CAIA,uEAME,+CAAmD,CAInD,eAAgC,CAAhC,+BAAgC,CAChC,oBAAkC,CAAlC,iCAAkC,CAClC,kBAAmB,CACnB,gBAAiB,CAEjB,iBAAkB,CAXlB,aAAc,CAId,eAAgB,CAMhB,SAAY,CATZ,iBAAkB,CAIlB,iBAOF,CAEA,kCAGE,UAAW,CADX,WAEF,CAEA,qCAGE,UAAW,CADX,SAEF,CAEA,gBAIE,6BACiC,CADjC,6IACiC,CAJjC,KAKF,CAEA,kCANE,QAAS,CACT,iCAA6B,CAA7B,yBAWF,CANA,kBAIE,6BACiC,CADjC,6IACiC,CAJjC,QAKF,CAEA,kBAIE,6BACiC,CADjC,6IACiC,CAHjC,MAIF,CAEA,qCAPE,OAAQ,CAER,kCAA6B,CAA7B,0BAUF,CALA,mBAIE,6BAAsF,CAAtF,qFAAsF,CAFtF,OAGF,CAGA,mGAQE,iBAAkB,CAClB,kBAAmB,CALnB,UAAW,CAGX,QAAS,CAFT,iBAAkB,CAKlB,mBAAqB,CAJrB,OAKF,CAEA,uBAKE,4CAAsE,CAAtE,qEAAsE,CADtE,wBAAwF,CAAxF,uFAAwF,CAFxF,QAAS,CADT,OAA+B,CAA/B,8BAA+B,CAE/B,sCAAgC,CAAhC,8BAGF,CAEA,yBAKE,4CAAsE,CAAtE,qEAAsE,CADtE,wBAA0F,CAA1F,uFAA0F,CAH1F,UAAkC,CAAlC,iCAAkC,CAClC,QAAS,CACT,qCAA+B,CAA/B,6BAGF,CAEA,yBAKE,wDAAsE,CAAtE,qEAAsE,CADtE,6BAAwF,CAAxF,uFAAwF,CAHxF,QAAgC,CAAhC,+BAAgC,CAChC,OAAQ,CACR,sCAAgC,CAAhC,8BAGF,CAEA,0BAKE,wDAAsE,CAAtE,qEAAsE,CADtE,6BAAwF,CAAxF,uFAAwF,CAHxF,SAAiC,CAAjC,gCAAiC,CACjC,OAAQ,CACR,qCAA+B,CAA/B,6BAGF,CAIA,+FAIE,eAAsC,CAAtC,qCAAsC,CACtC,oBAAwC,CAAxC,uCACF,CAEA,6BAEE,2BAA+C,CAA/C,8CAA+C,CAD/C,OAAqC,CAArC,oCAEF,CAEA,+BAEE,wBAA4C,CAA5C,2CAA4C,CAD5C,UAAwC,CAAxC,uCAEF,CAEA,+BAEE,0BAA8C,CAA9C,6CAA8C,CAD9C,QAAsC,CAAtC,qCAEF,CAEA,gCAEE,yBAA6C,CAA7C,4CAA6C,CAD7C,SAAuC,CAAvC,sCAEF,CAIA,sPAQE,eAAuC,CAAvC,sCAAuC,CACvC,oBAAyC,CAAzC,wCACF,CAEA,wEAEE,2BAAgD,CAAhD,+CACF,CAEA,4EAEE,wBAA6C,CAA7C,4CACF,CAEA,4EAEE,0BAA+C,CAA/C,8CACF,CAEA,8EAEE,yBAA8C,CAA9C,6CACF,CAIA,+JAIE,eAAyC,CAAzC,wCAAyC,CACzC,iBAA2C,CAA3C,0CACF,CAEA,6CAEE,wBAAkD,CAAlD,iDAAkD,CADlD,OAA+B,CAA/B,8BAEF,CAEA,+CAEE,qBAA+C,CAA/C,8CAA+C,CAD/C,UAAkC,CAAlC,iCAEF,CAEA,+CAEE,uBAAiD,CAAjD,gDAAiD,CADjD,QAAgC,CAAhC,+BAEF,CAEA,gDAEE,sBAAgD,CAAhD,+CAAgD,CADhD,SAAiC,CAAjC,gCAEF,CCtOA,MACE,6BAA8B,CAC9B,0BAA2B,CAE3B,wBAAyB,CACzB,+BAAgC,CAChC,8BACF,CAEA,0BAIE,kBAAmB,CAOnB,oBAAmC,CAAnC,kCAAmC,CADnC,kBAAmD,CAAnD,gDAAmD,CAGnD,kBAAmB,CADnB,gBAAiB,CAVjB,YAAa,CAMb,YAAoC,CAApC,mCAAoC,CALpC,sBAAuB,CAGvB,eAAgB,CALhB,iBAAkB,CAclB,mBAAqB,CANrB,WAAmC,CAAnC,kCAOF,CAEA,iDAKE,wBAAuC,CAAvC,sCAAuC,CADvC,iBAAkB,CAHlB,WAAkC,CAAlC,iCAAkC,CAMlC,mBAAqB,CALrB,UAAiC,CAAjC,gCAMF,CAEA,iDACE,oBAA0C,CAA1C,yCACF,CAEA,wEACE,wBAA8C,CAA9C,6CACF,CAEA,mDACE,iBAA4C,CAA5C,2CACF,CAEA,0EACE,qBAAgD,CAAhD,+CACF","sources":["index.css","components/RenderedScreen.module.css","components/DPad.module.css","components/Joystick.module.css"],"sourcesContent":["@tailwind base;\n@tailwind components;\n@tailwind utilities;\n\nbody {\n margin: 0;\n font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", \"Roboto\", \"Oxygen\", \"Ubuntu\", \"Cantarell\", \"Fira Sans\",\n \"Droid Sans\", \"Helvetica Neue\", sans-serif;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale;\n}\n\ncode {\n font-family: source-code-pro, Menlo, Monaco, Consolas, \"Courier New\", monospace;\n}\n",".container {\n position: relative;\n display: grid;\n}\n\n.canvas,\n.overlay {\n grid-area: 1/-1;\n z-index: 0;\n\n width: 100%;\n}\n\n.overlay {\n z-index: 1;\n\n display: flex;\n justify-content: center;\n align-items: center;\n}\n",":root {\n --dpad-bg-color: #fff;\n --dpad-bg-color-hover: #eee;\n --dpad-bg-color-active: #fff;\n --dpad-bg-color-disabled: #fff;\n --dpad-fg-color: #5217b8;\n --dpad-fg-color-hover: #5217b8;\n --dpad-fg-color-active: #ffb300;\n --dpad-fg-color-disabled: #bbb;\n\n --dpad-button-outer-radius: 15%;\n --dpad-button-inner-radius: 50%;\n\n --dpad-arrow-position: 40%;\n --dpad-arrow-position-hover: 35%;\n --dpad-arrow-base: 19px;\n --dpad-arrow-height: 13px;\n}\n\n.dpad {\n position: relative;\n display: inline-block;\n\n width: 200px;\n height: 200px;\n\n overflow: hidden;\n}\n\n/* Buttons background */\n\n.up,\n.right,\n.down,\n.left {\n display: block;\n position: absolute;\n -webkit-tap-highlight-color: rgba(255, 255, 255, 0);\n\n line-height: 40%;\n text-align: center;\n background: var(--dpad-bg-color);\n border-color: var(--dpad-fg-color);\n border-style: solid;\n border-width: 1px;\n padding: 0px;\n color: transparent;\n}\n\n.up,\n.down {\n width: 33.3%;\n height: 43%;\n}\n\n.left,\n.right {\n width: 43%;\n height: 33%;\n}\n\n.up {\n top: 0;\n left: 50%;\n transform: translate(-50%, 0);\n border-radius: var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) var(--dpad-button-inner-radius)\n var(--dpad-button-inner-radius);\n}\n\n.down {\n bottom: 0;\n left: 50%;\n transform: translate(-50%, 0);\n border-radius: var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius)\n var(--dpad-button-outer-radius);\n}\n\n.left {\n top: 50%;\n left: 0;\n transform: translate(0, -50%);\n border-radius: var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius)\n var(--dpad-button-outer-radius);\n}\n\n.right {\n top: 50%;\n right: 0;\n transform: translate(0, -50%);\n border-radius: 50% var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) 50%;\n}\n\n/* Buttons arrows */\n.up:before,\n.right:before,\n.down:before,\n.left:before {\n content: \"\";\n position: absolute;\n width: 0;\n height: 0;\n border-radius: 5px;\n border-style: solid;\n transition: all 0.25s;\n}\n\n.up:before {\n top: var(--dpad-arrow-position);\n left: 50%;\n transform: translate(-50%, -50%);\n border-width: 0 var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height);\n border-color: transparent transparent var(--dpad-fg-color) transparent;\n}\n\n.down:before {\n bottom: var(--dpad-arrow-position);\n left: 50%;\n transform: translate(-50%, 50%);\n border-width: var(--dpad-arrow-base) var(--dpad-arrow-height) 0px var(--dpad-arrow-height);\n border-color: var(--dpad-fg-color) transparent transparent transparent;\n}\n\n.left:before {\n left: var(--dpad-arrow-position);\n top: 50%;\n transform: translate(-50%, -50%);\n border-width: var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height) 0;\n border-color: transparent var(--dpad-fg-color) transparent transparent;\n}\n\n.right:before {\n right: var(--dpad-arrow-position);\n top: 50%;\n transform: translate(50%, -50%);\n border-width: var(--dpad-arrow-height) 0 var(--dpad-arrow-height) var(--dpad-arrow-base);\n border-color: transparent transparent transparent var(--dpad-fg-color);\n}\n\n/* Hover */\n\n.up:hover,\n.right:hover,\n.down:hover,\n.left:hover {\n background: var(--dpad-bg-color-hover);\n border-color: var(--dpad-fg-color-hover);\n}\n\n.up:hover:before {\n top: var(--dpad-arrow-position-hover);\n border-bottom-color: var(--dpad-fg-color-hover);\n}\n\n.down:hover:before {\n bottom: var(--dpad-arrow-position-hover);\n border-top-color: var(--dpad-fg-color-hover);\n}\n\n.left:hover:before {\n left: var(--dpad-arrow-position-hover);\n border-right-color: var(--dpad-fg-color-hover);\n}\n\n.right:hover:before {\n right: var(--dpad-arrow-position-hover);\n border-left-color: var(--dpad-fg-color-hover);\n}\n\n/* Active */\n\n.up:active,\n.right:active,\n.down:active,\n.left:active,\n.up.active,\n.right.active,\n.down.active,\n.left.active {\n background: var(--dpad-bg-color-active);\n border-color: var(--dpad-fg-color-active);\n}\n\n.up:active:before,\n.up.active:before {\n border-bottom-color: var(--dpad-fg-color-active);\n}\n\n.down:active:before,\n.down.active:before {\n border-top-color: var(--dpad-fg-color-active);\n}\n\n.left:active:before,\n.left.active:before {\n border-right-color: var(--dpad-fg-color-active);\n}\n\n.right:active:before,\n.right.active:before {\n border-left-color: var(--dpad-fg-color-active);\n}\n\n/* Disabled */\n\n.up.disabled,\n.right.disabled,\n.down.disabled,\n.left.disabled {\n background: var(--dpad-bg-color-disabled);\n border-color: var(--dpad-fg-color-disabled);\n}\n\n.up.disabled:before {\n top: var(--dpad-arrow-position);\n border-bottom-color: var(--dpad-fg-color-disabled);\n}\n\n.down.disabled:before {\n bottom: var(--dpad-arrow-position);\n border-top-color: var(--dpad-fg-color-disabled);\n}\n\n.left.disabled:before {\n left: var(--dpad-arrow-position);\n border-right-color: var(--dpad-fg-color-disabled);\n}\n\n.right.disabled:before {\n right: var(--dpad-arrow-position);\n border-left-color: var(--dpad-fg-color-disabled);\n}\n",":root {\n --joystick-surface-size: 200px;\n --joystick-stick-size: 50px;\n\n --joystick-color: #5217b8;\n --joystick-color-active: #ffb300;\n --joystick-color-disabled: #bbb;\n}\n\n.joystick {\n position: relative;\n display: flex;\n justify-content: center;\n align-items: center;\n\n overflow: hidden;\n\n height: var(--joystick-surface-size);\n width: var(--joystick-surface-size);\n border-radius: calc(var(--joystick-stick-size) / 2);\n border-color: var(--joystick-color);\n border-width: 1px;\n border-style: solid;\n\n transition: all 0.25s;\n}\n\n.joystick > .stick {\n height: var(--joystick-stick-size);\n width: var(--joystick-stick-size);\n\n border-radius: 50%;\n background-color: var(--joystick-color);\n\n transition: all 0.25s;\n}\n\n.joystick.active {\n border-color: var(--joystick-color-active);\n}\n\n.joystick.active > .stick {\n background-color: var(--joystick-color-active);\n}\n\n.joystick.disabled {\n border-color: var(--joystick-color-disabled);\n}\n\n.joystick.disabled > .stick {\n background-color: var(--joystick-color-disabled);\n}\n"],"names":[],"sourceRoot":""} \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/css/main.b6c8fc15.css b/cogment_verse/web/components/build/static/css/main.b6c8fc15.css new file mode 100644 index 00000000..6c11223f --- /dev/null +++ b/cogment_verse/web/components/build/static/css/main.b6c8fc15.css @@ -0,0 +1,4 @@ +/* +! tailwindcss v3.0.24 | MIT License | https://tailwindcss.com +*/*,:after,:before{border:0 solid #e5e7eb;box-sizing:border-box}:after,:before{--tw-content:""}html{-webkit-text-size-adjust:100%;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;line-height:1.5;tab-size:4}body{line-height:inherit}hr{border-top-width:1px;color:inherit;height:0}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{border-collapse:collapse;border-color:inherit;text-indent:0}button,input,optgroup,select,textarea{color:inherit;font-family:inherit;font-size:100%;line-height:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}textarea{resize:vertical}input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#9ca3af;opacity:1}input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#9ca3af;opacity:1}input::placeholder,textarea::placeholder{color:#9ca3af;opacity:1}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{height:auto;max-width:100%}[hidden]{display:none}*,:after,:before{--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.static{position:static}.mx-auto{margin-left:auto;margin-right:auto}.mb-2{margin-bottom:.5rem}.block{display:block}.flex{display:flex}.aspect-square{aspect-ratio:1/1}.min-h-screen{min-height:100vh}.w-full{width:100%}.w-fit{width:-webkit-fit-content;width:-moz-fit-content;width:fit-content}.max-w-screen-md{max-width:768px}.flex-1{flex:1 1}.flex-none{flex:none}.flex-initial{flex:0 1 auto}.transform{-webkit-transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y));transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y))}.list-inside{list-style-position:inside}.list-disc{list-style-type:disc}.flex-row{flex-direction:row}.flex-col{flex-direction:column}.items-center{align-items:center}.justify-center{justify-content:center}.gap-2{gap:.5rem}.gap-1{gap:.25rem}.gap-4{gap:1rem}.rounded{border-radius:.25rem}.rounded-full{border-radius:9999px}.bg-indigo-600{--tw-bg-opacity:1;background-color:rgb(79 70 229/var(--tw-bg-opacity))}.bg-slate-600{--tw-bg-opacity:1;background-color:rgb(71 85 105/var(--tw-bg-opacity))}.bg-indigo-200{--tw-bg-opacity:1;background-color:rgb(199 210 254/var(--tw-bg-opacity))}.p-2{padding:.5rem}.p-5{padding:1.25rem}.py-2{padding-bottom:.5rem;padding-top:.5rem}.px-5{padding-left:1.25rem;padding-right:1.25rem}.py-3{padding-bottom:.75rem;padding-top:.75rem}.px-10{padding-left:2.5rem;padding-right:2.5rem}.text-center{text-align:center}.text-right{text-align:right}.font-mono{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace}.text-xl{font-size:1.25rem;line-height:1.75rem}.text-base{font-size:1rem;line-height:1.5rem}.text-xs{font-size:.75rem;line-height:1rem}.text-sm{font-size:.875rem;line-height:1.25rem}.font-semibold{font-weight:600}.lowercase{text-transform:lowercase}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.ring-8{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(8px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),0 0 #0000;box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.ring-inset{--tw-ring-inset:inset}.blur{--tw-blur:blur(8px)}.blur,.filter{-webkit-filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow);filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}.duration-75{transition-duration:75ms}body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;margin:0}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}.hover\:bg-indigo-900:hover{--tw-bg-opacity:1;background-color:rgb(49 46 129/var(--tw-bg-opacity))}.disabled\:bg-gray-400:disabled{--tw-bg-opacity:1;background-color:rgb(156 163 175/var(--tw-bg-opacity))}.disabled\:text-gray-200:disabled{--tw-text-opacity:1;color:rgb(229 231 235/var(--tw-text-opacity))}.RenderedScreen_container__9inx9{display:grid;position:relative}.RenderedScreen_canvas__Zmrsu,.RenderedScreen_overlay__fApA7{grid-area:1/-1;width:100%;z-index:0}.RenderedScreen_overlay__fApA7{align-items:center;display:flex;justify-content:center;z-index:1}:root{--dpad-bg-color:#fff;--dpad-bg-color-hover:#eee;--dpad-bg-color-active:#fff;--dpad-bg-color-disabled:#fff;--dpad-fg-color:#5217b8;--dpad-fg-color-hover:#5217b8;--dpad-fg-color-active:#ffb300;--dpad-fg-color-disabled:#bbb;--dpad-button-outer-radius:15%;--dpad-button-inner-radius:50%;--dpad-arrow-position:40%;--dpad-arrow-position-hover:35%;--dpad-arrow-base:19px;--dpad-arrow-height:13px}.DPad_dpad__ZZdxt{display:inline-block;height:200px;overflow:hidden;position:relative;width:200px}.DPad_down__EeuUg,.DPad_left__1AYHI,.DPad_right__AqUBc,.DPad_up__dOfw6{-webkit-tap-highlight-color:rgba(255,255,255,0);background:#fff;background:var(--dpad-bg-color);border-color:#5217b8;border-color:var(--dpad-fg-color);border-style:solid;border-width:1px;color:transparent;display:block;line-height:40%;padding:0;position:absolute;text-align:center}.DPad_down__EeuUg,.DPad_up__dOfw6{height:43%;width:33.3%}.DPad_left__1AYHI,.DPad_right__AqUBc{height:33%;width:43%}.DPad_up__dOfw6{border-radius:15% 15% 50% 50%;border-radius:var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius);top:0}.DPad_down__EeuUg,.DPad_up__dOfw6{left:50%;-webkit-transform:translate(-50%);transform:translate(-50%)}.DPad_down__EeuUg{border-radius:50% 50% 15% 15%;border-radius:var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius) var(--dpad-button-outer-radius);bottom:0}.DPad_left__1AYHI{border-radius:15% 50% 50% 15%;border-radius:var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius);left:0}.DPad_left__1AYHI,.DPad_right__AqUBc{top:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%)}.DPad_right__AqUBc{border-radius:50% 15% 15% 50%;border-radius:50% var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) 50%;right:0}.DPad_down__EeuUg:before,.DPad_left__1AYHI:before,.DPad_right__AqUBc:before,.DPad_up__dOfw6:before{border-radius:5px;border-style:solid;content:"";height:0;position:absolute;transition:all .25s;width:0}.DPad_up__dOfw6:before{border-color:transparent transparent #5217b8;border-color:transparent transparent var(--dpad-fg-color) transparent;border-width:0 13px 19px;border-width:0 var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height);left:50%;top:40%;top:var(--dpad-arrow-position);-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}.DPad_down__EeuUg:before{border-color:#5217b8 transparent transparent;border-color:var(--dpad-fg-color) transparent transparent transparent;border-width:19px 13px 0;border-width:var(--dpad-arrow-base) var(--dpad-arrow-height) 0 var(--dpad-arrow-height);bottom:40%;bottom:var(--dpad-arrow-position);left:50%;-webkit-transform:translate(-50%,50%);transform:translate(-50%,50%)}.DPad_left__1AYHI:before{border-color:transparent #5217b8 transparent transparent;border-color:transparent var(--dpad-fg-color) transparent transparent;border-width:13px 19px 13px 0;border-width:var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height) 0;left:40%;left:var(--dpad-arrow-position);top:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}.DPad_right__AqUBc:before{border-color:transparent transparent transparent #5217b8;border-color:transparent transparent transparent var(--dpad-fg-color);border-width:13px 0 13px 19px;border-width:var(--dpad-arrow-height) 0 var(--dpad-arrow-height) var(--dpad-arrow-base);right:40%;right:var(--dpad-arrow-position);top:50%;-webkit-transform:translate(50%,-50%);transform:translate(50%,-50%)}.DPad_down__EeuUg:hover,.DPad_left__1AYHI:hover,.DPad_right__AqUBc:hover,.DPad_up__dOfw6:hover{background:#eee;background:var(--dpad-bg-color-hover);border-color:#5217b8;border-color:var(--dpad-fg-color-hover)}.DPad_up__dOfw6:hover:before{border-bottom-color:#5217b8;border-bottom-color:var(--dpad-fg-color-hover);top:35%;top:var(--dpad-arrow-position-hover)}.DPad_down__EeuUg:hover:before{border-top-color:#5217b8;border-top-color:var(--dpad-fg-color-hover);bottom:35%;bottom:var(--dpad-arrow-position-hover)}.DPad_left__1AYHI:hover:before{border-right-color:#5217b8;border-right-color:var(--dpad-fg-color-hover);left:35%;left:var(--dpad-arrow-position-hover)}.DPad_right__AqUBc:hover:before{border-left-color:#5217b8;border-left-color:var(--dpad-fg-color-hover);right:35%;right:var(--dpad-arrow-position-hover)}.DPad_down__EeuUg.DPad_active__OzzXu,.DPad_down__EeuUg:active,.DPad_left__1AYHI.DPad_active__OzzXu,.DPad_left__1AYHI:active,.DPad_right__AqUBc.DPad_active__OzzXu,.DPad_right__AqUBc:active,.DPad_up__dOfw6.DPad_active__OzzXu,.DPad_up__dOfw6:active{background:#fff;background:var(--dpad-bg-color-active);border-color:#ffb300;border-color:var(--dpad-fg-color-active)}.DPad_up__dOfw6.DPad_active__OzzXu:before,.DPad_up__dOfw6:active:before{border-bottom-color:#ffb300;border-bottom-color:var(--dpad-fg-color-active)}.DPad_down__EeuUg.DPad_active__OzzXu:before,.DPad_down__EeuUg:active:before{border-top-color:#ffb300;border-top-color:var(--dpad-fg-color-active)}.DPad_left__1AYHI.DPad_active__OzzXu:before,.DPad_left__1AYHI:active:before{border-right-color:#ffb300;border-right-color:var(--dpad-fg-color-active)}.DPad_right__AqUBc.DPad_active__OzzXu:before,.DPad_right__AqUBc:active:before{border-left-color:#ffb300;border-left-color:var(--dpad-fg-color-active)}.DPad_down__EeuUg.DPad_disabled__\+IfWl,.DPad_left__1AYHI.DPad_disabled__\+IfWl,.DPad_right__AqUBc.DPad_disabled__\+IfWl,.DPad_up__dOfw6.DPad_disabled__\+IfWl{background:#fff;background:var(--dpad-bg-color-disabled);border-color:#bbb;border-color:var(--dpad-fg-color-disabled)}.DPad_up__dOfw6.DPad_disabled__\+IfWl:before{border-bottom-color:#bbb;border-bottom-color:var(--dpad-fg-color-disabled);top:40%;top:var(--dpad-arrow-position)}.DPad_down__EeuUg.DPad_disabled__\+IfWl:before{border-top-color:#bbb;border-top-color:var(--dpad-fg-color-disabled);bottom:40%;bottom:var(--dpad-arrow-position)}.DPad_left__1AYHI.DPad_disabled__\+IfWl:before{border-right-color:#bbb;border-right-color:var(--dpad-fg-color-disabled);left:40%;left:var(--dpad-arrow-position)}.DPad_right__AqUBc.DPad_disabled__\+IfWl:before{border-left-color:#bbb;border-left-color:var(--dpad-fg-color-disabled);right:40%;right:var(--dpad-arrow-position)}:root{--joystick-surface-size:200px;--joystick-stick-size:50px;--joystick-color:#5217b8;--joystick-color-active:#ffb300;--joystick-color-disabled:#bbb}.Joystick_joystick__DwFj2{align-items:center;border-color:#5217b8;border-color:var(--joystick-color);border-radius:25px;border-radius:calc(var(--joystick-stick-size)/2);border-style:solid;border-width:1px;display:flex;height:200px;height:var(--joystick-surface-size);justify-content:center;overflow:hidden;position:relative;transition:all .25s;width:200px;width:var(--joystick-surface-size)}.Joystick_joystick__DwFj2>.Joystick_stick__z5P-5{background-color:#5217b8;background-color:var(--joystick-color);border-radius:50%;height:50px;height:var(--joystick-stick-size);transition:all .25s;width:50px;width:var(--joystick-stick-size)}.Joystick_joystick__DwFj2.Joystick_active__EOXxn{border-color:#ffb300;border-color:var(--joystick-color-active)}.Joystick_joystick__DwFj2.Joystick_active__EOXxn>.Joystick_stick__z5P-5{background-color:#ffb300;background-color:var(--joystick-color-active)}.Joystick_joystick__DwFj2.Joystick_disabled__WyYwu{border-color:#bbb;border-color:var(--joystick-color-disabled)}.Joystick_joystick__DwFj2.Joystick_disabled__WyYwu>.Joystick_stick__z5P-5{background-color:#bbb;background-color:var(--joystick-color-disabled)} +/*# sourceMappingURL=main.b6c8fc15.css.map*/ \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/css/main.b6c8fc15.css.map b/cogment_verse/web/components/build/static/css/main.b6c8fc15.css.map new file mode 100644 index 00000000..1407f751 --- /dev/null +++ b/cogment_verse/web/components/build/static/css/main.b6c8fc15.css.map @@ -0,0 +1 @@ +{"version":3,"file":"static/css/main.b6c8fc15.css","mappings":"AAAA;;CAAc,CAAd,uCAAc,CAAd,qBAAc,CAAd,8BAAc,CAAd,kCAAc,CAAd,gMAAc,CAAd,eAAc,CAAd,UAAc,CAAd,wBAAc,CAAd,uBAAc,CAAd,aAAc,CAAd,QAAc,CAAd,4DAAc,CAAd,gCAAc,CAAd,mCAAc,CAAd,mBAAc,CAAd,eAAc,CAAd,uBAAc,CAAd,2BAAc,CAAd,qHAAc,CAAd,aAAc,CAAd,mBAAc,CAAd,qBAAc,CAAd,aAAc,CAAd,iBAAc,CAAd,uBAAc,CAAd,iBAAc,CAAd,aAAc,CAAd,8BAAc,CAAd,oBAAc,CAAd,aAAc,CAAd,mDAAc,CAAd,mBAAc,CAAd,cAAc,CAAd,mBAAc,CAAd,QAAc,CAAd,SAAc,CAAd,iCAAc,CAAd,yEAAc,CAAd,4BAAc,CAAd,qBAAc,CAAd,4BAAc,CAAd,gCAAc,CAAd,gCAAc,CAAd,mEAAc,CAAd,0CAAc,CAAd,mBAAc,CAAd,mDAAc,CAAd,sDAAc,CAAd,YAAc,CAAd,yBAAc,CAAd,2DAAc,CAAd,iBAAc,CAAd,yBAAc,CAAd,0BAAc,CAAd,QAAc,CAAd,SAAc,CAAd,wBAAc,CAAd,kFAAc,CAAd,SAAc,CAAd,wEAAc,CAAd,SAAc,CAAd,sDAAc,CAAd,SAAc,CAAd,mCAAc,CAAd,wBAAc,CAAd,4DAAc,CAAd,qBAAc,CAAd,qBAAc,CAAd,cAAc,CAAd,qBAAc,CAAd,mCAAc,CAAd,kBAAc,CAAd,aAAc,CAAd,aAAc,CAAd,aAAc,CAAd,cAAc,CAAd,cAAc,CAAd,YAAc,CAAd,YAAc,CAAd,iBAAc,CAAd,qCAAc,CAAd,cAAc,CAAd,mBAAc,CAAd,qBAAc,CAAd,sBAAc,CAAd,uBAAc,CAAd,iBAAc,CAAd,0BAAc,CAAd,2BAAc,CAAd,mCAAc,CAAd,iCAAc,CAAd,0BAAc,CAAd,qBAAc,CAAd,6BAAc,CAAd,WAAc,CAAd,iBAAc,CAAd,eAAc,CAAd,gBAAc,CAAd,iBAAc,CAAd,aAAc,CAAd,eAAc,CAAd,YAAc,CAAd,kBAAc,CAAd,oBAAc,CAAd,0BAAc,CAAd,wBAAc,CAAd,yBAAc,CAAd,0BAAc,CAAd,sBAAc,CAAd,uBAAc,CAAd,wBAAc,CAAd,qBAAc,CAEd,uBAAmB,CAAnB,yBAAmB,CAAnB,iBAAmB,CAAnB,yBAAmB,CAAnB,oBAAmB,CAAnB,kBAAmB,CAAnB,+BAAmB,CAAnB,8BAAmB,CAAnB,kBAAmB,CAAnB,gCAAmB,CAAnB,sBAAmB,CAAnB,iBAAmB,CAAnB,gCAAmB,CAAnB,gBAAmB,CAAnB,oBAAmB,CAAnB,2BAAmB,CAAnB,gNAAmB,CAAnB,6LAAmB,CAAnB,uCAAmB,CAAnB,+BAAmB,CAAnB,4BAAmB,CAAnB,+BAAmB,CAAnB,gCAAmB,CAAnB,sCAAmB,CAAnB,gBAAmB,CAAnB,iBAAmB,CAAnB,eAAmB,CAAnB,6BAAmB,CAAnB,kCAAmB,CAAnB,gCAAmB,CAAnB,oDAAmB,CAAnB,+BAAmB,CAAnB,oDAAmB,CAAnB,gCAAmB,CAAnB,sDAAmB,CAAnB,kBAAmB,CAAnB,oBAAmB,CAAnB,4CAAmB,CAAnB,0BAAmB,CAAnB,qBAAmB,CAAnB,8CAAmB,CAAnB,0BAAmB,CAAnB,oBAAmB,CAAnB,8BAAmB,CAAnB,4BAAmB,CAAnB,8GAAmB,CAAnB,0BAAmB,CAAnB,mBAAmB,CAAnB,yBAAmB,CAAnB,kBAAmB,CAAnB,yBAAmB,CAAnB,gBAAmB,CAAnB,0BAAmB,CAAnB,mBAAmB,CAAnB,8BAAmB,CAAnB,mCAAmB,CAAnB,+BAAmB,CAAnB,6CAAmB,CAAnB,kHAAmB,CAAnB,wGAAmB,CAAnB,uEAAmB,CAAnB,wFAAmB,CAAnB,iCAAmB,CAAnB,yBAAmB,CAAnB,sMAAmB,CAAnB,gLAAmB,CAAnB,qCAAmB,CAEnB,KAIE,kCAAmC,CACnC,iCAAkC,CAHlC,mIAC4C,CAF5C,QAKF,CAEA,KACE,uEACF,CAdA,kG,CAAA,wG,CAAA,mG,CCAA,iCAEE,YAAa,CADb,iBAEF,CAEA,6DAEE,cAAe,CAGf,UAAW,CAFX,SAGF,CAEA,+BAKE,kBAAmB,CAFnB,YAAa,CACb,sBAAuB,CAHvB,SAKF,CCnBA,MACE,oBAAqB,CACrB,0BAA2B,CAC3B,2BAA4B,CAC5B,6BAA8B,CAC9B,uBAAwB,CACxB,6BAA8B,CAC9B,8BAA+B,CAC/B,6BAA8B,CAE9B,8BAA+B,CAC/B,8BAA+B,CAE/B,yBAA0B,CAC1B,+BAAgC,CAChC,sBAAuB,CACvB,wBACF,CAEA,kBAEE,oBAAqB,CAGrB,YAAa,CAEb,eAAgB,CANhB,iBAAkB,CAGlB,WAIF,CAIA,uEAME,+CAAmD,CAInD,eAAgC,CAAhC,+BAAgC,CAChC,oBAAkC,CAAlC,iCAAkC,CAClC,kBAAmB,CACnB,gBAAiB,CAEjB,iBAAkB,CAXlB,aAAc,CAId,eAAgB,CAMhB,SAAY,CATZ,iBAAkB,CAIlB,iBAOF,CAEA,kCAGE,UAAW,CADX,WAEF,CAEA,qCAGE,UAAW,CADX,SAEF,CAEA,gBAIE,6BACiC,CADjC,6IACiC,CAJjC,KAKF,CAEA,kCANE,QAAS,CACT,iCAA6B,CAA7B,yBAWF,CANA,kBAIE,6BACiC,CADjC,6IACiC,CAJjC,QAKF,CAEA,kBAIE,6BACiC,CADjC,6IACiC,CAHjC,MAIF,CAEA,qCAPE,OAAQ,CAER,kCAA6B,CAA7B,0BAUF,CALA,mBAIE,6BAAsF,CAAtF,qFAAsF,CAFtF,OAGF,CAGA,mGAQE,iBAAkB,CAClB,kBAAmB,CALnB,UAAW,CAGX,QAAS,CAFT,iBAAkB,CAKlB,mBAAqB,CAJrB,OAKF,CAEA,uBAKE,4CAAsE,CAAtE,qEAAsE,CADtE,wBAAwF,CAAxF,uFAAwF,CAFxF,QAAS,CADT,OAA+B,CAA/B,8BAA+B,CAE/B,sCAAgC,CAAhC,8BAGF,CAEA,yBAKE,4CAAsE,CAAtE,qEAAsE,CADtE,wBAA0F,CAA1F,uFAA0F,CAH1F,UAAkC,CAAlC,iCAAkC,CAClC,QAAS,CACT,qCAA+B,CAA/B,6BAGF,CAEA,yBAKE,wDAAsE,CAAtE,qEAAsE,CADtE,6BAAwF,CAAxF,uFAAwF,CAHxF,QAAgC,CAAhC,+BAAgC,CAChC,OAAQ,CACR,sCAAgC,CAAhC,8BAGF,CAEA,0BAKE,wDAAsE,CAAtE,qEAAsE,CADtE,6BAAwF,CAAxF,uFAAwF,CAHxF,SAAiC,CAAjC,gCAAiC,CACjC,OAAQ,CACR,qCAA+B,CAA/B,6BAGF,CAIA,+FAIE,eAAsC,CAAtC,qCAAsC,CACtC,oBAAwC,CAAxC,uCACF,CAEA,6BAEE,2BAA+C,CAA/C,8CAA+C,CAD/C,OAAqC,CAArC,oCAEF,CAEA,+BAEE,wBAA4C,CAA5C,2CAA4C,CAD5C,UAAwC,CAAxC,uCAEF,CAEA,+BAEE,0BAA8C,CAA9C,6CAA8C,CAD9C,QAAsC,CAAtC,qCAEF,CAEA,gCAEE,yBAA6C,CAA7C,4CAA6C,CAD7C,SAAuC,CAAvC,sCAEF,CAIA,sPAQE,eAAuC,CAAvC,sCAAuC,CACvC,oBAAyC,CAAzC,wCACF,CAEA,wEAEE,2BAAgD,CAAhD,+CACF,CAEA,4EAEE,wBAA6C,CAA7C,4CACF,CAEA,4EAEE,0BAA+C,CAA/C,8CACF,CAEA,8EAEE,yBAA8C,CAA9C,6CACF,CAIA,+JAIE,eAAyC,CAAzC,wCAAyC,CACzC,iBAA2C,CAA3C,0CACF,CAEA,6CAEE,wBAAkD,CAAlD,iDAAkD,CADlD,OAA+B,CAA/B,8BAEF,CAEA,+CAEE,qBAA+C,CAA/C,8CAA+C,CAD/C,UAAkC,CAAlC,iCAEF,CAEA,+CAEE,uBAAiD,CAAjD,gDAAiD,CADjD,QAAgC,CAAhC,+BAEF,CAEA,gDAEE,sBAAgD,CAAhD,+CAAgD,CADhD,SAAiC,CAAjC,gCAEF,CCtOA,MACE,6BAA8B,CAC9B,0BAA2B,CAE3B,wBAAyB,CACzB,+BAAgC,CAChC,8BACF,CAEA,0BAIE,kBAAmB,CAOnB,oBAAmC,CAAnC,kCAAmC,CADnC,kBAAmD,CAAnD,gDAAmD,CAGnD,kBAAmB,CADnB,gBAAiB,CAVjB,YAAa,CAMb,YAAoC,CAApC,mCAAoC,CALpC,sBAAuB,CAGvB,eAAgB,CALhB,iBAAkB,CAclB,mBAAqB,CANrB,WAAmC,CAAnC,kCAOF,CAEA,iDAKE,wBAAuC,CAAvC,sCAAuC,CADvC,iBAAkB,CAHlB,WAAkC,CAAlC,iCAAkC,CAMlC,mBAAqB,CALrB,UAAiC,CAAjC,gCAMF,CAEA,iDACE,oBAA0C,CAA1C,yCACF,CAEA,wEACE,wBAA8C,CAA9C,6CACF,CAEA,mDACE,iBAA4C,CAA5C,2CACF,CAEA,0EACE,qBAAgD,CAAhD,+CACF","sources":["index.css","components/RenderedScreen.module.css","components/DPad.module.css","components/Joystick.module.css"],"sourcesContent":["@tailwind base;\n@tailwind components;\n@tailwind utilities;\n\nbody {\n margin: 0;\n font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", \"Roboto\", \"Oxygen\", \"Ubuntu\", \"Cantarell\", \"Fira Sans\",\n \"Droid Sans\", \"Helvetica Neue\", sans-serif;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale;\n}\n\ncode {\n font-family: source-code-pro, Menlo, Monaco, Consolas, \"Courier New\", monospace;\n}\n",".container {\n position: relative;\n display: grid;\n}\n\n.canvas,\n.overlay {\n grid-area: 1/-1;\n z-index: 0;\n\n width: 100%;\n}\n\n.overlay {\n z-index: 1;\n\n display: flex;\n justify-content: center;\n align-items: center;\n}\n",":root {\n --dpad-bg-color: #fff;\n --dpad-bg-color-hover: #eee;\n --dpad-bg-color-active: #fff;\n --dpad-bg-color-disabled: #fff;\n --dpad-fg-color: #5217b8;\n --dpad-fg-color-hover: #5217b8;\n --dpad-fg-color-active: #ffb300;\n --dpad-fg-color-disabled: #bbb;\n\n --dpad-button-outer-radius: 15%;\n --dpad-button-inner-radius: 50%;\n\n --dpad-arrow-position: 40%;\n --dpad-arrow-position-hover: 35%;\n --dpad-arrow-base: 19px;\n --dpad-arrow-height: 13px;\n}\n\n.dpad {\n position: relative;\n display: inline-block;\n\n width: 200px;\n height: 200px;\n\n overflow: hidden;\n}\n\n/* Buttons background */\n\n.up,\n.right,\n.down,\n.left {\n display: block;\n position: absolute;\n -webkit-tap-highlight-color: rgba(255, 255, 255, 0);\n\n line-height: 40%;\n text-align: center;\n background: var(--dpad-bg-color);\n border-color: var(--dpad-fg-color);\n border-style: solid;\n border-width: 1px;\n padding: 0px;\n color: transparent;\n}\n\n.up,\n.down {\n width: 33.3%;\n height: 43%;\n}\n\n.left,\n.right {\n width: 43%;\n height: 33%;\n}\n\n.up {\n top: 0;\n left: 50%;\n transform: translate(-50%, 0);\n border-radius: var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) var(--dpad-button-inner-radius)\n var(--dpad-button-inner-radius);\n}\n\n.down {\n bottom: 0;\n left: 50%;\n transform: translate(-50%, 0);\n border-radius: var(--dpad-button-inner-radius) var(--dpad-button-inner-radius) var(--dpad-button-outer-radius)\n var(--dpad-button-outer-radius);\n}\n\n.left {\n top: 50%;\n left: 0;\n transform: translate(0, -50%);\n border-radius: var(--dpad-button-outer-radius) var(--dpad-button-inner-radius) var(--dpad-button-inner-radius)\n var(--dpad-button-outer-radius);\n}\n\n.right {\n top: 50%;\n right: 0;\n transform: translate(0, -50%);\n border-radius: 50% var(--dpad-button-outer-radius) var(--dpad-button-outer-radius) 50%;\n}\n\n/* Buttons arrows */\n.up:before,\n.right:before,\n.down:before,\n.left:before {\n content: \"\";\n position: absolute;\n width: 0;\n height: 0;\n border-radius: 5px;\n border-style: solid;\n transition: all 0.25s;\n}\n\n.up:before {\n top: var(--dpad-arrow-position);\n left: 50%;\n transform: translate(-50%, -50%);\n border-width: 0 var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height);\n border-color: transparent transparent var(--dpad-fg-color) transparent;\n}\n\n.down:before {\n bottom: var(--dpad-arrow-position);\n left: 50%;\n transform: translate(-50%, 50%);\n border-width: var(--dpad-arrow-base) var(--dpad-arrow-height) 0px var(--dpad-arrow-height);\n border-color: var(--dpad-fg-color) transparent transparent transparent;\n}\n\n.left:before {\n left: var(--dpad-arrow-position);\n top: 50%;\n transform: translate(-50%, -50%);\n border-width: var(--dpad-arrow-height) var(--dpad-arrow-base) var(--dpad-arrow-height) 0;\n border-color: transparent var(--dpad-fg-color) transparent transparent;\n}\n\n.right:before {\n right: var(--dpad-arrow-position);\n top: 50%;\n transform: translate(50%, -50%);\n border-width: var(--dpad-arrow-height) 0 var(--dpad-arrow-height) var(--dpad-arrow-base);\n border-color: transparent transparent transparent var(--dpad-fg-color);\n}\n\n/* Hover */\n\n.up:hover,\n.right:hover,\n.down:hover,\n.left:hover {\n background: var(--dpad-bg-color-hover);\n border-color: var(--dpad-fg-color-hover);\n}\n\n.up:hover:before {\n top: var(--dpad-arrow-position-hover);\n border-bottom-color: var(--dpad-fg-color-hover);\n}\n\n.down:hover:before {\n bottom: var(--dpad-arrow-position-hover);\n border-top-color: var(--dpad-fg-color-hover);\n}\n\n.left:hover:before {\n left: var(--dpad-arrow-position-hover);\n border-right-color: var(--dpad-fg-color-hover);\n}\n\n.right:hover:before {\n right: var(--dpad-arrow-position-hover);\n border-left-color: var(--dpad-fg-color-hover);\n}\n\n/* Active */\n\n.up:active,\n.right:active,\n.down:active,\n.left:active,\n.up.active,\n.right.active,\n.down.active,\n.left.active {\n background: var(--dpad-bg-color-active);\n border-color: var(--dpad-fg-color-active);\n}\n\n.up:active:before,\n.up.active:before {\n border-bottom-color: var(--dpad-fg-color-active);\n}\n\n.down:active:before,\n.down.active:before {\n border-top-color: var(--dpad-fg-color-active);\n}\n\n.left:active:before,\n.left.active:before {\n border-right-color: var(--dpad-fg-color-active);\n}\n\n.right:active:before,\n.right.active:before {\n border-left-color: var(--dpad-fg-color-active);\n}\n\n/* Disabled */\n\n.up.disabled,\n.right.disabled,\n.down.disabled,\n.left.disabled {\n background: var(--dpad-bg-color-disabled);\n border-color: var(--dpad-fg-color-disabled);\n}\n\n.up.disabled:before {\n top: var(--dpad-arrow-position);\n border-bottom-color: var(--dpad-fg-color-disabled);\n}\n\n.down.disabled:before {\n bottom: var(--dpad-arrow-position);\n border-top-color: var(--dpad-fg-color-disabled);\n}\n\n.left.disabled:before {\n left: var(--dpad-arrow-position);\n border-right-color: var(--dpad-fg-color-disabled);\n}\n\n.right.disabled:before {\n right: var(--dpad-arrow-position);\n border-left-color: var(--dpad-fg-color-disabled);\n}\n",":root {\n --joystick-surface-size: 200px;\n --joystick-stick-size: 50px;\n\n --joystick-color: #5217b8;\n --joystick-color-active: #ffb300;\n --joystick-color-disabled: #bbb;\n}\n\n.joystick {\n position: relative;\n display: flex;\n justify-content: center;\n align-items: center;\n\n overflow: hidden;\n\n height: var(--joystick-surface-size);\n width: var(--joystick-surface-size);\n border-radius: calc(var(--joystick-stick-size) / 2);\n border-color: var(--joystick-color);\n border-width: 1px;\n border-style: solid;\n\n transition: all 0.25s;\n}\n\n.joystick > .stick {\n height: var(--joystick-stick-size);\n width: var(--joystick-stick-size);\n\n border-radius: 50%;\n background-color: var(--joystick-color);\n\n transition: all 0.25s;\n}\n\n.joystick.active {\n border-color: var(--joystick-color-active);\n}\n\n.joystick.active > .stick {\n background-color: var(--joystick-color-active);\n}\n\n.joystick.disabled {\n border-color: var(--joystick-color-disabled);\n}\n\n.joystick.disabled > .stick {\n background-color: var(--joystick-color-disabled);\n}\n"],"names":[],"sourceRoot":""} \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/js/main.80bd9354.js b/cogment_verse/web/components/build/static/js/main.80bd9354.js deleted file mode 100644 index e09d2ae5..00000000 --- a/cogment_verse/web/components/build/static/js/main.80bd9354.js +++ /dev/null @@ -1,3 +0,0 @@ -/*! For license information please see main.80bd9354.js.LICENSE.txt */ -(function(){var __webpack_modules__={33:function(e,t,r){var n;t.CL=void 0;var o=r(358),i=function(){this.name="player",this.config=o.cogment_verse.AgentConfig,this.actionSpace=o.cogment_verse.PlayerAction,this.observationSpace=o.cogment_verse.Observation};var a=function(){this.name="teacher",this.config=o.cogment_verse.AgentConfig,this.actionSpace=o.cogment_verse.TeacherAction,this.observationSpace=o.cogment_verse.Observation};var s=function(){this.name="observer",this.config=o.cogment_verse.AgentConfig,this.actionSpace=o.cogment_verse.ObserverAction,this.observationSpace=o.cogment_verse.Observation};t.CL={messageUrlMap:(n={},n["type.googleapis.com/cogment_verse.NDArray"]=o.cogment_verse.NDArray,n["type.googleapis.com/cogment_verse.Space"]=o.cogment_verse.Space,n["type.googleapis.com/cogment_verse.Space.Discrete"]=o.cogment_verse.Space.Discrete,n["type.googleapis.com/cogment_verse.Space.Bound"]=o.cogment_verse.Space.Bound,n["type.googleapis.com/cogment_verse.Space.Box"]=o.cogment_verse.Space.Box,n["type.googleapis.com/cogment_verse.Space.Property"]=o.cogment_verse.Space.Property,n["type.googleapis.com/cogment_verse.SpaceValue"]=o.cogment_verse.SpaceValue,n["type.googleapis.com/cogment_verse.SpaceValue.SimpleBox"]=o.cogment_verse.SpaceValue.SimpleBox,n["type.googleapis.com/cogment_verse.SpaceValue.PropertyValue"]=o.cogment_verse.SpaceValue.PropertyValue,n["type.googleapis.com/cogment_verse.EnvironmentSpecs"]=o.cogment_verse.EnvironmentSpecs,n["type.googleapis.com/cogment_verse.EnvironmentConfig"]=o.cogment_verse.EnvironmentConfig,n["type.googleapis.com/cogment_verse.HFHubModel"]=o.cogment_verse.HFHubModel,n["type.googleapis.com/cogment_verse.AgentConfig"]=o.cogment_verse.AgentConfig,n["type.googleapis.com/cogment_verse.TrialConfig"]=o.cogment_verse.TrialConfig,n["type.googleapis.com/cogment_verse.Observation"]=o.cogment_verse.Observation,n["type.googleapis.com/cogment_verse.PlayerAction"]=o.cogment_verse.PlayerAction,n["type.googleapis.com/cogment_verse.TeacherAction"]=o.cogment_verse.TeacherAction,n["type.googleapis.com/cogment_verse.ObserverAction"]=o.cogment_verse.ObserverAction,n),actorClasses:{player:new i,teacher:new a,observer:new s},trial:{config:o.cogment_verse.TrialConfig},environment:{config:o.cogment_verse.EnvironmentConfig,class:{id:"env",config:o.cogment_verse.EnvironmentConfig}}}},358:function(e,t,r){var n,o,i;o=[r(886)],void 0===(i="function"===typeof(n=function(e){"use strict";var t=e.Reader,r=e.Writer,n=e.util,o=e.roots.default||(e.roots.default={});return o.cogment_verse=function(){var i={};return i.NDArray=function(){function i(e){if(this.shape=[],e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.dtype=e.string();break;case 2:if(i.shape&&i.shape.length||(i.shape=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>0}return null!=e.data&&("string"===typeof e.data?n.base64.decode(e.data,t.data=n.newBuffer(n.base64.length(e.data)),0):e.data.length>=0&&(t.data=e.data)),t},i.toObject=function(e,t){t||(t={});var r={};if((t.arrays||t.defaults)&&(r.shape=[]),t.defaults&&(r.dtype="",t.bytes===String?r.data="":(r.data=[],t.bytes!==Array&&(r.data=n.newBuffer(r.data)))),null!=e.dtype&&e.hasOwnProperty("dtype")&&(r.dtype=e.dtype),e.shape&&e.shape.length){r.shape=[];for(var o=0;o>>3===1?(i.properties&&i.properties.length||(i.properties=[]),i.properties.push(o.cogment_verse.Space.Property.decode(e,e.uint32()))):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.properties&&e.hasOwnProperty("properties")){if(!Array.isArray(e.properties))return"properties: array expected";for(var t=0;t>>3){case 1:i.num=e.int32();break;case 2:i.labels&&i.labels.length||(i.labels=[]),i.labels.push(e.string());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.num&&e.hasOwnProperty("num")&&!n.isInteger(e.num))return"num: integer expected";if(null!=e.labels&&e.hasOwnProperty("labels")){if(!Array.isArray(e.labels))return"labels: array expected";for(var t=0;t>>3===1?i.bound=e.float():e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};return null!=e.bound&&e.hasOwnProperty("bound")&&(t._bound=1,"number"!==typeof e.bound)?"bound: number expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.Space.Bound)return e;var t=new o.cogment_verse.Space.Bound;return null!=e.bound&&(t.bound=Number(e.bound)),t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.bound&&e.hasOwnProperty("bound")&&(r.bound=t.json&&!isFinite(e.bound)?String(e.bound):e.bound,t.oneofs&&(r._bound="bound")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.Space.Bound"},i}(),i.Box=function(){function i(e){if(this.shape=[],this.low=[],this.high=[],e)for(var t=Object.keys(e),r=0;r>>3){case 1:if(i.shape&&i.shape.length||(i.shape=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>0}if(e.low){if(!Array.isArray(e.low))throw TypeError(".cogment_verse.Space.Box.low: array expected");for(t.low=[],r=0;r>>3){case 1:i.key=e.string();break;case 2:i.discrete=o.cogment_verse.Space.Discrete.decode(e,e.uint32());break;case 3:i.box=o.cogment_verse.Space.Box.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.key&&e.hasOwnProperty("key")&&!n.isString(e.key))return"key: string expected";if(null!=e.discrete&&e.hasOwnProperty("discrete")&&(t.typeOneof=1,r=o.cogment_verse.Space.Discrete.verify(e.discrete)))return"discrete."+r;if(null!=e.box&&e.hasOwnProperty("box")){if(1===t.typeOneof)return"typeOneof: multiple values";var r;if(t.typeOneof=1,r=o.cogment_verse.Space.Box.verify(e.box))return"box."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.Space.Property)return e;var t=new o.cogment_verse.Space.Property;if(null!=e.key&&(t.key=String(e.key)),null!=e.discrete){if("object"!==typeof e.discrete)throw TypeError(".cogment_verse.Space.Property.discrete: object expected");t.discrete=o.cogment_verse.Space.Discrete.fromObject(e.discrete)}if(null!=e.box){if("object"!==typeof e.box)throw TypeError(".cogment_verse.Space.Property.box: object expected");t.box=o.cogment_verse.Space.Box.fromObject(e.box)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.key=""),null!=e.key&&e.hasOwnProperty("key")&&(r.key=e.key),null!=e.discrete&&e.hasOwnProperty("discrete")&&(r.discrete=o.cogment_verse.Space.Discrete.toObject(e.discrete,t),t.oneofs&&(r.typeOneof="discrete")),null!=e.box&&e.hasOwnProperty("box")&&(r.box=o.cogment_verse.Space.Box.toObject(e.box,t),t.oneofs&&(r.typeOneof="box")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.Space.Property"},i}(),i}(),i.SpaceValue=function(){function i(e){if(this.properties=[],e)for(var t=Object.keys(e),r=0;r>>3===1?(i.properties&&i.properties.length||(i.properties=[]),i.properties.push(o.cogment_verse.SpaceValue.PropertyValue.decode(e,e.uint32()))):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.properties&&e.hasOwnProperty("properties")){if(!Array.isArray(e.properties))return"properties: array expected";for(var t=0;t>>3===1)if(i.values&&i.values.length||(i.values=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>3){case 1:i.discrete=e.int32();break;case 2:i.box=o.cogment_verse.NDArray.decode(e,e.uint32());break;case 3:i.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.discrete&&e.hasOwnProperty("discrete")&&(t.value=1,!n.isInteger(e.discrete)))return"discrete: integer expected";if(null!=e.box&&e.hasOwnProperty("box")){if(1===t.value)return"value: multiple values";if(t.value=1,r=o.cogment_verse.NDArray.verify(e.box))return"box."+r}if(null!=e.simpleBox&&e.hasOwnProperty("simpleBox")){if(1===t.value)return"value: multiple values";var r;if(t.value=1,r=o.cogment_verse.SpaceValue.SimpleBox.verify(e.simpleBox))return"simpleBox."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.SpaceValue.PropertyValue)return e;var t=new o.cogment_verse.SpaceValue.PropertyValue;if(null!=e.discrete&&(t.discrete=0|e.discrete),null!=e.box){if("object"!==typeof e.box)throw TypeError(".cogment_verse.SpaceValue.PropertyValue.box: object expected");t.box=o.cogment_verse.NDArray.fromObject(e.box)}if(null!=e.simpleBox){if("object"!==typeof e.simpleBox)throw TypeError(".cogment_verse.SpaceValue.PropertyValue.simpleBox: object expected");t.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.fromObject(e.simpleBox)}return t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.discrete&&e.hasOwnProperty("discrete")&&(r.discrete=e.discrete,t.oneofs&&(r.value="discrete")),null!=e.box&&e.hasOwnProperty("box")&&(r.box=o.cogment_verse.NDArray.toObject(e.box,t),t.oneofs&&(r.value="box")),null!=e.simpleBox&&e.hasOwnProperty("simpleBox")&&(r.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.toObject(e.simpleBox,t),t.oneofs&&(r.value="simpleBox")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.SpaceValue.PropertyValue"},i}(),i}(),i.EnvironmentSpecs=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.implementation=e.string();break;case 2:i.numPlayers=e.int32();break;case 3:i.observationSpace=o.cogment_verse.Space.decode(e,e.uint32());break;case 4:i.actionSpace=o.cogment_verse.Space.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.implementation&&e.hasOwnProperty("implementation")&&!n.isString(e.implementation)?"implementation: string expected":null!=e.numPlayers&&e.hasOwnProperty("numPlayers")&&!n.isInteger(e.numPlayers)?"numPlayers: integer expected":null!=e.observationSpace&&e.hasOwnProperty("observationSpace")&&(t=o.cogment_verse.Space.verify(e.observationSpace))?"observationSpace."+t:null!=e.actionSpace&&e.hasOwnProperty("actionSpace")&&(t=o.cogment_verse.Space.verify(e.actionSpace))?"actionSpace."+t:null;var t},i.fromObject=function(e){if(e instanceof o.cogment_verse.EnvironmentSpecs)return e;var t=new o.cogment_verse.EnvironmentSpecs;if(null!=e.implementation&&(t.implementation=String(e.implementation)),null!=e.numPlayers&&(t.numPlayers=0|e.numPlayers),null!=e.observationSpace){if("object"!==typeof e.observationSpace)throw TypeError(".cogment_verse.EnvironmentSpecs.observationSpace: object expected");t.observationSpace=o.cogment_verse.Space.fromObject(e.observationSpace)}if(null!=e.actionSpace){if("object"!==typeof e.actionSpace)throw TypeError(".cogment_verse.EnvironmentSpecs.actionSpace: object expected");t.actionSpace=o.cogment_verse.Space.fromObject(e.actionSpace)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.implementation="",r.numPlayers=0,r.observationSpace=null,r.actionSpace=null),null!=e.implementation&&e.hasOwnProperty("implementation")&&(r.implementation=e.implementation),null!=e.numPlayers&&e.hasOwnProperty("numPlayers")&&(r.numPlayers=e.numPlayers),null!=e.observationSpace&&e.hasOwnProperty("observationSpace")&&(r.observationSpace=o.cogment_verse.Space.toObject(e.observationSpace,t)),null!=e.actionSpace&&e.hasOwnProperty("actionSpace")&&(r.actionSpace=o.cogment_verse.Space.toObject(e.actionSpace,t)),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.EnvironmentSpecs"},i}(),i.EnvironmentConfig=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.runId=e.string();break;case 2:i.render=e.bool();break;case 3:i.renderWidth=e.int32();break;case 4:i.seed=e.uint32();break;case 5:i.flatten=e.bool();break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.runId&&e.hasOwnProperty("runId")&&!n.isString(e.runId)?"runId: string expected":null!=e.render&&e.hasOwnProperty("render")&&"boolean"!==typeof e.render?"render: boolean expected":null!=e.renderWidth&&e.hasOwnProperty("renderWidth")&&!n.isInteger(e.renderWidth)?"renderWidth: integer expected":null!=e.seed&&e.hasOwnProperty("seed")&&!n.isInteger(e.seed)?"seed: integer expected":null!=e.flatten&&e.hasOwnProperty("flatten")&&"boolean"!==typeof e.flatten?"flatten: boolean expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.EnvironmentConfig)return e;var t=new o.cogment_verse.EnvironmentConfig;return null!=e.runId&&(t.runId=String(e.runId)),null!=e.render&&(t.render=Boolean(e.render)),null!=e.renderWidth&&(t.renderWidth=0|e.renderWidth),null!=e.seed&&(t.seed=e.seed>>>0),null!=e.flatten&&(t.flatten=Boolean(e.flatten)),t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.runId="",r.render=!1,r.renderWidth=0,r.seed=0,r.flatten=!1),null!=e.runId&&e.hasOwnProperty("runId")&&(r.runId=e.runId),null!=e.render&&e.hasOwnProperty("render")&&(r.render=e.render),null!=e.renderWidth&&e.hasOwnProperty("renderWidth")&&(r.renderWidth=e.renderWidth),null!=e.seed&&e.hasOwnProperty("seed")&&(r.seed=e.seed),null!=e.flatten&&e.hasOwnProperty("flatten")&&(r.flatten=e.flatten),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.EnvironmentConfig"},i}(),i.HFHubModel=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.repoId=e.string();break;case 2:i.filename=e.string();break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.repoId&&e.hasOwnProperty("repoId")&&!n.isString(e.repoId)?"repoId: string expected":null!=e.filename&&e.hasOwnProperty("filename")&&!n.isString(e.filename)?"filename: string expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.HFHubModel)return e;var t=new o.cogment_verse.HFHubModel;return null!=e.repoId&&(t.repoId=String(e.repoId)),null!=e.filename&&(t.filename=String(e.filename)),t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.repoId="",r.filename=""),null!=e.repoId&&e.hasOwnProperty("repoId")&&(r.repoId=e.repoId),null!=e.filename&&e.hasOwnProperty("filename")&&(r.filename=e.filename),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.HFHubModel"},i}(),i.AgentConfig=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.runId=e.string();break;case 2:i.environmentSpecs=o.cogment_verse.EnvironmentSpecs.decode(e,e.uint32());break;case 3:i.modelId=e.string();break;case 4:i.modelVersion=e.int32();break;case 5:i.actorIndex=e.int32();break;case 6:i.device=e.string();break;case 7:i.threadsPerWorker=e.uint32();break;case 8:i.hfHubModel=o.cogment_verse.HFHubModel.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.runId&&e.hasOwnProperty("runId")&&!n.isString(e.runId)?"runId: string expected":null!=e.environmentSpecs&&e.hasOwnProperty("environmentSpecs")&&(t=o.cogment_verse.EnvironmentSpecs.verify(e.environmentSpecs))?"environmentSpecs."+t:null!=e.modelId&&e.hasOwnProperty("modelId")&&!n.isString(e.modelId)?"modelId: string expected":null!=e.modelVersion&&e.hasOwnProperty("modelVersion")&&!n.isInteger(e.modelVersion)?"modelVersion: integer expected":null!=e.actorIndex&&e.hasOwnProperty("actorIndex")&&!n.isInteger(e.actorIndex)?"actorIndex: integer expected":null!=e.device&&e.hasOwnProperty("device")&&!n.isString(e.device)?"device: string expected":null!=e.threadsPerWorker&&e.hasOwnProperty("threadsPerWorker")&&!n.isInteger(e.threadsPerWorker)?"threadsPerWorker: integer expected":null!=e.hfHubModel&&e.hasOwnProperty("hfHubModel")&&(t=o.cogment_verse.HFHubModel.verify(e.hfHubModel))?"hfHubModel."+t:null;var t},i.fromObject=function(e){if(e instanceof o.cogment_verse.AgentConfig)return e;var t=new o.cogment_verse.AgentConfig;if(null!=e.runId&&(t.runId=String(e.runId)),null!=e.environmentSpecs){if("object"!==typeof e.environmentSpecs)throw TypeError(".cogment_verse.AgentConfig.environmentSpecs: object expected");t.environmentSpecs=o.cogment_verse.EnvironmentSpecs.fromObject(e.environmentSpecs)}if(null!=e.modelId&&(t.modelId=String(e.modelId)),null!=e.modelVersion&&(t.modelVersion=0|e.modelVersion),null!=e.actorIndex&&(t.actorIndex=0|e.actorIndex),null!=e.device&&(t.device=String(e.device)),null!=e.threadsPerWorker&&(t.threadsPerWorker=e.threadsPerWorker>>>0),null!=e.hfHubModel){if("object"!==typeof e.hfHubModel)throw TypeError(".cogment_verse.AgentConfig.hfHubModel: object expected");t.hfHubModel=o.cogment_verse.HFHubModel.fromObject(e.hfHubModel)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.runId="",r.environmentSpecs=null,r.modelId="",r.modelVersion=0,r.actorIndex=0,r.device="",r.threadsPerWorker=0,r.hfHubModel=null),null!=e.runId&&e.hasOwnProperty("runId")&&(r.runId=e.runId),null!=e.environmentSpecs&&e.hasOwnProperty("environmentSpecs")&&(r.environmentSpecs=o.cogment_verse.EnvironmentSpecs.toObject(e.environmentSpecs,t)),null!=e.modelId&&e.hasOwnProperty("modelId")&&(r.modelId=e.modelId),null!=e.modelVersion&&e.hasOwnProperty("modelVersion")&&(r.modelVersion=e.modelVersion),null!=e.actorIndex&&e.hasOwnProperty("actorIndex")&&(r.actorIndex=e.actorIndex),null!=e.device&&e.hasOwnProperty("device")&&(r.device=e.device),null!=e.threadsPerWorker&&e.hasOwnProperty("threadsPerWorker")&&(r.threadsPerWorker=e.threadsPerWorker),null!=e.hfHubModel&&e.hasOwnProperty("hfHubModel")&&(r.hfHubModel=o.cogment_verse.HFHubModel.toObject(e.hfHubModel,t)),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.AgentConfig"},i}(),i.TrialConfig=function(){function n(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32());break;case 2:i.renderedFrame=e.bytes();break;case 3:if(i.legalMovesAsInt&&i.legalMovesAsInt.length||(i.legalMovesAsInt=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos=0&&(t.renderedFrame=e.renderedFrame)),e.legalMovesAsInt){if(!Array.isArray(e.legalMovesAsInt))throw TypeError(".cogment_verse.Observation.legalMovesAsInt: array expected");t.legalMovesAsInt=[];for(var r=0;r>>3===1?i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32()):e.skipType(7&a)}return i},n.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},n.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.value&&e.hasOwnProperty("value")){var t=o.cogment_verse.SpaceValue.verify(e.value);if(t)return"value."+t}return null},n.fromObject=function(e){if(e instanceof o.cogment_verse.PlayerAction)return e;var t=new o.cogment_verse.PlayerAction;if(null!=e.value){if("object"!==typeof e.value)throw TypeError(".cogment_verse.PlayerAction.value: object expected");t.value=o.cogment_verse.SpaceValue.fromObject(e.value)}return t},n.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.value=null),null!=e.value&&e.hasOwnProperty("value")&&(r.value=o.cogment_verse.SpaceValue.toObject(e.value,t)),r},n.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},n.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.PlayerAction"},n}(),i.TeacherAction=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3===1?i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32()):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.value&&e.hasOwnProperty("value")){t._value=1;var r=o.cogment_verse.SpaceValue.verify(e.value);if(r)return"value."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.TeacherAction)return e;var t=new o.cogment_verse.TeacherAction;if(null!=e.value){if("object"!==typeof e.value)throw TypeError(".cogment_verse.TeacherAction.value: object expected");t.value=o.cogment_verse.SpaceValue.fromObject(e.value)}return t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.value&&e.hasOwnProperty("value")&&(r.value=o.cogment_verse.SpaceValue.toObject(e.value,t),t.oneofs&&(r._value="value")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.TeacherAction"},i}(),i.ObserverAction=function(){function n(e){if(e)for(var t=Object.keys(e),r=0;r0){var i=n.substring(0,o).trim(),a=n.substring(o+1).trim();this.append(i,a)}}},e.prototype.delete=function(e,t){var r=n.normalizeName(e);if(void 0===t)delete this.headersMap[r];else{var o=this.headersMap[r];if(o){var i=o.indexOf(t);i>=0&&o.splice(i,1),0===o.length&&delete this.headersMap[r]}}},e.prototype.append=function(e,t){var r=this,o=n.normalizeName(e);Array.isArray(this.headersMap[o])||(this.headersMap[o]=[]),Array.isArray(t)?t.forEach((function(e){r.headersMap[o].push(n.normalizeValue(e))})):this.headersMap[o].push(n.normalizeValue(t))},e.prototype.set=function(e,t){var r=n.normalizeName(e);if(Array.isArray(t)){var o=[];t.forEach((function(e){o.push(n.normalizeValue(e))})),this.headersMap[r]=o}else this.headersMap[r]=[n.normalizeValue(t)]},e.prototype.has=function(e,t){var r=this.headersMap[n.normalizeName(e)];if(!Array.isArray(r))return!1;if(void 0!==t){var o=n.normalizeValue(t);return r.indexOf(o)>=0}return!0},e.prototype.get=function(e){var t=this.headersMap[n.normalizeName(e)];return void 0!==t?t.concat():[]},e.prototype.forEach=function(e){var t=this;Object.getOwnPropertyNames(this.headersMap).forEach((function(r){e(r,t.headersMap[r])}),this)},e.prototype.toHeaders=function(){if("undefined"!=typeof Headers){var e=new Headers;return this.forEach((function(t,r){r.forEach((function(r){e.append(t,r)}))})),e}throw new Error("Headers class is not defined")},e}();t.BrowserHeaders=o},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(0);t.BrowserHeaders=n.BrowserHeaders},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.iterateHeaders=function(e,t){for(var r=e[Symbol.iterator](),n=r.next();!n.done;)t(n.value[0]),n=r.next()},t.iterateHeadersKeys=function(e,t){for(var r=e.keys(),n=r.next();!n.done;)t(n.value),n=r.next()}},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(2);t.normalizeName=function(e){if("string"!=typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.\^_`|~]/i.test(e))throw new TypeError("Invalid character in header field name");return e.toLowerCase()},t.normalizeValue=function(e){return"string"!=typeof e&&(e=String(e)),e},t.getHeaderValues=function(e,t){var r=e;if(r instanceof Headers&&r.getAll)return r.getAll(t);var n=r.get(t);return n&&"string"==typeof n?[n]:n},t.getHeaderKeys=function(e){var t=e,r={},o=[];return t.keys?n.iterateHeadersKeys(t,(function(e){r[e]||(r[e]=!0,o.push(e))})):t.forEach?t.forEach((function(e,t){r[t]||(r[t]=!0,o.push(t))})):n.iterateHeaders(t,(function(e){var t=e[0];r[t]||(r[t]=!0,o.push(t))})),o},t.splitHeaderValue=function(e){var t=[];return e.split(", ").forEach((function(e){e.split(",").forEach((function(e){t.push(e)}))})),t}}]))},617:function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.ChunkParser=t.ChunkType=t.encodeASCII=t.decodeASCII=void 0;var n,o=r(65);function i(e){return 9===(t=e)||10===t||13===t||e>=32&&e<=126;var t}function a(e){for(var t=0;t!==e.length;++t)if(!i(e[t]))throw new Error("Metadata is not valid (printable) ASCII");return String.fromCharCode.apply(String,Array.prototype.slice.call(e))}function s(e){return 128==(128&e.getUint8(0))}function l(e){return e.getUint32(1,!1)}function u(e,t,r){return e.byteLength-t>=r}function c(e,t,r){if(e.slice)return e.slice(t,r);var n=e.length;void 0!==r&&(n=r);for(var o=new Uint8Array(n-t),i=0,a=t;a=0?r:i.httpStatusToCode(t);this.props.debug&&a.debug("onHeaders.code",n);var o=e.get("grpc-message")||[];if(this.props.debug&&a.debug("onHeaders.gRPCMessage",o),this.rawOnHeaders(e),n!==i.Code.OK){var s=this.decodeGRPCStatus(o[0]);this.rawOnError(n,s,e)}}},e.prototype.onTransportChunk=function(e){var t=this;if(this.closed)this.props.debug&&a.debug("grpc.onChunk received after request was closed - ignoring");else{var r=[];try{r=this.parser.parse(e)}catch(e){return this.props.debug&&a.debug("onChunk.parsing error",e,e.message),void this.rawOnError(i.Code.Internal,"parsing error: "+e.message)}r.forEach((function(e){if(e.chunkType===o.ChunkType.MESSAGE){var r=t.methodDefinition.responseType.deserializeBinary(e.data);t.rawOnMessage(r)}else e.chunkType===o.ChunkType.TRAILERS&&(t.responseHeaders?(t.responseTrailers=new n.Metadata(e.trailers),t.props.debug&&a.debug("onChunk.trailers",t.responseTrailers)):(t.responseHeaders=new n.Metadata(e.trailers),t.rawOnHeaders(t.responseHeaders)))}))}},e.prototype.onTransportEnd=function(){if(this.props.debug&&a.debug("grpc.onEnd"),this.closed)this.props.debug&&a.debug("grpc.onEnd received after request was closed - ignoring");else if(void 0!==this.responseTrailers){var e=c(this.responseTrailers);if(null!==e){var t=this.responseTrailers.get("grpc-message"),r=this.decodeGRPCStatus(t[0]);this.rawOnEnd(e,r,this.responseTrailers)}else this.rawOnError(i.Code.Internal,"Response closed without grpc-status (Trailers provided)")}else{if(void 0===this.responseHeaders)return void this.rawOnError(i.Code.Unknown,"Response closed without headers");var n=c(this.responseHeaders),o=this.responseHeaders.get("grpc-message");if(this.props.debug&&a.debug("grpc.headers only response ",n,o),null===n)return void this.rawOnEnd(i.Code.Unknown,"Response closed without grpc-status (Headers only)",this.responseHeaders);var s=this.decodeGRPCStatus(o[0]);this.rawOnEnd(n,s,this.responseHeaders)}},e.prototype.decodeGRPCStatus=function(e){if(!e)return"";try{return decodeURIComponent(e)}catch(t){return e}},e.prototype.rawOnEnd=function(e,t,r){var n=this;this.props.debug&&a.debug("rawOnEnd",e,t,r),this.completed||(this.completed=!0,this.onEndCallbacks.forEach((function(o){if(!n.closed)try{o(e,t,r)}catch(e){setTimeout((function(){throw e}),0)}})))},e.prototype.rawOnHeaders=function(e){this.props.debug&&a.debug("rawOnHeaders",e),this.completed||this.onHeadersCallbacks.forEach((function(t){try{t(e)}catch(e){setTimeout((function(){throw e}),0)}}))},e.prototype.rawOnError=function(e,t,r){var o=this;void 0===r&&(r=new n.Metadata),this.props.debug&&a.debug("rawOnError",e,t),this.completed||(this.completed=!0,this.onEndCallbacks.forEach((function(n){if(!o.closed)try{n(e,t,r)}catch(e){setTimeout((function(){throw e}),0)}})))},e.prototype.rawOnMessage=function(e){var t=this;this.props.debug&&a.debug("rawOnMessage",e.toObject()),this.completed||this.closed||this.onMessageCallbacks.forEach((function(r){if(!t.closed)try{r(e)}catch(e){setTimeout((function(){throw e}),0)}}))},e.prototype.onHeaders=function(e){this.onHeadersCallbacks.push(e)},e.prototype.onMessage=function(e){this.onMessageCallbacks.push(e)},e.prototype.onEnd=function(e){this.onEndCallbacks.push(e)},e.prototype.start=function(e){if(this.started)throw new Error("Client already started - cannot .start()");this.started=!0;var t=new n.Metadata(e||{});t.set("content-type","application/grpc-web+proto"),t.set("x-grpc-web","1"),this.transport.start(t)},e.prototype.send=function(e){if(!this.started)throw new Error("Client not started - .start() must be called before .send()");if(this.closed)throw new Error("Client already closed - cannot .send()");if(this.finishedSending)throw new Error("Client already finished sending - cannot .send()");if(!this.methodDefinition.requestStream&&this.sentFirstMessage)throw new Error("Message already sent for non-client-streaming method - cannot .send()");this.sentFirstMessage=!0;var t=l.frameRequest(e);this.transport.sendMessage(t)},e.prototype.finishSend=function(){if(!this.started)throw new Error("Client not started - .finishSend() must be called before .close()");if(this.closed)throw new Error("Client already closed - cannot .send()");if(this.finishedSending)throw new Error("Client already finished sending - cannot .finishSend()");this.finishedSending=!0,this.transport.finishSend()},e.prototype.close=function(){if(!this.started)throw new Error("Client not started - .start() must be called before .close()");if(this.closed)throw new Error("Client already closed - cannot .close()");this.closed=!0,this.props.debug&&a.debug("request.abort aborting request"),this.transport.cancel()},e}();function c(e){var t=e.get("grpc-status")||[];if(t.length>0)try{var r=t[0];return parseInt(r,10)}catch(e){return null}return null}},346:function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.debug=void 0,t.debug=function(){for(var e=[],t=0;t=55296&&r<=56319){var n=e.charCodeAt(t+1);n>=56320&&n<=57343&&(r=65536+(r-55296<<10)+(n-56320))}return r}function g(e){for(var t=new Uint8Array(e.length),r=0,n=0;n1&&"="===e.charAt(t);)++r;return Math.ceil(3*e.length)/4-r};for(var n=new Array(64),o=new Array(123),i=0;i<64;)o[n[i]=i<26?i+65:i<52?i+71:i<62?i-4:i-59|43]=i++;r.encode=function(e,t,r){for(var o,i=null,a=[],s=0,l=0;t>2],o=(3&u)<<4,l=1;break;case 1:a[s++]=n[o|u>>4],o=(15&u)<<2,l=2;break;case 2:a[s++]=n[o|u>>6],a[s++]=n[63&u],l=0}s>8191&&((i||(i=[])).push(String.fromCharCode.apply(String,a)),s=0)}return l&&(a[s++]=n[o],a[s++]=61,1===l&&(a[s++]=61)),i?(s&&i.push(String.fromCharCode.apply(String,a.slice(0,s))),i.join("")):String.fromCharCode.apply(String,a.slice(0,s))};var a="invalid encoding";r.decode=function(e,t,r){for(var n,i=r,s=0,l=0;l1)break;if(void 0===(u=o[u]))throw Error(a);switch(s){case 0:n=u,s=1;break;case 1:t[r++]=n<<2|(48&u)>>4,n=u,s=2;break;case 2:t[r++]=(15&n)<<4|(60&u)>>2,n=u,s=3;break;case 3:t[r++]=(3&n)<<6|u,s=0}}if(1===s)throw Error(a);return r-i},r.test=function(e){return/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(e)}},"./node_modules/@protobufjs/eventemitter/index.js":function(e){"use strict";function t(){this._listeners={}}e.exports=t,t.prototype.on=function(e,t,r){return(this._listeners[e]||(this._listeners[e]=[])).push({fn:t,ctx:r||this}),this},t.prototype.off=function(e,t){if(void 0===e)this._listeners={};else if(void 0===t)this._listeners[e]=[];else for(var r=this._listeners[e],n=0;n0?0:2147483648,r,n);else if(isNaN(t))e(2143289344,r,n);else if(t>34028234663852886e22)e((o<<31|2139095040)>>>0,r,n);else if(t<11754943508222875e-54)e((o<<31|Math.round(t/1401298464324817e-60))>>>0,r,n);else{var i=Math.floor(Math.log(t)/Math.LN2);e((o<<31|i+127<<23|8388607&Math.round(t*Math.pow(2,-i)*8388608))>>>0,r,n)}}function a(e,t,r){var n=e(t,r),o=2*(n>>31)+1,i=n>>>23&255,a=8388607&n;return 255===i?a?NaN:o*(1/0):0===i?1401298464324817e-60*o*a:o*Math.pow(2,i-150)*(a+8388608)}e.writeFloatLE=t.bind(null,r),e.writeFloatBE=t.bind(null,n),e.readFloatLE=a.bind(null,o),e.readFloatBE=a.bind(null,i)}(),"undefined"!==typeof Float64Array?function(){var t=new Float64Array([-0]),r=new Uint8Array(t.buffer),n=128===r[7];function o(e,n,o){t[0]=e,n[o]=r[0],n[o+1]=r[1],n[o+2]=r[2],n[o+3]=r[3],n[o+4]=r[4],n[o+5]=r[5],n[o+6]=r[6],n[o+7]=r[7]}function i(e,n,o){t[0]=e,n[o]=r[7],n[o+1]=r[6],n[o+2]=r[5],n[o+3]=r[4],n[o+4]=r[3],n[o+5]=r[2],n[o+6]=r[1],n[o+7]=r[0]}function a(e,n){return r[0]=e[n],r[1]=e[n+1],r[2]=e[n+2],r[3]=e[n+3],r[4]=e[n+4],r[5]=e[n+5],r[6]=e[n+6],r[7]=e[n+7],t[0]}function s(e,n){return r[7]=e[n],r[6]=e[n+1],r[5]=e[n+2],r[4]=e[n+3],r[3]=e[n+4],r[2]=e[n+5],r[1]=e[n+6],r[0]=e[n+7],t[0]}e.writeDoubleLE=n?o:i,e.writeDoubleBE=n?i:o,e.readDoubleLE=n?a:s,e.readDoubleBE=n?s:a}():function(){function t(e,t,r,n,o,i){var a=n<0?1:0;if(a&&(n=-n),0===n)e(0,o,i+t),e(1/n>0?0:2147483648,o,i+r);else if(isNaN(n))e(0,o,i+t),e(2146959360,o,i+r);else if(n>17976931348623157e292)e(0,o,i+t),e((a<<31|2146435072)>>>0,o,i+r);else{var s;if(n<22250738585072014e-324)e((s=n/5e-324)>>>0,o,i+t),e((a<<31|s/4294967296)>>>0,o,i+r);else{var l=Math.floor(Math.log(n)/Math.LN2);1024===l&&(l=1023),e(4503599627370496*(s=n*Math.pow(2,-l))>>>0,o,i+t),e((a<<31|l+1023<<20|1048576*s&1048575)>>>0,o,i+r)}}}function a(e,t,r,n,o){var i=e(n,o+t),a=e(n,o+r),s=2*(a>>31)+1,l=a>>>20&2047,u=4294967296*(1048575&a)+i;return 2047===l?u?NaN:s*(1/0):0===l?5e-324*s*u:s*Math.pow(2,l-1075)*(u+4503599627370496)}e.writeDoubleLE=t.bind(null,r,0,4),e.writeDoubleBE=t.bind(null,n,4,0),e.readDoubleLE=a.bind(null,o,0,4),e.readDoubleBE=a.bind(null,i,4,0)}(),e}function r(e,t,r){t[r]=255&e,t[r+1]=e>>>8&255,t[r+2]=e>>>16&255,t[r+3]=e>>>24}function n(e,t,r){t[r]=e>>>24,t[r+1]=e>>>16&255,t[r+2]=e>>>8&255,t[r+3]=255&e}function o(e,t){return(e[t]|e[t+1]<<8|e[t+2]<<16|e[t+3]<<24)>>>0}function i(e,t){return(e[t]<<24|e[t+1]<<16|e[t+2]<<8|e[t+3])>>>0}e.exports=t(t)},"./node_modules/@protobufjs/inquire/index.js":function node_modulesProtobufjsInquireIndexJs(module){"use strict";function inquire(moduleName){try{var mod=eval("quire".replace(/^/,"re"))(moduleName);if(mod&&(mod.length||Object.keys(mod).length))return mod}catch(e){}return null}module.exports=inquire},"./node_modules/@protobufjs/pool/index.js":function(e){"use strict";e.exports=function(e,t,r){var n=r||8192,o=n>>>1,i=null,a=n;return function(r){if(r<1||r>o)return e(r);a+r>n&&(i=e(n),a=0);var s=t.call(i,a,a+=r);return 7&a&&(a=1+(7|a)),s}}},"./node_modules/@protobufjs/utf8/index.js":function(e,t){"use strict";var r=t;r.length=function(e){for(var t=0,r=0,n=0;n191&&n<224?i[a++]=(31&n)<<6|63&e[t++]:n>239&&n<365?(n=((7&n)<<18|(63&e[t++])<<12|(63&e[t++])<<6|63&e[t++])-65536,i[a++]=55296+(n>>10),i[a++]=56320+(1023&n)):i[a++]=(15&n)<<12|(63&e[t++])<<6|63&e[t++],a>8191&&((o||(o=[])).push(String.fromCharCode.apply(String,i)),a=0);return o?(a&&o.push(String.fromCharCode.apply(String,i.slice(0,a))),o.join("")):String.fromCharCode.apply(String,i.slice(0,a))},r.write=function(e,t,r){for(var n,o,i=r,a=0;a>6|192,t[r++]=63&n|128):55296===(64512&n)&&56320===(64512&(o=e.charCodeAt(a+1)))?(n=65536+((1023&n)<<10)+(1023&o),++a,t[r++]=n>>18|240,t[r++]=n>>12&63|128,t[r++]=n>>6&63|128,t[r++]=63&n|128):(t[r++]=n>>12|224,t[r++]=n>>6&63|128,t[r++]=63&n|128);return r-i}},"./node_modules/browser-headers/dist/browser-headers.umd.js":function(e){var t;t=function(){return function(e){var t={};function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}return r.m=e,r.c=t,r.i=function(e){return e},r.d=function(e,t,n){r.o(e,t)||Object.defineProperty(e,t,{configurable:!1,enumerable:!0,get:n})},r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,"a",t),t},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r.p="",r(r.s=1)}([function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(3),o=function(){function e(e,t){void 0===e&&(e={}),void 0===t&&(t={splitValues:!1});var r,o=this;this.headersMap={},e&&("undefined"!==typeof Headers&&e instanceof Headers?n.getHeaderKeys(e).forEach((function(r){n.getHeaderValues(e,r).forEach((function(e){t.splitValues?o.append(r,n.splitHeaderValue(e)):o.append(r,e)}))})):"object"===typeof(r=e)&&"object"===typeof r.headersMap&&"function"===typeof r.forEach?e.forEach((function(e,t){o.append(e,t)})):"undefined"!==typeof Map&&e instanceof Map?e.forEach((function(e,t){o.append(t,e)})):"string"===typeof e?this.appendFromString(e):"object"===typeof e&&Object.getOwnPropertyNames(e).forEach((function(t){var r=e[t];Array.isArray(r)?r.forEach((function(e){o.append(t,e)})):o.append(t,r)})))}return e.prototype.appendFromString=function(e){for(var t=e.split("\r\n"),r=0;r0){var i=n.substring(0,o).trim(),a=n.substring(o+1).trim();this.append(i,a)}}},e.prototype.delete=function(e,t){var r=n.normalizeName(e);if(void 0===t)delete this.headersMap[r];else{var o=this.headersMap[r];if(o){var i=o.indexOf(t);i>=0&&o.splice(i,1),0===o.length&&delete this.headersMap[r]}}},e.prototype.append=function(e,t){var r=this,o=n.normalizeName(e);Array.isArray(this.headersMap[o])||(this.headersMap[o]=[]),Array.isArray(t)?t.forEach((function(e){r.headersMap[o].push(n.normalizeValue(e))})):this.headersMap[o].push(n.normalizeValue(t))},e.prototype.set=function(e,t){var r=n.normalizeName(e);if(Array.isArray(t)){var o=[];t.forEach((function(e){o.push(n.normalizeValue(e))})),this.headersMap[r]=o}else this.headersMap[r]=[n.normalizeValue(t)]},e.prototype.has=function(e,t){var r=this.headersMap[n.normalizeName(e)];if(!Array.isArray(r))return!1;if(void 0!==t){var o=n.normalizeValue(t);return r.indexOf(o)>=0}return!0},e.prototype.get=function(e){var t=this.headersMap[n.normalizeName(e)];return void 0!==t?t.concat():[]},e.prototype.forEach=function(e){var t=this;Object.getOwnPropertyNames(this.headersMap).forEach((function(r){e(r,t.headersMap[r])}),this)},e.prototype.toHeaders=function(){if("undefined"!==typeof Headers){var e=new Headers;return this.forEach((function(t,r){r.forEach((function(r){e.append(t,r)}))})),e}throw new Error("Headers class is not defined")},e}();t.BrowserHeaders=o},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(0);t.BrowserHeaders=n.BrowserHeaders},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.iterateHeaders=function(e,t){for(var r=e[Symbol.iterator](),n=r.next();!n.done;)t(n.value[0]),n=r.next()},t.iterateHeadersKeys=function(e,t){for(var r=e.keys(),n=r.next();!n.done;)t(n.value),n=r.next()}},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(2);t.normalizeName=function(e){if("string"!==typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.\^_`|~]/i.test(e))throw new TypeError("Invalid character in header field name");return e.toLowerCase()},t.normalizeValue=function(e){return"string"!==typeof e&&(e=String(e)),e},t.getHeaderValues=function(e,t){var r=e;if(r instanceof Headers&&r.getAll)return r.getAll(t);var n=r.get(t);return n&&"string"===typeof n?[n]:n},t.getHeaderKeys=function(e){var t=e,r={},o=[];return t.keys?n.iterateHeadersKeys(t,(function(e){r[e]||(r[e]=!0,o.push(e))})):t.forEach?t.forEach((function(e,t){r[t]||(r[t]=!0,o.push(t))})):n.iterateHeaders(t,(function(e){var t=e[0];r[t]||(r[t]=!0,o.push(t))})),o},t.splitHeaderValue=function(e){var t=[];return e.split(", ").forEach((function(e){e.split(",").forEach((function(e){t.push(e)}))})),t}}])},e.exports=t()},"./node_modules/google-protobuf/google-protobuf.js":function node_modulesGoogleProtobufGoogleProtobufJs(__unused_webpack_module,exports,__nested_webpack_require_87310__){var $jscomp=$jscomp||{};$jscomp.scope={},$jscomp.findInternal=function(e,t,r){e instanceof String&&(e=String(e));for(var n=e.length,o=0;o=n}}),"es6","es3"),$jscomp.polyfill("Array.prototype.find",(function(e){return e||function(e,t){return $jscomp.findInternal(this,e,t).v}}),"es6","es3"),$jscomp.polyfill("String.prototype.startsWith",(function(e){return e||function(e,t){var r=$jscomp.checkStringArgs(this,e,"startsWith");e+="";var n=r.length,o=e.length;t=Math.max(0,Math.min(0|t,r.length));for(var i=0;i=o}}),"es6","es3"),$jscomp.polyfill("String.prototype.repeat",(function(e){return e||function(e){var t=$jscomp.checkStringArgs(this,null,"repeat");if(0>e||1342177279>>=1)&&(t+=t);return r}}),"es6","es3");var COMPILED=!0,goog=goog||{};goog.global=this||self,goog.isDef=function(e){return void 0!==e},goog.isString=function(e){return"string"==typeof e},goog.isBoolean=function(e){return"boolean"==typeof e},goog.isNumber=function(e){return"number"==typeof e},goog.exportPath_=function(e,t,r){e=e.split("."),r=r||goog.global,e[0]in r||"undefined"==typeof r.execScript||r.execScript("var "+e[0]);for(var n;e.length&&(n=e.shift());)!e.length&&goog.isDef(t)?r[n]=t:r=r[n]&&r[n]!==Object.prototype[n]?r[n]:r[n]={}},goog.define=function(e,t){if(!COMPILED){var r=goog.global.CLOSURE_UNCOMPILED_DEFINES,n=goog.global.CLOSURE_DEFINES;r&&void 0===r.nodeType&&Object.prototype.hasOwnProperty.call(r,e)?t=r[e]:n&&void 0===n.nodeType&&Object.prototype.hasOwnProperty.call(n,e)&&(t=n[e])}return t},goog.FEATURESET_YEAR=2012,goog.DEBUG=!0,goog.LOCALE="en",goog.TRUSTED_SITE=!0,goog.STRICT_MODE_COMPATIBLE=!1,goog.DISALLOW_TEST_ONLY_CODE=COMPILED&&!goog.DEBUG,goog.ENABLE_CHROME_APP_SAFE_SCRIPT_LOADING=!1,goog.provide=function(e){if(goog.isInModuleLoader_())throw Error("goog.provide cannot be used within a module.");if(!COMPILED&&goog.isProvided_(e))throw Error('Namespace "'+e+'" already declared.');goog.constructNamespace_(e)},goog.constructNamespace_=function(e,t){if(!COMPILED){delete goog.implicitNamespaces_[e];for(var r=e;(r=r.substring(0,r.lastIndexOf(".")))&&!goog.getObjectByName(r);)goog.implicitNamespaces_[r]=!0}goog.exportPath_(e,t)},goog.getScriptNonce=function(e){return e&&e!=goog.global?goog.getScriptNonce_(e.document):(null===goog.cspNonce_&&(goog.cspNonce_=goog.getScriptNonce_(goog.global.document)),goog.cspNonce_)},goog.NONCE_PATTERN_=/^[\w+/_-]+[=]{0,2}$/,goog.cspNonce_=null,goog.getScriptNonce_=function(e){return(e=e.querySelector&&e.querySelector("script[nonce]"))&&(e=e.nonce||e.getAttribute("nonce"))&&goog.NONCE_PATTERN_.test(e)?e:""},goog.VALID_MODULE_RE_=/^[a-zA-Z_$][a-zA-Z0-9._$]*$/,goog.module=function(e){if(!goog.isString(e)||!e||-1==e.search(goog.VALID_MODULE_RE_))throw Error("Invalid module identifier");if(!goog.isInGoogModuleLoader_())throw Error("Module "+e+" has been loaded incorrectly. Note, modules cannot be loaded as normal scripts. They require some kind of pre-processing step. You're likely trying to load a module via a script tag or as a part of a concatenated bundle without rewriting the module. For more info see: https://github.com/google/closure-library/wiki/goog.module:-an-ES6-module-like-alternative-to-goog.provide.");if(goog.moduleLoaderState_.moduleName)throw Error("goog.module may only be called once per module.");if(goog.moduleLoaderState_.moduleName=e,!COMPILED){if(goog.isProvided_(e))throw Error('Namespace "'+e+'" already declared.');delete goog.implicitNamespaces_[e]}},goog.module.get=function(e){return goog.module.getInternal_(e)},goog.module.getInternal_=function(e){if(!COMPILED){if(e in goog.loadedModules_)return goog.loadedModules_[e].exports;if(!goog.implicitNamespaces_[e])return null!=(e=goog.getObjectByName(e))?e:null}return null},goog.ModuleType={ES6:"es6",GOOG:"goog"},goog.moduleLoaderState_=null,goog.isInModuleLoader_=function(){return goog.isInGoogModuleLoader_()||goog.isInEs6ModuleLoader_()},goog.isInGoogModuleLoader_=function(){return!!goog.moduleLoaderState_&&goog.moduleLoaderState_.type==goog.ModuleType.GOOG},goog.isInEs6ModuleLoader_=function(){if(goog.moduleLoaderState_&&goog.moduleLoaderState_.type==goog.ModuleType.ES6)return!0;var e=goog.global.$jscomp;return!!e&&("function"==typeof e.getCurrentModulePath&&!!e.getCurrentModulePath())},goog.module.declareLegacyNamespace=function(){if(!COMPILED&&!goog.isInGoogModuleLoader_())throw Error("goog.module.declareLegacyNamespace must be called from within a goog.module");if(!COMPILED&&!goog.moduleLoaderState_.moduleName)throw Error("goog.module must be called prior to goog.module.declareLegacyNamespace.");goog.moduleLoaderState_.declareLegacyNamespace=!0},goog.declareModuleId=function(e){if(!COMPILED){if(!goog.isInEs6ModuleLoader_())throw Error("goog.declareModuleId may only be called from within an ES6 module");if(goog.moduleLoaderState_&&goog.moduleLoaderState_.moduleName)throw Error("goog.declareModuleId may only be called once per module.");if(e in goog.loadedModules_)throw Error('Module with namespace "'+e+'" already exists.')}if(goog.moduleLoaderState_)goog.moduleLoaderState_.moduleName=e;else{var t=goog.global.$jscomp;if(!t||"function"!=typeof t.getCurrentModulePath)throw Error('Module with namespace "'+e+'" has been loaded incorrectly.');t=t.require(t.getCurrentModulePath()),goog.loadedModules_[e]={exports:t,type:goog.ModuleType.ES6,moduleId:e}}},goog.setTestOnly=function(e){if(goog.DISALLOW_TEST_ONLY_CODE)throw e=e||"",Error("Importing test-only code into non-debug environment"+(e?": "+e:"."))},goog.forwardDeclare=function(e){},COMPILED||(goog.isProvided_=function(e){return e in goog.loadedModules_||!goog.implicitNamespaces_[e]&&goog.isDefAndNotNull(goog.getObjectByName(e))},goog.implicitNamespaces_={"goog.module":!0}),goog.getObjectByName=function(e,t){e=e.split("."),t=t||goog.global;for(var r=0;r>>0),goog.uidCounter_=0,goog.getHashCode=goog.getUid,goog.removeHashCode=goog.removeUid,goog.cloneObject=function(e){var t=goog.typeOf(e);if("object"==t||"array"==t){if("function"===typeof e.clone)return e.clone();for(var r in t="array"==t?[]:{},e)t[r]=goog.cloneObject(e[r]);return t}return e},goog.bindNative_=function(e,t,r){return e.call.apply(e.bind,arguments)},goog.bindJs_=function(e,t,r){if(!e)throw Error();if(2{"use strict";class X{constructor(){if(new.target!=String)throw 1;this.x=42}}let q=Reflect.construct(X,[],String);if(q.x!=42||!(q instanceof String))throw 1;for(const a of[2,3]){if(a==2)continue;function f(z={a}){let a=0;return z.a}{function f(){return 0;}}return f()==3}})()')})),a("es7",(function(){return b("2 ** 2 == 4")})),a("es8",(function(){return b("async () => 1, true")})),a("es9",(function(){return b("({...rest} = {}), true")})),a("es_next",(function(){return!1})),{target:c,map:d}},goog.Transpiler.prototype.needsTranspile=function(e,t){if("always"==goog.TRANSPILE)return!0;if("never"==goog.TRANSPILE)return!1;if(!this.requiresTranspilation_){var r=this.createRequiresTranspilation_();this.requiresTranspilation_=r.map,this.transpilationTarget_=this.transpilationTarget_||r.target}if(e in this.requiresTranspilation_)return!!this.requiresTranspilation_[e]||!(!goog.inHtmlDocument_()||"es6"!=t||"noModule"in goog.global.document.createElement("script"));throw Error("Unknown language mode: "+e)},goog.Transpiler.prototype.transpile=function(e,t){return goog.transpile_(e,t,this.transpilationTarget_)},goog.transpiler_=new goog.Transpiler,goog.protectScriptTag_=function(e){return e.replace(/<\/(SCRIPT)/gi,"\\x3c/$1")},goog.DebugLoader_=function(){this.dependencies_={},this.idToPath_={},this.written_={},this.loadingDeps_=[],this.depsToLoad_=[],this.paused_=!1,this.factory_=new goog.DependencyFactory(goog.transpiler_),this.deferredCallbacks_={},this.deferredQueue_=[]},goog.DebugLoader_.prototype.bootstrap=function(e,t){function r(){n&&(goog.global.setTimeout(n,0),n=null)}var n=t;if(e.length){t=[];for(var o=0;o<\/script>",t.write(goog.TRUSTED_TYPES_POLICY_?goog.TRUSTED_TYPES_POLICY_.createHTML(n):n)}else{var o=t.createElement("script");o.defer=goog.Dependency.defer_,o.async=!1,o.type="text/javascript",(n=goog.getScriptNonce())&&o.setAttribute("nonce",n),goog.DebugLoader_.IS_OLD_IE_?(e.pause(),o.onreadystatechange=function(){"loaded"!=o.readyState&&"complete"!=o.readyState||(e.loaded(),e.resume())}):o.onload=function(){o.onload=null,e.loaded()},o.src=goog.TRUSTED_TYPES_POLICY_?goog.TRUSTED_TYPES_POLICY_.createScriptURL(this.path):this.path,t.head.appendChild(o)}}else goog.logToConsole_("Cannot use default debug loader outside of HTML documents."),"deps.js"==this.relativePath?(goog.logToConsole_("Consider setting CLOSURE_IMPORT_SCRIPT before loading base.js, or setting CLOSURE_NO_DEPS to true."),e.loaded()):e.pause()},goog.Es6ModuleDependency=function(e,t,r,n,o){goog.Dependency.call(this,e,t,r,n,o)},goog.inherits(goog.Es6ModuleDependency,goog.Dependency),goog.Es6ModuleDependency.prototype.load=function(e){if(goog.global.CLOSURE_IMPORT_SCRIPT)goog.global.CLOSURE_IMPORT_SCRIPT(this.path)?e.loaded():e.pause();else if(goog.inHtmlDocument_()){var t=goog.global.document,r=this;if(goog.isDocumentLoading_()){var n=function(e,r){e=r?'
    \ No newline at end of file +Cogment Verse
    \ No newline at end of file diff --git a/cogment_verse/web/components/build/static/js/main.33a072ac.js b/cogment_verse/web/components/build/static/js/main.33a072ac.js new file mode 100644 index 00000000..869fa2ae --- /dev/null +++ b/cogment_verse/web/components/build/static/js/main.33a072ac.js @@ -0,0 +1,3 @@ +/*! For license information please see main.33a072ac.js.LICENSE.txt */ +(function(){var __webpack_modules__={33:function(e,t,r){var n;t.CL=void 0;var o=r(358),i=function(){this.name="player",this.config=o.cogment_verse.AgentConfig,this.actionSpace=o.cogment_verse.PlayerAction,this.observationSpace=o.cogment_verse.Observation};var a=function(){this.name="teacher",this.config=o.cogment_verse.AgentConfig,this.actionSpace=o.cogment_verse.TeacherAction,this.observationSpace=o.cogment_verse.Observation};var s=function(){this.name="observer",this.config=o.cogment_verse.AgentConfig,this.actionSpace=o.cogment_verse.ObserverAction,this.observationSpace=o.cogment_verse.Observation};t.CL={messageUrlMap:(n={},n["type.googleapis.com/cogment_verse.NDArray"]=o.cogment_verse.NDArray,n["type.googleapis.com/cogment_verse.Space"]=o.cogment_verse.Space,n["type.googleapis.com/cogment_verse.Space.Discrete"]=o.cogment_verse.Space.Discrete,n["type.googleapis.com/cogment_verse.Space.Bound"]=o.cogment_verse.Space.Bound,n["type.googleapis.com/cogment_verse.Space.Box"]=o.cogment_verse.Space.Box,n["type.googleapis.com/cogment_verse.Space.Property"]=o.cogment_verse.Space.Property,n["type.googleapis.com/cogment_verse.SpaceValue"]=o.cogment_verse.SpaceValue,n["type.googleapis.com/cogment_verse.SpaceValue.SimpleBox"]=o.cogment_verse.SpaceValue.SimpleBox,n["type.googleapis.com/cogment_verse.SpaceValue.PropertyValue"]=o.cogment_verse.SpaceValue.PropertyValue,n["type.googleapis.com/cogment_verse.SpaceMask"]=o.cogment_verse.SpaceMask,n["type.googleapis.com/cogment_verse.SpaceMask.PropertyMask"]=o.cogment_verse.SpaceMask.PropertyMask,n["type.googleapis.com/cogment_verse.EnvironmentSpecs"]=o.cogment_verse.EnvironmentSpecs,n["type.googleapis.com/cogment_verse.EnvironmentConfig"]=o.cogment_verse.EnvironmentConfig,n["type.googleapis.com/cogment_verse.HFHubModel"]=o.cogment_verse.HFHubModel,n["type.googleapis.com/cogment_verse.AgentConfig"]=o.cogment_verse.AgentConfig,n["type.googleapis.com/cogment_verse.TrialConfig"]=o.cogment_verse.TrialConfig,n["type.googleapis.com/cogment_verse.Observation"]=o.cogment_verse.Observation,n["type.googleapis.com/cogment_verse.PlayerAction"]=o.cogment_verse.PlayerAction,n["type.googleapis.com/cogment_verse.TeacherAction"]=o.cogment_verse.TeacherAction,n["type.googleapis.com/cogment_verse.ObserverAction"]=o.cogment_verse.ObserverAction,n),actorClasses:{player:new i,teacher:new a,observer:new s},trial:{config:o.cogment_verse.TrialConfig},environment:{config:o.cogment_verse.EnvironmentConfig,class:{id:"env",config:o.cogment_verse.EnvironmentConfig}}}},358:function(e,t,r){var n,o,i;o=[r(886)],void 0===(i="function"===typeof(n=function(e){"use strict";var t=e.Reader,r=e.Writer,n=e.util,o=e.roots.default||(e.roots.default={});return o.cogment_verse=function(){var i={};return i.NDArray=function(){function i(e){if(this.shape=[],e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.dtype=e.string();break;case 2:if(i.shape&&i.shape.length||(i.shape=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>0}return null!=e.data&&("string"===typeof e.data?n.base64.decode(e.data,t.data=n.newBuffer(n.base64.length(e.data)),0):e.data.length>=0&&(t.data=e.data)),t},i.toObject=function(e,t){t||(t={});var r={};if((t.arrays||t.defaults)&&(r.shape=[]),t.defaults&&(r.dtype="",t.bytes===String?r.data="":(r.data=[],t.bytes!==Array&&(r.data=n.newBuffer(r.data)))),null!=e.dtype&&e.hasOwnProperty("dtype")&&(r.dtype=e.dtype),e.shape&&e.shape.length){r.shape=[];for(var o=0;o>>3===1?(i.properties&&i.properties.length||(i.properties=[]),i.properties.push(o.cogment_verse.Space.Property.decode(e,e.uint32()))):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.properties&&e.hasOwnProperty("properties")){if(!Array.isArray(e.properties))return"properties: array expected";for(var t=0;t>>3){case 1:i.num=e.int32();break;case 2:i.labels&&i.labels.length||(i.labels=[]),i.labels.push(e.string());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.num&&e.hasOwnProperty("num")&&!n.isInteger(e.num))return"num: integer expected";if(null!=e.labels&&e.hasOwnProperty("labels")){if(!Array.isArray(e.labels))return"labels: array expected";for(var t=0;t>>3===1?i.bound=e.float():e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};return null!=e.bound&&e.hasOwnProperty("bound")&&(t._bound=1,"number"!==typeof e.bound)?"bound: number expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.Space.Bound)return e;var t=new o.cogment_verse.Space.Bound;return null!=e.bound&&(t.bound=Number(e.bound)),t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.bound&&e.hasOwnProperty("bound")&&(r.bound=t.json&&!isFinite(e.bound)?String(e.bound):e.bound,t.oneofs&&(r._bound="bound")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.Space.Bound"},i}(),i.Box=function(){function i(e){if(this.shape=[],this.low=[],this.high=[],e)for(var t=Object.keys(e),r=0;r>>3){case 1:if(i.shape&&i.shape.length||(i.shape=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>0}if(e.low){if(!Array.isArray(e.low))throw TypeError(".cogment_verse.Space.Box.low: array expected");for(t.low=[],r=0;r>>3){case 1:i.key=e.string();break;case 2:i.discrete=o.cogment_verse.Space.Discrete.decode(e,e.uint32());break;case 3:i.box=o.cogment_verse.Space.Box.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.key&&e.hasOwnProperty("key")&&!n.isString(e.key))return"key: string expected";if(null!=e.discrete&&e.hasOwnProperty("discrete")&&(t.type=1,r=o.cogment_verse.Space.Discrete.verify(e.discrete)))return"discrete."+r;if(null!=e.box&&e.hasOwnProperty("box")){if(1===t.type)return"type: multiple values";var r;if(t.type=1,r=o.cogment_verse.Space.Box.verify(e.box))return"box."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.Space.Property)return e;var t=new o.cogment_verse.Space.Property;if(null!=e.key&&(t.key=String(e.key)),null!=e.discrete){if("object"!==typeof e.discrete)throw TypeError(".cogment_verse.Space.Property.discrete: object expected");t.discrete=o.cogment_verse.Space.Discrete.fromObject(e.discrete)}if(null!=e.box){if("object"!==typeof e.box)throw TypeError(".cogment_verse.Space.Property.box: object expected");t.box=o.cogment_verse.Space.Box.fromObject(e.box)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.key=""),null!=e.key&&e.hasOwnProperty("key")&&(r.key=e.key),null!=e.discrete&&e.hasOwnProperty("discrete")&&(r.discrete=o.cogment_verse.Space.Discrete.toObject(e.discrete,t),t.oneofs&&(r.type="discrete")),null!=e.box&&e.hasOwnProperty("box")&&(r.box=o.cogment_verse.Space.Box.toObject(e.box,t),t.oneofs&&(r.type="box")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.Space.Property"},i}(),i}(),i.SpaceValue=function(){function i(e){if(this.properties=[],e)for(var t=Object.keys(e),r=0;r>>3===1?(i.properties&&i.properties.length||(i.properties=[]),i.properties.push(o.cogment_verse.SpaceValue.PropertyValue.decode(e,e.uint32()))):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.properties&&e.hasOwnProperty("properties")){if(!Array.isArray(e.properties))return"properties: array expected";for(var t=0;t>>3===1)if(i.values&&i.values.length||(i.values=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>3){case 1:i.discrete=e.int32();break;case 2:i.box=o.cogment_verse.NDArray.decode(e,e.uint32());break;case 3:i.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.discrete&&e.hasOwnProperty("discrete")&&(t.value=1,!n.isInteger(e.discrete)))return"discrete: integer expected";if(null!=e.box&&e.hasOwnProperty("box")){if(1===t.value)return"value: multiple values";if(t.value=1,r=o.cogment_verse.NDArray.verify(e.box))return"box."+r}if(null!=e.simpleBox&&e.hasOwnProperty("simpleBox")){if(1===t.value)return"value: multiple values";var r;if(t.value=1,r=o.cogment_verse.SpaceValue.SimpleBox.verify(e.simpleBox))return"simpleBox."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.SpaceValue.PropertyValue)return e;var t=new o.cogment_verse.SpaceValue.PropertyValue;if(null!=e.discrete&&(t.discrete=0|e.discrete),null!=e.box){if("object"!==typeof e.box)throw TypeError(".cogment_verse.SpaceValue.PropertyValue.box: object expected");t.box=o.cogment_verse.NDArray.fromObject(e.box)}if(null!=e.simpleBox){if("object"!==typeof e.simpleBox)throw TypeError(".cogment_verse.SpaceValue.PropertyValue.simpleBox: object expected");t.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.fromObject(e.simpleBox)}return t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.discrete&&e.hasOwnProperty("discrete")&&(r.discrete=e.discrete,t.oneofs&&(r.value="discrete")),null!=e.box&&e.hasOwnProperty("box")&&(r.box=o.cogment_verse.NDArray.toObject(e.box,t),t.oneofs&&(r.value="box")),null!=e.simpleBox&&e.hasOwnProperty("simpleBox")&&(r.simpleBox=o.cogment_verse.SpaceValue.SimpleBox.toObject(e.simpleBox,t),t.oneofs&&(r.value="simpleBox")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.SpaceValue.PropertyValue"},i}(),i}(),i.SpaceMask=function(){function i(e){if(this.properties=[],e)for(var t=Object.keys(e),r=0;r>>3===1?(i.properties&&i.properties.length||(i.properties=[]),i.properties.push(o.cogment_verse.SpaceMask.PropertyMask.decode(e,e.uint32()))):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.properties&&e.hasOwnProperty("properties")){if(!Array.isArray(e.properties))return"properties: array expected";for(var t=0;t>>3===1)if(i.discrete&&i.discrete.length||(i.discrete=[]),2===(7&a))for(var s=e.uint32()+e.pos;e.pos>>3){case 1:i.implementation=e.string();break;case 2:i.turnBased=e.bool();break;case 3:i.numPlayers=e.int32();break;case 4:i.observationSpace=o.cogment_verse.Space.decode(e,e.uint32());break;case 5:i.actionSpace=o.cogment_verse.Space.decode(e,e.uint32());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.implementation&&e.hasOwnProperty("implementation")&&!n.isString(e.implementation)?"implementation: string expected":null!=e.turnBased&&e.hasOwnProperty("turnBased")&&"boolean"!==typeof e.turnBased?"turnBased: boolean expected":null!=e.numPlayers&&e.hasOwnProperty("numPlayers")&&!n.isInteger(e.numPlayers)?"numPlayers: integer expected":null!=e.observationSpace&&e.hasOwnProperty("observationSpace")&&(t=o.cogment_verse.Space.verify(e.observationSpace))?"observationSpace."+t:null!=e.actionSpace&&e.hasOwnProperty("actionSpace")&&(t=o.cogment_verse.Space.verify(e.actionSpace))?"actionSpace."+t:null;var t},i.fromObject=function(e){if(e instanceof o.cogment_verse.EnvironmentSpecs)return e;var t=new o.cogment_verse.EnvironmentSpecs;if(null!=e.implementation&&(t.implementation=String(e.implementation)),null!=e.turnBased&&(t.turnBased=Boolean(e.turnBased)),null!=e.numPlayers&&(t.numPlayers=0|e.numPlayers),null!=e.observationSpace){if("object"!==typeof e.observationSpace)throw TypeError(".cogment_verse.EnvironmentSpecs.observationSpace: object expected");t.observationSpace=o.cogment_verse.Space.fromObject(e.observationSpace)}if(null!=e.actionSpace){if("object"!==typeof e.actionSpace)throw TypeError(".cogment_verse.EnvironmentSpecs.actionSpace: object expected");t.actionSpace=o.cogment_verse.Space.fromObject(e.actionSpace)}return t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.implementation="",r.turnBased=!1,r.numPlayers=0,r.observationSpace=null,r.actionSpace=null),null!=e.implementation&&e.hasOwnProperty("implementation")&&(r.implementation=e.implementation),null!=e.turnBased&&e.hasOwnProperty("turnBased")&&(r.turnBased=e.turnBased),null!=e.numPlayers&&e.hasOwnProperty("numPlayers")&&(r.numPlayers=e.numPlayers),null!=e.observationSpace&&e.hasOwnProperty("observationSpace")&&(r.observationSpace=o.cogment_verse.Space.toObject(e.observationSpace,t)),null!=e.actionSpace&&e.hasOwnProperty("actionSpace")&&(r.actionSpace=o.cogment_verse.Space.toObject(e.actionSpace,t)),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.EnvironmentSpecs"},i}(),i.EnvironmentConfig=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.runId=e.string();break;case 2:i.render=e.bool();break;case 3:i.renderWidth=e.int32();break;case 4:i.seed=e.uint32();break;case 5:i.flatten=e.bool();break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.runId&&e.hasOwnProperty("runId")&&!n.isString(e.runId)?"runId: string expected":null!=e.render&&e.hasOwnProperty("render")&&"boolean"!==typeof e.render?"render: boolean expected":null!=e.renderWidth&&e.hasOwnProperty("renderWidth")&&!n.isInteger(e.renderWidth)?"renderWidth: integer expected":null!=e.seed&&e.hasOwnProperty("seed")&&!n.isInteger(e.seed)?"seed: integer expected":null!=e.flatten&&e.hasOwnProperty("flatten")&&"boolean"!==typeof e.flatten?"flatten: boolean expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.EnvironmentConfig)return e;var t=new o.cogment_verse.EnvironmentConfig;return null!=e.runId&&(t.runId=String(e.runId)),null!=e.render&&(t.render=Boolean(e.render)),null!=e.renderWidth&&(t.renderWidth=0|e.renderWidth),null!=e.seed&&(t.seed=e.seed>>>0),null!=e.flatten&&(t.flatten=Boolean(e.flatten)),t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.runId="",r.render=!1,r.renderWidth=0,r.seed=0,r.flatten=!1),null!=e.runId&&e.hasOwnProperty("runId")&&(r.runId=e.runId),null!=e.render&&e.hasOwnProperty("render")&&(r.render=e.render),null!=e.renderWidth&&e.hasOwnProperty("renderWidth")&&(r.renderWidth=e.renderWidth),null!=e.seed&&e.hasOwnProperty("seed")&&(r.seed=e.seed),null!=e.flatten&&e.hasOwnProperty("flatten")&&(r.flatten=e.flatten),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.EnvironmentConfig"},i}(),i.HFHubModel=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.repoId=e.string();break;case 2:i.filename=e.string();break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){return"object"!==typeof e||null===e?"object expected":null!=e.repoId&&e.hasOwnProperty("repoId")&&!n.isString(e.repoId)?"repoId: string expected":null!=e.filename&&e.hasOwnProperty("filename")&&!n.isString(e.filename)?"filename: string expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.HFHubModel)return e;var t=new o.cogment_verse.HFHubModel;return null!=e.repoId&&(t.repoId=String(e.repoId)),null!=e.filename&&(t.filename=String(e.filename)),t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.repoId="",r.filename=""),null!=e.repoId&&e.hasOwnProperty("repoId")&&(r.repoId=e.repoId),null!=e.filename&&e.hasOwnProperty("filename")&&(r.filename=e.filename),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.HFHubModel"},i}(),i.AgentConfig=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.runId=e.string();break;case 2:i.environmentSpecs=o.cogment_verse.EnvironmentSpecs.decode(e,e.uint32());break;case 3:i.seed=e.uint32();break;case 4:i.modelId=e.string();break;case 5:i.modelVersion=e.int32();break;case 6:i.modelUpdateFrequency=e.int32();break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.runId&&e.hasOwnProperty("runId")&&!n.isString(e.runId))return"runId: string expected";if(null!=e.environmentSpecs&&e.hasOwnProperty("environmentSpecs")){var t=o.cogment_verse.EnvironmentSpecs.verify(e.environmentSpecs);if(t)return"environmentSpecs."+t}return null!=e.seed&&e.hasOwnProperty("seed")&&!n.isInteger(e.seed)?"seed: integer expected":null!=e.modelId&&e.hasOwnProperty("modelId")&&!n.isString(e.modelId)?"modelId: string expected":null!=e.modelVersion&&e.hasOwnProperty("modelVersion")&&!n.isInteger(e.modelVersion)?"modelVersion: integer expected":null!=e.modelUpdateFrequency&&e.hasOwnProperty("modelUpdateFrequency")&&!n.isInteger(e.modelUpdateFrequency)?"modelUpdateFrequency: integer expected":null},i.fromObject=function(e){if(e instanceof o.cogment_verse.AgentConfig)return e;var t=new o.cogment_verse.AgentConfig;if(null!=e.runId&&(t.runId=String(e.runId)),null!=e.environmentSpecs){if("object"!==typeof e.environmentSpecs)throw TypeError(".cogment_verse.AgentConfig.environmentSpecs: object expected");t.environmentSpecs=o.cogment_verse.EnvironmentSpecs.fromObject(e.environmentSpecs)}return null!=e.seed&&(t.seed=e.seed>>>0),null!=e.modelId&&(t.modelId=String(e.modelId)),null!=e.modelVersion&&(t.modelVersion=0|e.modelVersion),null!=e.modelUpdateFrequency&&(t.modelUpdateFrequency=0|e.modelUpdateFrequency),t},i.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.runId="",r.environmentSpecs=null,r.seed=0,r.modelId="",r.modelVersion=0,r.modelUpdateFrequency=0),null!=e.runId&&e.hasOwnProperty("runId")&&(r.runId=e.runId),null!=e.environmentSpecs&&e.hasOwnProperty("environmentSpecs")&&(r.environmentSpecs=o.cogment_verse.EnvironmentSpecs.toObject(e.environmentSpecs,t)),null!=e.seed&&e.hasOwnProperty("seed")&&(r.seed=e.seed),null!=e.modelId&&e.hasOwnProperty("modelId")&&(r.modelId=e.modelId),null!=e.modelVersion&&e.hasOwnProperty("modelVersion")&&(r.modelVersion=e.modelVersion),null!=e.modelUpdateFrequency&&e.hasOwnProperty("modelUpdateFrequency")&&(r.modelUpdateFrequency=e.modelUpdateFrequency),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.AgentConfig"},i}(),i.TrialConfig=function(){function n(e){if(e)for(var t=Object.keys(e),r=0;r>>3){case 1:i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32());break;case 2:i.currentPlayer=e.string();break;case 3:i.actionMask=o.cogment_verse.SpaceMask.decode(e,e.uint32());break;case 4:i.renderedFrame=e.bytes();break;case 5:i.overriddenPlayers&&i.overriddenPlayers.length||(i.overriddenPlayers=[]),i.overriddenPlayers.push(e.string());break;default:e.skipType(7&a)}}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t,r={};if(null!=e.value&&e.hasOwnProperty("value")&&(t=o.cogment_verse.SpaceValue.verify(e.value)))return"value."+t;if(null!=e.currentPlayer&&e.hasOwnProperty("currentPlayer")&&(r._currentPlayer=1,!n.isString(e.currentPlayer)))return"currentPlayer: string expected";if(null!=e.actionMask&&e.hasOwnProperty("actionMask")&&(r._actionMask=1,t=o.cogment_verse.SpaceMask.verify(e.actionMask)))return"actionMask."+t;if(null!=e.renderedFrame&&e.hasOwnProperty("renderedFrame")&&(r._renderedFrame=1,!(e.renderedFrame&&"number"===typeof e.renderedFrame.length||n.isString(e.renderedFrame))))return"renderedFrame: buffer expected";if(null!=e.overriddenPlayers&&e.hasOwnProperty("overriddenPlayers")){if(!Array.isArray(e.overriddenPlayers))return"overriddenPlayers: array expected";for(var i=0;i=0&&(t.renderedFrame=e.renderedFrame)),e.overriddenPlayers){if(!Array.isArray(e.overriddenPlayers))throw TypeError(".cogment_verse.Observation.overriddenPlayers: array expected");t.overriddenPlayers=[];for(var r=0;r>>3===1?i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32()):e.skipType(7&a)}return i},n.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},n.verify=function(e){if("object"!==typeof e||null===e)return"object expected";if(null!=e.value&&e.hasOwnProperty("value")){var t=o.cogment_verse.SpaceValue.verify(e.value);if(t)return"value."+t}return null},n.fromObject=function(e){if(e instanceof o.cogment_verse.PlayerAction)return e;var t=new o.cogment_verse.PlayerAction;if(null!=e.value){if("object"!==typeof e.value)throw TypeError(".cogment_verse.PlayerAction.value: object expected");t.value=o.cogment_verse.SpaceValue.fromObject(e.value)}return t},n.toObject=function(e,t){t||(t={});var r={};return t.defaults&&(r.value=null),null!=e.value&&e.hasOwnProperty("value")&&(r.value=o.cogment_verse.SpaceValue.toObject(e.value,t)),r},n.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},n.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.PlayerAction"},n}(),i.TeacherAction=function(){function i(e){if(e)for(var t=Object.keys(e),r=0;r>>3===1?i.value=o.cogment_verse.SpaceValue.decode(e,e.uint32()):e.skipType(7&a)}return i},i.decodeDelimited=function(e){return e instanceof t||(e=new t(e)),this.decode(e,e.uint32())},i.verify=function(e){if("object"!==typeof e||null===e)return"object expected";var t={};if(null!=e.value&&e.hasOwnProperty("value")){t._value=1;var r=o.cogment_verse.SpaceValue.verify(e.value);if(r)return"value."+r}return null},i.fromObject=function(e){if(e instanceof o.cogment_verse.TeacherAction)return e;var t=new o.cogment_verse.TeacherAction;if(null!=e.value){if("object"!==typeof e.value)throw TypeError(".cogment_verse.TeacherAction.value: object expected");t.value=o.cogment_verse.SpaceValue.fromObject(e.value)}return t},i.toObject=function(e,t){t||(t={});var r={};return null!=e.value&&e.hasOwnProperty("value")&&(r.value=o.cogment_verse.SpaceValue.toObject(e.value,t),t.oneofs&&(r._value="value")),r},i.prototype.toJSON=function(){return this.constructor.toObject(this,e.util.toJSONOptions)},i.getTypeUrl=function(){return"type.googleapis.com/cogment_verse.TeacherAction"},i}(),i.ObserverAction=function(){function n(e){if(e)for(var t=Object.keys(e),r=0;r0){var i=n.substring(0,o).trim(),a=n.substring(o+1).trim();this.append(i,a)}}},e.prototype.delete=function(e,t){var r=n.normalizeName(e);if(void 0===t)delete this.headersMap[r];else{var o=this.headersMap[r];if(o){var i=o.indexOf(t);i>=0&&o.splice(i,1),0===o.length&&delete this.headersMap[r]}}},e.prototype.append=function(e,t){var r=this,o=n.normalizeName(e);Array.isArray(this.headersMap[o])||(this.headersMap[o]=[]),Array.isArray(t)?t.forEach((function(e){r.headersMap[o].push(n.normalizeValue(e))})):this.headersMap[o].push(n.normalizeValue(t))},e.prototype.set=function(e,t){var r=n.normalizeName(e);if(Array.isArray(t)){var o=[];t.forEach((function(e){o.push(n.normalizeValue(e))})),this.headersMap[r]=o}else this.headersMap[r]=[n.normalizeValue(t)]},e.prototype.has=function(e,t){var r=this.headersMap[n.normalizeName(e)];if(!Array.isArray(r))return!1;if(void 0!==t){var o=n.normalizeValue(t);return r.indexOf(o)>=0}return!0},e.prototype.get=function(e){var t=this.headersMap[n.normalizeName(e)];return void 0!==t?t.concat():[]},e.prototype.forEach=function(e){var t=this;Object.getOwnPropertyNames(this.headersMap).forEach((function(r){e(r,t.headersMap[r])}),this)},e.prototype.toHeaders=function(){if("undefined"!=typeof Headers){var e=new Headers;return this.forEach((function(t,r){r.forEach((function(r){e.append(t,r)}))})),e}throw new Error("Headers class is not defined")},e}();t.BrowserHeaders=o},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(0);t.BrowserHeaders=n.BrowserHeaders},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.iterateHeaders=function(e,t){for(var r=e[Symbol.iterator](),n=r.next();!n.done;)t(n.value[0]),n=r.next()},t.iterateHeadersKeys=function(e,t){for(var r=e.keys(),n=r.next();!n.done;)t(n.value),n=r.next()}},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(2);t.normalizeName=function(e){if("string"!=typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.\^_`|~]/i.test(e))throw new TypeError("Invalid character in header field name");return e.toLowerCase()},t.normalizeValue=function(e){return"string"!=typeof e&&(e=String(e)),e},t.getHeaderValues=function(e,t){var r=e;if(r instanceof Headers&&r.getAll)return r.getAll(t);var n=r.get(t);return n&&"string"==typeof n?[n]:n},t.getHeaderKeys=function(e){var t=e,r={},o=[];return t.keys?n.iterateHeadersKeys(t,(function(e){r[e]||(r[e]=!0,o.push(e))})):t.forEach?t.forEach((function(e,t){r[t]||(r[t]=!0,o.push(t))})):n.iterateHeaders(t,(function(e){var t=e[0];r[t]||(r[t]=!0,o.push(t))})),o},t.splitHeaderValue=function(e){var t=[];return e.split(", ").forEach((function(e){e.split(",").forEach((function(e){t.push(e)}))})),t}}]))},617:function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.ChunkParser=t.ChunkType=t.encodeASCII=t.decodeASCII=void 0;var n,o=r(65);function i(e){return 9===(t=e)||10===t||13===t||e>=32&&e<=126;var t}function a(e){for(var t=0;t!==e.length;++t)if(!i(e[t]))throw new Error("Metadata is not valid (printable) ASCII");return String.fromCharCode.apply(String,Array.prototype.slice.call(e))}function s(e){return 128==(128&e.getUint8(0))}function l(e){return e.getUint32(1,!1)}function u(e,t,r){return e.byteLength-t>=r}function c(e,t,r){if(e.slice)return e.slice(t,r);var n=e.length;void 0!==r&&(n=r);for(var o=new Uint8Array(n-t),i=0,a=t;a=0?r:i.httpStatusToCode(t);this.props.debug&&a.debug("onHeaders.code",n);var o=e.get("grpc-message")||[];if(this.props.debug&&a.debug("onHeaders.gRPCMessage",o),this.rawOnHeaders(e),n!==i.Code.OK){var s=this.decodeGRPCStatus(o[0]);this.rawOnError(n,s,e)}}},e.prototype.onTransportChunk=function(e){var t=this;if(this.closed)this.props.debug&&a.debug("grpc.onChunk received after request was closed - ignoring");else{var r=[];try{r=this.parser.parse(e)}catch(e){return this.props.debug&&a.debug("onChunk.parsing error",e,e.message),void this.rawOnError(i.Code.Internal,"parsing error: "+e.message)}r.forEach((function(e){if(e.chunkType===o.ChunkType.MESSAGE){var r=t.methodDefinition.responseType.deserializeBinary(e.data);t.rawOnMessage(r)}else e.chunkType===o.ChunkType.TRAILERS&&(t.responseHeaders?(t.responseTrailers=new n.Metadata(e.trailers),t.props.debug&&a.debug("onChunk.trailers",t.responseTrailers)):(t.responseHeaders=new n.Metadata(e.trailers),t.rawOnHeaders(t.responseHeaders)))}))}},e.prototype.onTransportEnd=function(){if(this.props.debug&&a.debug("grpc.onEnd"),this.closed)this.props.debug&&a.debug("grpc.onEnd received after request was closed - ignoring");else if(void 0!==this.responseTrailers){var e=c(this.responseTrailers);if(null!==e){var t=this.responseTrailers.get("grpc-message"),r=this.decodeGRPCStatus(t[0]);this.rawOnEnd(e,r,this.responseTrailers)}else this.rawOnError(i.Code.Internal,"Response closed without grpc-status (Trailers provided)")}else{if(void 0===this.responseHeaders)return void this.rawOnError(i.Code.Unknown,"Response closed without headers");var n=c(this.responseHeaders),o=this.responseHeaders.get("grpc-message");if(this.props.debug&&a.debug("grpc.headers only response ",n,o),null===n)return void this.rawOnEnd(i.Code.Unknown,"Response closed without grpc-status (Headers only)",this.responseHeaders);var s=this.decodeGRPCStatus(o[0]);this.rawOnEnd(n,s,this.responseHeaders)}},e.prototype.decodeGRPCStatus=function(e){if(!e)return"";try{return decodeURIComponent(e)}catch(t){return e}},e.prototype.rawOnEnd=function(e,t,r){var n=this;this.props.debug&&a.debug("rawOnEnd",e,t,r),this.completed||(this.completed=!0,this.onEndCallbacks.forEach((function(o){if(!n.closed)try{o(e,t,r)}catch(e){setTimeout((function(){throw e}),0)}})))},e.prototype.rawOnHeaders=function(e){this.props.debug&&a.debug("rawOnHeaders",e),this.completed||this.onHeadersCallbacks.forEach((function(t){try{t(e)}catch(e){setTimeout((function(){throw e}),0)}}))},e.prototype.rawOnError=function(e,t,r){var o=this;void 0===r&&(r=new n.Metadata),this.props.debug&&a.debug("rawOnError",e,t),this.completed||(this.completed=!0,this.onEndCallbacks.forEach((function(n){if(!o.closed)try{n(e,t,r)}catch(e){setTimeout((function(){throw e}),0)}})))},e.prototype.rawOnMessage=function(e){var t=this;this.props.debug&&a.debug("rawOnMessage",e.toObject()),this.completed||this.closed||this.onMessageCallbacks.forEach((function(r){if(!t.closed)try{r(e)}catch(e){setTimeout((function(){throw e}),0)}}))},e.prototype.onHeaders=function(e){this.onHeadersCallbacks.push(e)},e.prototype.onMessage=function(e){this.onMessageCallbacks.push(e)},e.prototype.onEnd=function(e){this.onEndCallbacks.push(e)},e.prototype.start=function(e){if(this.started)throw new Error("Client already started - cannot .start()");this.started=!0;var t=new n.Metadata(e||{});t.set("content-type","application/grpc-web+proto"),t.set("x-grpc-web","1"),this.transport.start(t)},e.prototype.send=function(e){if(!this.started)throw new Error("Client not started - .start() must be called before .send()");if(this.closed)throw new Error("Client already closed - cannot .send()");if(this.finishedSending)throw new Error("Client already finished sending - cannot .send()");if(!this.methodDefinition.requestStream&&this.sentFirstMessage)throw new Error("Message already sent for non-client-streaming method - cannot .send()");this.sentFirstMessage=!0;var t=l.frameRequest(e);this.transport.sendMessage(t)},e.prototype.finishSend=function(){if(!this.started)throw new Error("Client not started - .finishSend() must be called before .close()");if(this.closed)throw new Error("Client already closed - cannot .send()");if(this.finishedSending)throw new Error("Client already finished sending - cannot .finishSend()");this.finishedSending=!0,this.transport.finishSend()},e.prototype.close=function(){if(!this.started)throw new Error("Client not started - .start() must be called before .close()");if(this.closed)throw new Error("Client already closed - cannot .close()");this.closed=!0,this.props.debug&&a.debug("request.abort aborting request"),this.transport.cancel()},e}();function c(e){var t=e.get("grpc-status")||[];if(t.length>0)try{var r=t[0];return parseInt(r,10)}catch(e){return null}return null}},346:function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.debug=void 0,t.debug=function(){for(var e=[],t=0;t=55296&&r<=56319){var n=e.charCodeAt(t+1);n>=56320&&n<=57343&&(r=65536+(r-55296<<10)+(n-56320))}return r}function g(e){for(var t=new Uint8Array(e.length),r=0,n=0;n1&&"="===e.charAt(t);)++r;return Math.ceil(3*e.length)/4-r};for(var n=new Array(64),o=new Array(123),i=0;i<64;)o[n[i]=i<26?i+65:i<52?i+71:i<62?i-4:i-59|43]=i++;r.encode=function(e,t,r){for(var o,i=null,a=[],s=0,l=0;t>2],o=(3&u)<<4,l=1;break;case 1:a[s++]=n[o|u>>4],o=(15&u)<<2,l=2;break;case 2:a[s++]=n[o|u>>6],a[s++]=n[63&u],l=0}s>8191&&((i||(i=[])).push(String.fromCharCode.apply(String,a)),s=0)}return l&&(a[s++]=n[o],a[s++]=61,1===l&&(a[s++]=61)),i?(s&&i.push(String.fromCharCode.apply(String,a.slice(0,s))),i.join("")):String.fromCharCode.apply(String,a.slice(0,s))};var a="invalid encoding";r.decode=function(e,t,r){for(var n,i=r,s=0,l=0;l1)break;if(void 0===(u=o[u]))throw Error(a);switch(s){case 0:n=u,s=1;break;case 1:t[r++]=n<<2|(48&u)>>4,n=u,s=2;break;case 2:t[r++]=(15&n)<<4|(60&u)>>2,n=u,s=3;break;case 3:t[r++]=(3&n)<<6|u,s=0}}if(1===s)throw Error(a);return r-i},r.test=function(e){return/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(e)}},"./node_modules/@protobufjs/eventemitter/index.js":function(e){"use strict";function t(){this._listeners={}}e.exports=t,t.prototype.on=function(e,t,r){return(this._listeners[e]||(this._listeners[e]=[])).push({fn:t,ctx:r||this}),this},t.prototype.off=function(e,t){if(void 0===e)this._listeners={};else if(void 0===t)this._listeners[e]=[];else for(var r=this._listeners[e],n=0;n0?0:2147483648,r,n);else if(isNaN(t))e(2143289344,r,n);else if(t>34028234663852886e22)e((o<<31|2139095040)>>>0,r,n);else if(t<11754943508222875e-54)e((o<<31|Math.round(t/1401298464324817e-60))>>>0,r,n);else{var i=Math.floor(Math.log(t)/Math.LN2);e((o<<31|i+127<<23|8388607&Math.round(t*Math.pow(2,-i)*8388608))>>>0,r,n)}}function a(e,t,r){var n=e(t,r),o=2*(n>>31)+1,i=n>>>23&255,a=8388607&n;return 255===i?a?NaN:o*(1/0):0===i?1401298464324817e-60*o*a:o*Math.pow(2,i-150)*(a+8388608)}e.writeFloatLE=t.bind(null,r),e.writeFloatBE=t.bind(null,n),e.readFloatLE=a.bind(null,o),e.readFloatBE=a.bind(null,i)}(),"undefined"!==typeof Float64Array?function(){var t=new Float64Array([-0]),r=new Uint8Array(t.buffer),n=128===r[7];function o(e,n,o){t[0]=e,n[o]=r[0],n[o+1]=r[1],n[o+2]=r[2],n[o+3]=r[3],n[o+4]=r[4],n[o+5]=r[5],n[o+6]=r[6],n[o+7]=r[7]}function i(e,n,o){t[0]=e,n[o]=r[7],n[o+1]=r[6],n[o+2]=r[5],n[o+3]=r[4],n[o+4]=r[3],n[o+5]=r[2],n[o+6]=r[1],n[o+7]=r[0]}function a(e,n){return r[0]=e[n],r[1]=e[n+1],r[2]=e[n+2],r[3]=e[n+3],r[4]=e[n+4],r[5]=e[n+5],r[6]=e[n+6],r[7]=e[n+7],t[0]}function s(e,n){return r[7]=e[n],r[6]=e[n+1],r[5]=e[n+2],r[4]=e[n+3],r[3]=e[n+4],r[2]=e[n+5],r[1]=e[n+6],r[0]=e[n+7],t[0]}e.writeDoubleLE=n?o:i,e.writeDoubleBE=n?i:o,e.readDoubleLE=n?a:s,e.readDoubleBE=n?s:a}():function(){function t(e,t,r,n,o,i){var a=n<0?1:0;if(a&&(n=-n),0===n)e(0,o,i+t),e(1/n>0?0:2147483648,o,i+r);else if(isNaN(n))e(0,o,i+t),e(2146959360,o,i+r);else if(n>17976931348623157e292)e(0,o,i+t),e((a<<31|2146435072)>>>0,o,i+r);else{var s;if(n<22250738585072014e-324)e((s=n/5e-324)>>>0,o,i+t),e((a<<31|s/4294967296)>>>0,o,i+r);else{var l=Math.floor(Math.log(n)/Math.LN2);1024===l&&(l=1023),e(4503599627370496*(s=n*Math.pow(2,-l))>>>0,o,i+t),e((a<<31|l+1023<<20|1048576*s&1048575)>>>0,o,i+r)}}}function a(e,t,r,n,o){var i=e(n,o+t),a=e(n,o+r),s=2*(a>>31)+1,l=a>>>20&2047,u=4294967296*(1048575&a)+i;return 2047===l?u?NaN:s*(1/0):0===l?5e-324*s*u:s*Math.pow(2,l-1075)*(u+4503599627370496)}e.writeDoubleLE=t.bind(null,r,0,4),e.writeDoubleBE=t.bind(null,n,4,0),e.readDoubleLE=a.bind(null,o,0,4),e.readDoubleBE=a.bind(null,i,4,0)}(),e}function r(e,t,r){t[r]=255&e,t[r+1]=e>>>8&255,t[r+2]=e>>>16&255,t[r+3]=e>>>24}function n(e,t,r){t[r]=e>>>24,t[r+1]=e>>>16&255,t[r+2]=e>>>8&255,t[r+3]=255&e}function o(e,t){return(e[t]|e[t+1]<<8|e[t+2]<<16|e[t+3]<<24)>>>0}function i(e,t){return(e[t]<<24|e[t+1]<<16|e[t+2]<<8|e[t+3])>>>0}e.exports=t(t)},"./node_modules/@protobufjs/inquire/index.js":function node_modulesProtobufjsInquireIndexJs(module){"use strict";function inquire(moduleName){try{var mod=eval("quire".replace(/^/,"re"))(moduleName);if(mod&&(mod.length||Object.keys(mod).length))return mod}catch(e){}return null}module.exports=inquire},"./node_modules/@protobufjs/pool/index.js":function(e){"use strict";e.exports=function(e,t,r){var n=r||8192,o=n>>>1,i=null,a=n;return function(r){if(r<1||r>o)return e(r);a+r>n&&(i=e(n),a=0);var s=t.call(i,a,a+=r);return 7&a&&(a=1+(7|a)),s}}},"./node_modules/@protobufjs/utf8/index.js":function(e,t){"use strict";var r=t;r.length=function(e){for(var t=0,r=0,n=0;n191&&n<224?i[a++]=(31&n)<<6|63&e[t++]:n>239&&n<365?(n=((7&n)<<18|(63&e[t++])<<12|(63&e[t++])<<6|63&e[t++])-65536,i[a++]=55296+(n>>10),i[a++]=56320+(1023&n)):i[a++]=(15&n)<<12|(63&e[t++])<<6|63&e[t++],a>8191&&((o||(o=[])).push(String.fromCharCode.apply(String,i)),a=0);return o?(a&&o.push(String.fromCharCode.apply(String,i.slice(0,a))),o.join("")):String.fromCharCode.apply(String,i.slice(0,a))},r.write=function(e,t,r){for(var n,o,i=r,a=0;a>6|192,t[r++]=63&n|128):55296===(64512&n)&&56320===(64512&(o=e.charCodeAt(a+1)))?(n=65536+((1023&n)<<10)+(1023&o),++a,t[r++]=n>>18|240,t[r++]=n>>12&63|128,t[r++]=n>>6&63|128,t[r++]=63&n|128):(t[r++]=n>>12|224,t[r++]=n>>6&63|128,t[r++]=63&n|128);return r-i}},"./node_modules/browser-headers/dist/browser-headers.umd.js":function(e){var t;t=function(){return function(e){var t={};function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}return r.m=e,r.c=t,r.i=function(e){return e},r.d=function(e,t,n){r.o(e,t)||Object.defineProperty(e,t,{configurable:!1,enumerable:!0,get:n})},r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,"a",t),t},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r.p="",r(r.s=1)}([function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(3),o=function(){function e(e,t){void 0===e&&(e={}),void 0===t&&(t={splitValues:!1});var r,o=this;this.headersMap={},e&&("undefined"!==typeof Headers&&e instanceof Headers?n.getHeaderKeys(e).forEach((function(r){n.getHeaderValues(e,r).forEach((function(e){t.splitValues?o.append(r,n.splitHeaderValue(e)):o.append(r,e)}))})):"object"===typeof(r=e)&&"object"===typeof r.headersMap&&"function"===typeof r.forEach?e.forEach((function(e,t){o.append(e,t)})):"undefined"!==typeof Map&&e instanceof Map?e.forEach((function(e,t){o.append(t,e)})):"string"===typeof e?this.appendFromString(e):"object"===typeof e&&Object.getOwnPropertyNames(e).forEach((function(t){var r=e[t];Array.isArray(r)?r.forEach((function(e){o.append(t,e)})):o.append(t,r)})))}return e.prototype.appendFromString=function(e){for(var t=e.split("\r\n"),r=0;r0){var i=n.substring(0,o).trim(),a=n.substring(o+1).trim();this.append(i,a)}}},e.prototype.delete=function(e,t){var r=n.normalizeName(e);if(void 0===t)delete this.headersMap[r];else{var o=this.headersMap[r];if(o){var i=o.indexOf(t);i>=0&&o.splice(i,1),0===o.length&&delete this.headersMap[r]}}},e.prototype.append=function(e,t){var r=this,o=n.normalizeName(e);Array.isArray(this.headersMap[o])||(this.headersMap[o]=[]),Array.isArray(t)?t.forEach((function(e){r.headersMap[o].push(n.normalizeValue(e))})):this.headersMap[o].push(n.normalizeValue(t))},e.prototype.set=function(e,t){var r=n.normalizeName(e);if(Array.isArray(t)){var o=[];t.forEach((function(e){o.push(n.normalizeValue(e))})),this.headersMap[r]=o}else this.headersMap[r]=[n.normalizeValue(t)]},e.prototype.has=function(e,t){var r=this.headersMap[n.normalizeName(e)];if(!Array.isArray(r))return!1;if(void 0!==t){var o=n.normalizeValue(t);return r.indexOf(o)>=0}return!0},e.prototype.get=function(e){var t=this.headersMap[n.normalizeName(e)];return void 0!==t?t.concat():[]},e.prototype.forEach=function(e){var t=this;Object.getOwnPropertyNames(this.headersMap).forEach((function(r){e(r,t.headersMap[r])}),this)},e.prototype.toHeaders=function(){if("undefined"!==typeof Headers){var e=new Headers;return this.forEach((function(t,r){r.forEach((function(r){e.append(t,r)}))})),e}throw new Error("Headers class is not defined")},e}();t.BrowserHeaders=o},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(0);t.BrowserHeaders=n.BrowserHeaders},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.iterateHeaders=function(e,t){for(var r=e[Symbol.iterator](),n=r.next();!n.done;)t(n.value[0]),n=r.next()},t.iterateHeadersKeys=function(e,t){for(var r=e.keys(),n=r.next();!n.done;)t(n.value),n=r.next()}},function(e,t,r){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=r(2);t.normalizeName=function(e){if("string"!==typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.\^_`|~]/i.test(e))throw new TypeError("Invalid character in header field name");return e.toLowerCase()},t.normalizeValue=function(e){return"string"!==typeof e&&(e=String(e)),e},t.getHeaderValues=function(e,t){var r=e;if(r instanceof Headers&&r.getAll)return r.getAll(t);var n=r.get(t);return n&&"string"===typeof n?[n]:n},t.getHeaderKeys=function(e){var t=e,r={},o=[];return t.keys?n.iterateHeadersKeys(t,(function(e){r[e]||(r[e]=!0,o.push(e))})):t.forEach?t.forEach((function(e,t){r[t]||(r[t]=!0,o.push(t))})):n.iterateHeaders(t,(function(e){var t=e[0];r[t]||(r[t]=!0,o.push(t))})),o},t.splitHeaderValue=function(e){var t=[];return e.split(", ").forEach((function(e){e.split(",").forEach((function(e){t.push(e)}))})),t}}])},e.exports=t()},"./node_modules/google-protobuf/google-protobuf.js":function node_modulesGoogleProtobufGoogleProtobufJs(__unused_webpack_module,exports,__nested_webpack_require_87310__){var $jscomp=$jscomp||{};$jscomp.scope={},$jscomp.findInternal=function(e,t,r){e instanceof String&&(e=String(e));for(var n=e.length,o=0;o=n}}),"es6","es3"),$jscomp.polyfill("Array.prototype.find",(function(e){return e||function(e,t){return $jscomp.findInternal(this,e,t).v}}),"es6","es3"),$jscomp.polyfill("String.prototype.startsWith",(function(e){return e||function(e,t){var r=$jscomp.checkStringArgs(this,e,"startsWith");e+="";var n=r.length,o=e.length;t=Math.max(0,Math.min(0|t,r.length));for(var i=0;i=o}}),"es6","es3"),$jscomp.polyfill("String.prototype.repeat",(function(e){return e||function(e){var t=$jscomp.checkStringArgs(this,null,"repeat");if(0>e||1342177279>>=1)&&(t+=t);return r}}),"es6","es3");var COMPILED=!0,goog=goog||{};goog.global=this||self,goog.isDef=function(e){return void 0!==e},goog.isString=function(e){return"string"==typeof e},goog.isBoolean=function(e){return"boolean"==typeof e},goog.isNumber=function(e){return"number"==typeof e},goog.exportPath_=function(e,t,r){e=e.split("."),r=r||goog.global,e[0]in r||"undefined"==typeof r.execScript||r.execScript("var "+e[0]);for(var n;e.length&&(n=e.shift());)!e.length&&goog.isDef(t)?r[n]=t:r=r[n]&&r[n]!==Object.prototype[n]?r[n]:r[n]={}},goog.define=function(e,t){if(!COMPILED){var r=goog.global.CLOSURE_UNCOMPILED_DEFINES,n=goog.global.CLOSURE_DEFINES;r&&void 0===r.nodeType&&Object.prototype.hasOwnProperty.call(r,e)?t=r[e]:n&&void 0===n.nodeType&&Object.prototype.hasOwnProperty.call(n,e)&&(t=n[e])}return t},goog.FEATURESET_YEAR=2012,goog.DEBUG=!0,goog.LOCALE="en",goog.TRUSTED_SITE=!0,goog.STRICT_MODE_COMPATIBLE=!1,goog.DISALLOW_TEST_ONLY_CODE=COMPILED&&!goog.DEBUG,goog.ENABLE_CHROME_APP_SAFE_SCRIPT_LOADING=!1,goog.provide=function(e){if(goog.isInModuleLoader_())throw Error("goog.provide cannot be used within a module.");if(!COMPILED&&goog.isProvided_(e))throw Error('Namespace "'+e+'" already declared.');goog.constructNamespace_(e)},goog.constructNamespace_=function(e,t){if(!COMPILED){delete goog.implicitNamespaces_[e];for(var r=e;(r=r.substring(0,r.lastIndexOf(".")))&&!goog.getObjectByName(r);)goog.implicitNamespaces_[r]=!0}goog.exportPath_(e,t)},goog.getScriptNonce=function(e){return e&&e!=goog.global?goog.getScriptNonce_(e.document):(null===goog.cspNonce_&&(goog.cspNonce_=goog.getScriptNonce_(goog.global.document)),goog.cspNonce_)},goog.NONCE_PATTERN_=/^[\w+/_-]+[=]{0,2}$/,goog.cspNonce_=null,goog.getScriptNonce_=function(e){return(e=e.querySelector&&e.querySelector("script[nonce]"))&&(e=e.nonce||e.getAttribute("nonce"))&&goog.NONCE_PATTERN_.test(e)?e:""},goog.VALID_MODULE_RE_=/^[a-zA-Z_$][a-zA-Z0-9._$]*$/,goog.module=function(e){if(!goog.isString(e)||!e||-1==e.search(goog.VALID_MODULE_RE_))throw Error("Invalid module identifier");if(!goog.isInGoogModuleLoader_())throw Error("Module "+e+" has been loaded incorrectly. Note, modules cannot be loaded as normal scripts. They require some kind of pre-processing step. You're likely trying to load a module via a script tag or as a part of a concatenated bundle without rewriting the module. For more info see: https://github.com/google/closure-library/wiki/goog.module:-an-ES6-module-like-alternative-to-goog.provide.");if(goog.moduleLoaderState_.moduleName)throw Error("goog.module may only be called once per module.");if(goog.moduleLoaderState_.moduleName=e,!COMPILED){if(goog.isProvided_(e))throw Error('Namespace "'+e+'" already declared.');delete goog.implicitNamespaces_[e]}},goog.module.get=function(e){return goog.module.getInternal_(e)},goog.module.getInternal_=function(e){if(!COMPILED){if(e in goog.loadedModules_)return goog.loadedModules_[e].exports;if(!goog.implicitNamespaces_[e])return null!=(e=goog.getObjectByName(e))?e:null}return null},goog.ModuleType={ES6:"es6",GOOG:"goog"},goog.moduleLoaderState_=null,goog.isInModuleLoader_=function(){return goog.isInGoogModuleLoader_()||goog.isInEs6ModuleLoader_()},goog.isInGoogModuleLoader_=function(){return!!goog.moduleLoaderState_&&goog.moduleLoaderState_.type==goog.ModuleType.GOOG},goog.isInEs6ModuleLoader_=function(){if(goog.moduleLoaderState_&&goog.moduleLoaderState_.type==goog.ModuleType.ES6)return!0;var e=goog.global.$jscomp;return!!e&&("function"==typeof e.getCurrentModulePath&&!!e.getCurrentModulePath())},goog.module.declareLegacyNamespace=function(){if(!COMPILED&&!goog.isInGoogModuleLoader_())throw Error("goog.module.declareLegacyNamespace must be called from within a goog.module");if(!COMPILED&&!goog.moduleLoaderState_.moduleName)throw Error("goog.module must be called prior to goog.module.declareLegacyNamespace.");goog.moduleLoaderState_.declareLegacyNamespace=!0},goog.declareModuleId=function(e){if(!COMPILED){if(!goog.isInEs6ModuleLoader_())throw Error("goog.declareModuleId may only be called from within an ES6 module");if(goog.moduleLoaderState_&&goog.moduleLoaderState_.moduleName)throw Error("goog.declareModuleId may only be called once per module.");if(e in goog.loadedModules_)throw Error('Module with namespace "'+e+'" already exists.')}if(goog.moduleLoaderState_)goog.moduleLoaderState_.moduleName=e;else{var t=goog.global.$jscomp;if(!t||"function"!=typeof t.getCurrentModulePath)throw Error('Module with namespace "'+e+'" has been loaded incorrectly.');t=t.require(t.getCurrentModulePath()),goog.loadedModules_[e]={exports:t,type:goog.ModuleType.ES6,moduleId:e}}},goog.setTestOnly=function(e){if(goog.DISALLOW_TEST_ONLY_CODE)throw e=e||"",Error("Importing test-only code into non-debug environment"+(e?": "+e:"."))},goog.forwardDeclare=function(e){},COMPILED||(goog.isProvided_=function(e){return e in goog.loadedModules_||!goog.implicitNamespaces_[e]&&goog.isDefAndNotNull(goog.getObjectByName(e))},goog.implicitNamespaces_={"goog.module":!0}),goog.getObjectByName=function(e,t){e=e.split("."),t=t||goog.global;for(var r=0;r>>0),goog.uidCounter_=0,goog.getHashCode=goog.getUid,goog.removeHashCode=goog.removeUid,goog.cloneObject=function(e){var t=goog.typeOf(e);if("object"==t||"array"==t){if("function"===typeof e.clone)return e.clone();for(var r in t="array"==t?[]:{},e)t[r]=goog.cloneObject(e[r]);return t}return e},goog.bindNative_=function(e,t,r){return e.call.apply(e.bind,arguments)},goog.bindJs_=function(e,t,r){if(!e)throw Error();if(2{"use strict";class X{constructor(){if(new.target!=String)throw 1;this.x=42}}let q=Reflect.construct(X,[],String);if(q.x!=42||!(q instanceof String))throw 1;for(const a of[2,3]){if(a==2)continue;function f(z={a}){let a=0;return z.a}{function f(){return 0;}}return f()==3}})()')})),a("es7",(function(){return b("2 ** 2 == 4")})),a("es8",(function(){return b("async () => 1, true")})),a("es9",(function(){return b("({...rest} = {}), true")})),a("es_next",(function(){return!1})),{target:c,map:d}},goog.Transpiler.prototype.needsTranspile=function(e,t){if("always"==goog.TRANSPILE)return!0;if("never"==goog.TRANSPILE)return!1;if(!this.requiresTranspilation_){var r=this.createRequiresTranspilation_();this.requiresTranspilation_=r.map,this.transpilationTarget_=this.transpilationTarget_||r.target}if(e in this.requiresTranspilation_)return!!this.requiresTranspilation_[e]||!(!goog.inHtmlDocument_()||"es6"!=t||"noModule"in goog.global.document.createElement("script"));throw Error("Unknown language mode: "+e)},goog.Transpiler.prototype.transpile=function(e,t){return goog.transpile_(e,t,this.transpilationTarget_)},goog.transpiler_=new goog.Transpiler,goog.protectScriptTag_=function(e){return e.replace(/<\/(SCRIPT)/gi,"\\x3c/$1")},goog.DebugLoader_=function(){this.dependencies_={},this.idToPath_={},this.written_={},this.loadingDeps_=[],this.depsToLoad_=[],this.paused_=!1,this.factory_=new goog.DependencyFactory(goog.transpiler_),this.deferredCallbacks_={},this.deferredQueue_=[]},goog.DebugLoader_.prototype.bootstrap=function(e,t){function r(){n&&(goog.global.setTimeout(n,0),n=null)}var n=t;if(e.length){t=[];for(var o=0;o<\/script>",t.write(goog.TRUSTED_TYPES_POLICY_?goog.TRUSTED_TYPES_POLICY_.createHTML(n):n)}else{var o=t.createElement("script");o.defer=goog.Dependency.defer_,o.async=!1,o.type="text/javascript",(n=goog.getScriptNonce())&&o.setAttribute("nonce",n),goog.DebugLoader_.IS_OLD_IE_?(e.pause(),o.onreadystatechange=function(){"loaded"!=o.readyState&&"complete"!=o.readyState||(e.loaded(),e.resume())}):o.onload=function(){o.onload=null,e.loaded()},o.src=goog.TRUSTED_TYPES_POLICY_?goog.TRUSTED_TYPES_POLICY_.createScriptURL(this.path):this.path,t.head.appendChild(o)}}else goog.logToConsole_("Cannot use default debug loader outside of HTML documents."),"deps.js"==this.relativePath?(goog.logToConsole_("Consider setting CLOSURE_IMPORT_SCRIPT before loading base.js, or setting CLOSURE_NO_DEPS to true."),e.loaded()):e.pause()},goog.Es6ModuleDependency=function(e,t,r,n,o){goog.Dependency.call(this,e,t,r,n,o)},goog.inherits(goog.Es6ModuleDependency,goog.Dependency),goog.Es6ModuleDependency.prototype.load=function(e){if(goog.global.CLOSURE_IMPORT_SCRIPT)goog.global.CLOSURE_IMPORT_SCRIPT(this.path)?e.loaded():e.pause();else if(goog.inHtmlDocument_()){var t=goog.global.document,r=this;if(goog.isDocumentLoading_()){var n=function(e,r){e=r?'