以图片为目标的视觉强化学习 -- 代码解析上篇(rlkit安装及部分代码及函数的解析和使用)

import os
from os.path import join
import rlkit
# 设定项目目录

rlkit_project_dir = join(os.path.dirname(rlkit.__file__), os.pardir)
LOCAL_LOG_DIR = join(rlkit_project_dir, 'data')


# 接下来设定大部分与远程服务器相关


# 代码目录，可能不止一个，不止一个就添加在后面
CODE_DIRS_TO_MOUNT = [
    rlkit_project_dir,
    # '/home/user/python/module/one', Add more paths as needed
]
# 工作需要用到mujoco模拟器，如果你mujoco安装的位置特殊，这里可能要改
DIR_AND_MOUNT_POINT_MAPPINGS = [
    dict(
        local_dir=join(os.getenv('HOME'), '.mujoco/'),
        mount_point='/root/.mujoco',
    ),
]
# 跑程序的代码地址
RUN_DOODAD_EXPERIMENT_SCRIPT_PATH = (
    join(rlkit_project_dir, 'scripts', 'run_experiment_from_doodad.py')
    # '/home/user/path/to/rlkit/scripts/run_experiment_from_doodad.py'
)


# 接下来是AWS，SLURM，和GCP的一些设定，
# 我目前都没用到，暂时不提，用到的小伙伴可以看一下

# If not set, default will be chosen by doodad
#AWS_S3_PATH = 's3://bucket/directory

# If not set, default will be chosen by doodad
AWS_S3_PATH = 's3://bucket/directory'

conda env create -f environment/linux-gpu-env.yml

name: rlkit
channels:
- kne  # for pybox2d
- pytorch
- anaconda  # for mkl
dependencies:
- cython
- ipython  # technically unnecessary

mkdir -p ~/.mujoco
cd ~/.mujoco
# 到mujoco官网->download页面->选2.1.0版本->下载linux版
wget https://github.com/deepmind/mujoco/releases/download/2.1.0/mujoco210-linux-x86_64.tar.gz
tar -xvf mujoco210-linux-x86_64.tar.gz
echo 'export LD_LIBRARY_PATH=$HOME/.mujoco/mujoco210/bin:$LD_LIBRARY_PATH' >> ~/.bashrc
source ~/.bashrc
# 以上mujoco就下载安置好了
# 接下来安装mujoco-py
git clone https://github.com/openai/mujoco-py.git
cd mujoco-py
pip3 install -r requirements.txt
pip3 install -r requirements.dev.txt
python3 setup.py install
#----------------------------------------
# 接下来是一些你可能import mujoco_py失败，甚至上面就没装成功，而需要进行的步骤
# 按错误提示选择性执行
sudo apt-get install gcc
sudo apt install libosmesa6-dev
sudo apt-get install python3-devel
sudo apt-get update -y
sudo apt-get install -y patchelf
sudo apt-get install libglew-dev
echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/nvidia' >> ~/.bashrc
echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.local/bin' >> ~/.bashrc
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libGLEW.so' >> ~/.bashrc
source ~/.bashrc

import rlkit.torch.pytorch_util as ptu
ptu.set_gpu_mode(True)

run_experiment(..., use_gpu=True)

from multiworld.envs.pygame.point2d import Point2DWallEnv
from rlkit.launchers.launcher_util import run_experiment
from rlkit.launchers.rig_experiments import grill_her_td3_full_experiment

git clone https://github.com/vitchyr/multiworld.git
cd multiworld
python setup.py install
pip install gym
pip install pygame
pip install boto3
pip install gtimer
pip install sk-video
pip install gitpython

    variant = dict(
        imsize=84,
        .......
        algorithm='RIG',
    )

    run_experiment(
        grill_her_td3_full_experiment, #rig算法实验的函数，以variant为参数
        exp_prefix='rlkit-pointmass-rig-example', #实验名称
        # 实验方式：默认是local，我们这里没装doodad。
        # 单机用here_no_doodad可以跑，也可以调用GPU
        # 其他方式暂时不研究了，有兴趣的可以研究
        # “local”，
        # “docker_local”，
        # “ec2”，
        # 'here_no_doodad'， 
        mode='here_no_doodad', 
        variant=variant, # 参数
        use_gpu=True,  # 要用GPU跑就打开
    )

    run_experiment_kwargs = dict(
        # 实验名称，默认会带上时间
        exp_prefix=exp_prefix,
        # 参数
        variant=variant,
        # 实验ID
        exp_id=exp_id,
        # 随机种子，不设定就随机产生
        seed=seed,
        use_gpu=use_gpu,
        # 默认是‘last’,作用之后说
        snapshot_mode=snapshot_mode,
        # 默认是1，作用之后说
        snapshot_gap=snapshot_gap,
        # 没装git就是None,暂时就None吧
        git_infos=git_infos,
        #这里应该是rig.py 
        script_name=main.__file__,
    )
    if mode == 'here_no_doodad':
        run_experiment_kwargs['base_log_dir'] = base_log_dir
        return run_experiment_here(
            method_call,
            **run_experiment_kwargs
        )

    # 这个函数里只有一行代码：
    # logger.reset()
    # logger也是他们自己开发的，reset就是回归初始状态
    reset_execution_environment()
    # 这里做的事情包括：创建log目录，将variant参数记录下来，
    # 以及添加各式各样的路径和参数
    actual_log_dir = setup_logger(
        exp_prefix=exp_prefix,
        variant=variant,
        exp_id=exp_id,
        seed=seed,
        snapshot_mode=snapshot_mode,
        snapshot_gap=snapshot_gap,
        base_log_dir=base_log_dir,
        log_dir=log_dir,
        git_infos=git_infos,
        script_name=script_name,
        **setup_logger_kwargs
    )
    # 启用seed
    set_seed(seed)
    # 启用GPU，默认用id为0的GPU
    # 如果想选择用其他gpu，可以set_gpu_mode(use_gpu,gpu_id)
    set_gpu_mode(use_gpu)
    
    run_experiment_here_kwargs = dict(
        variant=variant,
        exp_id=exp_id,
        seed=seed,
        use_gpu=use_gpu,
        exp_prefix=exp_prefix,
        snapshot_mode=snapshot_mode,
        snapshot_gap=snapshot_gap,
        git_infos=git_infos,
        script_name=script_name,
        base_log_dir=base_log_dir,
        **setup_logger_kwargs
    )
    # 把参数信息存成actual_log_dir下面的experiment.pkl文件
    save_experiment_data(
        dict(
            run_experiment_here_kwargs=run_experiment_here_kwargs
        ),
        actual_log_dir
    )
    # 好了，终于到我们的算法函数了。
    # 其实是这样的 grill_her_td3_full_experiment(variant)
    return experiment_function(variant)

def grill_her_td3_full_experiment(variant):
    # 看名字就知道功能了，预处理
    full_experiment_variant_preprocess(variant)
    # 训练vae，一边训练还会一边自己调参哦
    train_vae_and_update_variant(variant)
    # 这个看了再说。
    grill_her_td3_experiment(variant['grill_variant'])