基于深度强化学习的股票交易策略框架（代码+文档）

# Install the unstable development version in Jupyter notebook:
!pip install git+https://github.com/AI4Finance-LLC/FinRL-Library.git

import pkg_resources
import pip
installedPackages = {pkg.key for pkg in pkg_resources.working_set}
required = {'yfinance', 'pandas', 'matplotlib', 'stockstats','stable-baselines','gym','tensorflow'}
missing = required - installedPackages
if missing:
    !pip install yfinance
    !pip install pandas
    !pip install matplotlib
    !pip install stockstats
    !pip install gym
    !pip install stable-baselines[mpi]
    !pip install tensorflow==1.15.4

# import packages
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use('Agg')
import datetime
from finrl.config import config
from finrl.marketdata.yahoodownloader import YahooDownloader
from finrl.preprocessing.preprocessors import FeatureEngineer
from finrl.preprocessing.data import data_split
from finrl.env.environment import EnvSetup
from finrl.env.EnvMultipleStock_train import StockEnvTrain
from finrl.env.EnvMultipleStock_trade import StockEnvTrade
from finrl.model.models import DRLAgent
from finrl.trade.backtest import BackTestStats, BaselineStats, BackTestPlot

class YahooDownloader:
    """Provides methods for retrieving daily stock data from
    Yahoo Finance API
Attributes
    ----------
        start_date : str
            start date of the data (modified from config.py)
        end_date : str
            end date of the data (modified from config.py)
        ticker_list : list
            a list of stock tickers (modified from config.py)
Methods
    -------
    fetch_data()
        Fetches data from yahoo API
"""

data_df = YahooDownloader(start_date = '2009-01-01', 
                          end_date = '2020-09-30', 
                          ticker_list = ['AAPL']).fetch_data()
view raw

class FeatureEngineer:
    """Provides methods for preprocessing the stock price data
Attributes
    ----------
        df: DataFrame
            data downloaded from Yahoo API
        feature_number : int
            number of features we used
        use_technical_indicator : boolean
            we technical indicator or not
        use_turbulence : boolean
            use turbulence index or not
Methods
    -------
    preprocess_data()
        main method to do the feature engineering
"""

data_df = FeatureEngineer(data_df.copy(),
                          use_technical_indicator=True,
                          tech_indicator_list = 
                          tech_indicator_list,use_turbulence=False,
                          user_defined_feature = True).preprocess_data()

class EnvSetup:
    """Provides methods for retrieving daily stock data from
    Yahoo Finance API
Attributes
    ----------
    stock_dim: int
        number of unique stocks
    hmax : int
        maximum number of shares to trade
    initial_amount: int
        start money
    transaction_cost_pct : float
        transaction cost percentage per trade
    reward_scaling: float
        scaling factor for reward, good for training
    tech_indicator_list: list
        a list of technical indicator names (modified from config.py)
Methods
    -------
    fetch_data()
        Fetches data from yahoo API
"""

env_setup = EnvSetup(stock_dim = stock_dimension,
                     state_space = state_space,
                     hmax = 200,
                     initial_amount = 100000,
                     transaction_cost_pct = 0.001,
                     tech_indicator_list = tech_indicator_list)
env_train = env_setup.create_env_training(data = train, 
                                         env_class = SingleStockEnv)

class SingleStockEnv(gym.Env):
    """A single stock trading environment for OpenAI gym
    Attributes
    ----------
        df: DataFrame
            input data
        stock_dim : int
            number of unique stocks
        hmax : int
            maximum number of shares to trade
        initial_amount : int
            start money
        transaction_cost_pct: float
            transaction cost percentage per trade
        reward_scaling: float
            scaling factor for reward, good for training
        state_space: int
            the dimension of input features
        action_space: int
            equals stock dimension
        tech_indicator_list: list
            a list of technical indicator names
        turbulence_threshold: int
            a threshold to control risk aversion
        day: int
            an increment number to control date
    Methods
    -------
    _sell_stock()
        perform sell action based on the sign of the action
    _buy_stock()
        perform buy action based on the sign of the action
    step()
        at each step the agent will return actions, then 
        we will calculate the reward, and return the next 
        observation.
    reset()
        reset the environment
    render()
        use render to return other functions
    save_asset_memory()
        return account value at each time step
    save_action_memory()
        return actions/positions at each time step
"""

class DRLAgent:
    """Provides implementations for DRL algorithms
Attributes
    ----------
    env: gym environment class
         user-defined class
Methods
    -------
    train_PPO()
        the implementation for PPO algorithm
    train_A2C()
        the implementation for A2C algorithm
    train_DDPG()
        the implementation for DDPG algorithm
    train_TD3()
        the implementation for TD3 algorithm 
    DRL_prediction() 
        make a prediction in a test dataset and get results
    """

print("==============Model Training===========")
now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')
td3_params_tuning = {'batch_size': 128,
                    'buffer_size':50000,
                    'learning_rate': 0.0001,
                    'verbose':0,
                    'timesteps':20000}
                    
agent = DRLAgent(env = env_train)

model_td3 = agent.train_TD3(model_name = "TD3_{}".format(now), 
                            model_params = td3_params_tuning)

# cd to the tensorboard_log folder, run the following command 
tensorboard --logdir ./A2C_20201127-19h01/
# you can also add past logging folder
tensorboard --logdir ./a2c_tensorboard/;./ppo2_tensorboard/

env_trade, obs_trade = env_setup.create_env_trading(data = trade,
                                       env_class = SingleStockEnv)
## make a prediction and get the account value change
df_account_value = DRLAgent.DRL_prediction(model=model_td3,
                                           test_data = trade,
                                           test_env = env_trade,
                                           test_obs = obs_trade)

print("==============Get Backtest Results===========")
perf_stats_all = BackTestStats(account_value = df_account_value)
perf_stats_all = pd.DataFrame(perf_stats_all)
perf_stats_all.to_csv("./"+config.RESULTS_DIR+"/perf_stats_all_"+now+'.csv')


# BackTestPlot
# pass the account value memory into the backtest functions
# and select a baseline ticker
print("=====Compare to AAPL itself buy-and-hold=======")
%matplotlib inline
BackTestPlot(account_value = df_account_value, 
             baseline_ticker = 'AAPL')