Back to Community
DollarNeutral not working?

Hey,

I built a simple trading algorithm, that uses Linear Regression to predict future returns of an equity.
I am using the optimiziation API to do the trading and added the DollarNeutral Constraint, however, no matter how low I set the tolerance, the Net Dollar Exposure peaks at over 10% during the Backtest (see attached). What am I missing?

Clone Algorithm
2
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
from sklearn.linear_model import LinearRegression, ridge_regression
import pandas as pd
import numpy as np
import time
from sklearn import svm
from sklearn.ensemble import GradientBoostingRegressor

from quantopian.pipeline.filters import QTradableStocksUS
from quantopian.pipeline.experimental import risk_loading_pipeline  
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import SimpleMovingAverage, Returns, CustomFactor
from quantopian.pipeline.data import USEquityPricing
import quantopian.algorithm as algo
import quantopian.optimize as opt
import sklearn

def initialize(context):
    context.optimize_portfolio=True
    context.trained = False
    context.max_leverage = 1.0
    context.max_pos_size = 0.015
    context.max_turnover = 0.65
    context.train_days_count = 2000
    log.info(context.train_days_count)
    context.input_fields = ['close','open','high','low','volume','sma_10','sma_30']
    context.model = LinearRegression()
    
    #ridge_regression()#svm.SVR()#GradientBoostingRegressor()
    #context.model = tree.DecisionTreeRegressor    
    log.info(context.model)
    context.asset = sid(24)
    algo.attach_pipeline(
        make_pipeline(),
        'data_pipe'
    )
    if context.optimize_portfolio:
        algo.attach_pipeline(risk_loading_pipeline(), 'risk_loading_pipeline')
    algo.schedule_function(
        rebalance,
        date_rules.every_day(),#week_start(),
        time_rule=time_rules.market_open(hours=1,minutes=45)
    )

        
def train_model_once(context,data):
    if not context.trained:
        pipe_data = context.train_data
        X = pipe_data[context.input_fields].values
        y = np.array(pipe_data['returns'].values).reshape((-1,))
        log.info("Now training!")
        context.model.fit(X,y)
        print(context.model.score(X,y))
        pipe_data, context.train_data = "0", "0"
        context.trained = True
    
def before_trading_start(context, data):
    context.pipeline_data = algo.pipeline_output('data_pipe')
    if context.optimize_portfolio:
        context.risk_loading_pipeline = algo.pipeline_output('risk_loading_pipeline')
    if not context.trained:
        context.assets = context.pipeline_data.index.values.tolist()
        context.train_data =  get_historical_pipeline_data(context,data).dropna()
        train_model_once(context,data)

def get_historical_pipeline_data(context,data):
    all_stock_results = []
    log.info(len(context.assets))
    all_stock_data = data.history(context.assets, 
                                  fields=["price","open","high","low","volume"], 
                                  bar_count=context.train_days_count, frequency="1d")
    for asset in context.assets:
        try:
            hist_data = all_stock_data.minor_xs(asset)
            hist_data["close"] = hist_data["price"]
            sma_10 = pd.DataFrame(hist_data["price"].rolling(10).mean())
            sma_10["sma_10"] = sma_10['price']
            sma_10 = sma_10.drop(["price"],axis=1)
            sma_30 = pd.DataFrame(hist_data["price"].rolling(30).mean())
            sma_30["sma_30"] = sma_30["price"]
            sma_30 = sma_30.drop(["price"],axis=1)
            pcchange = pd.DataFrame(hist_data["price"].pct_change(periods=2)).shift(-2)
            pcchange["returns"] = pcchange["price"]
            pcchange = pcchange.drop(["price"],axis=1)
            hist_data = hist_data.drop(["price"],axis=1)
            res = pd.concat([hist_data, sma_10, sma_30,pcchange], axis=1)
            asset_col = []
            for x in range(context.train_days_count):
                asset_col.append(asset)
            res = res.assign(asset=asset_col)
            all_stock_results.append(res)
        except TypeError as e:
            log.error(e)
            log.error(asset, "This Stock might not have sufficient historical data, to be suitable for training.")
            
    pipe_data = pd.concat(all_stock_results)
    pipe_data["date"] = pipe_data.index
    pipe_data = pipe_data.set_index('asset')
    return pipe_data


def make_pipeline():
    yesterday_close = USEquityPricing.close.latest
    yesterday_open = USEquityPricing.open.latest
    yesterday_high = USEquityPricing.high.latest
    yesterday_low = USEquityPricing.low.latest
    yesterday_volume = USEquityPricing.volume.latest
    sma_10 = SimpleMovingAverage(inputs=[USEquityPricing.close], window_length=10)
    sma_30 = SimpleMovingAverage(inputs=[USEquityPricing.close], window_length=30)
    returns = Returns(window_length=2)   
    pipe = Pipeline(
        columns={
            'close': yesterday_close,
            'open':yesterday_open,
            'high': yesterday_high,
            'low': yesterday_low,
            'volume': yesterday_volume,
            'sma_10':sma_10,
            'sma_30':sma_30,
            'returns':returns
        },
        screen = QTradableStocksUS()
    )
    return pipe


def rebalance(context, data):
    X = context.pipeline_data[context.input_fields]
    print(X.shape)
    X = X.dropna().values
    print(len(X))
    y = context.model.predict(X)
    alphas = pd.Series(data=y,index=context.pipeline_data.index.values)#.sort_values(ascending=True)

    objective = opt.MaximizeAlpha(alphas)
    
    sector_style_risk = opt.experimental.RiskModelExposure(risk_model_loadings=context.risk_loading_pipeline)
    max_leverage = opt.MaxGrossExposure(context.max_leverage)
    constrain_pos_size = opt.PositionConcentration.with_equal_bounds(
        -context.max_pos_size,
        context.max_pos_size
    )
    dollar_neutral = opt.DollarNeutral(tolerance=0.000001)
    max_turnover = opt.MaxTurnover(context.max_turnover)
    
    constraints = [max_turnover,max_leverage,sector_style_risk,dollar_neutral]
    
    algo.order_optimal_portfolio(objective,constraints)
There was a runtime error.
3 responses

I would guess that because of the other constraints it is impossible for the optimizer to keep the algorithm exactly dollar neutral

The optimize method is working, and does calculate weights, which meet the constraints -including the DollarNeutral constraint. However, the trades aren't being executed. Looking at the logs, there are a lot of unfilled orders at the end of each day. Many of the stocks being traded have low trading volume and the slippage model is limiting the shares which can be ordered.

The easy way to check this is to lower the initial portfolio balance from $10M to $100k and run a backtest. Check the logs and one won't see all the 'unfilled order' messages. The backtest also now meets the 'Net Dollar Exposure' criteria.

One technique I like in debugging is to use calculate_optimal_portfolio. This way one can record the output of of the optimizer. Something like this

    objective = opt.MaximizeAlpha(alphas)  
    constraints = [max_turnover,max_leverage,sector_style_risk,dollar_neutral]

    opt_weights = opt.calculate_optimal_portfolio(objective,constraints)

    # Record the output of the optimizer to see whats going on  
    gross_exposure = opt_weights.abs().sum()  
    net_exposure = opt_weights.sum()  
    record(gross=gross_exposure, net=net_exposure)

    # Order using the calculated opt_weights  
    algo.order_optimal_portfolio(opt.TargetWeights(opt_weights),[])

The attached algo has this change. In debugging, one can look at the gross exposure and the net exposure calculated by the optimizer. The values are 1 and 0 throughout the backtest. This is what's expected. Since the target weights should result in a 'Net Dollar Exposure' close to zero, there must be an issue with the execution.

Hope that helps. Good algo.

Clone Algorithm
4
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
from sklearn.linear_model import LinearRegression, ridge_regression
import pandas as pd
import numpy as np
import time
from sklearn import svm
from sklearn.ensemble import GradientBoostingRegressor

from quantopian.pipeline.filters import QTradableStocksUS
from quantopian.pipeline.experimental import risk_loading_pipeline  
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import SimpleMovingAverage, Returns, CustomFactor
from quantopian.pipeline.data import USEquityPricing
import quantopian.algorithm as algo
import quantopian.optimize as opt
import sklearn

def initialize(context):
    context.optimize_portfolio=True
    context.trained = False
    context.max_leverage = 1.0
    context.max_pos_size = 0.015
    context.max_turnover = 0.65
    context.train_days_count = 2000
    log.info(context.train_days_count)
    context.input_fields = ['close','open','high','low','volume','sma_10','sma_30']
    context.model = LinearRegression()
    
    #ridge_regression()#svm.SVR()#GradientBoostingRegressor()
    #context.model = tree.DecisionTreeRegressor    
    log.info(context.model)
    context.asset = sid(24)
    algo.attach_pipeline(
        make_pipeline(),
        'data_pipe'
    )
    if context.optimize_portfolio:
        algo.attach_pipeline(risk_loading_pipeline(), 'risk_loading_pipeline')
    algo.schedule_function(
        rebalance,
        date_rules.every_day(),#week_start(),
        time_rule=time_rules.market_open(hours=1,minutes=45)
    )

        
def train_model_once(context,data):
    if not context.trained:
        pipe_data = context.train_data
        X = pipe_data[context.input_fields].values
        y = np.array(pipe_data['returns'].values).reshape((-1,))
        log.info("Now training!")
        context.model.fit(X,y)
        print(context.model.score(X,y))
        pipe_data, context.train_data = "0", "0"
        context.trained = True
    
def before_trading_start(context, data):
    context.pipeline_data = algo.pipeline_output('data_pipe')
    if context.optimize_portfolio:
        context.risk_loading_pipeline = algo.pipeline_output('risk_loading_pipeline')
    if not context.trained:
        context.assets = context.pipeline_data.index.values.tolist()
        context.train_data =  get_historical_pipeline_data(context,data).dropna()
        train_model_once(context,data)

def get_historical_pipeline_data(context,data):
    all_stock_results = []
    log.info(len(context.assets))
    all_stock_data = data.history(context.assets, 
                                  fields=["price","open","high","low","volume"], 
                                  bar_count=context.train_days_count, frequency="1d")
    for asset in context.assets:
        try:
            hist_data = all_stock_data.minor_xs(asset)
            hist_data["close"] = hist_data["price"]
            sma_10 = pd.DataFrame(hist_data["price"].rolling(10).mean())
            sma_10["sma_10"] = sma_10['price']
            sma_10 = sma_10.drop(["price"],axis=1)
            sma_30 = pd.DataFrame(hist_data["price"].rolling(30).mean())
            sma_30["sma_30"] = sma_30["price"]
            sma_30 = sma_30.drop(["price"],axis=1)
            pcchange = pd.DataFrame(hist_data["price"].pct_change(periods=2)).shift(-2)
            pcchange["returns"] = pcchange["price"]
            pcchange = pcchange.drop(["price"],axis=1)
            hist_data = hist_data.drop(["price"],axis=1)
            res = pd.concat([hist_data, sma_10, sma_30,pcchange], axis=1)
            asset_col = []
            for x in range(context.train_days_count):
                asset_col.append(asset)
            res = res.assign(asset=asset_col)
            all_stock_results.append(res)
        except TypeError as e:
            log.error(e)
            log.error(asset, "This Stock might not have sufficient historical data, to be suitable for training.")
            
    pipe_data = pd.concat(all_stock_results)
    pipe_data["date"] = pipe_data.index
    pipe_data = pipe_data.set_index('asset')
    return pipe_data


def make_pipeline():
    yesterday_close = USEquityPricing.close.latest
    yesterday_open = USEquityPricing.open.latest
    yesterday_high = USEquityPricing.high.latest
    yesterday_low = USEquityPricing.low.latest
    yesterday_volume = USEquityPricing.volume.latest
    sma_10 = SimpleMovingAverage(inputs=[USEquityPricing.close], window_length=10)
    sma_30 = SimpleMovingAverage(inputs=[USEquityPricing.close], window_length=30)
    returns = Returns(window_length=2)   
    pipe = Pipeline(
        columns={
            'close': yesterday_close,
            'open':yesterday_open,
            'high': yesterday_high,
            'low': yesterday_low,
            'volume': yesterday_volume,
            'sma_10':sma_10,
            'sma_30':sma_30,
            'returns':returns
        },
        screen = QTradableStocksUS()
    )
    return pipe


def rebalance(context, data):
    X = context.pipeline_data[context.input_fields]
    print(X.shape)
    X = X.dropna().values
    print(len(X))
    y = context.model.predict(X)
    alphas = pd.Series(data=y,index=context.pipeline_data.index.values)#.sort_values(ascending=True)

    objective = opt.MaximizeAlpha(alphas)
    
    sector_style_risk = opt.experimental.RiskModelExposure(risk_model_loadings=context.risk_loading_pipeline)
    max_leverage = opt.MaxGrossExposure(context.max_leverage)
    constrain_pos_size = opt.PositionConcentration.with_equal_bounds(
        -context.max_pos_size,
        context.max_pos_size
    )
    dollar_neutral = opt.DollarNeutral(tolerance=0.000001)
    max_turnover = opt.MaxTurnover(context.max_turnover)
    
    constraints = [max_turnover,max_leverage,sector_style_risk,dollar_neutral]
    
    opt_weights = opt.calculate_optimal_portfolio(objective,constraints)
    
    # Record the output of the optimizer to see whats going on
    gross_exposure = opt_weights.abs().sum()
    net_exposure = opt_weights.sum()
    
    record(gross=gross_exposure, net=net_exposure)
    
    # Order using the calculated opt_weights 
    algo.order_optimal_portfolio(opt.TargetWeights(opt_weights),[])
There was a runtime error.
Disclaimer

The material on this website is provided for informational purposes only and does not constitute an offer to sell, a solicitation to buy, or a recommendation or endorsement for any security or strategy, nor does it constitute an offer to provide investment advisory services by Quantopian. In addition, the material offers no opinion with respect to the suitability of any security or specific investment. No information contained herein should be regarded as a suggestion to engage in or refrain from any investment-related course of action as none of Quantopian nor any of its affiliates is undertaking to provide investment advice, act as an adviser to any plan or entity subject to the Employee Retirement Income Security Act of 1974, as amended, individual retirement account or individual retirement annuity, or give advice in a fiduciary capacity with respect to the materials presented herein. If you are an individual retirement or other investor, contact your financial advisor or other fiduciary unrelated to Quantopian about whether any given investment idea, strategy, product or service described herein may be appropriate for your circumstances. All investments involve risk, including loss of principal. Quantopian makes no guarantees as to the accuracy or completeness of the views expressed in the website. The views are subject to change, and may have become unreliable for various reasons, including changes in market conditions or economic circumstances.

@Dan Whitnable Thank you so much!