Back to Community
Uncorrelate those models - Seattle/Portland meetup talk

Something that I found myself running into when modeling is that sometimes my explanatory variables were explaining the same thing, and appeared to be correlated over time. I figured that I wasn't the only person running into this problem, aware of it or not, and decided to do a little research into it. I'll post a notebook shortly with the code for some of the methods I used. Hopefully these backtests demonstrate the power of adding uncorrelated factors to your models.

Here is the code for the backtests that I talked about during the Seattle and Portland meetups. The final model does require access to the paid Sentdex dataset though the theory should still hold with the free data. Note that these algos are for demonstration/education purpose so commission and slippage have been disabled.

2 Correlated Factors

Quality, Value

Clone Algorithm
27
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import CustomFactor, SimpleMovingAverage, AverageDollarVolume, Latest, RSI
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.filters.morningstar import IsPrimaryShare

import numpy as np
import pandas as pd

class Momentum(CustomFactor):
    
    inputs = [USEquityPricing.close]
    window_length = 252
    
    def compute(self, today, assets, out, close):
        out[:] = close[-20] / close[0]


class Value(CustomFactor):
    
    inputs = [morningstar.income_statement.ebitda, morningstar.valuation.enterprise_value]

    window_length = 1
    
    def compute(self, today, assets, out, ebitda, ev):
        out[:] = ebitda[-1] / ev[-1]
        
        
class Quality(CustomFactor):
    
    inputs = [morningstar.operation_ratios.roe]
    window_length = 1
    
    def compute(self, today, assets, out, roe):  
        out[:] = roe[-1]
        
class Volatility(CustomFactor):
    
    inputs = [USEquityPricing.close]
    window_length = 252
    
    def compute(self, today, assets, out, close):  
        close = pd.DataFrame(data=close, columns=assets)
        out[:] = -close.pct_change().std()
        
def make_pipeline():
    """
    Create and return our pipeline.
    
    We break this piece of logic out into its own function to make it easier to
    test and modify in isolation.
    
    In particular, this function can be copy/pasted into research and run by itself.
    """
    pipe = Pipeline()
    
    dollar_volume = AverageDollarVolume(window_length=20)    
    initial_screen = dollar_volume.top(2000) & IsPrimaryShare()

    factors = {"Quality": Quality(mask=initial_screen),
               "Value": Value(mask=initial_screen)
              }
    
    clean_factors = None
    for name, factor in factors.items():
        if not clean_factors:
            clean_factors = factor.isfinite()
        else:
            clean_factors = clean_factors & factor.isfinite()  
            
    combined_rank = None
    for name, factor in factors.items():
        if not combined_rank:
            combined_rank = factor.rank(mask=clean_factors)
        else:
            combined_rank += factor.rank(mask=clean_factors)
    pipe.add(combined_rank, 'factor')

    # Build Filters representing the top and bottom 200 stocks by our combined ranking system.
    # We'll use these as our tradeable universe each day.
    longs = combined_rank.percentile_between(90, 100)
    shorts = combined_rank.percentile_between(0, 10)
    
    pipe.set_screen(longs | shorts)
    
    pipe.add(longs, 'longs')
    pipe.add(shorts, 'shorts')
    
    return pipe


def initialize(context):
    
    # Set slippage and commission to zero to evaulate the signal generating 
    # ability of the algorithm 
    set_commission(commission.PerTrade(cost=0))
    set_slippage(slippage.FixedSlippage(spread=0.00))

    context.long_leverage = 0.50
    context.short_leverage = -0.50
    context.spy = sid(8554)
    
    attach_pipeline(make_pipeline(), 'ranking_example')
    
    # Used to avoid purchasing any leveraged ETFs 
    context.dont_buys = security_lists.leveraged_etf_list
     
    # Schedule my rebalance function
    schedule_function(func=rebalance, 
                      date_rule=date_rules.month_start(days_offset=0), 
                      time_rule=time_rules.market_open(hours=0,minutes=30), 
                      half_days=True)
    
    # Schedule a function to plot leverage and position count
    schedule_function(func=record_vars, 
                      date_rule=date_rules.every_day(), 
                      time_rule=time_rules.market_close(), 
                      half_days=True)

def before_trading_start(context, data):
    # Call pipeline_output to get the output
    # Note this is a dataframe where the index is the SIDs for all 
    # securities to pass my screen and the columns are the factors which
    output = pipeline_output('ranking_example')
    ranks = output['factor']
    
    long_ranks = ranks[output['longs']].rank()
    short_ranks = ranks[output['shorts']].rank()

    context.long_weights = (long_ranks / long_ranks.sum())
    log.info("Long Weights:")
    log.info(context.long_weights)
    
    context.short_weights = (short_ranks / short_ranks.sum())
    log.info("Short Weights:")
    log.info(context.short_weights)
    
    context.active_portfolio = context.long_weights.index.union(context.short_weights.index)


def record_vars(context, data):  
    
    # Record and plot the leverage, number of positions, and expsoure of our portfolio over time. 
    record(num_positions=len(context.portfolio.positions),
           exposure=context.account.net_leverage, 
           leverage=context.account.leverage)
    

# This function is scheduled to run at the start of each month.
def rebalance(context, data):
    """
    Allocate our long/short portfolio based on the weights supplied by
    context.long_weights and context.short_weights.
    """
    # Order our longs.
    log.info("ordering longs")
    for long_stock, long_weight in context.long_weights.iterkv():
        if data.can_trade(long_stock):
            if get_open_orders(long_stock):
                continue
            if long_stock in context.dont_buys:
                continue
            order_target_percent(long_stock, context.long_leverage * long_weight)
    
    # Order our shorts.
    log.info("ordering shorts")
    for short_stock, short_weight in context.short_weights.iterkv():
        if data.can_trade(short_stock):
            if get_open_orders(short_stock):
                continue
            if short_stock in context.dont_buys:
                continue
            order_target_percent(short_stock, context.short_leverage * short_weight)
    
    # Sell any positions in assets that are no longer in our target portfolio.
    for security in context.portfolio.positions:
        if get_open_orders(security):
            continue
        if data.can_trade(security):  # Work around inability to sell de-listed stocks.
            if security not in context.active_portfolio:
                order_target_percent(security, 0)
There was a runtime error.
Disclaimer

The material on this website is provided for informational purposes only and does not constitute an offer to sell, a solicitation to buy, or a recommendation or endorsement for any security or strategy, nor does it constitute an offer to provide investment advisory services by Quantopian. In addition, the material offers no opinion with respect to the suitability of any security or specific investment. No information contained herein should be regarded as a suggestion to engage in or refrain from any investment-related course of action as none of Quantopian nor any of its affiliates is undertaking to provide investment advice, act as an adviser to any plan or entity subject to the Employee Retirement Income Security Act of 1974, as amended, individual retirement account or individual retirement annuity, or give advice in a fiduciary capacity with respect to the materials presented herein. If you are an individual retirement or other investor, contact your financial advisor or other fiduciary unrelated to Quantopian about whether any given investment idea, strategy, product or service described herein may be appropriate for your circumstances. All investments involve risk, including loss of principal. Quantopian makes no guarantees as to the accuracy or completeness of the views expressed in the website. The views are subject to change, and may have become unreliable for various reasons, including changes in market conditions or economic circumstances.

5 responses

2 Uncorrelated Factors

Value, Momentum

Clone Algorithm
27
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import CustomFactor, SimpleMovingAverage, AverageDollarVolume, Latest, RSI
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.filters.morningstar import IsPrimaryShare

import numpy as np
import pandas as pd

class Momentum(CustomFactor):
    
    inputs = [USEquityPricing.close]
    window_length = 252
    
    def compute(self, today, assets, out, close):
        out[:] = close[-20] / close[0]


class Value(CustomFactor):
    
    inputs = [morningstar.income_statement.ebitda, morningstar.valuation.enterprise_value]

    window_length = 1
    
    def compute(self, today, assets, out, ebitda, ev):
        out[:] = ebitda[-1] / ev[-1]
        
        
class Quality(CustomFactor):
    
    inputs = [morningstar.operation_ratios.roe]
    window_length = 1
    
    def compute(self, today, assets, out, roe):  
        out[:] = roe[-1]
        
class Volatility(CustomFactor):
    
    inputs = [USEquityPricing.close]
    window_length = 252
    
    def compute(self, today, assets, out, close):  
        close = pd.DataFrame(data=close, columns=assets)
        out[:] = -close.pct_change().std()
        
def make_pipeline():
    """
    Create and return our pipeline.
    
    We break this piece of logic out into its own function to make it easier to
    test and modify in isolation.
    
    In particular, this function can be copy/pasted into research and run by itself.
    """
    pipe = Pipeline()
    
    dollar_volume = AverageDollarVolume(window_length=20)    
    initial_screen = dollar_volume.top(2000) & IsPrimaryShare()

    factors = {"Momentum": Momentum(mask=initial_screen),
               "Value": Value(mask=initial_screen)
              }
    
    clean_factors = None
    for name, factor in factors.items():
        if not clean_factors:
            clean_factors = factor.isfinite()
        else:
            clean_factors = clean_factors & factor.isfinite()  
            
    combined_rank = None
    for name, factor in factors.items():
        if not combined_rank:
            combined_rank = factor.rank(mask=clean_factors)
        else:
            combined_rank += factor.rank(mask=clean_factors)
    pipe.add(combined_rank, 'factor')

    # Build Filters representing the top and bottom 200 stocks by our combined ranking system.
    # We'll use these as our tradeable universe each day.
    longs = combined_rank.percentile_between(90, 100)
    shorts = combined_rank.percentile_between(0, 10)
    
    pipe.set_screen(longs | shorts)
    
    pipe.add(longs, 'longs')
    pipe.add(shorts, 'shorts')
    
    return pipe


def initialize(context):
    
    # Set slippage and commission to zero to evaulate the signal generating 
    # ability of the algorithm 
    set_commission(commission.PerTrade(cost=0))
    set_slippage(slippage.FixedSlippage(spread=0.00))

    context.long_leverage = 0.50
    context.short_leverage = -0.50
    context.spy = sid(8554)
    
    attach_pipeline(make_pipeline(), 'ranking_example')
    
    # Used to avoid purchasing any leveraged ETFs 
    context.dont_buys = security_lists.leveraged_etf_list
     
    # Schedule my rebalance function
    schedule_function(func=rebalance, 
                      date_rule=date_rules.month_start(days_offset=0), 
                      time_rule=time_rules.market_open(hours=0,minutes=30), 
                      half_days=True)
    
    # Schedule a function to plot leverage and position count
    schedule_function(func=record_vars, 
                      date_rule=date_rules.every_day(), 
                      time_rule=time_rules.market_close(), 
                      half_days=True)

def before_trading_start(context, data):
    # Call pipeline_output to get the output
    # Note this is a dataframe where the index is the SIDs for all 
    # securities to pass my screen and the columns are the factors which
    output = pipeline_output('ranking_example')
    ranks = output['factor']
    
    long_ranks = ranks[output['longs']].rank()
    short_ranks = ranks[output['shorts']].rank()

    context.long_weights = (long_ranks / long_ranks.sum())
    log.info("Long Weights:")
    log.info(context.long_weights)
    
    context.short_weights = (short_ranks / short_ranks.sum())
    log.info("Short Weights:")
    log.info(context.short_weights)
    
    context.active_portfolio = context.long_weights.index.union(context.short_weights.index)


def record_vars(context, data):  
    
    # Record and plot the leverage, number of positions, and expsoure of our portfolio over time. 
    record(num_positions=len(context.portfolio.positions),
           exposure=context.account.net_leverage, 
           leverage=context.account.leverage)
    

# This function is scheduled to run at the start of each month.
def rebalance(context, data):
    """
    Allocate our long/short portfolio based on the weights supplied by
    context.long_weights and context.short_weights.
    """
    # Order our longs.
    log.info("ordering longs")
    for long_stock, long_weight in context.long_weights.iterkv():
        if data.can_trade(long_stock):
            if get_open_orders(long_stock):
                continue
            if long_stock in context.dont_buys:
                continue
            order_target_percent(long_stock, context.long_leverage * long_weight)
    
    # Order our shorts.
    log.info("ordering shorts")
    for short_stock, short_weight in context.short_weights.iterkv():
        if data.can_trade(short_stock):
            if get_open_orders(short_stock):
                continue
            if short_stock in context.dont_buys:
                continue
            order_target_percent(short_stock, context.short_leverage * short_weight)
    
    # Sell any positions in assets that are no longer in our target portfolio.
    for security in context.portfolio.positions:
        if get_open_orders(security):
            continue
        if data.can_trade(security):  # Work around inability to sell de-listed stocks.
            if security not in context.active_portfolio:
                order_target_percent(security, 0)
There was a runtime error.

3 Uncorrelated Factors + new datasource

Value, Momentum, Sentiment

As mentioned above this algo uses the paid version of the Sentdex data, though you can easily run the algo with the free version. Just change the import statement from

from quantopian.pipeline.data.sentdex import sentiment as sentdex  

to

from quantopian.pipeline.data.sentdex import sentiment_free as sentdex  

also don't forget to adjust the date ranges if using the free dataset.

Clone Algorithm
27
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import CustomFactor, SimpleMovingAverage, AverageDollarVolume, Latest, RSI
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.filters.morningstar import IsPrimaryShare
from quantopian.pipeline.data.sentdex import sentiment as sentdex

import numpy as np
import pandas as pd

class Momentum(CustomFactor):
    
    inputs = [USEquityPricing.close]
    window_length = 252
    
    def compute(self, today, assets, out, close):
        out[:] = close[-20] / close[0]


class Value(CustomFactor):
    
    inputs = [morningstar.income_statement.ebitda, morningstar.valuation.enterprise_value]

    window_length = 1
    
    def compute(self, today, assets, out, ebitda, ev):
        out[:] = ebitda[-1] / ev[-1]
        
        
class Quality(CustomFactor):
    
    inputs = [morningstar.operation_ratios.roe]
    window_length = 1
    
    def compute(self, today, assets, out, roe):  
        out[:] = roe[-1]
        
class Volatility(CustomFactor):
    
    inputs = [USEquityPricing.close]
    window_length = 252
    
    def compute(self, today, assets, out, close):  
        close = pd.DataFrame(data=close, columns=assets)
        out[:] = -close.pct_change().std()
        
class AvgSentiment(CustomFactor):
    inputs=[sentdex.sentiment_signal]
    window_length=7
    
    def compute(self, today, assets, out, impact):
        np.mean(impact, axis=0, out=out)
        
def make_pipeline():
    """
    Create and return our pipeline.
    
    We break this piece of logic out into its own function to make it easier to
    test and modify in isolation.
    
    In particular, this function can be copy/pasted into research and run by itself.
    """
    pipe = Pipeline()
    
    dollar_volume = AverageDollarVolume(window_length=20)    
    initial_screen = dollar_volume.top(2000) & IsPrimaryShare()

    factors = {"Momentum": Momentum(mask=initial_screen),
               "Value": Value(mask=initial_screen),
               "AvgSentiment": AvgSentiment(mask=initial_screen)
              }
    
    clean_factors = None
    for name, factor in factors.items():
        if not clean_factors:
            clean_factors = factor.isfinite()
        else:
            clean_factors = clean_factors & factor.isfinite()  
            
    combined_rank = None
    for name, factor in factors.items():
        if not combined_rank:
            combined_rank = factor.rank(mask=clean_factors)
        else:
            combined_rank += factor.rank(mask=clean_factors)
    pipe.add(combined_rank, 'factor')

    # Build Filters representing the top and bottom 200 stocks by our combined ranking system.
    # We'll use these as our tradeable universe each day.
    longs = combined_rank.percentile_between(90, 100)
    shorts = combined_rank.percentile_between(0, 10)
    
    pipe.set_screen(longs | shorts)
    
    pipe.add(longs, 'longs')
    pipe.add(shorts, 'shorts')
    
    return pipe


def initialize(context):
    
    # Set slippage and commission to zero to evaulate the signal generating 
    # ability of the algorithm 
    set_commission(commission.PerTrade(cost=0))
    set_slippage(slippage.FixedSlippage(spread=0.00))

    context.long_leverage = 0.50
    context.short_leverage = -0.50
    context.spy = sid(8554)
    
    attach_pipeline(make_pipeline(), 'ranking_example')
    
    # Used to avoid purchasing any leveraged ETFs 
    context.dont_buys = security_lists.leveraged_etf_list
     
    # Schedule my rebalance function
    schedule_function(func=rebalance, 
                      date_rule=date_rules.month_start(days_offset=0), 
                      time_rule=time_rules.market_open(hours=0,minutes=30), 
                      half_days=True)
    
    # Schedule a function to plot leverage and position count
    schedule_function(func=record_vars, 
                      date_rule=date_rules.every_day(), 
                      time_rule=time_rules.market_close(), 
                      half_days=True)

def before_trading_start(context, data):
    # Call pipeline_output to get the output
    # Note this is a dataframe where the index is the SIDs for all 
    # securities to pass my screen and the columns are the factors which
    output = pipeline_output('ranking_example')
    ranks = output['factor']
    
    long_ranks = ranks[output['longs']].rank()
    short_ranks = ranks[output['shorts']].rank()

    context.long_weights = (long_ranks / long_ranks.sum())
    log.info("Long Weights:")
    log.info(context.long_weights)
    
    context.short_weights = (short_ranks / short_ranks.sum())
    log.info("Short Weights:")
    log.info(context.short_weights)
    
    context.active_portfolio = context.long_weights.index.union(context.short_weights.index)


def record_vars(context, data):  
    
    # Record and plot the leverage, number of positions, and expsoure of our portfolio over time. 
    record(num_positions=len(context.portfolio.positions),
           exposure=context.account.net_leverage, 
           leverage=context.account.leverage)
    

# This function is scheduled to run at the start of each month.
def rebalance(context, data):
    """
    Allocate our long/short portfolio based on the weights supplied by
    context.long_weights and context.short_weights.
    """
    # Order our longs.
    log.info("ordering longs")
    for long_stock, long_weight in context.long_weights.iterkv():
        if data.can_trade(long_stock):
            if get_open_orders(long_stock):
                continue
            if long_stock in context.dont_buys:
                continue
            order_target_percent(long_stock, context.long_leverage * long_weight)
    
    # Order our shorts.
    log.info("ordering shorts")
    for short_stock, short_weight in context.short_weights.iterkv():
        if data.can_trade(short_stock):
            if get_open_orders(short_stock):
                continue
            if short_stock in context.dont_buys:
                continue
            order_target_percent(short_stock, context.short_leverage * short_weight)
    
    # Sell any positions in assets that are no longer in our target portfolio.
    for security in context.portfolio.positions:
        if get_open_orders(security):
            continue
        if data.can_trade(security):  # Work around inability to sell de-listed stocks.
            if security not in context.active_portfolio:
                order_target_percent(security, 0)
There was a runtime error.

James,
Nice talk in Portland...thanks!
Looking forward to your notebook posting. I found that very interesting.
alan

Looking forward to learning from this. Thanks for sharing.

Hey All - really dropped the ball on this one! Sorry for the delay I spent most of the summer heads down on Alphalens. Here is the NB for the Seattle and Portland talks it is pretty slow to execute (lots of permutations and correlations) so it may be better to just view the preview, otherwise enjoy.

And I'll be in Seattle with Max on October 10th (same venue) for another talk!

Loading notebook preview...