Back to Community
ETF market rotation strategy

ETF market rotation strategy provides steady positive results and small drawdown

Clone Algorithm
1575
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#
# USA Market Sector Rotation Strategy
#
# This strategy rotates between Vanguard market sector ETFs on a monthly
# basis.  Each month the performance and mean 20-day volitility over
# the last 13 weeks are used to rank which ETF should be invested 
# in for the coming month.
import math
import pandas


def initialize(context):
    context.stocks = {
        # 25904: sid(25904), # VFH (Vanguard Financials ETF)
        25906: sid(25906), # VHT (Vanguard Health Care ETF)
        25905: sid(25905), # VGT (Vanguard Information Technology ETF)
        # 26667: sid(26667), # VDE (Vanguard Energy ETF)
        25902: sid(25902), # VCR (Vanguard Consumer Discretionary ETF)
        22445: sid(22445), # IBB (iShares Nasdaq Biotechnology Index Fund)
        # 39479: sid(39479), # IBB (iShares Nasdaq Biotechnology Index Fund)
        22887: sid(22887), # EDV VANGUARD treasury
        25899: sid(25899), # VB = Vanguard small cap
        25898: sid(25898)  # VAW (Vanguard Materials ETF)
    } 

    # Keep track of the current month.
    context.currentMonth = None
    
    # compare algorithm's performance versus small cap EFT buy and hold
    set_benchmark(symbol('vb'))
    
    # The order ID of the sell order currently being filled
    context.oid = None
    
    # The current stock being held
    context.currentStock = None
    
    # The next stock that needs to get purchased (once the sell order
    # on the current stock is filled
    context.nextStock = None
    
    # The 3-month lookback period.  Calculated based on there being
    # an average of 21 trading days in a month
    context.lookback = 63

'''
  Gets the minimum and maximum values of an array of values
'''
def getMinMax(arr):
   return min(arr.values()), max(arr.values()) 

'''
  Calculates the n-day historical volatility given a set of
  n+1 prices. 

  @param period The number of days for which to calculate volatility
  @param prices An array of price information.  Must be of length
    period + 1.
'''
def historicalVolatility(period, prices):
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)
    
    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    # Find the average of all returns
    rMean = sum(r) / period;
    
    # Determine the difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))
    
    # Take the square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year
    vol = math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)
    
    return vol

'''
  Gets the performance and average 20-day volatility of a security
  over a given period

  @param prices
  @param period The time period for which to find
'''
def getStockMetrics(prices, period):
    # Get the prices
    #prices = data['close_price'][security][-period-1:]
    start = prices[-period] # First item
    end = prices[-1] # Last item
    
    performance = (end - start) / start
    
    # Calculate 20-day volatility for the given period
    v = []
    x = 0
    for i in xrange(-period, 0):
        v.append(historicalVolatility(20, prices[i-21:21+x]))
        x += 1
    
    volatility = sum(v) / period
    
    return performance, volatility

'''
  Picks the best stock from a group of stocks based on the given
  data over a specified period using the stocks' performance and
  volatility

  @param data The datapanel with data of all the stocks
  @param stocks A list of stocks to rank
  @param period The time period over which the stocks will be
    analyzed
'''
def getBestStock(data, stocks, period):
    best = None
    
    performances = {}
    volatilities = {}
        
    # Get performance and volatility for all the stocks
    for s in stocks:
        p, v = getStockMetrics(data['price'][s.sid], period)
        performances[s.sid] = p
        volatilities[s.sid] = v
    
    # Determine min/max of each.  NOTE: volatility is switched
    # since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)
    
    # Normalize the performance and volatility values to a range
    # between [0..1] then rank them based on a 70/30 weighting.
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        rank = p * 0.7 + v * 0.3
        
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))
        
        # If the new rank is greater than the old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    
    return best[0]

'''
  Sells all the currently held positions in the context's portfolio
'''
def sellHoldings(context):
    positions = context.portfolio.positions
    
    oid = None
    for p in positions.values():
        if (p.amount > 0):
            #log.debug('ordering %s' % p)
            oid = order(p.sid, -p.amount)
            
    return oid

'''
  Utilize the batch_transform decorator to accumulate multiple days
  of data into one datapanel  Need the window length to be 20 longer
  than lookback period to allow for a 20-day volatility calculation
'''
@batch_transform(window_length=83)
def accumulateData(data):
    # return price_history = history(bar_count=20, frequency='1d', field='price')
    return data
  

'''
  The main proccessing function.  This is called and passed data
'''
def handle_data(context, data):
    # Accumulate data until there is enough days worth of data
    # to process without having outOfBounds issues.
    datapanel = accumulateData(data)
    
    if datapanel is None:
        # There is insufficient data accumulated to process
        return
    
    
    # If there is an order ID, check the status of the order.
    # If there is an order and it is filled, the next stock
    # can be purchased.
    if context.oid is not None:
        orderObj = get_order(context.oid)
        if orderObj.filled == orderObj.amount:
            # Good to buy next holding
            amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
            log.info('Sell order complete, buying %s of %s (%s of %s)' % \
                 (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
            order(context.nextStock, amount)
            context.currentStock = context.nextStock
            context.oid = None
            context.nextStock = None
    
    
    date = get_datetime()
    month = date.month    
    
    if not context.currentMonth:
        # Set the month initially
        context.currentMonth = month
    
    if context.currentMonth == month:
        # If the current month is unchanged, nothing further to do
        return
    
    context.currentMonth = month
    
    # At this point, a new month has been reached.  The stocks
    # need to be
    
    # Ensure stocks are only traded if possible.  
    # (e.g) EDV doesn't start trading until late 2007, without
    # this, any backtest run before that date would fail.
    stocks = []
    for s in context.stocks.values():
        if date > s.security_start_date:
            stocks.append(s)
    
    
    # Determine which stock should be used for the next month
    best = getBestStock(datapanel, stocks, context.lookback)

    if best:
        if (context.currentStock is not None and context.currentStock == best):
            # If there is a stock currently held and it is the same as
            # the new 'best' stock, nothing needs to be done
            return
        else:
            # Otherwise, the current stock needs to be sold and the new
            # stock bought
            context.oid = sellHoldings(context)
            context.nextStock = best
            
            # Purchase will not occur until the next call of handle_data
            # and only when the order has been filled.

    # If there is no stock currently held, it needs to be bought.
    # This only happend
    if context.currentStock is None:
        amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
        order(context.nextStock, amount)
        context.currentStock = context.nextStock
        context.oid = None
        context.nextStock = None
We have migrated this algorithm to work with a new version of the Quantopian API. The code is different than the original version, but the investment rationale of the algorithm has not changed. We've put everything you need to know here on one page.
There was a runtime error.
37 responses

Very interesting, particularly because you pick a best one and not a group of etf's. One question as I look for curve fitting when I look at an algo and I see

 rank = p * 0.7 + v * 0.3  

Why this ratio and did you test other ratio's?

Best, Peter

Good job! Also excellent to have largely avoided some of the remarkable downturns during that period.

sir,
Why not using history function?

@batch_transform(window_length=83)  
def accumulateData(data):  
    # return price_history = history(bar_count=20, frequency='1d', field='price')  
    return data  

I may have munged it a bit... Here's a compact, version. Doesn't get the same results, so your entry/exit technique must impart spin of some sort...

[Edit: Updated the whole ranking mechanism. getting consistent results now.]

import math  
import numpy

lookbackPeriods = 63  
historicPeriods = 83  
volatilityPeriods = 20

def initialize(context):  
    set_symbol_lookup_date('2007-12-31')  
    #context.stocks = symbols("EDV","VGT","VCR","VB","VAW","VHT","IBB")  
    context.stocks = symbols("VAW","VHT","IBB","EDV","VGT","VCR","VB")  
    context.stockMetrics = {}  
    for stock in context.stocks:  
        context.stockMetrics[stock] = Metric(0,0,0)  
    context.currentMonth = 0  
def handle_data(context, data):  
    date = get_datetime()  
    month = date.month  
    if context.currentMonth == month:  
        return  
    context.currentMonth = month

    historic = history(historicPeriods, "1d", "close_price")  
    historic = historic.dropna(axis=1)  
    for stock in context.stocks:  
        if (stock in historic):  
            context.stockMetrics[stock] = GetStockMetrics(historic[stock], lookbackPeriods)  
        else:  
            context.stockMetrics[stock] = Metric(0,0,0)  
    best = GetBestRanked(context.stockMetrics)  
    if (best == None):  
        print("All securities ranked 0.0")  
    for stock in context.stocks:  
        if (stock == best):  
            order_target_percent(stock, 1)  
            print(stock.symbol, context.stockMetrics[stock].ToString())  
        else:  
            order_target_percent(stock, 0)  
def GetBestRanked(stockMetrics):  
    list = [metric.Performance for metric in stockMetrics.values()]  
    minP, maxP = GetMinMax(list)  
    list = [metric.Volatility for metric in stockMetrics.values()]  
    minV, maxV = GetMinMax(list)  
    isValid = False  
    for stock in stockMetrics:  
        p = stockMetrics[stock].Performance  
        v = stockMetrics[stock].Volatility  
        if (p != 0.0 and v != 0.0):  
            p = .7 * (p - minP) / (maxP - minP)  
            v = .3 * (v - minV) / (maxV - minV)  
            rank = p - v  
            stockMetrics[stock].Performance = p  
            stockMetrics[stock].Volatility = v  
            stockMetrics[stock].Rank = rank  
            isValid = True  
    if (isValid):  
        keyValuesList = sorted(stockMetrics.items(), key=lambda x: x[1].Rank)  
        if (keyValuesList[-1][1].Rank != 0.0):  
            return keyValuesList[-1][0]  
        else:  
            return None  
    else:  
        return None  
def GetStockMetrics(prices, period):  
    volatilities = []  
    x = 0  
    volatilityWindow = volatilityPeriods + 1  
    for i in xrange(-period, 0):  
        volatilities.append(GetHistoricalVolatility(prices[i - volatilityWindow : volatilityWindow + x], volatilityPeriods))  
        x += 1  
    meanVolatility = sum(volatilities) / period  
    performance = (prices[-1] - prices[-period]) / prices[-period]  
    return Metric(0, performance, meanVolatility)

def GetHistoricalVolatility(prices, period):  
    normalizedPrices = numpy.asarray(prices) / prices[0]  
    volatility = numpy.std(normalizedPrices)  
    return volatility

def GetMinMax(arr):  
   return min(arr), max(arr) 

class Metric(object):  
    def __init__(this, r, p, v):  
        this.Rank = r  
        this.Performance = p  
        this.Volatility = v  
    def ToString(this):  
        toString = "Rank:{0:<7.5f} performance:{1:<7.5f} volatility:{2:<7.5f}".format(this.Rank, this.Performance, this.Volatility)  
        return toString     

Print log:

2014-07-01 PRINT('EDV', 'Rank:0.45876 performance:0.51824 volatility:0.05949')  
2014-08-01 PRINT('VGT', 'Rank:0.46813 performance:0.62635 volatility:0.15823')  
2014-09-02 PRINT('IBB', 'Rank:0.40000 performance:0.70000 volatility:0.30000')  
2014-10-01 PRINT('VHT', 'Rank:0.51983 performance:0.69815 volatility:0.17832')  
2014-11-03 PRINT('IBB', 'Rank:0.40000 performance:0.70000 volatility:0.30000')  
2014-12-01 PRINT('VHT', 'Rank:0.58249 performance:0.66101 volatility:0.07853')  
2015-01-02 PRINT('EDV', 'Rank:0.64355 performance:0.70000 volatility:0.05645')  

the differnce might be explained by p * 0.5 + v * 0.5 vs p * 0.7 + v * 0.3 ?

@Dave M.'s use of volatility + return performance is good to know about and study. I thank ye kindly for your efforts and sharing.

I did swap out the flip-flopped min/max @Dave M. had going and simply subtracted the volatility instead. Plus, when used with .7 and .3, the numbers still don't come out like his. I used all of his securities too, but even when duplicated exactly, the entry/exit change alters things. I'd have to guess the extra delay in the exit first then enter one period later (which ends up being 2 periods) of @Dave's code may have made the difference.

Reworking other quant's code is good practice for learning this sometimes tricksy language. That's the only reason I even attempted it (and others). Python'ers will no doubt be able to eliminate many of the loops with embedded matrix algebra -- a thing I continue to struggle with.

Do any portfolios model tax consequences? The question is not specific to this strategy, but this one looks interesting enough to continue refining the model.

not automatically, its always possible to do via extending the Run Summary or a custom logging function. IB has a tax report buyt thats very US oriented. As this is country dependent and personal circumstances dependent it's hard to model this in a more general fashion, however Quantopian could create a configurable module that allows a user to state the tax on transactions (UK), tax on profits (AUS,US, UK), compensation of losses (AUS,US), tax on total average holding (NL), tax on holding at 31st Dec... etc etc

Return for the last 2 years (period 1/1/2013 to end of 2014) is all along below SPY.

How did you pick the ETFs? Some sector ETFs are commented out.

Just for reference, this is Dave's backtest with the financial ETF plugged back in. The return is a little lower but the drawdown is the same. (I had wondered if the drawdown would be increased)

Clone Algorithm
83
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#
# USA Market Sector Rotation Strategy
#
# This strategy rotates between Vanguard market sector ETFs on a monthly
# basis.  Each month the performance and mean 20-day volitility over
# the last 13 weeks are used to rank which ETF should be invested 
# in for the coming month.
import math
import pandas


def initialize(context):
    context.stocks = {
        25904: sid(25904), # VFH (Vanguard Financials ETF)
        25906: sid(25906), # VHT (Vanguard Health Care ETF)
        25905: sid(25905), # VGT (Vanguard Information Technology ETF)
        # 26667: sid(26667), # VDE (Vanguard Energy ETF)
        25902: sid(25902), # VCR (Vanguard Consumer Discretionary ETF)
        22445: sid(22445), # IBB (iShares Nasdaq Biotechnology Index Fund)
        # 39479: sid(39479), # IBB (iShares Nasdaq Biotechnology Index Fund)
        22887: sid(22887), # EDV VANGUARD treasury
        25899: sid(25899), # VB = Vanguard small cap
        25898: sid(25898)  # VAW (Vanguard Materials ETF)
    } 

    # Keep track of the current month.
    context.currentMonth = None
    
    # compare algorithm's performance versus small cap EFT buy and hold
    set_benchmark(symbol('vb'))
    
    # The order ID of the sell order currently being filled
    context.oid = None
    
    # The current stock being held
    context.currentStock = None
    
    # The next stock that needs to get purchased (once the sell order
    # on the current stock is filled
    context.nextStock = None
    
    # The 3-month lookback period.  Calculated based on there being
    # an average of 21 trading days in a month
    context.lookback = 63

'''
  Gets the minimum and maximum values of an array of values
'''
def getMinMax(arr):
   return min(arr.values()), max(arr.values()) 

'''
  Calculates the n-day historical volatility given a set of
  n+1 prices. 

  @param period The number of days for which to calculate volatility
  @param prices An array of price information.  Must be of length
    period + 1.
'''
def historicalVolatility(period, prices):
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)
    
    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    # Find the average of all returns
    rMean = sum(r) / period;
    
    # Determine the difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))
    
    # Take the square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year
    vol = math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)
    
    return vol

'''
  Gets the performance and average 20-day volatility of a security
  over a given period

  @param prices
  @param period The time period for which to find
'''
def getStockMetrics(prices, period):
    # Get the prices
    #prices = data['close_price'][security][-period-1:]
    start = prices[-period] # First item
    end = prices[-1] # Last item
    
    performance = (end - start) / start
    
    # Calculate 20-day volatility for the given period
    v = []
    x = 0
    for i in xrange(-period, 0):
        v.append(historicalVolatility(20, prices[i-21:21+x]))
        x += 1
    
    volatility = sum(v) / period
    
    return performance, volatility

'''
  Picks the best stock from a group of stocks based on the given
  data over a specified period using the stocks' performance and
  volatility

  @param data The datapanel with data of all the stocks
  @param stocks A list of stocks to rank
  @param period The time period over which the stocks will be
    analyzed
'''
def getBestStock(data, stocks, period):
    best = None
    
    performances = {}
    volatilities = {}
        
    # Get performance and volatility for all the stocks
    for s in stocks:
        p, v = getStockMetrics(data['price'][s.sid], period)
        performances[s.sid] = p
        volatilities[s.sid] = v
    
    # Determine min/max of each.  NOTE: volatility is switched
    # since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)
    
    # Normalize the performance and volatility values to a range
    # between [0..1] then rank them based on a 70/30 weighting.
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        rank = p * 0.7 + v * 0.3
        
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))
        
        # If the new rank is greater than the old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    
    return best[0]

'''
  Sells all the currently held positions in the context's portfolio
'''
def sellHoldings(context):
    positions = context.portfolio.positions
    
    oid = None
    for p in positions.values():
        if (p.amount > 0):
            #log.debug('ordering %s' % p)
            oid = order(p.sid, -p.amount)
            
    return oid

'''
  Utilize the batch_transform decorator to accumulate multiple days
  of data into one datapanel  Need the window length to be 20 longer
  than lookback period to allow for a 20-day volatility calculation
'''
@batch_transform(window_length=83)
def accumulateData(data):
    # return price_history = history(bar_count=20, frequency='1d', field='price')
    return data
  

'''
  The main proccessing function.  This is called and passed data
'''
def handle_data(context, data):
    # Accumulate data until there is enough days worth of data
    # to process without having outOfBounds issues.
    datapanel = accumulateData(data)
    
    if datapanel is None:
        # There is insufficient data accumulated to process
        return
    
    
    # If there is an order ID, check the status of the order.
    # If there is an order and it is filled, the next stock
    # can be purchased.
    if context.oid is not None:
        orderObj = get_order(context.oid)
        if orderObj.filled == orderObj.amount:
            # Good to buy next holding
            amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
            log.info('Sell order complete, buying %s of %s (%s of %s)' % \
                 (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
            order(context.nextStock, amount)
            context.currentStock = context.nextStock
            context.oid = None
            context.nextStock = None
    
    
    date = get_datetime()
    month = date.month    
    
    if not context.currentMonth:
        # Set the month initially
        context.currentMonth = month
    
    if context.currentMonth == month:
        # If the current month is unchanged, nothing further to do
        return
    
    context.currentMonth = month
    
    # At this point, a new month has been reached.  The stocks
    # need to be
    
    # Ensure stocks are only traded if possible.  
    # (e.g) EDV doesn't start trading until late 2007, without
    # this, any backtest run before that date would fail.
    stocks = []
    for s in context.stocks.values():
        if date > s.security_start_date:
            stocks.append(s)
    
    
    # Determine which stock should be used for the next month
    best = getBestStock(datapanel, stocks, context.lookback)

    if best:
        if (context.currentStock is not None and context.currentStock == best):
            # If there is a stock currently held and it is the same as
            # the new 'best' stock, nothing needs to be done
            return
        else:
            # Otherwise, the current stock needs to be sold and the new
            # stock bought
            context.oid = sellHoldings(context)
            context.nextStock = best
            
            # Purchase will not occur until the next call of handle_data
            # and only when the order has been filled.

    # If there is no stock currently held, it needs to be bought.
    # This only happend
    if context.currentStock is None:
        amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
        order(context.nextStock, amount)
        context.currentStock = context.nextStock
        context.oid = None
        context.nextStock = None
There was a runtime error.

Looking at Market Tech's re-factoring of this algorithm, if you replace this context.stocks line

context.stocks = symbols('VHT','VGT','VCR','IBB','EDV','VB','VAW')

with this:

context.stocks = { sid(25906), sid(25905), sid(25902), sid(22445), sid(22887), sid(25899), sid(25898)}

Then the performance is in line with Dave's original one. But I'm not sure why this makes such a big change. Originally, I thought maybe the symbols weren't resolving to the correct ETFs, but that doesn't seem it. Anyone know why this makes such a difference? A set vs. a list? Oddly (because the first two should be identical) all of the below give wildly different results:

context.stocks = symbols('VHT','VGT','VCR','IBB','EDV','VB','VAW')
context.stocks = [ sid(25906), sid(25905), sid(25902), sid(22445), sid(22887), sid(25899), sid(25898) ]
context.stocks = { sid(25906), sid(25905), sid(25902), sid(22445), sid(22887), sid(25899), sid(25898) }

@Alan B. I tell ya what, I'm gonna use dictionaries from now on 'cuz they give much better results!

It looks like the first two in your list are equivalent; the both are "list"s.
That last one though, the dictionary object {} yeah, I like that one much better. It's a "set".
Set, Game, Match!

Apparently the treatment for the "set" and the sids/symbols therein must be selecting something differently than the "symbols()" method returns. That would seem to require some explanation from those in the Q'Know.

[EDIT: Try this list:

context.stocks = symbols('EDV','VB','VAW', 'VHT','VGT','VCR','IBB')  

And you'll find that it matches the "set" version above.
Which means that ORDER now has something to do with the logic in this strategy. It's not all rose smelling in Denmark me thinks.
]

Hello Dave & all,

Skimming over the original algo posted above, I gather that in a nutshell, it is a matter of applying this, and finding the top-ranked ETF:

 Normalize the performance and volatility values to a range  
    # between [0..1] then rank them based on a 70/30 weighting.  
    for s in stocks:  
        p = (performances[s.sid] - minP) / (maxP - minP)  
        v = (volatilities[s.sid] - minV) / (maxV - minV)  
        rank = p * 0.7 + v * 0.3  

If I understand correctly, the performance is simply the overall relative gain/loss over the trailing window:

performance = (end - start) / start  

Correct?

The volatility calculation is more involved, though:

    # Calculate 20-day volatility for the given period  
    v = []  
    x = 0  
    for i in xrange(-period, 0):  
        v.append(historicalVolatility(20, prices[i-21:21+x]))  
        x += 1  
    volatility = sum(v) / period

with historicalVolatility using the log of the day-to-day returns, etc. Is there a justification for this approach? A quick google search provides http://quantivity.wordpress.com/2011/02/21/why-log-returns/, which concludes with some references discussing the use of logarithmic versus linear returns. Also, I've wondered if a proper volatility calculation should somehow incorporate all of the OHLCV values contained within a bar (be it daily or minutely), rather than just the closing price value.

Grant

Dear Quants,

For even better performance, perhaps the use of leveraged ETFs could be considered.
When a 3X small cap fund is substituted for all non-Treasury EFTs, the performance increased to over 600% over the same period.

The obvious drawback is a larger draw down (51%). and high volatility.

Is this progress in the desired direction?

Dave

Clone Algorithm
1575
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
# USA Market Sector Rotation Strategy
#
# This strategy rotates between Vanguard market sector ETFs on a monthly
# basis.  Each month the performance and mean 20-day volitility over
# the last 13 weeks are used to rank which ETF should be invested 
# in for the coming month.
#
# start date: 11/05/2008, end date: 01/02/2015, returns 401% running daily using EFTs
# VHT, VGT, VCR, IBB, TNA EDV, VB, VAW
#
import math
import pandas


def initialize(context):
    context.stocks = {
        # 25900: sid(25900), # Vbk (Vanguard small cap growth ETF)
        # 25906: sid(25906), # VHT (Vanguard Health Care ETF)
        # 25905: sid(25905), # VGT (Vanguard Information Technology ETF)
        # 25902: sid(25902), # VCR (Vanguard Consumer Discretionary ETF)
        # 22445: sid(22445), # IBB (iShares Nasdaq Biotechnology Index Fund)
        37515: sid(37515), # TNA (Direxion Small Cap Bull 3X Shares (ETF))
        22887: sid(22887), # EDV VANGUARD treasury
        # 25899: sid(25899), # VB = Vanguard small cap
        # 25898: sid(25898)  # VAW (Vanguard Materials ETF)
    } 

    # Keep track of the current month.
    context.currentMonth = None
    
    # compare algorithm's performance versus small cap EFT buy and hold
    set_benchmark(symbol('vb'))
    
    # The order ID of the sell order currently being filled
    context.oid = None
    
    # The current stock being held
    context.currentStock = None
    
    # The next stock that needs to get purchased (once the sell order
    # on the current stock is filled
    context.nextStock = None
    
    # The 3-month lookback period.  Calculated based on there being
    # an average of 21 trading days in a month
    context.lookback = 63

'''
  Gets the minimum and maximum values of an array of values
'''
def getMinMax(arr):
   return min(arr.values()), max(arr.values()) 

'''
  Calculates the n-day historical volatility given a set of
  n+1 prices. 

  @param period The number of days for which to calculate volatility
  @param prices An array of price information.  Must be of length
    period + 1.
'''
def historicalVolatility(period, prices):
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)
    
    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    # Find the average of all returns
    rMean = sum(r) / period;
    
    # Determine the difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))
    
    # Take the square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year
    vol = math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)
    
    return vol

'''
  Gets the performance and average 20-day volatility of a security
  over a given period

  @param prices
  @param period The time period for which to find
'''
def getStockMetrics(prices, period):
    # Get the prices
    #prices = data['close_price'][security][-period-1:]
    start = prices[-period] # First item
    end = prices[-1] # Last item
    
    performance = (end - start) / start
    
    # Calculate 20-day volatility for the given period
    v = []
    x = 0
    for i in xrange(-period, 0):
        v.append(historicalVolatility(20, prices[i-21:21+x]))
        x += 1
    
    volatility = sum(v) / period
    
    return performance, volatility

'''
  Picks the best stock from a group of stocks based on the given
  data over a specified period using the stocks' performance and
  volatility

  @param data The datapanel with data of all the stocks
  @param stocks A list of stocks to rank
  @param period The time period over which the stocks will be
    analyzed
'''
def getBestStock(data, stocks, period):
    best = None
    
    performances = {}
    volatilities = {}
        
    # Get performance and volatility for all the stocks
    for s in stocks:
        p, v = getStockMetrics(data['price'][s.sid], period)
        performances[s.sid] = p
        volatilities[s.sid] = v
    
    # Determine min/max of each.  NOTE: volatility is switched
    # since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)
    
    # Normalize the performance and volatility values to a range
    # between [0..1] then rank them based on a 70/30 weighting.
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        rank = p * 0.7 + v * 0.3
        
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))
        
        # If the new rank is greater than the old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    
    return best[0]

'''
  Sells all the currently held positions in the context's portfolio
'''
def sellHoldings(context):
    positions = context.portfolio.positions
    
    oid = None
    for p in positions.values():
        if (p.amount > 0):
            #log.debug('ordering %s' % p)
            oid = order(p.sid, -p.amount)
            
    return oid

'''
  Utilize the batch_transform decorator to accumulate multiple days
  of data into one datapanel  Need the window length to be 20 longer
  than lookback period to allow for a 20-day volatility calculation
'''
@batch_transform(window_length=83)
def accumulateData(data):
    # return price_history = history(bar_count=20, frequency='1d', field='price')
    return data
  

'''
  The main proccessing function.  This is called and passed data
'''
def handle_data(context, data):
    # Accumulate data until there is enough days worth of data
    # to process without having outOfBounds issues.
    datapanel = accumulateData(data)
    
    if datapanel is None:
        # There is insufficient data accumulated to process
        return
    
    
    # If there is an order ID, check the status of the order.
    # If there is an order and it is filled, the next stock
    # can be purchased.
    if context.oid is not None:
        orderObj = get_order(context.oid)
        if orderObj.filled == orderObj.amount:
            # Good to buy next holding
            amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
            log.info('Sell order complete, buying %s of %s (%s of %s)' % \
                 (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
            order(context.nextStock, amount)
            context.currentStock = context.nextStock
            context.oid = None
            context.nextStock = None
    
    
    date = get_datetime()
    month = date.month    
    
    if not context.currentMonth:
        # Set the month initially
        context.currentMonth = month
    
    if context.currentMonth == month:
        # If the current month is unchanged, nothing further to do
        return
    
    context.currentMonth = month
    
    # At this point, a new month has been reached.  The stocks
    # need to be
    
    # Ensure stocks are only traded if possible.  
    # (e.g) EDV doesn't start trading until late 2007, without
    # this, any backtest run before that date would fail.
    stocks = []
    for s in context.stocks.values():
        if date > s.security_start_date:
            stocks.append(s)
    
    
    # Determine which stock should be used for the next month
    best = getBestStock(datapanel, stocks, context.lookback)

    if best:
        if (context.currentStock is not None and context.currentStock == best):
            # If there is a stock currently held and it is the same as
            # the new 'best' stock, nothing needs to be done
            return
        else:
            # Otherwise, the current stock needs to be sold and the new
            # stock bought
            context.oid = sellHoldings(context)
            context.nextStock = best
            
            # Purchase will not occur until the next call of handle_data
            # and only when the order has been filled.

    # If there is no stock currently held, it needs to be bought.
    # This only happend
    if context.currentStock is None:
        amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
        order(context.nextStock, amount)
        context.currentStock = context.nextStock
        context.oid = None
        context.nextStock = None
We have migrated this algorithm to work with a new version of the Quantopian API. The code is different than the original version, but the investment rationale of the algorithm has not changed. We've put everything you need to know here on one page.
There was a runtime error.

@dave macpherson: definitely better one, but it outperforms SPY significantly in the last two years only in the last year.
The problem is when market is steadily going up, a lot of strategies perform well as well.

Also there are many strategies that could have avoided the 2008-2009 downturn and once you stop including 2008-2009 the strategies fall apart. It's not necessarily a bad thing that it performs better in a massive downturn, but it always feels a little bit like data mining.

There's still some unanswered questions about this algo. See Alan Baer's post above.

Take @Dave M. original strat, Swap out only his stocks collection mechanism and retest. Yeah you have to make some minor changes to the iterations within the body of code but this does not change logic in anyway.

You can try a "set" {} or a "list" [] but either should work no? The code just iterates the collection right? Then change the order of the list version... Different results???

This,
context.stocks = [sid(22887),sid(25905),sid(25902),sid(25899),sid(25898),sid(25906),sid(22445)]

gives different results than this: (just rearranged in order)
context.stocks = [sid(25898),sid(25906),sid(22445),sid(22887),sid(25905),sid(25902),sid(25899)]

Which may give different results than (using a set not a list)
context.stocks = {sid(25898),sid(25906),sid(22445),sid(22887),sid(25905),sid(25902),sid(25899)}
(note the use of [] vs {})

Thanks for your question:

Q. You can try a "set" {} or a "list" [] but either should work no?
A. algorithm works with "set" {} only. What is the advantage of switching to the list data structure?

Q. The code just iterates the collection right?
A. Yes. Picks best ETF based on simple performance criteria.

Q.Then change the order of the list version... Different results???
A. Not when using the original code with set notation. Still investigating the "rearranged order" differences. I DON'T THINK the order of the ETFs within the set should matter.

I always express the ETFs using the sid number as follows:

def initialize(context):
context.stocks = {
22887: sid(22887), # EDV VANGUARD treasury
25906: sid(25906), # VHT (Vanguard Health Care ETF)
25905: sid(25905), # VGT (Vanguard Information Technology ETF)
25902: sid(25902), # VCR (Vanguard Consumer Discretionary ETF)
22445: sid(22445), # IBB (iShares Nasdaq Biotechnology Index Fund)
37515: sid(37515), # TNA (Direxion Small Cap Bull 3X Shares (ETF))
25899: sid(25899), # VB = Vanguard small cap
25898: sid(25898) # VAW (Vanguard Materials ETF)
}

================================== experimental results changing ETF order within the set =============== these experiments (using the set {} notation) all produced the same performance results with ETFs listed in different order as expected.
start date: 11/05/2008, end date: 01/06/2015, returns 400.7% running daily using EFTs in this order: VHT, VGT, VCR, IBB, TNA, EDV, VB, VAW
start date: 11/05/2008, end date: 01/06/2015, returns 400.7% running daily using EFTs in this order: VB, VHT, VGT, VCR, IBB, TNA, EDV, VAW
start date: 11/05/2008, end date: 01/06/2015, returns 400.7% running daily using EFTs in this order: EDV, VHT, VGT, VCR, IBB, TNA, VB, VAW
start date: 11/05/2008, end date: 01/06/2015, returns 400.7% running daily using EFTs in this order: VHT, VGT, IBB, TNA, EDV, VB, VAW, VCR

....

Dave M

The reason to use a list "[]" is to use the easier to understand implementation of context.stocks = symbols()

Here's two differently ordered lists of your ETFs that produce different results. Still trying to figure out why.

    context.stocks = symbols("EDV","VGT","VCR","VB","VAW","VHT","IBB") # returns: 287.6%  
    context.stocks = symbols("VAW","VHT","IBB","EDV","VGT","VCR","VB") # returns: 87.1%  

Greetings, first post to Quantopian forums...

This algo caught my eye and I got sucked into figuring out what is going on with the rest of you.

Seems this variable result is somewhat due to the EDV instrument and perhaps the fact that there is no data for EDV until sometime in 2013. If you remove EDV from the list, the performance is consistent regardless of order.

I'm not yet up to speed enough with the platform to figure out how to ignore an instrument in the list that does not have valid data. Found this after putting the following check in around line 53 since GetStockMetrics was returning NaN values:

if math.isnan(p) or math.isnan(v):  
            continue  

Without EDV, the algo is returning 80% for me with 42.9%DD.

Randy,

Thanks -- Very interesting discovery. Nice method provided for detecting NaNs. A more rigorous approach [for trading real money] could test all ETF data for NaN.

The inception date for the EDV ETF is 6 Dec 2007 [ https://institutional.vanguard.com/iippdf/pdfs/FS930R.pdf ] , so perhaps some of the Quantopian data is missing?
We also expected the Quantopian wrapper to adjust / report when EDV's data was not available.

I guess Quantopian is using us to beta test stuff like this???

Dave

Sets and Lists behave differently when it comes to internal order.

Given this code:

def handle_data(context, data):  
    set = {'a','b','c','d'}  
    list = ['a','b','c','d']  
    print('test1')  
    print(set)  
    print(list)

    set = {'c','d','a','b'}  
    list = ['c','d','a','b']  
    print('test2')  
    print(set)  
    print(list)  

You'll get this print result:

2011-01-04 PRINT test1  
2011-01-04 PRINT set(['a', 'c', 'b', 'd'])  
2011-01-04 PRINT ['a', 'b', 'c', 'd']  
2011-01-04 PRINT test2  
2011-01-04 PRINT set(['a', 'c', 'b', 'd'])  
2011-01-04 PRINT ['c', 'd', 'a', 'b']  

The sets do not change their order based on their constructors. The lists do.
This means @Dave M. that your algo for applying performance and volatility during ranking appears to be imperfect. You get the same order of securities no matter what by using a set. But using a list changes the order which should not change the results -- but it does.

The above code block of the refactoring effort has been updated to reflect this issue.

Fails for Minute data; index out of bounds error on 230.

Hey Noah,

There are a couple version of the algorithm posted on this thread, do you mind sharing the one you're working with so we can help debug?

Thanks,
Seong

Disclaimer

The material on this website is provided for informational purposes only and does not constitute an offer to sell, a solicitation to buy, or a recommendation or endorsement for any security or strategy, nor does it constitute an offer to provide investment advisory services by Quantopian. In addition, the material offers no opinion with respect to the suitability of any security or specific investment. No information contained herein should be regarded as a suggestion to engage in or refrain from any investment-related course of action as none of Quantopian nor any of its affiliates is undertaking to provide investment advice, act as an adviser to any plan or entity subject to the Employee Retirement Income Security Act of 1974, as amended, individual retirement account or individual retirement annuity, or give advice in a fiduciary capacity with respect to the materials presented herein. If you are an individual retirement or other investor, contact your financial advisor or other fiduciary unrelated to Quantopian about whether any given investment idea, strategy, product or service described herein may be appropriate for your circumstances. All investments involve risk, including loss of principal. Quantopian makes no guarantees as to the accuracy or completeness of the views expressed in the website. The views are subject to change, and may have become unreliable for various reasons, including changes in market conditions or economic circumstances.

The main one at the very top.

Hi Noah,

Take a look at the code below, I think a part of the error was coming from the batch_transform use so the algo here has been converted into history()

Clone Algorithm
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#
# USA Market Sector Rotation Strategy
#
# This strategy rotates between Vanguard market sector ETFs on a monthly
# basis.  Each month the performance and mean 20-day volitility over
# the last 13 weeks are used to rank which ETF should be invested 
# in for the coming month.
import math
import pandas


def initialize(context):
    context.stocks = {
        # 25904: sid(25904), # VFH (Vanguard Financials ETF)
        25906: sid(25906), # VHT (Vanguard Health Care ETF)
        25905: sid(25905), # VGT (Vanguard Information Technology ETF)
        # 26667: sid(26667), # VDE (Vanguard Energy ETF)
        25902: sid(25902), # VCR (Vanguard Consumer Discretionary ETF)
        22445: sid(22445), # IBB (iShares Nasdaq Biotechnology Index Fund)
        # 39479: sid(39479), # IBB (iShares Nasdaq Biotechnology Index Fund)
        22887: sid(22887), # EDV VANGUARD treasury
        25899: sid(25899), # VB = Vanguard small cap
        25898: sid(25898)  # VAW (Vanguard Materials ETF)
    } 

    # Keep track of the current month.
    context.currentMonth = None
    
    # compare algorithm's performance versus small cap EFT buy and hold
    set_benchmark(symbol('vb'))
    
    # The order ID of the sell order currently being filled
    context.oid = None
    
    # The current stock being held
    context.currentStock = None
    
    # The next stock that needs to get purchased (once the sell order
    # on the current stock is filled
    context.nextStock = None
    
    # The 3-month lookback period.  Calculated based on there being
    # an average of 21 trading days in a month
    context.lookback = 63

'''
  Gets the minimum and maximum values of an array of values
'''
def getMinMax(arr):
   return min(arr.values()), max(arr.values()) 

'''
  Calculates the n-day historical volatility given a set of
  n+1 prices. 

  @param period The number of days for which to calculate volatility
  @param prices An array of price information.  Must be of length
    period + 1.
'''
def historicalVolatility(period, prices):
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)
    
    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    # Find the average of all returns
    rMean = sum(r) / period;
    
    # Determine the difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))
    
    # Take the square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year
    vol = math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)
    
    return vol

'''
  Gets the performance and average 20-day volatility of a security
  over a given period

  @param prices
  @param period The time period for which to find
'''
def getStockMetrics(prices, period):
    # Get the prices
    #prices = data['close_price'][security][-period-1:]
    start = prices[-period] # First item
    end = prices[-1] # Last item
    
    performance = (end - start) / start
    
    # Calculate 20-day volatility for the given period
    v = []
    x = 0
    for i in xrange(-period, 0):
        v.append(historicalVolatility(20, prices[i-21:21+x]))
        x += 1
    
    volatility = sum(v) / period
    
    return performance, volatility

'''
  Picks the best stock from a group of stocks based on the given
  data over a specified period using the stocks' performance and
  volatility

  @param data The datapanel with data of all the stocks
  @param stocks A list of stocks to rank
  @param period The time period over which the stocks will be
    analyzed
'''
def getBestStock(data, stocks, period):
    best = None
    
    performances = {}
    volatilities = {}
        
    # Get performance and volatility for all the stocks
    for s in stocks:
        p, v = getStockMetrics(data[s.sid], period)
        performances[s.sid] = p
        volatilities[s.sid] = v
    
    # Determine min/max of each.  NOTE: volatility is switched
    # since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)
    
    # Normalize the performance and volatility values to a range
    # between [0..1] then rank them based on a 70/30 weighting.
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        rank = p * 0.7 + v * 0.3
        
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))
        
        # If the new rank is greater than the old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    
    return best[0]

'''
  Sells all the currently held positions in the context's portfolio
'''
def sellHoldings(context):
    positions = context.portfolio.positions
    
    oid = None
    for p in positions.values():
        if (p.amount > 0):
            #log.debug('ordering %s' % p)
            oid = order(p.sid, -p.amount)
            
    return oid

'''
  Utilize the history function to accumulate 83 days worth of data
'''
def accumulateData(data):
    price_history = history(bar_count=83, frequency='1d', field='price')
    return price_history
    
  

'''
  The main proccessing function.  This is called and passed data
'''
def handle_data(context, data):
    # Accumulate data until there is enough days worth of data
    # to process without having outOfBounds issues.
    datapanel = accumulateData(data)
    
    if datapanel is None:
        # There is insufficient data accumulated to process
        return
    
    
    # If there is an order ID, check the status of the order.
    # If there is an order and it is filled, the next stock
    # can be purchased.
    if context.oid is not None:
        orderObj = get_order(context.oid)
        if orderObj.filled == orderObj.amount:
            # Good to buy next holding
            amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
            log.info('Sell order complete, buying %s of %s (%s of %s)' % \
                 (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
            order(context.nextStock, amount)
            context.currentStock = context.nextStock
            context.oid = None
            context.nextStock = None
    
    
    date = get_datetime()
    month = date.month    
    
    if not context.currentMonth:
        # Set the month initially
        context.currentMonth = month
    
    if context.currentMonth == month:
        # If the current month is unchanged, nothing further to do
        return
    
    context.currentMonth = month
    
    # At this point, a new month has been reached.  The stocks
    # need to be
    
    # Ensure stocks are only traded if possible.  
    # (e.g) EDV doesn't start trading until late 2007, without
    # this, any backtest run before that date would fail.
    stocks = []
    for s in context.stocks.values():
        if date > s.security_start_date:
            stocks.append(s)
    
    
    # Determine which stock should be used for the next month
    best = getBestStock(datapanel, stocks, context.lookback)

    if best:
        if (context.currentStock is not None and context.currentStock == best):
            # If there is a stock currently held and it is the same as
            # the new 'best' stock, nothing needs to be done
            return
        else:
            # Otherwise, the current stock needs to be sold and the new
            # stock bought
            context.oid = sellHoldings(context)
            context.nextStock = best
            
            # Purchase will not occur until the next call of handle_data
            # and only when the order has been filled.

    # If there is no stock currently held, it needs to be bought.
    # This only happend
    if context.currentStock is None:
        amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
        order(context.nextStock, amount)
        context.currentStock = context.nextStock
        context.oid = None
        context.nextStock = None
There was a runtime error.

Sorry for digging up an old thread, but I don't understand the concept of this code. It seems to take the best performance per volatility ETF, but what is the theory and math behind it? Can anyone point to a paper or tutorial that could explain this to me?

Thanks

Hi Johnny,
The theory is based on empirical evidence that stocks and ETFs which have performed well in the past, tend to outperform. This is an empirically observed break from the efficient markets hypothesis which assumes stock returns are independently and identically distributed (i.e. today's return can tell you nothing about tomorrow's return). The profitability of this and similar strategies is based on the idea that past returns do contain information about future returns.
There has been much written about this in the literature. Feel fee to browse some of the articles on Google Scholar:
https://scholar.google.com/scholar?as_vis=1&q=finance+momentum&hl=en&as_sdt=1,9

Thanks for your reply, Reed.
There is something other than momentum at play here because momentum is simple to calculate while Dave's code is rather complex. My simple momentum algo fares a lot worse than Dave's code. There is something else at play here other than comparing x month return.

I don't understand the code so it has an almost magical quality to it. I was hoping that the technique (or magic!) was sufficiently robust to apply to other rotation strategies such as asset rotation.

It's quite impressive until about 2009...

Clone Algorithm
25
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#
# USA Market Sector Rotation Strategy
#
# This strategy rotates between Vanguard market sector ETFs on a monthly
# basis.  Each month the performance and mean 20-day volitility over
# the last 13 weeks are used to rank which ETF should be invested 
# in for the coming month.
import math
import pandas


def initialize(context):
    context.stocks = symbols('SPY', 'EFA', 'VNQ', 'GSG', 'TLT')

    # Keep track of the current month.
    context.currentMonth = None
    
    # compare algorithm's performance versus small cap EFT buy and hold
    set_benchmark(symbol('vb'))
    
    # The order ID of the sell order currently being filled
    context.oid = None
    
    # The current stock being held
    context.currentStock = None
    
    # The next stock that needs to get purchased (once the sell order
    # on the current stock is filled
    context.nextStock = None
    
    # The 3-month lookback period.  Calculated based on there being
    # an average of 21 trading days in a month
    context.lookback = 63

'''
  Gets the minimum and maximum values of an array of values
'''
def getMinMax(arr):
   return min(arr.values()), max(arr.values()) 

'''
  Calculates the n-day historical volatility given a set of
  n+1 prices. 

  @param period The number of days for which to calculate volatility
  @param prices An array of price information.  Must be of length
    period + 1.
'''
def historicalVolatility(period, prices):
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)
    
    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    # Find the average of all returns
    rMean = sum(r) / period;
    
    # Determine the difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))
    
    # Take the square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year
    vol = math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)
    
    return vol

'''
  Gets the performance and average 20-day volatility of a security
  over a given period

  @param prices
  @param period The time period for which to find
'''
def getStockMetrics(prices, period):
    # Get the prices
    #prices = data['close_price'][security][-period-1:]
    start = prices[-period] # First item
    end = prices[-1] # Last item
    
    performance = (end - start) / start
    
    # Calculate 20-day volatility for the given period
    v = []
    x = 0
    for i in xrange(-period, 0):
        v.append(historicalVolatility(20, prices[i-21:21+x]))
        x += 1
    
    volatility = sum(v) / period
    
    return performance, volatility

'''
  Picks the best stock from a group of stocks based on the given
  data over a specified period using the stocks' performance and
  volatility

  @param data The datapanel with data of all the stocks
  @param stocks A list of stocks to rank
  @param period The time period over which the stocks will be
    analyzed
'''
def getBestStock(data, stocks, period):
    best = None
    
    performances = {}
    volatilities = {}
        
    # Get performance and volatility for all the stocks
    for s in stocks:
        p, v = getStockMetrics(data['price'][s.sid], period)
        performances[s.sid] = p
        volatilities[s.sid] = v
    
    # Determine min/max of each.  NOTE: volatility is switched
    # since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)
    
    # Normalize the performance and volatility values to a range
    # between [0..1] then rank them based on a 70/30 weighting.
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        rank = p * 0.7 + v * 0.3
        
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))
        
        # If the new rank is greater than the old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    
    return best[0]

'''
  Sells all the currently held positions in the context's portfolio
'''
def sellHoldings(context):
    positions = context.portfolio.positions
    
    oid = None
    for p in positions.values():
        if (p.amount > 0):
            #log.debug('ordering %s' % p)
            oid = order(p.sid, -p.amount)
            
    return oid

'''
  Utilize the batch_transform decorator to accumulate multiple days
  of data into one datapanel  Need the window length to be 20 longer
  than lookback period to allow for a 20-day volatility calculation
'''
@batch_transform(window_length=83)
def accumulateData(data):
    # return price_history = history(bar_count=20, frequency='1d', field='price')
    return data
  

'''
  The main proccessing function.  This is called and passed data
'''
def handle_data(context, data):
    # Accumulate data until there is enough days worth of data
    # to process without having outOfBounds issues.
    datapanel = accumulateData(data)
    
    if datapanel is None:
        # There is insufficient data accumulated to process
        return
    
    
    # If there is an order ID, check the status of the order.
    # If there is an order and it is filled, the next stock
    # can be purchased.
    if context.oid is not None:
        orderObj = get_order(context.oid)
        if orderObj.filled == orderObj.amount:
            # Good to buy next holding
            amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
            log.info('Sell order complete, buying %s of %s (%s of %s)' % \
                 (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
            order(context.nextStock, amount)
            context.currentStock = context.nextStock
            context.oid = None
            context.nextStock = None
    
    
    date = get_datetime()
    month = date.month    
    
    if not context.currentMonth:
        # Set the month initially
        context.currentMonth = month
    
    if context.currentMonth == month:
        # If the current month is unchanged, nothing further to do
        return
    
    context.currentMonth = month
    
    # At this point, a new month has been reached.  The stocks
    # need to be
    
    # Ensure stocks are only traded if possible.  
    # (e.g) EDV doesn't start trading until late 2007, without
    # this, any backtest run before that date would fail.
    stocks = []
    for s in context.stocks:
        if date > s.security_start_date:
            stocks.append(s)
    
    
    # Determine which stock should be used for the next month
    best = getBestStock(datapanel, stocks, context.lookback)

    if best:
        if (context.currentStock is not None and context.currentStock == best):
            # If there is a stock currently held and it is the same as
            # the new 'best' stock, nothing needs to be done
            return
        else:
            # Otherwise, the current stock needs to be sold and the new
            # stock bought
            context.oid = sellHoldings(context)
            context.nextStock = best
            
            # Purchase will not occur until the next call of handle_data
            # and only when the order has been filled.

    # If there is no stock currently held, it needs to be bought.
    # This only happend
    if context.currentStock is None:
        amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
        order(context.nextStock, amount)
        context.currentStock = context.nextStock
        context.oid = None
        context.nextStock = None
There was a runtime error.

The order magic likely comes from the rank being equal. Then the first listed ETF is picked.

@ maket tech how were you able... to quantify ... those results context.stocks = symbols("EDV","VGT","VCR","VB","VAW","VHT","IBB") # returns: 287.6% context.stocks = symbols("VAW","VHT","IBB","EDV","VGT","VCR","VB") # returns: 87.1% can you show us the code... we will check on how it is computed...

Hi,
Is there a reason why this algorithm works on equity sectors, but not all 13 of them like for example the standard morningstar sector classification?
I mean, is there some logic behind?
Plus, In the original list of stocks IBB and EDV have not enough data, you can tell buy adding a check like:

backtest_start = '2006-1-1'  
context.no_data = [sec for sec in context.stocks.itervalues() if sec.start_date.replace(tzinfo=None) > context.backtest_start]  
if len(context.no_data) > 0:  
    context.enough_history= False  

By the way, unfortunately adding all sectors cause the equity line to be very poor

Backtest in minute mode of the same algo utilizing the history function to accumulate data is for now 29% in drowdown.

Clone Algorithm
25
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#  ETF market rotation strategy by Seong Lee
# USA Market Sector Rotation Strategy
#
# This strategy rotates between Vanguard market sector ETFs on a monthly
# basis.  Each month the performance and mean 20-day volitility over
# the last 13 weeks are used to rank which ETF should be invested 
# in for the coming month.
import math
import pandas


def initialize(context):
    context.stocks = {
        # 25904: sid(25904), # VFH (Vanguard Financials ETF)
        25906: sid(25906), # VHT (Vanguard Health Care ETF)
        25905: sid(25905), # VGT (Vanguard Information Technology ETF)
        # 26667: sid(26667), # VDE (Vanguard Energy ETF)
        25902: sid(25902), # VCR (Vanguard Consumer Discretionary ETF)
        22445: sid(22445), # IBB (iShares Nasdaq Biotechnology Index Fund)
        # 39479: sid(39479), # IBB (iShares Nasdaq Biotechnology Index Fund)
        22887: sid(22887), # EDV VANGUARD treasury
        25899: sid(25899), # VB = Vanguard small cap
        25898: sid(25898)  # VAW (Vanguard Materials ETF)
    } 

    # Keep track of the current month.
    context.currentMonth = None
    
    # compare algorithm's performance versus small cap EFT buy and hold
    set_benchmark(symbol('vb'))
    
    # The order ID of the sell order currently being filled
    context.oid = None
    
    # The current stock being held
    context.currentStock = None
    
    # The next stock that needs to get purchased (once the sell order
    # on the current stock is filled
    context.nextStock = None
    
    # The 3-month lookback period.  Calculated based on there being
    # an average of 21 trading days in a month
    context.lookback = 63

'''
  Gets the minimum and maximum values of an array of values
'''
def getMinMax(arr):
   return min(arr.values()), max(arr.values()) 

'''
  Calculates the n-day historical volatility given a set of
  n+1 prices. 

  @param period The number of days for which to calculate volatility
  @param prices An array of price information.  Must be of length
    period + 1.
'''
def historicalVolatility(period, prices):
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)
    
    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    # Find the average of all returns
    rMean = sum(r) / period;
    
    # Determine the difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))
    
    # Take the square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year
    vol = math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)
    
    return vol

'''
  Gets the performance and average 20-day volatility of a security
  over a given period

  @param prices
  @param period The time period for which to find
'''
def getStockMetrics(prices, period):
    # Get the prices
    #prices = data['close_price'][security][-period-1:]
    start = prices[-period] # First item
    end = prices[-1] # Last item
    
    performance = (end - start) / start
    
    # Calculate 20-day volatility for the given period
    v = []
    x = 0
    for i in xrange(-period, 0):
        v.append(historicalVolatility(20, prices[i-21:21+x]))
        x += 1
    
    volatility = sum(v) / period
    
    return performance, volatility

'''
  Picks the best stock from a group of stocks based on the given
  data over a specified period using the stocks' performance and
  volatility

  @param data The datapanel with data of all the stocks
  @param stocks A list of stocks to rank
  @param period The time period over which the stocks will be
    analyzed
'''
def getBestStock(data, stocks, period):
    best = None
    
    performances = {}
    volatilities = {}
        
    # Get performance and volatility for all the stocks
    for s in stocks:
        p, v = getStockMetrics(data[s.sid], period)
        performances[s.sid] = p
        volatilities[s.sid] = v
    
    # Determine min/max of each.  NOTE: volatility is switched
    # since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)
    
    # Normalize the performance and volatility values to a range
    # between [0..1] then rank them based on a 70/30 weighting.
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        rank = p * 0.7 + v * 0.3
        
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))
        
        # If the new rank is greater than the old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    
    return best[0]

'''
  Sells all the currently held positions in the context's portfolio
'''
def sellHoldings(context):
    positions = context.portfolio.positions
    
    oid = None
    for p in positions.values():
        if (p.amount > 0):
            #log.debug('ordering %s' % p)
            oid = order(p.sid, -p.amount)
            
    return oid

'''
  Utilize the history function to accumulate 83 days worth of data
'''
def accumulateData(data):
    price_history = history(bar_count=83, frequency='1d', field='price')
    return price_history
    
  

'''
  The main proccessing function.  This is called and passed data
'''
def handle_data(context, data):
    # Accumulate data until there is enough days worth of data
    # to process without having outOfBounds issues.
    datapanel = accumulateData(data)
    
    if datapanel is None:
        # There is insufficient data accumulated to process
        return
    
    
    # If there is an order ID, check the status of the order.
    # If there is an order and it is filled, the next stock
    # can be purchased.
    if context.oid is not None:
        orderObj = get_order(context.oid)
        if orderObj.filled == orderObj.amount:
            # Good to buy next holding
            amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
            log.info('Sell order complete, buying %s of %s (%s of %s)' % \
                 (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
            order(context.nextStock, amount)
            context.currentStock = context.nextStock
            context.oid = None
            context.nextStock = None
    
    
    date = get_datetime()
    month = date.month    
    
    if not context.currentMonth:
        # Set the month initially
        context.currentMonth = month
    
    if context.currentMonth == month:
        # If the current month is unchanged, nothing further to do
        return
    
    context.currentMonth = month
    
    # At this point, a new month has been reached.  The stocks
    # need to be
    
    # Ensure stocks are only traded if possible.  
    # (e.g) EDV doesn't start trading until late 2007, without
    # this, any backtest run before that date would fail.
    stocks = []
    for s in context.stocks.values():
        if date > s.security_start_date:
            stocks.append(s)
    
    
    # Determine which stock should be used for the next month
    best = getBestStock(datapanel, stocks, context.lookback)

    if best:
        if (context.currentStock is not None and context.currentStock == best):
            # If there is a stock currently held and it is the same as
            # the new 'best' stock, nothing needs to be done
            return
        else:
            # Otherwise, the current stock needs to be sold and the new
            # stock bought
            context.oid = sellHoldings(context)
            context.nextStock = best
            
            # Purchase will not occur until the next call of handle_data
            # and only when the order has been filled.

    # If there is no stock currently held, it needs to be bought.
    # This only happend
    if context.currentStock is None:
        amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
        order(context.nextStock, amount)
        context.currentStock = context.nextStock
        context.oid = None
        context.nextStock = None
There was a runtime error.

Over 850% at one point. How on earth to address DD. Quite a few changes here ...

Clone Algorithm
83
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
'''
 ETF market rotation strategy by Seong Lee [modified]
 USA Market Sector Rotation Strategy

This strategy rotates between [various stocks] on a monthly basis.  
Each month the performance and mean x-day volitility over the last y-weeks 
are used to rank which ETF should be invested in for the coming month.
'''
import math
import numpy as np

def initialize(context):
    stocks = [
        sid(8554),  # SPY
        sid(37515), # TNA (Direxion Small Cap Bull 3X Shares (ETF))
        sid(22887), # EDV Vanguard Treasury
        #sid(25900), # VBK Vanguard Small cap growth ETF
        #sid(25899), # VB  Vanguard Small cap
        #sid(25906), # VHT Vanguard Health Care ETF
        #sid(25905), # VGT Vanguard Information Technology ETF
        #sid(25902), # VCR Vanguard Consumer Discretionary ETF
        #sid(25898), # VAW Vanguard Materials ETF
        #sid(26667), # VDE Vanguard Energy ETF
        #sid(25904), # VFH Vanguard Financials ETF
        #sid(22445), # IBB iShares Nasdaq Biotechnology Index Fund
    ]
    context.stocks = {}
    for s in stocks:
        context.stocks[s.sid] = s
    context.oid          = None  # Order ID of sell order currently being filled
    context.currentMonth = None  # Keep track of current month.
    context.currentStock = None  # Current stock being held
    context.nextStock    = None  # Next stock to purchased once sell order on current filled
    context.lookback     = 60    # Lookback period
    context.wndw_vol     = 25    # Volatility window
    context.pf_prv       = context.portfolio.portfolio_value  # experiment
    
    schedule_function(trade, date_rules.month_start(0), time_rules.market_open())
    schedule_function(trade, date_rules.month_start(10), time_rules.market_open())
    schedule_function(trade, date_rules.month_start(20), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(1), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(11), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(21), time_rules.market_open())
    '''
    schedule_function(trade, date_rules.month_start(0), time_rules.market_open())
    schedule_function(trade, date_rules.month_start(5), time_rules.market_open())
    schedule_function(trade, date_rules.month_start(10), time_rules.market_open())
    schedule_function(trade, date_rules.month_start(15), time_rules.market_open())
    schedule_function(trade, date_rules.month_start(20), time_rules.market_open())
    schedule_function(trade, date_rules.month_start(25), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(0), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(6), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(11), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(16), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(21), time_rules.market_open())
    schedule_function(track_orders, date_rules.month_start(25), time_rules.market_open())
    '''
    
def trade(context, data):
    c = context
    datapanel = history(bar_count=83, frequency='1d', field='price')

    # If an order is filled, next stock can be purchased.
    if c.oid is not None:    # If an order ID, check status of order.
        orderObj = get_order(c.oid)
        if orderObj.filled == orderObj.amount:      # Good to buy next holding
            amount = math.floor((c.portfolio.cash) / data[c.nextStock.sid].price) - 1
            log.info('Sell order made, buy {} {} ({} of {})'.format(amount, c.nextStock.symbol,
                   int(amount*data[c.nextStock.sid].price), int(c.portfolio.cash)))
            order(c.nextStock, amount)
            track_orders(context, data)
            c.currentStock = c.nextStock
            c.oid          = None
            c.nextStock    = None

    # Determine which stock should be used for next month
    best = getBestStock(context, datapanel, c.stocks.values(), c.lookback)

    if best:
        # experiment
        if  0 and  context.portfolio.portfolio_value < context.pf_prv * .9: # If drop, dump, reduce drawdown?
            log.info('pf drop, sell')
            c.oid = sellHoldings(context, data)
            c.nextStock = best
            return
        context.pf_prv = context.portfolio.portfolio_value

        if (c.currentStock is not None and c.currentStock == best):
            return # If stock currently held is same as new 'best' stock, nothing to do
        else:      # Otherwise, current stock needs to be sold and new stock bought
            c.oid = sellHoldings(context, data)
            c.nextStock = best
            # Purchase will not occur until next call of handle_data
            # and only when order has been filled.
    else:
        pass # or something

    if c.currentStock is None:    # If no stock currently held, it needs to be bought.
        amount = math.floor((c.portfolio.cash) / data[c.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, c.nextStock, amount*data[c.nextStock.sid].price, c.portfolio.cash))
        order(c.nextStock, amount)
        track_orders(context, data)
        c.currentStock = c.nextStock
        c.nextStock    = None
        c.oid          = None

def getBestStock(context, datapanel, stocks, period):
    '''  Picks the best stock using the stocks' performance and volatility
      @param datapanel  Datapanel with data of all the stocks
      @param stocks     List of stocks to rank
      @param period     Time period over which the stocks will be analyzed
    '''
    best = None
    performances = {}
    volatilities = {}

    for s in stocks:    # Performance and volatility for all the stocks
        p, v = getStockMetrics(context, datapanel[s], period)
        p = 0 if np.isnan(p) else p
        v = 0 if np.isnan(v) else v
        #track_values(context, v)    # avg 0.09  lo 0.00  hi 0.34
        performances[s.sid] = p
        volatilities[s.sid] = v

    # min/max of each. Volatility is switched since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)

    # Normalize performance, volatility values to a range between [0..1], rank based on 70/30 weighting
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        '''
        p    = (performances[s.sid]) ; v = (volatilities[s.sid]) # experiment and info
        rank = p * 0.7 * (1.0 - v * 0.3)
        #track_values(context, rank)    # avg 0.05  lo -0.40  hi 1.03
        '''
        rank = p * 0.7 - v * 0.3
        #rank = p * 0.7 + v * 0.3
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))

        # If new rank is greater than old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    return best[0]

def getStockMetrics(context, prices, period):
    '''  Performance and average x-day volatility over a given period
      @param prices
      @param period The time period for which to find
    '''
    start = prices[0]  # First item
    end   = prices[-1] # Last item
    performance = (end - start) / start
    v = []
    x = 0
    offset = context.wndw_vol + 1
    for i in xrange(-period, 0):    # x-day volatility for the given period
        v.append(historicalVolatility(context.wndw_vol, prices[i-offset:offset+x]))
        x += 1
    volatility = sum(v) / period
    return performance, volatility

def historicalVolatility(period, prices):
    '''  The n-day historical volatility given a set of n+1 prices.
      @param period  Number of days for which to calculate volatility
      @param prices  Array of price information, of length period + 1
    '''
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)

    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    #period = len(prices)
    #for i in range(-period, 0):
    #    r.append(math.log(prices[i] / prices[i+1]))
    
    #for i in xrange(1, period + 1):
    #    r.append(math.log(prices[i] / prices[i-1]))
    
    period = len(prices) - 1
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    rMean = sum(r) / period    # average of all returns

    # Difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))

    # Square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year, for vol.
    return math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)

def sellHoldings(context, data):
    '''  Sells all currently held positions in context's portfolio
    '''
    positions = context.portfolio.positions
    oid = None
    for p in positions.values():
        if (p.amount != 0):
            #log.debug('ordering %s' % p)
            oid = order_target(p.sid, 0)
            track_orders(context, data)
    return oid

def getMinMax(arr): #  Minimum and maximum values of an array of values
   return min(arr.values()), max(arr.values())

def handle_data(context, data):
    pvr(context, data)
    #track_orders(context, data)

def track_orders(context, data):  # Log orders created, filled, unfilled or canceled.
    '''      https://www.quantopian.com/posts/track-orders
    Status:
       0 - Unfilled
       1 - Filled (can be partial)
       2 - Canceled
    '''
    c = context
    log_cash = 0    # Show cash values in logging window or not.
    log_ids  = 0    # Include order id's in logging window or not.

    ''' Start and stop date options ...
    To not overwhelm the logging window, start/stop dates can be entered
      either below or in initialize() if you move to there for better efficiency.
    Example:
        c.dates  = {
            'active': 0,
            'start' : ['2007-05-07', '2010-04-26'],
            'stop'  : ['2008-02-13', '2010-11-15']
        }
    '''
    if 'orders' not in c:
        c.orders = {}               # Move these to initialize() for better efficiency.
        c.dates  = {
            'active': 0,
            'start' : [],           # Start dates, option
            'stop'  : []            # Stop  dates, option
        }
    from pytz import timezone       # Python only does once, makes this portable.
                                    #   Move to top of algo for better efficiency.

    # If the dates 'start' or 'stop' lists have something in them, sets them.
    if c.dates['start'] or c.dates['stop']:
        date = str(get_datetime().date())
        if   date in c.dates['start']:    # See if there's a match to start
            c.dates['active'] = 1
        elif date in c.dates['stop']:     #   ... or to stop
            c.dates['active'] = 0
    else:
        c.dates['active'] = 1  # Set to active b/c no conditions

    if c.dates['active'] == 0:
        return                 # Skip if off

    def _minute():   # To preface each line with the minute of the day.
        if get_environment('data_frequency') == 'minute':
            bar_dt = get_datetime().astimezone(timezone('US/Eastern'))
            minute = (bar_dt.hour * 60) + bar_dt.minute - 570  # (-570 = 9:31a)
            return str(minute).rjust(3)
        return ''    # Daily mode, just leave it out.

    def _orders(to_log):    # So all logging comes from the same line number,
        log.info(to_log)    #   for vertical alignment in the logging window.

    to_delete = []
    for id in c.orders:
        o     = get_order(id)
        sec   = o.sid ; sym = sec.symbol
        oid   = o.id if log_ids else ''
        cash  = 'cash {}'.format(int(c.portfolio.cash)) if log_cash else ''
        price = '%.2f' % data[sec].price if sec in data else '<sym absent in frame>'
        if o.filled:        # Filled at least some
            trade = 'Bot' if o.amount > 0 else 'Sold'
            _orders(' {}      {} {} {} at {}   {} {}'.format(_minute(),
                trade, o.filled, sym, price, cash, oid))
            to_delete.append(o.id)
        else:
            canceled = 'canceled' if o.status == 2 else ''
            _orders(' {}         {} {} unfilled {} {}'.format(_minute(),
                    o.sid.symbol, o.amount, canceled, oid))
            if canceled: to_delete.append(o.id)

    for oo_list in get_open_orders().values(): # Open orders list
        for o in oo_list:
            sec   = o.sid ; sym = sec.symbol
            oid   = o.id if log_ids else ''
            cash  = 'cash {}'.format(int(c.portfolio.cash)) if log_cash else ''
            price = '%.2f' % data[sec].price if sec in data else '<sym absent in frame>'
            if o.id in to_delete:
                continue
            if o.status == 2:                  # Canceled
                _orders(' {}    Canceled {} {} order   {} {}'.format(_minute(),
                        trade, o.amount, sym, price, cash, oid))
                to_delete.append(o.id)
            elif o.id not in c.orders:         # New
                c.orders[o.id] = 1
                trade = 'Buy' if o.amount > 0 else 'Sell'
                if o.limit:                    # Limit order
                    _orders(' {}   {} {} {} now {} limit {}   {} {}'.format(_minute(),
                        trade, o.amount, sym, price, o.limit, cash, oid))
                elif o.stop:                   # Stop order
                    _orders(' {}   {} {} {} now {} stop {}   {} {}'.format(_minute(),
                        trade, o.amount, sym, price, o.stop, cash, oid))
                else:                          # Market order
                    _orders(' {}   {} {} {} at {}   {} {}'.format(_minute(),
                        trade, o.amount, sym, price, cash, oid))
    for d in to_delete:
        del c.orders[d]
        
def pvr(context, data):
    ''' Custom chart and/or log of profit_vs_risk returns and related information
    '''
    # # # # # # # # # #  Options  # # # # # # # # # #
    record_max_lvrg = 1         # Maximum leverage encountered
    record_leverage = 0         # Leverage (context.account.leverage)
    record_q_return = 0         # Quantopian returns (percentage)
    record_pvr      = 1         # Profit vs Risk returns (percentage)
    record_pnl      = 0         # Profit-n-Loss
    record_shorting = 0         # Total value of any shorts
    record_overshrt = 0         # Shorts beyond longs+cash
    record_risk     = 0         # Risked, max cash spent or shorts beyond longs+cash
    record_risk_hi  = 1         # Highest risk overall
    record_cash     = 0         # Cash available
    record_cash_low = 1         # Any new lowest cash level
    logging         = 0         # Also to logging window conditionally (1) or not (0)
    log_method      = 'risk_hi' # 'daily' or 'risk_hi'

    from pytz import timezone   # Python will only do once, makes this portable.
                                #   Move to top of algo for better efficiency.
    c = context  # Brevity is the soul of wit -- Shakespeare [for efficiency, readability]
    if 'pvr' not in c:
        date_strt = get_environment('start').date()
        date_end  = get_environment('end').date()
        cash_low  = c.portfolio.starting_cash
        mode      = get_environment('data_frequency')
        c.pvr = {
            'max_lvrg': 0,
            'risk_hi' : 0,
            'date_prv': '',
            'cash_low': cash_low,
            'date_end': date_end,
            'mode'    : mode,
            'run_str' : '{} to {}  {}  {}'.format(date_strt,date_end,int(cash_low),mode)
        }
        log.info(c.pvr['run_str'])
    pvr_rtrn     = 0            # Profit vs Risk returns based on maximum spent
    profit_loss  = 0            # Profit-n-loss
    shorts       = 0            # Shorts value
    longs        = 0            # Longs  value
    overshorts   = 0            # Shorts value beyond longs plus cash
    new_risk_hi  = 0
    new_cash_low = 0                           # To trigger logging in cash_low case
    lvrg         = c.account.leverage          # Standard leverage, in-house
    date         = get_datetime().date()       # To trigger logging in daily case
    cash         = c.portfolio.cash
    start        = c.portfolio.starting_cash
    cash_dip     = int(max(0, start - cash))
    q_rtrn       = 100 * (c.portfolio.portfolio_value - start) / start

    if int(cash) < c.pvr['cash_low']:                # New cash low
        new_cash_low = 1
        c.pvr['cash_low']   = int(cash)
        if record_cash_low:
            record(CashLow = int(c.pvr['cash_low'])) # Lowest cash level hit

    if record_max_lvrg:
        if c.account.leverage > c.pvr['max_lvrg']:
            c.pvr['max_lvrg'] = c.account.leverage
            record(MaxLv = c.pvr['max_lvrg'])        # Maximum leverage

    if record_pnl:
        profit_loss = c.portfolio.pnl
        record(PnL = profit_loss)                    # "Profit and Loss" in dollars

    for p in c.portfolio.positions:
        shrs = c.portfolio.positions[p].amount
        if shrs < 0:
            shorts += int(abs(shrs * data[p].price))
        if shrs > 0:
            longs  += int(shrs * data[p].price)

    if shorts > longs + cash: overshorts = shorts             # Shorts when too high
    if record_shorting: record(Shorts  = shorts)              # Shorts value as a positve
    if record_overshrt: record(OvrShrt = overshorts)          # Shorts value as a positve
    if record_cash:     record(Cash = int(c.portfolio.cash))  # Cash
    if record_leverage: record(Lvrg = c.account.leverage)     # Leverage

    risk = int(max(cash_dip, shorts))
    if record_risk: record(Risk = risk)       # Amount in play, maximum of shorts or cash used

    if risk > c.pvr['risk_hi']:
        c.pvr['risk_hi'] = risk
        new_risk_hi = 1

        if record_risk_hi:
            record(RiskHi = c.pvr['risk_hi']) # Highest risk overall

    if record_pvr:      # Profit_vs_Risk returns based on max amount actually spent (risk high)
        if c.pvr['risk_hi'] != 0:     # Avoid zero-divide
            pvr_rtrn = 100 * (c.portfolio.portfolio_value - start) / c.pvr['risk_hi']
            record(PvR = pvr_rtrn)            # Profit_vs_Risk returns

    if record_q_return:
        record(QRet = q_rtrn)                 # Quantopian returns to compare to pvr returns curve

    def _minute():   # To preface each line with the minute of the day.
        if get_environment('data_frequency') == 'minute':
            bar_dt = get_datetime().astimezone(timezone('US/Eastern'))
            minute = (bar_dt.hour * 60) + bar_dt.minute - 570  # (-570 = 9:31a)
            return str(minute).rjust(3)
        return ''    # Daily mode, just leave it out.

    if logging:
        if log_method == 'risk_hi' and new_risk_hi \
          or log_method == 'daily' and c.date_prv != date \
          or new_cash_low:
            qret    = ' QRet '   + '%.1f' % q_rtrn
            lv      = ' Lv '     + '%.1f' % lvrg              if record_leverage else ''
            pvr     = ' PvR '    + '%.1f' % pvr_rtrn          if record_pvr      else ''
            pnl     = ' PnL '    + '%.0f' % profit_loss       if record_pnl      else ''
            csh     = ' Cash '   + '%.0f' % cash              if record_cash     else ''
            shrt    = ' Shrt '   + '%.0f' % shorts            if record_shorting else ''
            ovrshrt = ' Shrt '   + '%.0f' % overshorts        if record_overshrt else ''
            risk    = ' Risk '   + '%.0f' % risk              if record_risk     else ''
            mxlv    = ' MaxLv '  + '%.2f' % c.pvr['max_lvrg'] if record_max_lvrg else ''
            csh_lw  = ' CshLw '  + '%.0f' % c.pvr['cash_low'] if record_cash_low else ''
            rsk_hi  = ' RskHi '  + '%.0f' % c.pvr['risk_hi']  if record_risk_hi  else ''
            log.info('{}{}{}{}{}{}{}{}{}{}{}{}'.format(_minute(),
               lv, mxlv, qret, pvr, pnl, csh, csh_lw, shrt, ovrshrt, risk, rsk_hi))
    c.date_prv = date
    if c.pvr['date_end'] == date:
        # Summary on last minute of last day.
        # If using schedule_function(), backtest last day/time may need to match for this to execute.
        if 'pvr_summary_done' not in c: c.pvr_summary_done = 0
        log_summary = 0
        if c.pvr['mode'] == 'daily' and get_datetime().date() == c.pvr['date_end']:
            log_summary = 1
        elif c.pvr['mode'] == 'minute' and get_datetime() == get_environment('end'):
            log_summary = 1
        if log_summary and not c.pvr_summary_done:
            log.info('PvR {} ... {}'.format('%.1f' % pvr_rtrn, c.pvr['run_str']))
            log.info('  Profited {} on {} activated/transacted for PvR of {}%'.format(
                '%.0f' % (c.portfolio.portfolio_value - start), '%.0f' % c.pvr['risk_hi'], '%.1f' % pvr_rtrn))
            log.info('  QRet {} PvR {} CshLw {} MxLv {} RskHi {}'.format(
                '%.2f' % q_rtrn, '%.2f' % pvr_rtrn, '%.0f' % c.pvr['cash_low'],
                '%.2f' % c.pvr['max_lvrg'], '%.0f' % c.pvr['risk_hi']))
            c.pvr_summary_done = 1

def  track_values(c, var):
    ''' Log current, average, high and low for a particular value.
        Incoming 'c' for brevity is context
    '''
    if not (float('-inf') < float(var) < float('inf')): var = 0  # nan to 0  
    if 'val' not in c:
        c.val = {
            'hi' : var,
            'lo' : var,
            'avg': [0, 0],
        }
    avg, num     = c.val['avg']
    avg_new      = ((avg * num) + var) / (num + 1)
    c.val['avg'] = [avg_new, num + 1]
    if var > c.val['hi']: c.val['hi'] = var
    if var < c.val['lo']: c.val['lo'] = var
    log.info('now {}  avg {}  lo {}  hi {} '.format(
        '%.2f' % var, '%.2f' % avg_new, '%.2f' % c.val['lo'], '%.2f' % c.val['hi']))
There was a runtime error.

Only change to previous test here about Backtest for last 2 years. The results below Show the difficulty of picking of a " ETF universe" to trade and how returns changes over a different market behavior(what type of news is driving the market now VS 5 years ago). eg- 2015 Jan down market is now clear retrospective , however difficult to read the market message with a live systematic traded strategy or in hindsight
1,Although we are doing systematic trading.. its constant tuning effort by using a lot common sense read the market.
2, picking a 'universe of instrument' to trade changes for the same strategy over different market will be very different (essentially the news drives the market and our algo/instrument should reflect the view )
3, Never trade live with real money until you perform your due diligence and have some money to loose.
4, Never think you or your algo are the smartest thing created by us (academic or Geek or .. ) ; the random nature of the market will humble you (it at the end , same as going playing in a casino, at the end of day )

play safe , smart and self-aware even if we are the smartest dudes on the planet !
cheers

Clone Algorithm
26
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#  ETF market rotation strategy by Seong Lee
# USA Market Sector Rotation Strategy
#
# This strategy rotates between Vanguard market sector ETFs on a monthly
# basis.  Each month the performance and mean 20-day volitility over
# the last 13 weeks are used to rank which ETF should be invested 
# in for the coming month.
import math
import pandas


def initialize(context):
    context.stocks = {
        # 25904: sid(25904), # VFH (Vanguard Financials ETF)
        25906: sid(25906), # VHT (Vanguard Health Care ETF)
        25905: sid(25905), # VGT (Vanguard Information Technology ETF)
        # 26667: sid(26667), # VDE (Vanguard Energy ETF)
        25902: sid(25902), # VCR (Vanguard Consumer Discretionary ETF)
        22445: sid(22445), # IBB (iShares Nasdaq Biotechnology Index Fund)
        # 39479: sid(39479), # IBB (iShares Nasdaq Biotechnology Index Fund)
        22887: sid(22887), # EDV VANGUARD treasury
        25899: sid(25899), # VB = Vanguard small cap
        25898: sid(25898)  # VAW (Vanguard Materials ETF)
    } 

    # Keep track of the current month.
    context.currentMonth = None
    
    # compare algorithm's performance versus small cap EFT buy and hold
    set_benchmark(symbol('vb'))
    
    # The order ID of the sell order currently being filled
    context.oid = None
    
    # The current stock being held
    context.currentStock = None
    
    # The next stock that needs to get purchased (once the sell order
    # on the current stock is filled
    context.nextStock = None
    
    # The 3-month lookback period.  Calculated based on there being
    # an average of 21 trading days in a month
    context.lookback = 63

'''
  Gets the minimum and maximum values of an array of values
'''
def getMinMax(arr):
   return min(arr.values()), max(arr.values()) 

'''
  Calculates the n-day historical volatility given a set of
  n+1 prices. 

  @param period The number of days for which to calculate volatility
  @param prices An array of price information.  Must be of length
    period + 1.
'''
def historicalVolatility(period, prices):
    # HVdaily = sqrt( sum[1..n](x_t - Xbar)^2 / n - 1)
    
    # Start by calculating Xbar = 1/n sum[1..n] (ln(P_t / P_t-1))
    r = []
    for i in xrange(1, period + 1):
        r.append(math.log(prices[i] / prices[i-1]))

    # Find the average of all returns
    rMean = sum(r) / period;
    
    # Determine the difference of each return from the mean, then square
    d = []
    for i in xrange(0, period):
        d.append(math.pow((r[i] - rMean), 2))
    
    # Take the square root of the sum over the period - 1.  Then mulitply
    # that by the square root of the number of trading days in a year
    vol = math.sqrt(sum(d) / (period - 1)) * math.sqrt(252/period)
    
    return vol

'''
  Gets the performance and average 20-day volatility of a security
  over a given period

  @param prices
  @param period The time period for which to find
'''
def getStockMetrics(prices, period):
    # Get the prices
    #prices = data['close_price'][security][-period-1:]
    start = prices[-period] # First item
    end = prices[-1] # Last item
    
    performance = (end - start) / start
    
    # Calculate 20-day volatility for the given period
    v = []
    x = 0
    for i in xrange(-period, 0):
        v.append(historicalVolatility(20, prices[i-21:21+x]))
        x += 1
    
    volatility = sum(v) / period
    
    return performance, volatility

'''
  Picks the best stock from a group of stocks based on the given
  data over a specified period using the stocks' performance and
  volatility

  @param data The datapanel with data of all the stocks
  @param stocks A list of stocks to rank
  @param period The time period over which the stocks will be
    analyzed
'''
def getBestStock(data, stocks, period):
    best = None
    
    performances = {}
    volatilities = {}
        
    # Get performance and volatility for all the stocks
    for s in stocks:
        p, v = getStockMetrics(data[s.sid], period)
        performances[s.sid] = p
        volatilities[s.sid] = v
    
    # Determine min/max of each.  NOTE: volatility is switched
    # since a low volatility should be weighted highly.
    minP, maxP = getMinMax(performances)
    maxV, minV = getMinMax(volatilities)
    
    # Normalize the performance and volatility values to a range
    # between [0..1] then rank them based on a 70/30 weighting.
    for s in stocks:
        p = (performances[s.sid] - minP) / (maxP - minP)
        v = (volatilities[s.sid] - minV) / (maxV - minV)
        rank = p * 0.7 + v * 0.3
        
        #log.info('Rank info for %s: p=%s, v=%s, r=%s' % (s,p,v,rank))
        
        # If the new rank is greater than the old best rank, pick it.
        if best is None or rank > best[1]:
            best = s, rank
    
    return best[0]

'''
  Sells all the currently held positions in the context's portfolio
'''
def sellHoldings(context):
    positions = context.portfolio.positions
    
    oid = None
    for p in positions.values():
        if (p.amount > 0):
            #log.debug('ordering %s' % p)
            oid = order(p.sid, -p.amount)
            
    return oid

'''
  Utilize the history function to accumulate 83 days worth of data
'''
def accumulateData(data):
    price_history = history(bar_count=83, frequency='1d', field='price')
    return price_history
    
  

'''
  The main proccessing function.  This is called and passed data
'''
def handle_data(context, data):
    # Accumulate data until there is enough days worth of data
    # to process without having outOfBounds issues.
    datapanel = accumulateData(data)
    
    if datapanel is None:
        # There is insufficient data accumulated to process
        return
    
    
    # If there is an order ID, check the status of the order.
    # If there is an order and it is filled, the next stock
    # can be purchased.
    if context.oid is not None:
        orderObj = get_order(context.oid)
        if orderObj.filled == orderObj.amount:
            # Good to buy next holding
            amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
            log.info('Sell order complete, buying %s of %s (%s of %s)' % \
                 (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
            order(context.nextStock, amount)
            context.currentStock = context.nextStock
            context.oid = None
            context.nextStock = None
    
    
    date = get_datetime()
    month = date.month    
    
    if not context.currentMonth:
        # Set the month initially
        context.currentMonth = month
    
    if context.currentMonth == month:
        # If the current month is unchanged, nothing further to do
        return
    
    context.currentMonth = month
    
    # At this point, a new month has been reached.  The stocks
    # need to be
    
    # Ensure stocks are only traded if possible.  
    # (e.g) EDV doesn't start trading until late 2007, without
    # this, any backtest run before that date would fail.
    stocks = []
    for s in context.stocks.values():
        if date > s.security_start_date:
            stocks.append(s)
    
    
    # Determine which stock should be used for the next month
    best = getBestStock(datapanel, stocks, context.lookback)

    if best:
        if (context.currentStock is not None and context.currentStock == best):
            # If there is a stock currently held and it is the same as
            # the new 'best' stock, nothing needs to be done
            return
        else:
            # Otherwise, the current stock needs to be sold and the new
            # stock bought
            context.oid = sellHoldings(context)
            context.nextStock = best
            
            # Purchase will not occur until the next call of handle_data
            # and only when the order has been filled.

    # If there is no stock currently held, it needs to be bought.
    # This only happend
    if context.currentStock is None:
        amount = math.floor((context.portfolio.cash) / data[context.nextStock.sid].price) - 1
        log.info('First purchase, buying %s of %s (%s of %s)' % \
             (amount, context.nextStock, amount*data[context.nextStock.sid].price, context.portfolio.cash))
        order(context.nextStock, amount)
        context.currentStock = context.nextStock
        context.oid = None
        context.nextStock = None
There was a runtime error.