Back to Community
The Dogs of the Dow

This is the "famous" Dogs of the Dow strategy. I mainly continued the version posted by @ Jack Wood.

What's the Dogs of the Dow? Investing in the top 10 highest dividend yielding Dow stocks and hold them for a year. You can find out more about it here:
https://www.investopedia.com/terms/d/dogsofthedow.asp

UPDATE: Working version at the end of the post

What works:
Everything works fine if the backtest is run year by year, so the rebalance is made just once during a single backtest.

Problem:
If the backtest is run over multiple years, then the stocks are not chosen correctly.

Does anyone have a clue of why that happens??

Thanks.

Clone Algorithm
9
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
"""
IT RETURNS THE RIGHT STOCKS IF BACKTEST IS LAUNCHED EVERY YEAR SEPARATELY.
IF BACKTEST IS OVER MULTIPLE YEARS, IT DOESN'T.
"""

from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline import Pipeline, CustomFilter, CustomFactor
from quantopian.pipeline.data import Fundamentals
from quantopian.pipeline.filters import Q500US, Q1500US, QTradableStocksUS
from quantopian.pipeline.filters import StaticSids, StaticAssets
from quantopian.pipeline.factors import AverageDollarVolume

from pytz import timezone as tz
import datetime
import pandas as pd
import numpy as np
import math
import scipy.stats as stats


class Dow30:
    def __init__(self):
        '''
        this is a utility class that can return the list of Dow30 stocks for a given date.
        the input date must be >= 1999-11-01.
        source: http://www.djindexes.com/mdsidx/downloads/brochure_info/Dow_Jones_Industrial_Average_Historical_Components.pdf
        '''
        
         # 0 is replaced by 1
        raw_constituents = [
            ["1999-11-01",sid(4922),1],    # Minnesota Mining & Manufacturing/3M Company
            ["1999-11-01",sid(50428),1],    # AA
            ["1999-11-01",sid(679),1],
            ["1999-11-01",sid(7289),1],    # AT&T Corporation
            ["1999-11-01",sid(698),1],
            ["1999-11-01",sid(1267),1],
            ["1999-11-01",sid(1335),1],
            ["1999-11-01",sid(4283),1],
            ["1999-11-01",sid(2119),1],    # DD
            ["1999-11-01",sid(2482),1],    # EK
            ["1999-11-01",sid(8347),1],    # XOM
            ["1999-11-01",sid(3149),1],    # GE
            ["1999-11-01",sid(3246),1],
            ["1999-11-01",sid(3735),1],
            ["1999-11-01",sid(3496),1],
            ["1999-11-01",sid(25090),1],    # AlliedSignal Incorporated/Honeywell
            ["1999-11-01",sid(3951),1],
            ["1999-11-01",sid(3766),1],    # IBM
            ["1999-11-01",sid(3971),1],
            ["1999-11-01",sid(25006),1],
            ["1999-11-01",sid(4151),1],
            ["1999-11-01",sid(4707),1],    # MCD
            ["1999-11-01",sid(5029),1],
            ["1999-11-01",sid(5061),1],
            ["1999-11-01",sid(4954),1],    # MO
            ["1999-11-01",sid(5938),1],    # PG
            ["1999-11-01",sid(6653),1],    # SBC Communications Incorporated/AT&T Incorporated
            ["1999-11-01",sid(7883),1],
            ["1999-11-01",sid(8229),1],    # WMT
            ["1999-11-01",sid(2190),1],
            ["2004-04-08",sid(7289),0],
            ["2004-04-08",sid(2482),0],
            ["2004-04-08",sid(3971),0],
            ["2004-04-08",sid(239),1],
            ["2004-04-08",sid(5923),1],
            ["2004-04-08",sid(21839),1],
            ["2008-02-19",sid(4954),0],
            ["2008-02-19",sid(25090),0],
            ["2008-02-19",sid(700),1],
            ["2008-02-19",sid(23112),1],
            ["2008-09-22",sid(239),0],
            ["2008-09-22",sid(22802),1],    # KRFT/MDLZ
            ["2009-06-08",sid(1335),0],
            ["2009-06-08",sid(3246),0],
            ["2009-06-08",sid(7041),1],
            ["2009-06-08",sid(1900),1],
            ["2012-09-24",sid(22802),0],
            ["2012-09-24",sid(7792),1],
            ["2013-09-23",sid(50428),0],
            ["2013-09-23",sid(700),0],
            ["2013-09-23",sid(3735),0],
            ["2013-09-23",sid(20088),1],
            ["2013-09-23",sid(5328),1],
            ["2013-09-23",sid(35920),1],
            ["2015-03-19",sid(6653),0],
            ["2015-03-19",sid(24),1],
            ["2017-09-01",sid(2119),0],
            ["2017-09-01",sid(51157),1],
            ["2018-06-26",sid(3149),0],    # GE
            ["2018-06-26",sid(8089),1]    # WBA
            ]
        
        # translate the date string into a datetime.date
        self.data = []
        for equity in raw_constituents:
            self.data.append((
                datetime.datetime.strptime(equity[0], "%Y-%m-%d").date(),
                equity[1],
                equity[2]
            ))
            
    def get(self, dt):
        '''
        for a given datetime, return a list of the 30 dow stocks as Equity objects.
        dt must be >= 1999-11-01
        '''
        d = dt.date()
        ret = []
        for equity in self.data:
            if equity[0] > d: 
                break
            if equity[2] == 1:
                ret.append(equity[1])
            elif equity[2] == 0:
                ret.remove(equity[1])
            else:
                raise Exception('unknown membership')
        return ret
            


def rebalance(context, data):
    if context.trade == True:
        context.trade = False

        
        dogs = context.fundamentals_df.index
        current_portfolio = context.portfolio.positions

        # Check if we are holding an old position that is no longer in the dogs, needs to be closed
        for s in current_portfolio:
            if s not in dogs:
                order_target_percent(s, 0)

        # Check if we are holding each of the dogs, if not buy to 10% of our total portfolio
        for s in dogs:
            if data.can_trade(s):
                order_target_percent(s, .1)


def initialize(context):
    # Benchmark against the Dow Jones Industrial Average (DIA)
    set_benchmark(symbol('DIA'))
    
    # These are the default commission and slippage settings.  Change them to fit your
    # brokerage fees. These settings only matter for backtesting.  When you trade this 
    # algorithm, they are moot - the brokerage and real market takes over.
    set_commission(commission.PerShare(cost=0.0035, min_trade_cost=0.35))
    set_slippage(slippage.VolumeShareSlippage(volume_limit=0.025, price_impact=0.1))
    
    # create an instance of the Dow30 class and set it within context
    context.dow30 = Dow30()
    
    # next trade year
    context.year = 0
    
    # set to True to trigger a rebalance
    context.trade = False
    
    # for tracking max leverage
    context.mx_lvrg = 0
    
    # check for possible trade, daily
    schedule_function(func=rebalance, date_rule=date_rules.every_day(), time_rule=time_rules.market_open(hours=1))
    
    attach_pipeline(make_pipeline(), 'fundamentals_pipeline')

 
def before_trading_start(context, data):
    dt = get_datetime()
    
    # rebalance at the beginning or at the beginning of a new year
    if context.year == 0 or context.year == dt.year or len(context.portfolio.positions) != 10:
        context.trade = True
        context.year = dt.year + 1
    
    
        context.pipe_output = pipeline_output('fundamentals_pipeline')
        
        fundamentals_df = context.pipe_output.sort_values(['forward_dividend_yield'], ascending=False)
        
        
        context.fundamentals_df = fundamentals_df[:10]    # save to context to use during rebalance()


def handle_data(context, data):
    record(positions=len(context.portfolio.positions),leverage=context.account.leverage)
    
    if context.account.leverage > context.mx_lvrg:
        context.mx_lvrg = context.account.leverage  
        record(mx_lvrg = context.mx_lvrg)    # Record maximum leverage encountered

        
def make_pipeline():
    dt = get_datetime()
    
    members = Dow30().get( dt )
    sids = [m.sid for m in members]
    
    
    universe = StaticSids(sids)
    
    
    # Get dividend yield of the current year
    forward_dividend_yield = Fundamentals.forward_dividend_yield.latest
    
    
    #Build Pipeline
    pipe = Pipeline(screen=universe, columns = {
            'forward_dividend_yield' : forward_dividend_yield
        })

    return pipe
There was a runtime error.
7 responses

Some logging that might help.

Click a line number in the margin for line{s} of code to set breakpoint(s), run it, then type or copy variable names in the console prompt that appears and hit [Enter] to examine them.

1969-12-31 16:00 get:110 INFO get(self, 2016-12-27 00:00:00+00:00),    year 2016  
stocks ['MMM', 'AXP', 'BA', 'CAT', 'KO', 'DD', 'XOM', 'GE', 'HD', 'INTC', 'IBM', 'JPM', 'JNJ', 'MCD', 'MRK', 'MSFT', 'PG', 'UTX', 'WMT', 'DIS', 'PFE', 'VZ', 'CVX', 'TRV', 'CSCO', 'UNH', 'GS', 'NKE', 'V', 'AAPL']  
2016-12-27 05:45  PRINT .

2016  
    do_pipe, context.year 2016 == dt.year 2016  
    do_pipe, len(context.portfolio.positions) 0  
2016-12-27 05:45 before_trading_start:162 INFO context.year 2016 to dt.year + 1 = 2017  
2016-12-27 07:30 trade:119 INFO trade(),    context.year 2017  
stocks ['VZ', 'PFE', 'CVX', 'BA', 'CSCO', 'KO', 'IBM', 'XOM', 'CAT', 'MRK']  
2017-01-03 05:45  PRINT .

2017  
    do_pipe, context.year 2017 == dt.year 2017  
2017-01-03 05:45 before_trading_start:162 INFO context.year 2017 to dt.year + 1 = 2018  
2017-01-03 07:30 trade:119 INFO trade(),    context.year 2018  
stocks ['VZ', 'PFE', 'CVX', 'BA', 'CSCO', 'KO', 'IBM', 'CAT', 'XOM', 'MRK']  
2018-01-02 05:45  PRINT .

2018  
    do_pipe, context.year 2018 == dt.year 2018  
2018-01-02 05:45 before_trading_start:162 INFO context.year 2018 to dt.year + 1 = 2019  
2018-01-02 07:30 trade:119 INFO trade(),    context.year 2019  
stocks ['VZ', 'IBM', 'PFE', 'XOM', 'CVX', 'MRK', 'KO', 'CSCO', 'PG', 'GE']  
Clone Algorithm
2
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
'''
1969-12-31 16:00 get:110 INFO get(self, 2016-12-27 00:00:00+00:00),    year 2016  
stocks ['MMM', 'AXP', 'BA', 'CAT', 'KO', 'DD', 'XOM', 'GE', 'HD', 'INTC', 'IBM', 'JPM', 'JNJ', 'MCD', 'MRK', 'MSFT', 'PG', 'UTX', 'WMT', 'DIS', 'PFE', 'VZ', 'CVX', 'TRV', 'CSCO', 'UNH', 'GS', 'NKE', 'V', 'AAPL']
2016-12-27 05:45  PRINT .

2016 
    do_pipe, context.year 2016 == dt.year 2016
    do_pipe, len(context.portfolio.positions) 0
2016-12-27 05:45 before_trading_start:162 INFO context.year 2016 to dt.year + 1 = 2017
2016-12-27 07:30 trade:119 INFO trade(),    context.year 2017  
stocks ['VZ', 'PFE', 'CVX', 'BA', 'CSCO', 'KO', 'IBM', 'XOM', 'CAT', 'MRK']
2017-01-03 05:45  PRINT .

2017 
    do_pipe, context.year 2017 == dt.year 2017
2017-01-03 05:45 before_trading_start:162 INFO context.year 2017 to dt.year + 1 = 2018
2017-01-03 07:30 trade:119 INFO trade(),    context.year 2018  
stocks ['VZ', 'PFE', 'CVX', 'BA', 'CSCO', 'KO', 'IBM', 'CAT', 'XOM', 'MRK']
2018-01-02 05:45  PRINT .

2018 
    do_pipe, context.year 2018 == dt.year 2018
2018-01-02 05:45 before_trading_start:162 INFO context.year 2018 to dt.year + 1 = 2019
2018-01-02 07:30 trade:119 INFO trade(),    context.year 2019  
stocks ['VZ', 'IBM', 'PFE', 'XOM', 'CVX', 'MRK', 'KO', 'CSCO', 'PG', 'GE']



IT RETURNS THE RIGHT STOCKS IF BACKTEST IS LAUNCHED EVERY YEAR SEPARATELY.
IF BACKTEST IS OVER MULTIPLE YEARS, IT DOESN'T.
'''

from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline  import Pipeline, CustomFactor
from quantopian.pipeline.data    import Fundamentals
from quantopian.pipeline.filters import StaticSids, StaticAssets
import datetime

class Dow30:
    def __init__(self):
        '''
        this is a utility class that can return the list of Dow30 stocks for a given date.
        the input date must be >= 1999-11-01.
        source: http://www.djindexes.com/mdsidx/downloads/brochure_info/Dow_Jones_Industrial_Average_Historical_Components.pdf
        '''

         # 0 is replaced by 1
        raw_constituents = [
            ["1999-11-01",sid(4922),1],    # Minnesota Mining & Manufacturing/3M Company
            ["1999-11-01",sid(50428),1],   # AA
            ["1999-11-01",sid(679),1],
            ["1999-11-01",sid(7289),1],    # AT&T Corporation
            ["1999-11-01",sid(698),1],
            ["1999-11-01",sid(1267),1],
            ["1999-11-01",sid(1335),1],
            ["1999-11-01",sid(4283),1],
            ["1999-11-01",sid(2119),1],    # DD
            ["1999-11-01",sid(2482),1],    # EK
            ["1999-11-01",sid(8347),1],    # XOM
            ["1999-11-01",sid(3149),1],    # GE
            ["1999-11-01",sid(3246),1],
            ["1999-11-01",sid(3735),1],
            ["1999-11-01",sid(3496),1],
            ["1999-11-01",sid(25090),1],   # AlliedSignal Incorporated/Honeywell
            ["1999-11-01",sid(3951),1],
            ["1999-11-01",sid(3766),1],    # IBM
            ["1999-11-01",sid(3971),1],
            ["1999-11-01",sid(25006),1],
            ["1999-11-01",sid(4151),1],
            ["1999-11-01",sid(4707),1],    # MCD
            ["1999-11-01",sid(5029),1],
            ["1999-11-01",sid(5061),1],
            ["1999-11-01",sid(4954),1],    # MO
            ["1999-11-01",sid(5938),1],    # PG
            ["1999-11-01",sid(6653),1],    # SBC Communications Incorporated/AT&T Incorporated
            ["1999-11-01",sid(7883),1],
            ["1999-11-01",sid(8229),1],    # WMT
            ["1999-11-01",sid(2190),1],
            ["2004-04-08",sid(7289),0],
            ["2004-04-08",sid(2482),0],
            ["2004-04-08",sid(3971),0],
            ["2004-04-08",sid(239),1],
            ["2004-04-08",sid(5923),1],
            ["2004-04-08",sid(21839),1],
            ["2008-02-19",sid(4954),0],
            ["2008-02-19",sid(25090),0],
            ["2008-02-19",sid(700),1],
            ["2008-02-19",sid(23112),1],
            ["2008-09-22",sid(239),0],
            ["2008-09-22",sid(22802),1],    # KRFT/MDLZ
            ["2009-06-08",sid(1335),0],
            ["2009-06-08",sid(3246),0],
            ["2009-06-08",sid(7041),1],
            ["2009-06-08",sid(1900),1],
            ["2012-09-24",sid(22802),0],
            ["2012-09-24",sid(7792),1],
            ["2013-09-23",sid(50428),0],
            ["2013-09-23",sid(700),0],
            ["2013-09-23",sid(3735),0],
            ["2013-09-23",sid(20088),1],
            ["2013-09-23",sid(5328),1],
            ["2013-09-23",sid(35920),1],
            ["2015-03-19",sid(6653),0],
            ["2015-03-19",sid(24),1],
            ["2017-09-01",sid(2119),0],
            ["2017-09-01",sid(51157),1],
            ["2018-06-26",sid(3149),0],    # GE
            ["2018-06-26",sid(8089),1]     # WBA
            ]

        # translate the date string into a datetime.date
        self.data = []
        for equity in raw_constituents:
            self.data.append((
                datetime.datetime.strptime(equity[0], "%Y-%m-%d").date(),
                equity[1],
                equity[2]
            ))

    def get(self, dt):
        '''
        for a given datetime, return a list of the 30 dow stocks as Equity objects.
        dt must be >= 1999-11-01
        '''
        d   = dt.date()
        ret = []
        for equity in self.data:
            if equity[0] > d:
                break
            if equity[2] == 1:
                ret.append(equity[1])
            elif equity[2] == 0:
                ret.remove(equity[1])
            else:
                raise Exception('unknown membership')
        log.info('get(self, {}),    year {}  \nstocks {}'.format(str(dt), dt.year,
            str([str(s.symbol) for s in ret])
        ))
        return ret

def trade(context, data):
    if context.trade == True:
        dogs = context.out.index

        log.info('trade(),    context.year {}  \nstocks {}'.format(context.year,
            str([str(s.symbol) for s in dogs])
        ))
        context.trade = False

        # Check ifare holding an old position that is no longer in the dogs, needs to be closed
        for s in context.portfolio.positions:
            if s not in dogs:
                order_target(s, 0)

        # Check ifare holding each of the dogs, if not buy to 10% of the total portfolio
        for s in dogs:
            if data.can_trade(s):
                order_target_percent(s, .1)

def initialize(context):
    set_benchmark(symbol('DIA'))
    set_commission(commission.PerShare(cost=0.0035, min_trade_cost=0.35))
    set_slippage(slippage.VolumeShareSlippage(volume_limit=0.025, price_impact=0.1))
    context.dow30   = Dow30()
    context.year    = get_datetime().year
    context.trade   = False
    context.mx_lvrg = 0
    schedule_function(trade, date_rules.every_day(), time_rules.market_open(hours=1))
    attach_pipeline(make_pipeline(), 'pipe')

def before_trading_start(context, data):
    dt = get_datetime()
    do_pipe = 0
    msg = ''

    if context.year == 0:
        msg += ('\n    do_pipe, context.year {}'.format(context.year))
        do_pipe = 1
    if context.year == dt.year:
        msg += ('\n    do_pipe, context.year {} == dt.year {}'.format(context.year, dt.year))
        do_pipe = 1
    if len(context.portfolio.positions) != 10:
        msg += ('\n    do_pipe, len(context.portfolio.positions) {}'.format(len(context.portfolio.positions)))
        do_pipe = 1

    if do_pipe:
        context.trade = True
        print '.\n\n{} {}'.format(dt.year, msg)  # separator b/t pipe runs
        log.info('context.year {} to dt.year + 1 = {}'.format(context.year, dt.year + 1))
        context.year  = dt.year + 1
        # save to context to use during trade()
        context.out = pipeline_output('pipe').sort_values(['forward_dividend_yield'], ascending=False)[:10]

    record(positions=len(context.portfolio.positions), leverage=context.account.leverage)
    record(mx_lvrg = context.mx_lvrg)    # Record maximum leverage encountered

def make_pipeline():
    members = Dow30().get( get_datetime() )

    return Pipeline(
        screen = StaticSids( [m.sid for m in members] ),
        columns = {
            'forward_dividend_yield' : Fundamentals.forward_dividend_yield.latest
        })

def handle_data(context, data):
    if context.account.leverage > context.mx_lvrg:
        context.mx_lvrg = context.account.leverage
There was a runtime error.

Thanks for the logging and for your consideration.

The attached backtest is run with 11 stocks instead of 10 on purpose.
It's clear that the algo choses the correct stocks the first year (2015). The following years would be correct if "T" wasn't returned as a dog. And that's exactly the problem.

The algo is not broken if run over multiple years, but if "T" is bought. In 2015 "T" was a dog, but in 2016 it wasn't, but somehow it's still returned as a dog. The backtest you attached before never had "T" and the dogs are always correct, indeed.

I have no idea why that happens.

Clone Algorithm
1
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
'''
1969-12-31 16:00 get:110 INFO get(self, 2016-12-27 00:00:00+00:00),    year 2016  
stocks ['MMM', 'AXP', 'BA', 'CAT', 'KO', 'DD', 'XOM', 'GE', 'HD', 'INTC', 'IBM', 'JPM', 'JNJ', 'MCD', 'MRK', 'MSFT', 'PG', 'UTX', 'WMT', 'DIS', 'PFE', 'VZ', 'CVX', 'TRV', 'CSCO', 'UNH', 'GS', 'NKE', 'V', 'AAPL']
2016-12-27 05:45  PRINT .

2016 
    do_pipe, context.year 2016 == dt.year 2016
    do_pipe, len(context.portfolio.positions) 0
2016-12-27 05:45 before_trading_start:162 INFO context.year 2016 to dt.year + 1 = 2017
2016-12-27 07:30 trade:119 INFO trade(),    context.year 2017  
stocks ['VZ', 'PFE', 'CVX', 'BA', 'CSCO', 'KO', 'IBM', 'XOM', 'CAT', 'MRK']
2017-01-03 05:45  PRINT .

2017 
    do_pipe, context.year 2017 == dt.year 2017
2017-01-03 05:45 before_trading_start:162 INFO context.year 2017 to dt.year + 1 = 2018
2017-01-03 07:30 trade:119 INFO trade(),    context.year 2018  
stocks ['VZ', 'PFE', 'CVX', 'BA', 'CSCO', 'KO', 'IBM', 'CAT', 'XOM', 'MRK']
2018-01-02 05:45  PRINT .

2018 
    do_pipe, context.year 2018 == dt.year 2018
2018-01-02 05:45 before_trading_start:162 INFO context.year 2018 to dt.year + 1 = 2019
2018-01-02 07:30 trade:119 INFO trade(),    context.year 2019  
stocks ['VZ', 'IBM', 'PFE', 'XOM', 'CVX', 'MRK', 'KO', 'CSCO', 'PG', 'GE']



IT RETURNS THE RIGHT STOCKS IF BACKTEST IS LAUNCHED EVERY YEAR SEPARATELY.
IF BACKTEST IS OVER MULTIPLE YEARS, IT DOESN'T.
'''

from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline  import Pipeline, CustomFactor
from quantopian.pipeline.data    import Fundamentals
from quantopian.pipeline.filters import StaticSids, StaticAssets
import datetime

class Dow30:
    def __init__(self):
        '''
        this is a utility class that can return the list of Dow30 stocks for a given date.
        the input date must be >= 1999-11-01.
        source: http://www.djindexes.com/mdsidx/downloads/brochure_info/Dow_Jones_Industrial_Average_Historical_Components.pdf
        '''

         # 0 is replaced by 1
        raw_constituents = [
            ["1999-11-01",sid(4922),1],    # Minnesota Mining & Manufacturing/3M Company
            ["1999-11-01",sid(50428),1],   # AA
            ["1999-11-01",sid(679),1],
            ["1999-11-01",sid(7289),1],    # AT&T Corporation
            ["1999-11-01",sid(698),1],
            ["1999-11-01",sid(1267),1],
            ["1999-11-01",sid(1335),1],
            ["1999-11-01",sid(4283),1],
            ["1999-11-01",sid(2119),1],    # DD
            ["1999-11-01",sid(2482),1],    # EK
            ["1999-11-01",sid(8347),1],    # XOM
            ["1999-11-01",sid(3149),1],    # GE
            ["1999-11-01",sid(3246),1],
            ["1999-11-01",sid(3735),1],
            ["1999-11-01",sid(3496),1],
            ["1999-11-01",sid(25090),1],   # AlliedSignal Incorporated/Honeywell
            ["1999-11-01",sid(3951),1],
            ["1999-11-01",sid(3766),1],    # IBM
            ["1999-11-01",sid(3971),1],
            ["1999-11-01",sid(25006),1],
            ["1999-11-01",sid(4151),1],
            ["1999-11-01",sid(4707),1],    # MCD
            ["1999-11-01",sid(5029),1],
            ["1999-11-01",sid(5061),1],
            ["1999-11-01",sid(4954),1],    # MO
            ["1999-11-01",sid(5938),1],    # PG
            ["1999-11-01",sid(6653),1],    # SBC Communications Incorporated/AT&T Incorporated
            ["1999-11-01",sid(7883),1],
            ["1999-11-01",sid(8229),1],    # WMT
            ["1999-11-01",sid(2190),1],
            ["2004-04-08",sid(7289),0],
            ["2004-04-08",sid(2482),0],
            ["2004-04-08",sid(3971),0],
            ["2004-04-08",sid(239),1],
            ["2004-04-08",sid(5923),1],
            ["2004-04-08",sid(21839),1],
            ["2008-02-19",sid(4954),0],
            ["2008-02-19",sid(25090),0],
            ["2008-02-19",sid(700),1],
            ["2008-02-19",sid(23112),1],
            ["2008-09-22",sid(239),0],
            ["2008-09-22",sid(22802),1],    # KRFT/MDLZ
            ["2009-06-08",sid(1335),0],
            ["2009-06-08",sid(3246),0],
            ["2009-06-08",sid(7041),1],
            ["2009-06-08",sid(1900),1],
            ["2012-09-24",sid(22802),0],
            ["2012-09-24",sid(7792),1],
            ["2013-09-23",sid(50428),0],
            ["2013-09-23",sid(700),0],
            ["2013-09-23",sid(3735),0],
            ["2013-09-23",sid(20088),1],
            ["2013-09-23",sid(5328),1],
            ["2013-09-23",sid(35920),1],
            ["2015-03-19",sid(6653),0],
            ["2015-03-19",sid(24),1],
            ["2017-09-01",sid(2119),0],
            ["2017-09-01",sid(51157),1],
            ["2018-06-26",sid(3149),0],    # GE
            ["2018-06-26",sid(8089),1]     # WBA
            ]

        # translate the date string into a datetime.date
        self.data = []
        for equity in raw_constituents:
            self.data.append((
                datetime.datetime.strptime(equity[0], "%Y-%m-%d").date(),
                equity[1],
                equity[2]
            ))

    def get(self, dt):
        '''
        for a given datetime, return a list of the 30 dow stocks as Equity objects.
        dt must be >= 1999-11-01
        '''
        d   = dt.date()
        ret = []
        for equity in self.data:
            if equity[0] > d:
                break
            if equity[2] == 1:
                ret.append(equity[1])
            elif equity[2] == 0:
                ret.remove(equity[1])
            else:
                raise Exception('unknown membership')
        log.info('get(self, {}),    year {}  \nstocks {}'.format(str(dt), dt.year,
            str([str(s.symbol) for s in ret])
        ))
        return ret

def trade(context, data):
    if context.trade == True:
        dogs = context.out.index

        log.info('trade(),    context.year {}  \nstocks {}'.format(context.year,
            str([str(s.symbol) for s in dogs])
        ))
        context.trade = False

        # Check ifare holding an old position that is no longer in the dogs, needs to be closed
        for s in context.portfolio.positions:
            if s not in dogs:
                order_target(s, 0)

        # Check ifare holding each of the dogs, if not buy to 10% of the total portfolio
        for s in dogs:
            if data.can_trade(s):
                order_target_percent(s, .1)

def initialize(context):
    set_benchmark(symbol('DIA'))
    set_commission(commission.PerShare(cost=0.0035, min_trade_cost=0.35))
    set_slippage(slippage.VolumeShareSlippage(volume_limit=0.025, price_impact=0.1))
    context.dow30   = Dow30()
    context.year    = get_datetime().year
    context.trade   = False
    context.mx_lvrg = 0
    schedule_function(trade, date_rules.every_day(), time_rules.market_open(hours=1))
    attach_pipeline(make_pipeline(), 'pipe')

def before_trading_start(context, data):
    dt = get_datetime()
    do_pipe = 0
    msg = ''

    if context.year == 0:
        msg += ('\n    do_pipe, context.year {}'.format(context.year))
        do_pipe = 1
    if context.year == dt.year:
        msg += ('\n    do_pipe, context.year {} == dt.year {}'.format(context.year, dt.year))
        do_pipe = 1
    if len(context.portfolio.positions) != 11:
        msg += ('\n    do_pipe, len(context.portfolio.positions) {}'.format(len(context.portfolio.positions)))
        do_pipe = 1

    if do_pipe:
        context.trade = True
        print '.\n\n{} {}'.format(dt.year, msg)  # separator b/t pipe runs
        log.info('context.year {} to dt.year + 1 = {}'.format(context.year, dt.year + 1))
        context.year  = dt.year + 1
        # save to context to use during trade()
        context.out = pipeline_output('pipe').sort_values(['forward_dividend_yield'], ascending=False)[:11]
        
        log.info(context.out)

    record(positions=len(context.portfolio.positions), leverage=context.account.leverage)
    record(mx_lvrg = context.mx_lvrg)    # Record maximum leverage encountered

def make_pipeline():
    members = Dow30().get( get_datetime() )

    return Pipeline(
        screen = StaticSids( [m.sid for m in members] ),
        columns = {
            'forward_dividend_yield' : Fundamentals.forward_dividend_yield.latest
        })

def handle_data(context, data):
    if context.account.leverage > context.mx_lvrg:
        context.mx_lvrg = context.account.leverage
There was a runtime error.

Maybe closer. 'T' drops out in 2016. Main changes marked with #.#

Clone Algorithm
2
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
''' Modified.

IT RETURNS THE RIGHT STOCKS IF BACKTEST IS LAUNCHED EVERY YEAR SEPARATELY.
IF BACKTEST IS OVER MULTIPLE YEARS, IT DOESN'T.
'''

from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline  import Pipeline, CustomFactor
from quantopian.pipeline.data    import Fundamentals
from quantopian.pipeline.filters import StaticSids, StaticAssets
import datetime

class Dow30:
    def __init__(self):
        '''
        this is a utility class that can return the list of Dow30 stocks for a given date.
        the input date must be >= 1999-11-01.
        source: http://www.djindexes.com/mdsidx/downloads/brochure_info/Dow_Jones_Industrial_Average_Historical_Components.pdf
        '''

         # 0 is replaced by 1
        raw_constituents = [
            ["1999-11-01",sid(4922),1],    # Minnesota Mining & Manufacturing/3M Company
            ["1999-11-01",sid(50428),1],   # AA
            ["1999-11-01",sid(679),1],
            ["1999-11-01",sid(7289),1],    # AT&T Corporation
            ["1999-11-01",sid(698),1],
            ["1999-11-01",sid(1267),1],
            ["1999-11-01",sid(1335),1],
            ["1999-11-01",sid(4283),1],
            ["1999-11-01",sid(2119),1],    # DD
            ["1999-11-01",sid(2482),1],    # EK
            ["1999-11-01",sid(8347),1],    # XOM
            ["1999-11-01",sid(3149),1],    # GE
            ["1999-11-01",sid(3246),1],
            ["1999-11-01",sid(3735),1],
            ["1999-11-01",sid(3496),1],
            ["1999-11-01",sid(25090),1],   # AlliedSignal Incorporated/Honeywell
            ["1999-11-01",sid(3951),1],
            ["1999-11-01",sid(3766),1],    # IBM
            ["1999-11-01",sid(3971),1],
            ["1999-11-01",sid(25006),1],
            ["1999-11-01",sid(4151),1],
            ["1999-11-01",sid(4707),1],    # MCD
            ["1999-11-01",sid(5029),1],
            ["1999-11-01",sid(5061),1],
            ["1999-11-01",sid(4954),1],    # MO
            ["1999-11-01",sid(5938),1],    # PG
            ["1999-11-01",sid(6653),1],    # SBC Communications Incorporated/AT&T Incorporated
            ["1999-11-01",sid(7883),1],
            ["1999-11-01",sid(8229),1],    # WMT
            ["1999-11-01",sid(2190),1],
            ["2004-04-08",sid(7289),0],
            ["2004-04-08",sid(2482),0],
            ["2004-04-08",sid(3971),0],
            ["2004-04-08",sid(239),1],
            ["2004-04-08",sid(5923),1],
            ["2004-04-08",sid(21839),1],
            ["2008-02-19",sid(4954),0],
            ["2008-02-19",sid(25090),0],
            ["2008-02-19",sid(700),1],
            ["2008-02-19",sid(23112),1],
            ["2008-09-22",sid(239),0],
            ["2008-09-22",sid(22802),1],    # KRFT/MDLZ
            ["2009-06-08",sid(1335),0],
            ["2009-06-08",sid(3246),0],
            ["2009-06-08",sid(7041),1],
            ["2009-06-08",sid(1900),1],
            ["2012-09-24",sid(22802),0],
            ["2012-09-24",sid(7792),1],
            ["2013-09-23",sid(50428),0],
            ["2013-09-23",sid(700),0],
            ["2013-09-23",sid(3735),0],
            ["2013-09-23",sid(20088),1],
            ["2013-09-23",sid(5328),1],
            ["2013-09-23",sid(35920),1],
            ["2015-03-19",sid(6653),0],    # T  AT&T Inc. Com
            ["2015-03-19",sid(24),1],
            ["2017-09-01",sid(2119),0],
            ["2017-09-01",sid(51157),1],
            ["2018-06-26",sid(3149),0],    # GE
            ["2018-06-26",sid(8089),1]     # WBA
            ]

        # translate the date string into a datetime.date
        self.data = []
        for equity in raw_constituents:
            self.data.append((
                datetime.datetime.strptime(equity[0], "%Y-%m-%d").date(),
                equity[1],
                equity[2]
            ))

    def get(self, dt):
        '''
        for a given datetime, return a list of the 30 dow stocks as Equity objects.
        dt must be >= 1999-11-01
        '''
        d   = dt.date()
        ret = []
        for equity in self.data:
            if equity[0] > d:
                break
            if equity[2] == 1:
                ret.append(equity[1])
            elif equity[2] == 0:
                ret.remove(equity[1])
            else:
                raise Exception('unknown membership')
        #log.info('get(self, {}),    year {}  \nstocks {}'.format(str(dt), dt.year,
        #    str([str(s.symbol) for s in ret])
        #))
        return ret

def trade(context, data):
    if context.trade == True:
        #.#dogs = context.out.index
        dogs = Dow30().get( get_datetime() )  #.# rerunning with current date

        log_data(context, data, context.out.loc[dogs], 4)   # show selected pipe info

        log.info('trade(),    context.year {}  stocks ...\n{}'.format(context.year,
            str([str(s.symbol) for s in dogs])
        ))
        context.trade = False

        # Check if holding an old position no longer in the dogs, needs to be closed
        for s in context.portfolio.positions:
            if s not in dogs:
                order_target(s, 0)

        # Check if holding each of the dogs, if not buy to 10% of the total portfolio
        for s in dogs:
            if data.can_trade(s):
                order_target_percent(s, 1.0 / len(dogs))
                #.#order_target_percent(s, .1)
            else:
                log.info('Cant trade {}'.format(s))
                assert(0)   # Deliberate crash to call attention to it. Would be a delist surely.

def initialize(context):
    set_benchmark(symbol('DIA'))
    set_commission(commission.PerShare(cost=0.0035, min_trade_cost=0.35))
    set_slippage(slippage.VolumeShareSlippage(volume_limit=0.025, price_impact=0.1))
    context.dow30   = Dow30()
    context.year    = get_datetime().year
    context.trade   = False
    context.mx_lvrg = 0
    schedule_function(trade, date_rules.every_day(), time_rules.market_open(hours=1))
    attach_pipeline(make_pipeline(), 'pipe')

    for i in range(1, 391):
        schedule_function(track_orders, date_rules.every_day(), time_rules.market_open(minutes=i))

def before_trading_start(context, data):
    dt = get_datetime()
    do_pipe = 0
    msg = ''

    if context.year == 0:
        msg += ('\n    do_pipe, context.year {}'.format(context.year))
        do_pipe = 1
    if context.year == dt.year:
        msg += ('\n    do_pipe, context.year {} == dt.year {}'.format(context.year, dt.year))
        do_pipe = 1
    if len(context.portfolio.positions) < 11:  #.# unsure here (says blue)
        msg += ('\n    do_pipe, len(context.portfolio.positions) {}'.format(len(context.portfolio.positions)))
        do_pipe = 1

    if do_pipe:
        context.trade = True
        print '.\n\n{} {}'.format(dt.year, msg)  # separator b/t pipe runs
        log.info('context.year {} to dt.year + 1 = {}'.format(context.year, dt.year + 1))
        context.year  = dt.year + 1
        # save to context to use during trade()
        context.out = pipeline_output('pipe').sort_values(['forward_dividend_yield'], ascending=False)  #.#[:11]

        #log.info(context.out.tail())
        #log.info(context.out.head())

        #log_data(context, data, context.out, 4)        # show pipe info every time

        '''
        do_log_preview = 1    # a way to toggle this off when it becomes annoying
        if do_log_preview:
            try: context.log_data_done
            except:
                log_data(context, data, context.out, 4)        # show pipe info once
        '''

    record(positions=len(context.portfolio.positions), leverage=context.account.leverage)
    record(mx_lvrg = context.mx_lvrg)    # Record maximum leverage encountered

def make_pipeline():
    members = Dow30().get( get_datetime() )

    return Pipeline(
        screen = StaticSids( [m.sid for m in members] ),
        columns = {
            'forward_dividend_yield' : Fundamentals.forward_dividend_yield.latest
        })

    # There are nans even for AAPL in forward_dividend_yield
    # Consider dividend_per_share_earnings_reports

def handle_data(context, data):
    if context.account.leverage > context.mx_lvrg:
        context.mx_lvrg = context.account.leverage

def log_data(context, data, z, num, fields=None):
    ''' Log info about pipeline output or, z can be any DataFrame or Series
    https://www.quantopian.com/posts/overview-of-pipeline-content-easy-to-add-to-your-backtest

    try: context.log_data_done
    except:
        # {:,} is magic for adding commas
        log.info('cash ${:,}   portfolio ${:,}     {} positions ...'.format(
            int(context.portfolio.cash),
            int(context.portfolio.portfolio_value),
            len(context.portfolio.positions),
        ))
        context.log_data_done = 1
    '''
    # {:,} is magic for adding commas
    log.info('starting_cash ${:,}   portfolio ${:,}     {} positions ...'.format(
        int(context.portfolio.cash),
        int(context.portfolio.portfolio_value),
        len(context.portfolio.positions),
    ))

    if not len(z):
        log.info('Empty')
        return

    # Options
    log_nan_only = 0          # Only log if nans are present
    show_sectors = 0          # If sectors, do you want to see them or not
    show_sorted_details = 1   # [num] high & low securities sorted, each column
    padmax = 6                # num characters for each field, starting point

    # Series ......
    if 'Series' in str(type(z)):    # is Series, not DataFrame
        nan_count = len(z[z != z])
        nan_count = 'NaNs {}/{}'.format(nan_count, len(z)) if nan_count else ''
        if (log_nan_only and nan_count) or not log_nan_only:
            pad = max( padmax, len('%.5f' % z.max()) )
            log.info('{}{}{}   Series  len {}'.format('min'.rjust(pad+5),
                'mean'.rjust(pad+5), 'max'.rjust(pad+5), len(z)))
            log.info('{}{}{} {}'.format(
                ('%.5f' % z.min()) .rjust(pad+5),
                ('%.5f' % z.mean()).rjust(pad+5),
                ('%.5f' % z.max()) .rjust(pad+5),
                nan_count
            ))
            log.info('High\n{}'.format(z.sort_values(ascending=False).head(num)))
            log.info('Low\n{}' .format(z.sort_values(ascending=False).tail(num)))
        return

    # DataFrame ......
    content_min_max = [ ['','min','mid','max',''] ] ; content = ''
    for col in z.columns:
        #try: z[col].max()
        #except:
        #    log.info('{} non-numeric'.format(col))
        #    #continue   # skip non-numeric
        if col == 'sector' and not show_sectors: continue
        nan_count = len(z[col][z[col] != z[col]])
        nan_count = 'NaNs {}/{}'.format(nan_count, len(z)) if nan_count else ''
        # known bug, not always sorting strings alphabetically ...
        srt       = z[col].sort_values() if type(z[col][0]) != str else z.iloc[z[col].str.lower().argsort()]
        padmax    = max( padmax, len(str(srt[-1])) )
        content_min_max.append([col, str(srt[0]), str(srt[len(srt)//2]), str(srt[-1]), nan_count])
    if log_nan_only and nan_count or not log_nan_only:
        if len(z.columns) == 1: content = 'Stocks: {}'.format(z.shape[0])
        if len(z.columns)  > 1: content = 'Stocks: {}  Columns: {}'.format(z.shape[0], z.shape[1])
        if len(z.columns):
            paddings = [6 for i in range(4)]
            for lst in content_min_max:    # set max lengths
                i = 0
                for val in lst[:4]:        # value in each sub-list
                    paddings[i] = max(paddings[i], len(str(val)))
                    i += 1
            headr = content_min_max[0]
            content += ('\n{}{}{}{}{}'.format(
                 headr[0] .rjust(paddings[0]),
                (headr[1]).rjust(paddings[1]+5),
                (headr[2]).rjust(paddings[2]+5),
                (headr[3]).rjust(paddings[3]+5),
                ''
            ))
            for lst in content_min_max[1:]:    # populate content using max lengths
                content += ('\n{}{}{}{}     {}'.format(
                    lst[0].rjust(paddings[0]),
                    lst[1].rjust(paddings[1]+5),
                    lst[2].rjust(paddings[2]+5),
                    lst[3].rjust(paddings[3]+5),
                    lst[4],
                ))
        log.info(content)

    if not show_sorted_details: return
    #if len(z.columns) == 1:     return     # skip detail if only 1 column
    if fields == None: details = z.columns
    for detail in details:
        if detail == 'sector' and not show_sectors: continue
        hi = z[details].sort_values(by=detail, ascending=False).head(num)
        lo = z[details].sort_values(by=detail, ascending=False).tail(num)
        content  = ''
        content += ('_ _ _   {}   _ _ _'  .format(detail))
        content += ('\n\t... {} highs\n{}'.format(detail, str(hi)))
        content += ('\n\t... {} lows \n{}'.format(detail, str(lo)))
        if log_nan_only and not len(lo[lo[detail] != lo[detail]]):
            continue  # skip if no nans
        log.info(content)

def track_orders(context, data):
    '''  Show orders when made and filled.
           Info: https://www.quantopian.com/posts/track-orders
    '''
    c = context
    if 'trac' not in c:
        c.t_opts = {        # __________    O P T I O N S    __________
            'symbols'     : [],   # List of symbols to filter for, like ['TSLA', 'SPY']
            'log_neg_cash': 1,    # Show cash only when negative.
            'log_cash'    : 1,    # Show cash values in logging window or not.
            'log_ids'     : 1,    # Include order id's in logging window or not.
            'log_unfilled': 1,    # When orders are unfilled. (stop & limit excluded).
            'log_cancels' : 0,    # When orders are canceled.
        }    # Move these to initialize() for better efficiency.
        c.trac = {}
        c.t_dates  = {  # To not overwhelm the log window, start/stop dates can be entered.
            'active': 0,
            'start' : [],   # Start dates, option like ['2007-05-07', '2010-04-26']
            'stop'  : []    # Stop  dates, option like ['2008-02-13', '2010-11-15']
        }
        log.info('track_orders active. Headers ...')
        log.info('             Shares     Shares')
        log.info('Min   Action Order  Sym  Now   at Price   PnL   Stop or Limit   Cash  Id')
    from pytz import timezone as _tz  # Python only does once, makes this portable.
                                      #   Move to top of algo for better efficiency.
    # If 'start' or 'stop' lists have something in them, triggers ...
    if c.t_dates['start'] or c.t_dates['stop']:
        _date = str(get_datetime().date())
        if   _date in c.t_dates['start']:    # See if there's a match to start
            c.t_dates['active'] = 1
        elif _date in c.t_dates['stop']:     #   ... or to stop
            c.t_dates['active'] = 0
    else: c.t_dates['active'] = 1           # Set to active b/c no conditions.
    if c.t_dates['active'] == 0: return     # Skip if not active.
    def _minute():   # To preface each line with the minute of the day.
        bar_dt = get_datetime().astimezone(_tz('US/Eastern'))
        return (bar_dt.hour * 60) + bar_dt.minute - 570 # (-570 = 9:31a)
    def _trac(to_log):      # So all logging comes from the same line number,
        log.info(' {}   {}'.format(str(_minute()).rjust(3), to_log))  # for vertical alignment in the logging window.

    for oid in c.trac.copy():               # Existing known orders
      o = get_order(oid)
      if c.t_opts['symbols'] and (o.sid.symbol not in c.t_opts['symbols']): continue
      if o.dt == o.created: continue        # No chance of fill yet.
      cash = ''
      prc  = data.current(o.sid, 'price') if data.can_trade(o.sid) else c.portfolio.positions[o.sid].last_sale_price
      if (c.t_opts['log_neg_cash'] and c.portfolio.cash < 0) or c.t_opts['log_cash']:
        cash = str(int(c.portfolio.cash))
      if o.status == 2:                     # Canceled
        do = 'Buy' if o.amount > 0 else 'Sell' ; style = ''
        if o.stop:
          style = ' stop {}'.format(o.stop)
          if o.limit: style = ' stop {} limit {}'.format(o.stop, o.limit)
        elif o.limit: style = ' limit {}'.format(o.limit)
        if c.t_opts['log_cancels']:
          _trac('  Canceled {} {} {}{} at {}   {}  {}'.format(do, o.amount,
             o.sid.symbol, style, prc, cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
        del c.trac[o.id]
      elif o.filled:                        # Filled at least some.
        filled = '{}'.format(o.amount)
        filled_amt = 0
        if o.status == 1:                   # Complete
          if 0 < c.trac[o.id] < o.amount:
            filled   = 'all {}/{}'.format(o.filled - c.trac[o.id], o.amount)
          filled_amt = o.filled
        else:                                    # c.trac[o.id] value is previously filled total
          filled_amt = o.filled - c.trac[o.id]   # filled this time, can be 0
          c.trac[o.id] = o.filled                # save fill value for increments math
          filled = '{}/{}'.format(filled_amt, o.amount)
        if filled_amt:
          now = ' ({})'.format(c.portfolio.positions[o.sid].amount) if c.portfolio.positions[o.sid].amount else ' _'
          pnl = ''  # for the trade only
          amt = c.portfolio.positions[o.sid].amount ; style = ''
          if (amt - o.filled) * o.filled < 0:    # Profit-taking scenario including short-buyback
            cb = c.portfolio.positions[o.sid].cost_basis
            if cb:
              pnl  = -filled_amt * (prc - cb)
              sign = '+' if pnl > 0 else '-'
              pnl  = '  ({}{})'.format(sign, '%.0f' % abs(pnl))
          if o.stop:
            style = ' stop {}'.format(o.stop)
            if o.limit: style = ' stop () limit {}'.format(o.stop, o.limit)
          elif o.limit: style = ' limit {}'.format(o.limit)
          if o.filled == o.amount: del c.trac[o.id]
          _trac('   {} {} {}{} at {}{}{}'.format(
            'Bot' if o.amount > 0 else 'Sold', filled, o.sid.symbol, now,
            '%.2f' % prc, pnl, style).ljust(52) + '  {}  {}'.format(cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
      elif c.t_opts['log_unfilled'] and not (o.stop or o.limit):
        _trac('      {} {}{} unfilled  {}'.format(o.sid.symbol, o.amount,
         ' limit' if o.limit else '', o.id[-4:] if c.t_opts['log_ids'] else ''))

    oo = get_open_orders().values()
    if not oo: return                       # Handle new orders
    cash = ''
    if (c.t_opts['log_neg_cash'] and c.portfolio.cash < 0) or c.t_opts['log_cash']:
      cash = str(int(c.portfolio.cash))
    for oo_list in oo:
      for o in oo_list:
        if c.t_opts['symbols'] and (o.sid.symbol not in c.t_opts['symbols']): continue
        if o.id in c.trac: continue         # Only new orders beyond this point
        prc = data.current(o.sid, 'price') if data.can_trade(o.sid) else c.portfolio.positions[o.sid].last_sale_price
        c.trac[o.id] = 0 ; style = ''
        now  = ' ({})'.format(c.portfolio.positions[o.sid].amount) if c.portfolio.positions[o.sid].amount else ' _'
        if o.stop:
          style = ' stop {}'.format(o.stop)
          if o.limit: style = ' stop {} limit {}'.format(o.stop, o.limit)
        elif o.limit: style = ' limit {}'.format(o.limit)
        _trac('{} {} {}{} at {}{}'.format('Buy' if o.amount > 0 else 'Sell',
          o.amount, o.sid.symbol, now, '%.2f' % prc, style).ljust(52) + '  {}  {}'.format(cash, o.id[-4:] if c.t_opts['log_ids'] else ''))

There was a runtime error.

We're definitely getting closer, but there are still some issues.

The actual strategy is to go long the first 10 stocks of the Dow (sorted by forward_dividend_yield), but having a look at the whole Dow is a great starting point to figure out why 'T' is returned and why there are some NaN values - these 2 issues are actually connected.
In the backtest I will hold 30 stocks. Once we address how to fix the universe from which to select the 10 stocks, I will change the algo to hold just 10 positions.

I noticed that in the 'trade' function, 'context.out.index' (which is the output of the pipeline) returns different stocks than 'context.out.loc[dogs].index' (which is the output of Dow30().get( get_datetime() ) called in 'trade').
I don't know why they return 2 different set of stocks, since they should work with the same universe - because they call the same function 'Dow30().get( get_datetime() )' - and since they are both called on the first day of the year. Or am I missing something, maybe about when point in time they are called? Because it looks like the 'Dow30().get( get_datetime() )' returns one set of sids in 'make_pipeline', and a different set of sids in the 'trade' function.

In 2016, actually the only difference is that 'make_pipeline' returns 'T' and not 'AAPL' - which is wrong - while 'trade' returns 'AAPL' and not 'T' - which is correct. However, the fundamentals data are fetched in 'make_pipeline' and that's why 'AAPL' shows NaN values (thanks for the function you added). So the NaN values are not an issue of 'forward_dividend_yield', but they are Nan because 'context.out.loc[dogs].index' - I think - substitues 'T' with 'AAPL' after 'make_pipeline' is called, therefore it's impossible to have fundamental data for 'AAPL'.

So, it looks to me that 'Dow30().get( get_datetime() )' returns the wrong stocks in 'make_pipeline', but it returns the right stocks in 'trade' - only 'AAPL' and 'T' differ.
Do you think this is the issue of the whole algo?

I hope to have been clear until now, if not I'll try to explain myself better (let me know).

N.B.
In the algo, I added some log.info at the beginning of the 'trade' function.
Also, the values returned in context.out are not sorted, but I will sort them in 'trade'.

Clone Algorithm
0
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
''' Modified.

IT RETURNS THE RIGHT STOCKS IF BACKTEST IS LAUNCHED EVERY YEAR SEPARATELY.
IF BACKTEST IS OVER MULTIPLE YEARS, IT DOESN'T.
'''

from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline  import Pipeline, CustomFactor
from quantopian.pipeline.data    import Fundamentals
from quantopian.pipeline.filters import StaticSids, StaticAssets
import datetime

class Dow30:
    def __init__(self):
        '''
        this is a utility class that can return the list of Dow30 stocks for a given date.
        the input date must be >= 1999-11-01.
        source: http://www.djindexes.com/mdsidx/downloads/brochure_info/Dow_Jones_Industrial_Average_Historical_Components.pdf
        '''

         # 0 is replaced by 1
        raw_constituents = [
            ["1999-11-01",sid(4922),1],    # Minnesota Mining & Manufacturing/3M Company
            ["1999-11-01",sid(50428),1],   # AA
            ["1999-11-01",sid(679),1],
            ["1999-11-01",sid(7289),1],    # AT&T Corporation
            ["1999-11-01",sid(698),1],
            ["1999-11-01",sid(1267),1],
            ["1999-11-01",sid(1335),1],
            ["1999-11-01",sid(4283),1],
            ["1999-11-01",sid(2119),1],    # DD
            ["1999-11-01",sid(2482),1],    # EK
            ["1999-11-01",sid(8347),1],    # XOM
            ["1999-11-01",sid(3149),1],    # GE
            ["1999-11-01",sid(3246),1],
            ["1999-11-01",sid(3735),1],
            ["1999-11-01",sid(3496),1],
            ["1999-11-01",sid(25090),1],   # AlliedSignal Incorporated/Honeywell
            ["1999-11-01",sid(3951),1],
            ["1999-11-01",sid(3766),1],    # IBM
            ["1999-11-01",sid(3971),1],
            ["1999-11-01",sid(25006),1],
            ["1999-11-01",sid(4151),1],
            ["1999-11-01",sid(4707),1],    # MCD
            ["1999-11-01",sid(5029),1],
            ["1999-11-01",sid(5061),1],
            ["1999-11-01",sid(4954),1],    # MO
            ["1999-11-01",sid(5938),1],    # PG
            ["1999-11-01",sid(6653),1],    # SBC Communications Incorporated/AT&T Incorporated
            ["1999-11-01",sid(7883),1],
            ["1999-11-01",sid(8229),1],    # WMT
            ["1999-11-01",sid(2190),1],
            ["2004-04-08",sid(7289),0],
            ["2004-04-08",sid(2482),0],
            ["2004-04-08",sid(3971),0],
            ["2004-04-08",sid(239),1],
            ["2004-04-08",sid(5923),1],
            ["2004-04-08",sid(21839),1],
            ["2008-02-19",sid(4954),0],
            ["2008-02-19",sid(25090),0],
            ["2008-02-19",sid(700),1],
            ["2008-02-19",sid(23112),1],
            ["2008-09-22",sid(239),0],
            ["2008-09-22",sid(22802),1],    # KRFT/MDLZ
            ["2009-06-08",sid(1335),0],
            ["2009-06-08",sid(3246),0],
            ["2009-06-08",sid(7041),1],
            ["2009-06-08",sid(1900),1],
            ["2012-09-24",sid(22802),0],
            ["2012-09-24",sid(7792),1],
            ["2013-09-23",sid(50428),0],
            ["2013-09-23",sid(700),0],
            ["2013-09-23",sid(3735),0],
            ["2013-09-23",sid(20088),1],
            ["2013-09-23",sid(5328),1],
            ["2013-09-23",sid(35920),1],
            ["2015-03-19",sid(6653),0],    # T  AT&T Inc. Com
            ["2015-03-19",sid(24),1],
            ["2017-09-01",sid(2119),0],
            ["2017-09-01",sid(51157),1],
            ["2018-06-26",sid(3149),0],    # GE
            ["2018-06-26",sid(8089),1]     # WBA
            ]

        # translate the date string into a datetime.date
        self.data = []
        for equity in raw_constituents:
            self.data.append((
                datetime.datetime.strptime(equity[0], "%Y-%m-%d").date(),
                equity[1],
                equity[2]
            ))

    def get(self, dt):
        '''
        for a given datetime, return a list of the 30 dow stocks as Equity objects.
        dt must be >= 1999-11-01
        '''
        d   = dt.date()
        ret = []
        for equity in self.data:
            if equity[0] > d:
                break
            if equity[2] == 1:
                ret.append(equity[1])
            elif equity[2] == 0:
                ret.remove(equity[1])
            else:
                raise Exception('unknown membership')
        #log.info('get(self, {}),    year {}  \nstocks {}'.format(str(dt), dt.year,
        #    str([str(s.symbol) for s in ret])
        #))
        return ret

def trade(context, data):
    if context.trade == True:
        #.#dogs = context.out.index
        dogs = Dow30().get( get_datetime() )  #.# rerunning with current date
        
        members = context.out.loc[dogs].index
        log.info(members)
        log.info(len(members))
        
        members = context.out.index
        log.info(members)
        log.info(len(members))
        
        members = context.out.loc[dogs].sort_values(['forward_dividend_yield'], ascending=False).index
        log.info(members)
        log.info(len(members))
        
        

        log_data(context, data, context.out.loc[dogs], 4)   # show selected pipe info

        log.info('trade(),    context.year {}  stocks ...\n{}'.format(context.year,
            str([str(s.symbol) for s in dogs])
        ))
        context.trade = False

        # Check if holding an old position no longer in the dogs, needs to be closed
        for s in context.portfolio.positions:
            if s not in dogs:
                order_target(s, 0)

        # Check if holding each of the dogs, if not buy to 10% of the total portfolio
        for s in dogs:
            if data.can_trade(s):
                order_target_percent(s, 1.0 / len(dogs))
                #.#order_target_percent(s, .1)
            else:
                log.info('Cant trade {}'.format(s))
                assert(0)   # Deliberate crash to call attention to it. Would be a delist surely.

def initialize(context):
    set_benchmark(symbol('DIA'))
    set_commission(commission.PerShare(cost=0.0035, min_trade_cost=0.35))
    set_slippage(slippage.VolumeShareSlippage(volume_limit=0.025, price_impact=0.1))
    context.dow30   = Dow30()
    context.year    = get_datetime().year
    context.trade   = False
    context.mx_lvrg = 0
    schedule_function(trade, date_rules.every_day(), time_rules.market_open(hours=1))
    attach_pipeline(make_pipeline(), 'pipe')

    for i in range(1, 391):
        schedule_function(track_orders, date_rules.every_day(), time_rules.market_open(minutes=i))

def before_trading_start(context, data):
    dt = get_datetime()
    do_pipe = 0
    msg = ''

    if context.year == 0:
        msg += ('\n    do_pipe, context.year {}'.format(context.year))
        do_pipe = 1
    if context.year == dt.year:
        msg += ('\n    do_pipe, context.year {} == dt.year {}'.format(context.year, dt.year))
        do_pipe = 1
    if len(context.portfolio.positions) < 11:  #.# unsure here (says blue)
        msg += ('\n    do_pipe, len(context.portfolio.positions) {}'.format(len(context.portfolio.positions)))
        do_pipe = 1

    if do_pipe:
        context.trade = True
        print '.\n\n{} {}'.format(dt.year, msg)  # separator b/t pipe runs
        log.info('context.year {} to dt.year + 1 = {}'.format(context.year, dt.year + 1))
        context.year  = dt.year + 1
        # save to context to use during trade()
        context.out = pipeline_output('pipe') #.sort_values(['forward_dividend_yield'], ascending=False)  #.#[:11]
        log.info(context.out)
        log.info(len(context.out))

        #log.info(context.out.tail())
        #log.info(context.out.head())

        #log_data(context, data, context.out, 4)        # show pipe info every time

        '''
        do_log_preview = 1    # a way to toggle this off when it becomes annoying
        if do_log_preview:
            try: context.log_data_done
            except:
                log_data(context, data, context.out, 4)        # show pipe info once
        '''

    record(positions=len(context.portfolio.positions), leverage=context.account.leverage)
    record(mx_lvrg = context.mx_lvrg)    # Record maximum leverage encountered

def make_pipeline():
    members = Dow30().get( get_datetime() )
    log.info(members)

    return Pipeline(
        screen = StaticSids( [m.sid for m in members] ),
        columns = {
            'forward_dividend_yield' : Fundamentals.forward_dividend_yield.latest
            #'dividend_per_share_earnings_reports' : Fundamentals.dividend_per_share_earnings_reports.latest
        })

    # There are nans even for AAPL in forward_dividend_yield
    # Consider dividend_per_share_earnings_reports

def handle_data(context, data):
    if context.account.leverage > context.mx_lvrg:
        context.mx_lvrg = context.account.leverage

def log_data(context, data, z, num, fields=None):
    ''' Log info about pipeline output or, z can be any DataFrame or Series
    https://www.quantopian.com/posts/overview-of-pipeline-content-easy-to-add-to-your-backtest

    try: context.log_data_done
    except:
        # {:,} is magic for adding commas
        log.info('cash ${:,}   portfolio ${:,}     {} positions ...'.format(
            int(context.portfolio.cash),
            int(context.portfolio.portfolio_value),
            len(context.portfolio.positions),
        ))
        context.log_data_done = 1
    '''
    # {:,} is magic for adding commas
    log.info('starting_cash ${:,}   portfolio ${:,}     {} positions ...'.format(
        int(context.portfolio.cash),
        int(context.portfolio.portfolio_value),
        len(context.portfolio.positions),
    ))

    if not len(z):
        log.info('Empty')
        return

    # Options
    log_nan_only = 0          # Only log if nans are present
    show_sectors = 0          # If sectors, do you want to see them or not
    show_sorted_details = 1   # [num] high & low securities sorted, each column
    padmax = 6                # num characters for each field, starting point

    # Series ......
    if 'Series' in str(type(z)):    # is Series, not DataFrame
        nan_count = len(z[z != z])
        nan_count = 'NaNs {}/{}'.format(nan_count, len(z)) if nan_count else ''
        if (log_nan_only and nan_count) or not log_nan_only:
            pad = max( padmax, len('%.5f' % z.max()) )
            log.info('{}{}{}   Series  len {}'.format('min'.rjust(pad+5),
                'mean'.rjust(pad+5), 'max'.rjust(pad+5), len(z)))
            log.info('{}{}{} {}'.format(
                ('%.5f' % z.min()) .rjust(pad+5),
                ('%.5f' % z.mean()).rjust(pad+5),
                ('%.5f' % z.max()) .rjust(pad+5),
                nan_count
            ))
            log.info('High\n{}'.format(z.sort_values(ascending=False).head(num)))
            log.info('Low\n{}' .format(z.sort_values(ascending=False).tail(num)))
        return

    # DataFrame ......
    content_min_max = [ ['','min','mid','max',''] ] ; content = ''
    for col in z.columns:
        #try: z[col].max()
        #except:
        #    log.info('{} non-numeric'.format(col))
        #    #continue   # skip non-numeric
        if col == 'sector' and not show_sectors: continue
        nan_count = len(z[col][z[col] != z[col]])
        nan_count = 'NaNs {}/{}'.format(nan_count, len(z)) if nan_count else ''
        # known bug, not always sorting strings alphabetically ...
        srt       = z[col].sort_values() if type(z[col][0]) != str else z.iloc[z[col].str.lower().argsort()]
        padmax    = max( padmax, len(str(srt[-1])) )
        content_min_max.append([col, str(srt[0]), str(srt[len(srt)//2]), str(srt[-1]), nan_count])
    if log_nan_only and nan_count or not log_nan_only:
        if len(z.columns) == 1: content = 'Stocks: {}'.format(z.shape[0])
        if len(z.columns)  > 1: content = 'Stocks: {}  Columns: {}'.format(z.shape[0], z.shape[1])
        if len(z.columns):
            paddings = [6 for i in range(4)]
            for lst in content_min_max:    # set max lengths
                i = 0
                for val in lst[:4]:        # value in each sub-list
                    paddings[i] = max(paddings[i], len(str(val)))
                    i += 1
            headr = content_min_max[0]
            content += ('\n{}{}{}{}{}'.format(
                 headr[0] .rjust(paddings[0]),
                (headr[1]).rjust(paddings[1]+5),
                (headr[2]).rjust(paddings[2]+5),
                (headr[3]).rjust(paddings[3]+5),
                ''
            ))
            for lst in content_min_max[1:]:    # populate content using max lengths
                content += ('\n{}{}{}{}     {}'.format(
                    lst[0].rjust(paddings[0]),
                    lst[1].rjust(paddings[1]+5),
                    lst[2].rjust(paddings[2]+5),
                    lst[3].rjust(paddings[3]+5),
                    lst[4],
                ))
        log.info(content)

    if not show_sorted_details: return
    #if len(z.columns) == 1:     return     # skip detail if only 1 column
    if fields == None: details = z.columns
    for detail in details:
        if detail == 'sector' and not show_sectors: continue
        hi = z[details].sort_values(by=detail, ascending=False).head(num)
        lo = z[details].sort_values(by=detail, ascending=False).tail(num)
        content  = ''
        content += ('_ _ _   {}   _ _ _'  .format(detail))
        content += ('\n\t... {} highs\n{}'.format(detail, str(hi)))
        content += ('\n\t... {} lows \n{}'.format(detail, str(lo)))
        if log_nan_only and not len(lo[lo[detail] != lo[detail]]):
            continue  # skip if no nans
        log.info(content)

def track_orders(context, data):
    '''  Show orders when made and filled.
           Info: https://www.quantopian.com/posts/track-orders
    '''
    c = context
    if 'trac' not in c:
        c.t_opts = {        # __________    O P T I O N S    __________
            'symbols'     : [],   # List of symbols to filter for, like ['TSLA', 'SPY']
            'log_neg_cash': 1,    # Show cash only when negative.
            'log_cash'    : 1,    # Show cash values in logging window or not.
            'log_ids'     : 1,    # Include order id's in logging window or not.
            'log_unfilled': 1,    # When orders are unfilled. (stop & limit excluded).
            'log_cancels' : 0,    # When orders are canceled.
        }    # Move these to initialize() for better efficiency.
        c.trac = {}
        c.t_dates  = {  # To not overwhelm the log window, start/stop dates can be entered.
            'active': 0,
            'start' : [],   # Start dates, option like ['2007-05-07', '2010-04-26']
            'stop'  : []    # Stop  dates, option like ['2008-02-13', '2010-11-15']
        }
        log.info('track_orders active. Headers ...')
        log.info('             Shares     Shares')
        log.info('Min   Action Order  Sym  Now   at Price   PnL   Stop or Limit   Cash  Id')
    from pytz import timezone as _tz  # Python only does once, makes this portable.
                                      #   Move to top of algo for better efficiency.
    # If 'start' or 'stop' lists have something in them, triggers ...
    if c.t_dates['start'] or c.t_dates['stop']:
        _date = str(get_datetime().date())
        if   _date in c.t_dates['start']:    # See if there's a match to start
            c.t_dates['active'] = 1
        elif _date in c.t_dates['stop']:     #   ... or to stop
            c.t_dates['active'] = 0
    else: c.t_dates['active'] = 1           # Set to active b/c no conditions.
    if c.t_dates['active'] == 0: return     # Skip if not active.
    def _minute():   # To preface each line with the minute of the day.
        bar_dt = get_datetime().astimezone(_tz('US/Eastern'))
        return (bar_dt.hour * 60) + bar_dt.minute - 570 # (-570 = 9:31a)
    def _trac(to_log):      # So all logging comes from the same line number,
        log.info(' {}   {}'.format(str(_minute()).rjust(3), to_log))  # for vertical alignment in the logging window.

    for oid in c.trac.copy():               # Existing known orders
      o = get_order(oid)
      if c.t_opts['symbols'] and (o.sid.symbol not in c.t_opts['symbols']): continue
      if o.dt == o.created: continue        # No chance of fill yet.
      cash = ''
      prc  = data.current(o.sid, 'price') if data.can_trade(o.sid) else c.portfolio.positions[o.sid].last_sale_price
      if (c.t_opts['log_neg_cash'] and c.portfolio.cash < 0) or c.t_opts['log_cash']:
        cash = str(int(c.portfolio.cash))
      if o.status == 2:                     # Canceled
        do = 'Buy' if o.amount > 0 else 'Sell' ; style = ''
        if o.stop:
          style = ' stop {}'.format(o.stop)
          if o.limit: style = ' stop {} limit {}'.format(o.stop, o.limit)
        elif o.limit: style = ' limit {}'.format(o.limit)
        if c.t_opts['log_cancels']:
          _trac('  Canceled {} {} {}{} at {}   {}  {}'.format(do, o.amount,
             o.sid.symbol, style, prc, cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
        del c.trac[o.id]
      elif o.filled:                        # Filled at least some.
        filled = '{}'.format(o.amount)
        filled_amt = 0
        if o.status == 1:                   # Complete
          if 0 < c.trac[o.id] < o.amount:
            filled   = 'all {}/{}'.format(o.filled - c.trac[o.id], o.amount)
          filled_amt = o.filled
        else:                                    # c.trac[o.id] value is previously filled total
          filled_amt = o.filled - c.trac[o.id]   # filled this time, can be 0
          c.trac[o.id] = o.filled                # save fill value for increments math
          filled = '{}/{}'.format(filled_amt, o.amount)
        if filled_amt:
          now = ' ({})'.format(c.portfolio.positions[o.sid].amount) if c.portfolio.positions[o.sid].amount else ' _'
          pnl = ''  # for the trade only
          amt = c.portfolio.positions[o.sid].amount ; style = ''
          if (amt - o.filled) * o.filled < 0:    # Profit-taking scenario including short-buyback
            cb = c.portfolio.positions[o.sid].cost_basis
            if cb:
              pnl  = -filled_amt * (prc - cb)
              sign = '+' if pnl > 0 else '-'
              pnl  = '  ({}{})'.format(sign, '%.0f' % abs(pnl))
          if o.stop:
            style = ' stop {}'.format(o.stop)
            if o.limit: style = ' stop () limit {}'.format(o.stop, o.limit)
          elif o.limit: style = ' limit {}'.format(o.limit)
          if o.filled == o.amount: del c.trac[o.id]
          _trac('   {} {} {}{} at {}{}{}'.format(
            'Bot' if o.amount > 0 else 'Sold', filled, o.sid.symbol, now,
            '%.2f' % prc, pnl, style).ljust(52) + '  {}  {}'.format(cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
      elif c.t_opts['log_unfilled'] and not (o.stop or o.limit):
        _trac('      {} {}{} unfilled  {}'.format(o.sid.symbol, o.amount,
         ' limit' if o.limit else '', o.id[-4:] if c.t_opts['log_ids'] else ''))

    oo = get_open_orders().values()
    if not oo: return                       # Handle new orders
    cash = ''
    if (c.t_opts['log_neg_cash'] and c.portfolio.cash < 0) or c.t_opts['log_cash']:
      cash = str(int(c.portfolio.cash))
    for oo_list in oo:
      for o in oo_list:
        if c.t_opts['symbols'] and (o.sid.symbol not in c.t_opts['symbols']): continue
        if o.id in c.trac: continue         # Only new orders beyond this point
        prc = data.current(o.sid, 'price') if data.can_trade(o.sid) else c.portfolio.positions[o.sid].last_sale_price
        c.trac[o.id] = 0 ; style = ''
        now  = ' ({})'.format(c.portfolio.positions[o.sid].amount) if c.portfolio.positions[o.sid].amount else ' _'
        if o.stop:
          style = ' stop {}'.format(o.stop)
          if o.limit: style = ' stop {} limit {}'.format(o.stop, o.limit)
        elif o.limit: style = ' limit {}'.format(o.limit)
        _trac('{} {} {}{} at {}{}'.format('Buy' if o.amount > 0 else 'Sell',
          o.amount, o.sid.symbol, now, '%.2f' % prc, style).ljust(52) + '  {}  {}'.format(cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
There was a runtime error.

I think I found the "glitch". Correct me if I'm wrong.

Basically, Quantopian doesn't provide neither DJIA index nor its members, so we call a function each time we need to get the Dow members.
'attach_pipeline' is only called in 'initialize', while 'make_pipeline' is called whenever we want. I think that, however, 'make_pipeline' only uses the data that is fetched at the time 'initialize' is run. E.g. if we run the backtest starting in 2015, 'make_pipeline' will always use the Dow members as of 2015, and will never update in the future. Instead, running 'Dow30()' outside the pipeline will return the correct members.

One solution would be to pass Q500US to Pipeline, so we always get the data for all the stocks, then we select only the members of the Dow outside of pipeline and we should have the members with the fundamental data.

Is there a better solution?
Can @Quantopian create a "dynamic pipeline"?

I will update the algo asap.

We're DONE!
Thanks Blue!

P.S.
Obviously some improvements can be made, but the logic is second to none.

Clone Algorithm
20
Loading...
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline import Pipeline, CustomFactor
from quantopian.pipeline.data import Fundamentals
from quantopian.pipeline.filters import StaticSids, StaticAssets
from quantopian.pipeline.filters import Q500US
import datetime

class Dow30:
    def __init__(self):
        '''
        this is a utility class that can return the list of Dow30 stocks for a given date.
        the input date must be >= 1999-11-01.
        source: http://www.djindexes.com/mdsidx/downloads/brochure_info/Dow_Jones_Industrial_Average_Historical_Components.pdf
        '''

         # 0 is replaced by 1
        raw_constituents = [
            ["1999-11-01",sid(4922),1],    # Minnesota Mining & Manufacturing/3M Company
            ["1999-11-01",sid(50428),1],   # AA
            ["1999-11-01",sid(679),1],
            ["1999-11-01",sid(7289),1],    # AT&T Corporation
            ["1999-11-01",sid(698),1],
            ["1999-11-01",sid(1267),1],
            ["1999-11-01",sid(1335),1],
            ["1999-11-01",sid(4283),1],
            ["1999-11-01",sid(2119),1],    # DD
            ["1999-11-01",sid(2482),1],    # EK
            ["1999-11-01",sid(8347),1],    # XOM
            ["1999-11-01",sid(3149),1],    # GE
            ["1999-11-01",sid(3246),1],
            ["1999-11-01",sid(3735),1],
            ["1999-11-01",sid(3496),1],
            ["1999-11-01",sid(25090),1],   # AlliedSignal Incorporated/Honeywell
            ["1999-11-01",sid(3951),1],
            ["1999-11-01",sid(3766),1],    # IBM
            ["1999-11-01",sid(3971),1],
            ["1999-11-01",sid(25006),1],
            ["1999-11-01",sid(4151),1],
            ["1999-11-01",sid(4707),1],    # MCD
            ["1999-11-01",sid(5029),1],
            ["1999-11-01",sid(5061),1],
            ["1999-11-01",sid(4954),1],    # MO
            ["1999-11-01",sid(5938),1],    # PG
            ["1999-11-01",sid(6653),1],    # SBC Communications Incorporated/AT&T Incorporated
            ["1999-11-01",sid(7883),1],
            ["1999-11-01",sid(8229),1],    # WMT
            ["1999-11-01",sid(2190),1],
            ["2004-04-08",sid(7289),0],
            ["2004-04-08",sid(2482),0],
            ["2004-04-08",sid(3971),0],
            ["2004-04-08",sid(239),1],
            ["2004-04-08",sid(5923),1],
            ["2004-04-08",sid(21839),1],
            ["2008-02-19",sid(4954),0],
            ["2008-02-19",sid(25090),0],
            ["2008-02-19",sid(700),1],
            ["2008-02-19",sid(23112),1],
            ["2008-09-22",sid(239),0],
            ["2008-09-22",sid(22802),1],    # KRFT/MDLZ
            ["2009-06-08",sid(1335),0],
            ["2009-06-08",sid(3246),0],
            ["2009-06-08",sid(7041),1],
            ["2009-06-08",sid(1900),1],
            ["2012-09-24",sid(22802),0],
            ["2012-09-24",sid(7792),1],
            ["2013-09-23",sid(50428),0],
            ["2013-09-23",sid(700),0],
            ["2013-09-23",sid(3735),0],
            ["2013-09-23",sid(20088),1],
            ["2013-09-23",sid(5328),1],
            ["2013-09-23",sid(35920),1],
            ["2015-03-19",sid(6653),0],    # T  AT&T Inc. Com
            ["2015-03-19",sid(24),1],
            ["2017-09-01",sid(2119),0],
            ["2017-09-01",sid(51157),1],
            ["2018-06-26",sid(3149),0],    # GE
            ["2018-06-26",sid(8089),1]     # WBA
            ]

        # translate the date string into a datetime.date
        self.data = []
        for equity in raw_constituents:
            self.data.append((
                datetime.datetime.strptime(equity[0], "%Y-%m-%d").date(),
                equity[1],
                equity[2]
            ))

    def get(self, dt):
        '''
        for a given datetime, return a list of the 30 dow stocks as Equity objects.
        dt must be >= 1999-11-01
        '''
        d   = dt.date()
        ret = []
        for equity in self.data:
            if equity[0] > d:
                break
            if equity[2] == 1:
                ret.append(equity[1])
            elif equity[2] == 0:
                ret.remove(equity[1])
            else:
                raise Exception('unknown membership')
        #log.info('get(self, {}),    year {}  \nstocks {}'.format(str(dt), dt.year,
        #    str([str(s.symbol) for s in ret])
        #))
        return ret
    
    
    
def initialize(context):
    set_benchmark(symbol('DIA'))
    set_commission(commission.PerShare(cost=0.0035, min_trade_cost=0.35))
    set_slippage(slippage.VolumeShareSlippage(volume_limit=0.025, price_impact=0.1))
    context.dow30   = Dow30()
    context.year    = get_datetime().year
    context.trade   = False
    context.mx_lvrg = 0
    schedule_function(trade, date_rules.every_day(), time_rules.market_open(hours=1))
    attach_pipeline(make_pipeline(), 'pipe')

    for i in range(1, 391):
        schedule_function(track_orders, date_rules.every_day(), time_rules.market_open(minutes=i))
                
def make_pipeline():

    return Pipeline(
        screen = Q500US(),
        columns = {
            'forward_dividend_yield' : Fundamentals.forward_dividend_yield.latest
            #'dividend_per_share_earnings_reports' : Fundamentals.dividend_per_share_earnings_reports.latest
        })
        
def before_trading_start(context, data):
    dt = get_datetime()
    do_pipe = 0
    msg = ''

    if context.year == 0:
        msg += ('\n    do_pipe, context.year {}'.format(context.year))
        do_pipe = 1
    if context.year == dt.year:
        msg += ('\n    do_pipe, context.year {} == dt.year {}'.format(context.year, dt.year))
        do_pipe = 1
    if len(context.portfolio.positions) < 10:  #.# unsure here (says blue)
        msg += ('\n    do_pipe, len(context.portfolio.positions) {}'.format(len(context.portfolio.positions)))
        do_pipe = 1

    if do_pipe:
        context.trade = True
        print '.\n\n{} {}'.format(dt.year, msg)  # separator b/t pipe runs
        log.info('context.year {} to dt.year + 1 = {}'.format(context.year, dt.year + 1))
        context.year  = dt.year + 1
        # save to context to use during trade()
        context.out = pipeline_output('pipe') #.sort_values(['forward_dividend_yield'], ascending=False)  #.#[:11]
        log.info(context.out)
        log.info(len(context.out))

        #log.info(context.out.tail())
        #log.info(context.out.head())

        #log_data(context, data, context.out, 4)        # show pipe info every time

        '''
        do_log_preview = 1    # a way to toggle this off when it becomes annoying
        if do_log_preview:
            try: context.log_data_done
            except:
                log_data(context, data, context.out, 4)        # show pipe info once
        '''

    record(positions=len(context.portfolio.positions), leverage=context.account.leverage)
    record(mx_lvrg = context.mx_lvrg)    # Record maximum leverage encountered
        

def trade(context, data):
    if context.trade == True:
        
        #.#dogs = context.out.index
        dow = Dow30().get( get_datetime() )  #.# rerunning with current date
        
        members = context.out.loc[dow].sort_values(['forward_dividend_yield'], ascending=False).index
        #log.info(members)
        #log.info(len(members))
        
        dogs = members[:10]
        log.info(dogs)
        log.info(len(dogs))
        
        
        log_data(context, data, context.out.loc[dogs], 4)   # show selected pipe info

        log.info('trade(),    context.year {}  stocks ...\n{}'.format(context.year,
            str([str(s.symbol) for s in dogs])
        ))
        context.trade = False

        # Check if holding an old position no longer in the dogs, needs to be closed
        for s in context.portfolio.positions:
            if s not in dogs:
                order_target(s, 0)

        # Check if holding each of the dogs, if not buy to 10% of the total portfolio
        for s in dogs:
            if data.can_trade(s):
                order_target_percent(s, 1.0 / len(dogs))
                #.#order_target_percent(s, .1)
            else:
                log.info('Cant trade {}'.format(s))
                assert(0)   # Deliberate crash to call attention to it. Would be a delist surely.


    
    
    
    
def handle_data(context, data):
    if context.account.leverage > context.mx_lvrg:
        context.mx_lvrg = context.account.leverage

def log_data(context, data, z, num, fields=None):
    ''' Log info about pipeline output or, z can be any DataFrame or Series
    https://www.quantopian.com/posts/overview-of-pipeline-content-easy-to-add-to-your-backtest

    try: context.log_data_done
    except:
        # {:,} is magic for adding commas
        log.info('cash ${:,}   portfolio ${:,}     {} positions ...'.format(
            int(context.portfolio.cash),
            int(context.portfolio.portfolio_value),
            len(context.portfolio.positions),
        ))
        context.log_data_done = 1
    '''
    # {:,} is magic for adding commas
    log.info('starting_cash ${:,}   portfolio ${:,}     {} positions ...'.format(
        int(context.portfolio.cash),
        int(context.portfolio.portfolio_value),
        len(context.portfolio.positions),
    ))

    if not len(z):
        log.info('Empty')
        return

    # Options
    log_nan_only = 0          # Only log if nans are present
    show_sectors = 0          # If sectors, do you want to see them or not
    show_sorted_details = 1   # [num] high & low securities sorted, each column
    padmax = 6                # num characters for each field, starting point

    # Series ......
    if 'Series' in str(type(z)):    # is Series, not DataFrame
        nan_count = len(z[z != z])
        nan_count = 'NaNs {}/{}'.format(nan_count, len(z)) if nan_count else ''
        if (log_nan_only and nan_count) or not log_nan_only:
            pad = max( padmax, len('%.5f' % z.max()) )
            log.info('{}{}{}   Series  len {}'.format('min'.rjust(pad+5),
                'mean'.rjust(pad+5), 'max'.rjust(pad+5), len(z)))
            log.info('{}{}{} {}'.format(
                ('%.5f' % z.min()) .rjust(pad+5),
                ('%.5f' % z.mean()).rjust(pad+5),
                ('%.5f' % z.max()) .rjust(pad+5),
                nan_count
            ))
            log.info('High\n{}'.format(z.sort_values(ascending=False).head(num)))
            log.info('Low\n{}' .format(z.sort_values(ascending=False).tail(num)))
        return

    # DataFrame ......
    content_min_max = [ ['','min','mid','max',''] ] ; content = ''
    for col in z.columns:
        #try: z[col].max()
        #except:
        #    log.info('{} non-numeric'.format(col))
        #    #continue   # skip non-numeric
        if col == 'sector' and not show_sectors: continue
        nan_count = len(z[col][z[col] != z[col]])
        nan_count = 'NaNs {}/{}'.format(nan_count, len(z)) if nan_count else ''
        # known bug, not always sorting strings alphabetically ...
        srt       = z[col].sort_values() if type(z[col][0]) != str else z.iloc[z[col].str.lower().argsort()]
        padmax    = max( padmax, len(str(srt[-1])) )
        content_min_max.append([col, str(srt[0]), str(srt[len(srt)//2]), str(srt[-1]), nan_count])
    if log_nan_only and nan_count or not log_nan_only:
        if len(z.columns) == 1: content = 'Stocks: {}'.format(z.shape[0])
        if len(z.columns)  > 1: content = 'Stocks: {}  Columns: {}'.format(z.shape[0], z.shape[1])
        if len(z.columns):
            paddings = [6 for i in range(4)]
            for lst in content_min_max:    # set max lengths
                i = 0
                for val in lst[:4]:        # value in each sub-list
                    paddings[i] = max(paddings[i], len(str(val)))
                    i += 1
            headr = content_min_max[0]
            content += ('\n{}{}{}{}{}'.format(
                 headr[0] .rjust(paddings[0]),
                (headr[1]).rjust(paddings[1]+5),
                (headr[2]).rjust(paddings[2]+5),
                (headr[3]).rjust(paddings[3]+5),
                ''
            ))
            for lst in content_min_max[1:]:    # populate content using max lengths
                content += ('\n{}{}{}{}     {}'.format(
                    lst[0].rjust(paddings[0]),
                    lst[1].rjust(paddings[1]+5),
                    lst[2].rjust(paddings[2]+5),
                    lst[3].rjust(paddings[3]+5),
                    lst[4],
                ))
        log.info(content)

    if not show_sorted_details: return
    #if len(z.columns) == 1:     return     # skip detail if only 1 column
    if fields == None: details = z.columns
    for detail in details:
        if detail == 'sector' and not show_sectors: continue
        hi = z[details].sort_values(by=detail, ascending=False).head(num)
        lo = z[details].sort_values(by=detail, ascending=False).tail(num)
        content  = ''
        content += ('_ _ _   {}   _ _ _'  .format(detail))
        content += ('\n\t... {} highs\n{}'.format(detail, str(hi)))
        content += ('\n\t... {} lows \n{}'.format(detail, str(lo)))
        if log_nan_only and not len(lo[lo[detail] != lo[detail]]):
            continue  # skip if no nans
        log.info(content)

def track_orders(context, data):
    '''  Show orders when made and filled.
           Info: https://www.quantopian.com/posts/track-orders
    '''
    c = context
    if 'trac' not in c:
        c.t_opts = {        # __________    O P T I O N S    __________
            'symbols'     : [],   # List of symbols to filter for, like ['TSLA', 'SPY']
            'log_neg_cash': 1,    # Show cash only when negative.
            'log_cash'    : 1,    # Show cash values in logging window or not.
            'log_ids'     : 1,    # Include order id's in logging window or not.
            'log_unfilled': 1,    # When orders are unfilled. (stop & limit excluded).
            'log_cancels' : 0,    # When orders are canceled.
        }    # Move these to initialize() for better efficiency.
        c.trac = {}
        c.t_dates  = {  # To not overwhelm the log window, start/stop dates can be entered.
            'active': 0,
            'start' : [],   # Start dates, option like ['2007-05-07', '2010-04-26']
            'stop'  : []    # Stop  dates, option like ['2008-02-13', '2010-11-15']
        }
        log.info('track_orders active. Headers ...')
        log.info('             Shares     Shares')
        log.info('Min   Action Order  Sym  Now   at Price   PnL   Stop or Limit   Cash  Id')
    from pytz import timezone as _tz  # Python only does once, makes this portable.
                                      #   Move to top of algo for better efficiency.
    # If 'start' or 'stop' lists have something in them, triggers ...
    if c.t_dates['start'] or c.t_dates['stop']:
        _date = str(get_datetime().date())
        if   _date in c.t_dates['start']:    # See if there's a match to start
            c.t_dates['active'] = 1
        elif _date in c.t_dates['stop']:     #   ... or to stop
            c.t_dates['active'] = 0
    else: c.t_dates['active'] = 1           # Set to active b/c no conditions.
    if c.t_dates['active'] == 0: return     # Skip if not active.
    def _minute():   # To preface each line with the minute of the day.
        bar_dt = get_datetime().astimezone(_tz('US/Eastern'))
        return (bar_dt.hour * 60) + bar_dt.minute - 570 # (-570 = 9:31a)
    def _trac(to_log):      # So all logging comes from the same line number,
        log.info(' {}   {}'.format(str(_minute()).rjust(3), to_log))  # for vertical alignment in the logging window.

    for oid in c.trac.copy():               # Existing known orders
      o = get_order(oid)
      if c.t_opts['symbols'] and (o.sid.symbol not in c.t_opts['symbols']): continue
      if o.dt == o.created: continue        # No chance of fill yet.
      cash = ''
      prc  = data.current(o.sid, 'price') if data.can_trade(o.sid) else c.portfolio.positions[o.sid].last_sale_price
      if (c.t_opts['log_neg_cash'] and c.portfolio.cash < 0) or c.t_opts['log_cash']:
        cash = str(int(c.portfolio.cash))
      if o.status == 2:                     # Canceled
        do = 'Buy' if o.amount > 0 else 'Sell' ; style = ''
        if o.stop:
          style = ' stop {}'.format(o.stop)
          if o.limit: style = ' stop {} limit {}'.format(o.stop, o.limit)
        elif o.limit: style = ' limit {}'.format(o.limit)
        if c.t_opts['log_cancels']:
          _trac('  Canceled {} {} {}{} at {}   {}  {}'.format(do, o.amount,
             o.sid.symbol, style, prc, cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
        del c.trac[o.id]
      elif o.filled:                        # Filled at least some.
        filled = '{}'.format(o.amount)
        filled_amt = 0
        if o.status == 1:                   # Complete
          if 0 < c.trac[o.id] < o.amount:
            filled   = 'all {}/{}'.format(o.filled - c.trac[o.id], o.amount)
          filled_amt = o.filled
        else:                                    # c.trac[o.id] value is previously filled total
          filled_amt = o.filled - c.trac[o.id]   # filled this time, can be 0
          c.trac[o.id] = o.filled                # save fill value for increments math
          filled = '{}/{}'.format(filled_amt, o.amount)
        if filled_amt:
          now = ' ({})'.format(c.portfolio.positions[o.sid].amount) if c.portfolio.positions[o.sid].amount else ' _'
          pnl = ''  # for the trade only
          amt = c.portfolio.positions[o.sid].amount ; style = ''
          if (amt - o.filled) * o.filled < 0:    # Profit-taking scenario including short-buyback
            cb = c.portfolio.positions[o.sid].cost_basis
            if cb:
              pnl  = -filled_amt * (prc - cb)
              sign = '+' if pnl > 0 else '-'
              pnl  = '  ({}{})'.format(sign, '%.0f' % abs(pnl))
          if o.stop:
            style = ' stop {}'.format(o.stop)
            if o.limit: style = ' stop () limit {}'.format(o.stop, o.limit)
          elif o.limit: style = ' limit {}'.format(o.limit)
          if o.filled == o.amount: del c.trac[o.id]
          _trac('   {} {} {}{} at {}{}{}'.format(
            'Bot' if o.amount > 0 else 'Sold', filled, o.sid.symbol, now,
            '%.2f' % prc, pnl, style).ljust(52) + '  {}  {}'.format(cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
      elif c.t_opts['log_unfilled'] and not (o.stop or o.limit):
        _trac('      {} {}{} unfilled  {}'.format(o.sid.symbol, o.amount,
         ' limit' if o.limit else '', o.id[-4:] if c.t_opts['log_ids'] else ''))

    oo = get_open_orders().values()
    if not oo: return                       # Handle new orders
    cash = ''
    if (c.t_opts['log_neg_cash'] and c.portfolio.cash < 0) or c.t_opts['log_cash']:
      cash = str(int(c.portfolio.cash))
    for oo_list in oo:
      for o in oo_list:
        if c.t_opts['symbols'] and (o.sid.symbol not in c.t_opts['symbols']): continue
        if o.id in c.trac: continue         # Only new orders beyond this point
        prc = data.current(o.sid, 'price') if data.can_trade(o.sid) else c.portfolio.positions[o.sid].last_sale_price
        c.trac[o.id] = 0 ; style = ''
        now  = ' ({})'.format(c.portfolio.positions[o.sid].amount) if c.portfolio.positions[o.sid].amount else ' _'
        if o.stop:
          style = ' stop {}'.format(o.stop)
          if o.limit: style = ' stop {} limit {}'.format(o.stop, o.limit)
        elif o.limit: style = ' limit {}'.format(o.limit)
        _trac('{} {} {}{} at {}{}'.format('Buy' if o.amount > 0 else 'Sell',
          o.amount, o.sid.symbol, now, '%.2f' % prc, style).ljust(52) + '  {}  {}'.format(cash, o.id[-4:] if c.t_opts['log_ids'] else ''))
There was a runtime error.