Back to Community
Machine Learing|Mean Reversion|The Bottom Of The Q500

Basic Machine Learning Models and Mean Reversion Techniques are rather popular on Quantopian's forums right now. I went ahead and tweaked the ideas (for now [I'm busy] this uses a modified version of "Enhancing Mean Reversion"- that will soon change), then combined them (then optimized them some more). The bottom of the Q500 just comes from: (pricing < 6 ) and is vital to the algorithms success. You can get similar success with the bottom of the Q1500. There is also some success to be found in limiting the pricing to 10 < pricing < 30 with certain sectors. The sector codes can be annoying to find so here they are:

1 Cyclical
101 Basic Materials 102 Consumer Cyclical 103 Financial Services 104 Real Estate
2 Defensive
205 Consumer Defensive 206 Healthcare 207 Utilities
3 Sensitive
308 Communication Services 309 Energy 310 Industrials 311 Technology
[This is from page 9][1]

This is still very much a work in progress. The algo will not run if you run it through may of 2016 and I have a fix (but it's compute time is stupid high I'm working on reducing it)

I understand that most are not ok with loading all money on one stock. The whole point of algorithms is to increase returns whilst reducing risk and thus, I have no issue with it. Also, this is being built specifically to grow a small amount of money.

Thanks for stopping by
-J

[1]: http://corporate.morningstar.com/us/documents/methodologydocuments/methodologypapers/equityclassmethodology.pdfhttp://

Clone Algorithm
119
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#MACHINE LEARNING, MEAN REVERSION, & THE BOTTOM OF THE Q500
import numpy as np
import pandas as pd
from quantopian.pipeline import Pipeline
from quantopian.pipeline import CustomFactor
from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.factors import SimpleMovingAverage, AverageDollarVolume
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.filters import Q500US
from sklearn.ensemble import ExtraTreesRegressor
import numpy as np

def initialize(context):
    set_benchmark(sid(8554))
    schedule_function(my_rebalance, date_rules.every_day(), time_rules.market_close(minutes = 1))
    schedule_function(close_, date_rules.every_day(), time_rules.market_close(minutes = 1))
    schedule_function(open_, date_rules.every_day(), time_rules.market_close())
    schedule_function(my_record_vars, date_rules.every_day(), time_rules.market_close())
    schedule_function(get_prices,date_rules.every_day(), time_rules.market_close(minutes=5))
# I Use RobinHood
    set_commission(commission.PerShare(cost=0.00, min_trade_cost=0.00))

    context.model = ExtraTreesRegressor()
    # NOT NEEDED HERE BUT IS USEFULL IN SLOWING DOWN THE ALGORITHM (FOR THOSE WHO WANT TO SEE EVERY DATA POINT AS IT HAPPENS)
#================================================================================================    
    # context.model.n_estimators = 1000
#================================================================================================ 
    # ALSO NOT NEEDED, WILL DO NOTHING FOR YOU.
    # context.model.min_samples_leaf = 10
#================================================================================================ 
    context.lookback = 3 
    context.history_range = 5

    context.nq=5
    context.nq_vol=3
    
    context.existing_longs=0
    context.existing_shorts=0

    my_pipe = make_pipeline()
    attach_pipeline(my_pipe, 'my_pipeline')

class Volatility(CustomFactor):  
    inputs = [USEquityPricing.close]
    window_length=132
    
    def compute(self, today, assets, out, close):
        daily_returns = np.log(close[1:-6]) - np.log(close[0:-7])
        out[:] = daily_returns.std(axis = 0)           

class Liquidity(CustomFactor):   
    inputs = [USEquityPricing.volume, morningstar.valuation.shares_outstanding] 
    window_length = 1
    
    def compute(self, today, assets, out, volume, shares):       
        out[:] = volume[-1]/shares[-1]        
        
class Sector(CustomFactor):
    inputs=[morningstar.asset_classification.morningstar_sector_code]
    window_length=1
    
    def compute(self, today, assets, out, sector):
        out[:] = sector[-1]   
   
def make_pipeline():
    pricing=USEquityPricing.close.latest
    profitable = morningstar.valuation_ratios.ev_to_ebitda.latest > 0  
    # sector = morningstar.asset_classification.morningstar_sector_code.latest
    # vol=Volatility(mask=Q500US())
    # vol=vol.zscore(groupby=sector)
    # vol_filter=vol.percentile_between(0,100)
    #liquidity=Liquidity(mask=Q1500US())
    #liquidity_filter=liquidity.percentile_between(0,75) | liquidity.isnan()
    universe = (
        Q500US()
        & ( pricing < 6) & profitable
        )
    return Pipeline(
        screen=universe
    )
def before_trading_start(context, data):

    context.output = pipeline_output('my_pipeline')
       
def get_prices(context, data):

    Universe500=context.output.index.tolist()
    # ORIGINAL MEAN REVERSION ALGORITHM AUTHOR ARGUED FOR 6 : NOTEBOOKS HAVE SHOWN THAT 9 WORKS BETTER FOR THIS SITUATION
    prices = data.history(Universe500,'price',9,'1d')
    daily_rets=np.log(prices/prices.shift(1))
    
    rets=(prices.iloc[-3] - prices.iloc[0]) / prices.iloc[0]
    
    stdevs=daily_rets.std(axis=0)

    rets_df=pd.DataFrame(rets,columns=['five_day_ret'])
    stdevs_df=pd.DataFrame(stdevs,columns=['stdev_ret'])
    
    context.output=context.output.join(rets_df,how='outer')
    context.output=context.output.join(stdevs_df,how='outer')
    
    context.output['ret_quantile']=pd.qcut(context.output['five_day_ret'],context.nq,labels=False)+1
    context.output['stdev_quantile']=pd.qcut(context.output['stdev_ret'],3,labels=False)+1

    context.longs=context.output[(context.output['ret_quantile']==1) & 
                                (context.output['stdev_quantile']<context.nq_vol)].index.tolist()
    context.shorts=context.output[(context.output['ret_quantile']==context.nq) & 
                                 (context.output['stdev_quantile']<context.nq_vol)].index.tolist()       
def my_rebalance(context, data):
    Universe500=context.output.index.tolist()
    context.existing_longs=0
    context.existing_shorts=0
    for security in context.portfolio.positions:
        if security not in Universe500 and data.can_trade(security): 
            order_target_percent(security, 0)
        else:
            if data.can_trade(security):
                current_quantile=context.output['ret_quantile'].loc[security]
                if context.portfolio.positions[security].amount>0:
                    if (current_quantile==1) and (security not in context.longs):
                        context.existing_longs += 1
                    elif (current_quantile>1) and (security not in context.shorts):
                        order_target_percent(security, 0)
                elif context.portfolio.positions[security].amount<0:
                    if (current_quantile==context.nq) and (security not in context.shorts):
                        context.existing_shorts += 1
                    elif (current_quantile<context.nq) and (security not in context.longs):
                        order_target_percent(security, 0)
def close_(context, data):
    results = context.shorts
    log.info(results)
    securities_in_results = results.index
    passed = []
    for sec in context.shorts:
                                     #Stock, Price, lookback 5, Days
        recent_prices =  data.history(sec, 'price', context.history_range, '1d').values
                                     #Stock, Volume, Lookback 5, Days
        recent_volumes = data.history(sec, 'volume', context.history_range, '1d').values
        price_changes = np.diff(recent_prices).tolist()
        volume_changes = np.diff(recent_volumes).tolist()
        
#X = Input, Y= Output 
        X,Y = [],[]
    
        for i in range(context.history_range-context.lookback-1): 
            X.append(price_changes[i:i+context.lookback] + volume_changes[i:i+context.lookback])
            Y.append(price_changes[i+context.lookback])
            
        context.model.fit(X, Y) 
        
#Check Model Accuracy:
        recent_prices = data.history(sec, 'price', context.lookback+1, '1d').values
        recent_volumes = data.history(sec, 'volume', context.lookback+1, '1d').values
        price_changes =  np.diff(recent_prices).tolist()
        volume_changes = np.diff(recent_volumes).tolist()
        
#Make Prediction 
        prediction = context.model.predict(price_changes + volume_changes)
    #PREDICTION OF SHORT LIST STOCKS!!!
        if prediction < 0.0: 
            passed.append(sec)
            
    for sec in context.shorts:
        if data.can_trade(sec) and sec in passed:
            order_target_percent(sec, -.95/(len(context.shorts)+context.existing_shorts))
        elif data.can_trade(sec) and sec not in passed:
            order_target_percent(sec, 0.0/(len(context.shorts)+context.existing_shorts))
        else:
            print("WTF")
                        
def open_(context, data):    
    results = context.longs
    log.info(results)
    securities_in_results = results.index
    passed = []
    for sec in context.longs:
                                     #Stock, Price, lookback 5, Days
        recent_prices =  data.history(sec, 'price', context.history_range, '1d').values
                                     #Stock, Volume, Lookback 5, Days
        recent_volumes = data.history(sec, 'volume', context.history_range, '1d').values
        price_changes = np.diff(recent_prices).tolist()
        volume_changes = np.diff(recent_volumes).tolist()
        
#X = Input, Y= Output 
        X,Y = [],[]
    
        for i in range(context.history_range-context.lookback-1): 
            X.append(price_changes[i:i+context.lookback] + volume_changes[i:i+context.lookback])
            Y.append(price_changes[i+context.lookback])
            
        context.model.fit(X, Y) 
        
#Check Model Accuracy:
        recent_prices = data.history(sec, 'price', context.lookback+1, '1d').values
        recent_volumes = data.history(sec, 'volume', context.lookback+1, '1d').values
        price_changes =  np.diff(recent_prices).tolist()
        volume_changes = np.diff(recent_volumes).tolist()
        
        prediction = context.model.predict(price_changes + volume_changes)
        
        print (sec.symbol, prediction)
        #PREDICTION OF LONG LIST STOCKS!!!
        if prediction > 0.0: 
            passed.append(sec)
            
    for sec in context.longs:
        if data.can_trade(sec) and sec in passed:
            order_target_percent(sec, 1.0/(len(context.longs)+context.existing_longs) )
# If you dont care how high your beta is. uncomment this line and play around. you will see higher returns but with much higher drawdowns
       # elif data.can_trade(sec): 
         #   order_target_percent(sec, 1.0/(len(context.longs)+context.existing_longs) )
            
def my_record_vars(context, data):
    """
    Record variables at the end of each day.
    """
    longs = shorts = 0
    for position in context.portfolio.positions.itervalues():
        if position.amount > 0:
            longs += 1
        elif position.amount < 0:
            shorts += 1
    # Record our variables.
    record(leverage=context.account.leverage, long_count=longs, short_count=shorts)
    
    log.info("Today's shorts: "  +", ".join([short_.symbol for short_ in context.shorts]))
    log.info("Today's longs: "  +", ".join([long_.symbol for long_ in context.longs]))
There was a runtime error.
3 responses

IGNORE THE TITLE OF THE NOTEBOOK that was its previous purpose. wooops

Loading notebook preview...
Notebook previews are currently unavailable.

Still very much a WIP

Clone Algorithm
45
Loading...
Backtest from to with initial capital
Total Returns
--
Alpha
--
Beta
--
Sharpe
--
Sortino
--
Max Drawdown
--
Benchmark Returns
--
Volatility
--
Returns 1 Month 3 Month 6 Month 12 Month
Alpha 1 Month 3 Month 6 Month 12 Month
Beta 1 Month 3 Month 6 Month 12 Month
Sharpe 1 Month 3 Month 6 Month 12 Month
Sortino 1 Month 3 Month 6 Month 12 Month
Volatility 1 Month 3 Month 6 Month 12 Month
Max Drawdown 1 Month 3 Month 6 Month 12 Month
#MACHINE LEARNING, MEAN REVERSION, & THE BOTTOM OF THE Q500
import numpy as np
import pandas as pd
from quantopian.pipeline import Pipeline
from quantopian.pipeline import CustomFactor
from quantopian.algorithm import attach_pipeline, pipeline_output
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.factors import SimpleMovingAverage, AverageDollarVolume
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.filters import Q500US
from sklearn.ensemble import ExtraTreesClassifier
import numpy as np

def initialize(context):
    set_benchmark(sid(8554))

    schedule_function(my_rebalance, date_rules.every_day(), time_rules.market_close(minutes = 1))
    schedule_function(close_, date_rules.every_day(), time_rules.market_close(minutes = 1))
    schedule_function(open_, date_rules.every_day(), time_rules.market_close())
    schedule_function(my_record_vars, date_rules.every_day(), time_rules.market_close())
    schedule_function(get_prices,date_rules.every_day(), time_rules.market_close(minutes=5))
    schedule_function(check_switch,date_rules.every_day(),time_rules.market_close())
    set_commission(commission.PerShare(cost=0.00, min_trade_cost=0.00))

    context.model = ExtraTreesClassifier()
    # NOT NEEDED HERE BUT IS USEFULL IN SLOWING DOWN THE ALGORITHM (FOR THOSE WHO WANT TO SEE EVERY DATA POINT AS IT HAPPENS)
#================================================================================================    
    # context.model.n_estimators = 1000
#================================================================================================ 
    # ALSO NOT NEEDED, WILL DO NOTHING FOR YOU.
    # context.model.min_samples_leaf = 10
#================================================================================================ 
    context.lookback = 3
    context.history_range = 5

    context.nq=5
    context.nq_vol=3
    
    context.existing_longs=0
    context.existing_shorts=0
    
    context.switch = True
    
    context.trade = []

    my_pipe = make_pipeline()
    attach_pipeline(my_pipe, 'my_pipeline')

class Volatility(CustomFactor):  
    inputs = [USEquityPricing.close]
    window_length=132
    
    def compute(self, today, assets, out, close):
        daily_returns = np.log(close[1:-6]) - np.log(close[0:-7])
        out[:] = daily_returns.std(axis = 0)           

class Liquidity(CustomFactor):   
    inputs = [USEquityPricing.volume, morningstar.valuation.shares_outstanding] 
    window_length = 1
    
    def compute(self, today, assets, out, volume, shares):       
        out[:] = volume[-1]/shares[-1]        
        
class Sector(CustomFactor):
    inputs=[morningstar.asset_classification.morningstar_sector_code]
    window_length=1
    
    def compute(self, today, assets, out, sector):
        out[:] = sector[-1]   
   
def make_pipeline():
    pricing=USEquityPricing.close.latest
    profitable = morningstar.valuation_ratios.ev_to_ebitda.latest > 0  
    # sector = morningstar.asset_classification.morningstar_sector_code.latest
    # vol=Volatility(mask=Q500US())
    # vol=vol.zscore(groupby=sector)
    # vol_filter=vol.percentile_between(0,100)
    #liquidity=Liquidity(mask=Q1500US())
    #liquidity_filter=liquidity.percentile_between(0,75) | liquidity.isnan()
    universe = (
        Q500US()
        & ( pricing < 6) & profitable
        )
    return Pipeline(
        screen=universe
    )
def before_trading_start(context, data):
    context.trade = []
    context.output = pipeline_output('my_pipeline')
       
def get_prices(context, data):
    
    #THE GLOBAL SWITCH WILL START "OFF". IF WE MAKE IT ALL THE WAY THROUGH EACH "TRY"
    #WE WILL FLIP THE SWTICH ON
    context.switch = False
    
    #IF SWITCH_TWO IS TRUE, WE WILL EXIT THE LOOP
    switch_two = False
    while switch_two == False:
        Universe500=context.output.index.tolist()
    # ORIGINAL MEAN REVERSION ALGORITHM AUTHOR ARGUED FOR 6 : NOTEBOOKS HAVE SHOWN THAT 9 WORKS BETTER FOR THIS SITUATION
        prices = data.history(Universe500,'price',9,'1d')
        daily_rets=np.log(prices/prices.shift(1))
    
        rets=(prices.iloc[-3] - prices.iloc[0]) / prices.iloc[0]
    
        stdevs=daily_rets.std(axis=0)

        rets_df=pd.DataFrame(rets,columns=['five_day_ret'])
        stdevs_df=pd.DataFrame(stdevs,columns=['stdev_ret'])
        try:
            context.output=context.output.join(rets_df,how='outer')
            context.output=context.output.join(stdevs_df,how='outer')
        except:
            switch_two = True
    
    
    
        try:
            context.output['ret_quantile']=pd.qcut(context.output['five_day_ret'],context.nq,labels=False)+1
        except:
            switch_two = True                              
                                  
    
        try:                          
            context.output['stdev_quantile']=pd.qcut(context.output['stdev_ret'],3,labels=False)+1
        except:
            #FAIL: TURN THE LOCAL SWITCH ON AND EXIT THE LOOP.
            switch_two = True
        try:
            context.longs=context.output[(context.output['ret_quantile']==1) & 
                                (context.output['stdev_quantile']<context.nq_vol)].index.tolist()
        except:
            #FAIL: TURN THE LOCAL SWITCH ON AND EXIT THE LOOP.
            switch_two = True
        try:
            context.shorts=context.output[(context.output['ret_quantile']==context.nq) & 
                                 (context.output['stdev_quantile']<context.nq_vol)].index.tolist() 
        except:
            #FAIL: TURN THE LOCAL SWITCH ON AND EXIT THE LOOP.
            switch_two = True
        
        
        #IF WE HAVE MADE IT THROUGH THE WHOLE LOOP, TURN THE GLOBAL SWITCH ON
        #AND TURN THE LOCAL SWITCH OFF. 
        if switch_two == False:
            context.switch = True
        #THE LOOP HAS SUCCESSFULLY BEEN COMPLETED IN ENTIRETY, EXIT THE LOOP.
        switch_two = True
                                      
def my_rebalance(context, data):
    if context.switch == True:
        Universe500=context.output.index.tolist()
        context.existing_longs=0
        context.existing_shorts=0
        for security in context.portfolio.positions:
            if security not in Universe500 and data.can_trade(security): 
                order_target_percent(security, 0)
            else:
                if data.can_trade(security):
                    current_quantile=context.output['ret_quantile'].loc[security]
                    if context.portfolio.positions[security].amount>0:
                        if (current_quantile==1) and (security not in context.longs):
                            context.existing_longs += 1
                        elif (current_quantile>1) and (security not in context.shorts):
                            order_target_percent(security, 0)
                    elif context.portfolio.positions[security].amount<0:
                        if (current_quantile==context.nq) and (security not in context.shorts):
                            context.existing_shorts += 1
                        elif (current_quantile<context.nq) and (security not in context.longs):
                            order_target_percent(security, 0)
def close_(context, data):
    if context.switch == True:
        results = context.shorts
        log.info(results)
        securities_in_results = results.index
        passed = []
        for sec in context.shorts:
                                     #Stock, Price, lookback 5, Days
            recent_prices =  data.history(sec, 'price', context.history_range, '1d').values
                                     #Stock, Volume, Lookback 5, Days
            recent_volumes = data.history(sec, 'volume', context.history_range, '1d').values
            price_changes = np.diff(recent_prices).tolist()
            volume_changes = np.diff(recent_volumes).tolist()
        
#X = Input, Y= Output 
            X,Y = [],[]
    
            for i in range(context.history_range-context.lookback-1): 
                X.append(price_changes[i:i+context.lookback] + volume_changes[i:i+context.lookback])
                Y.append(price_changes[i+context.lookback])
            
            context.model.fit(X, Y) 
        
#Check Model Accuracy:
            recent_prices = data.history(sec, 'price', context.lookback+1, '1d').values
            recent_volumes = data.history(sec, 'volume', context.lookback+1, '1d').values
            price_changes =  np.diff(recent_prices).tolist()
            volume_changes = np.diff(recent_volumes).tolist()
        
#Make Prediction 
            prediction = context.model.predict(price_changes + volume_changes)
    #PREDICTION OF SHORT LIST STOCKS!!!
            if -2.0 < prediction < -0.05: 
                passed.append(sec)
            
        for sec in context.shorts:
            if data.can_trade(sec) and sec in passed and sec not in context.trade:
                order_target_percent(sec, -.95/(len(context.shorts)+context.existing_shorts))
                context.trade.append(sec)
            elif data.can_trade(sec) and sec not in passed and sec not in context.trade:
                order_target_percent(sec, 0.0/(len(context.shorts)+context.existing_shorts))
                context.trade.append(sec)
            else:
                print("WTF")
                        
def open_(context, data):  
    if context.switch == True:
        results = context.longs
        log.info(results)
        securities_in_results = results.index
        passed = []
        for sec in context.longs:
                                     #Stock, Price, lookback 5, Days
            recent_prices =  data.history(sec, 'price', context.history_range, '1d').values
                                     #Stock, Volume, Lookback 5, Days
            recent_volumes = data.history(sec, 'volume', context.history_range, '1d').values
            price_changes = np.diff(recent_prices).tolist()
            volume_changes = np.diff(recent_volumes).tolist()
        
#X = Input, Y= Output 
            X,Y = [],[]
    
            for i in range(context.history_range-context.lookback-1): 
                X.append(price_changes[i:i+context.lookback] + volume_changes[i:i+context.lookback])
                Y.append(price_changes[i+context.lookback])
            
            context.model.fit(X, Y) 
        
#Check Model Accuracy:
            recent_prices = data.history(sec, 'price', context.lookback+1, '1d').values
            recent_volumes = data.history(sec, 'volume', context.lookback+1, '1d').values
            price_changes =  np.diff(recent_prices).tolist()
            volume_changes = np.diff(recent_volumes).tolist()
        
            prediction = context.model.predict(price_changes + volume_changes)
        
            print (sec.symbol, prediction)
        #PREDICTION OF LONG LIST STOCKS!!!
            if 2.0 > prediction > 0.05: 
                passed.append(sec)
            
        for sec in context.longs:
            if data.can_trade(sec) and sec in passed and sec not in context.trade:
                order_target_percent(sec, 1.0/(len(context.longs)+context.existing_longs) )
                context.trade.append(sec)
# If you dont care how high your beta is. uncomment this line and play around. you will see higher returns but with much higher drawdowns
       # elif data.can_trade(sec): 
         #   order_target_percent(sec, 1.0/(len(context.longs)+context.existing_longs) )

def check_switch(context, data):
    if context.switch == False:
        print("Global Switch Is Off! Closing All Positions!")
        for security in context.portfolio.positions:
            order_target_percent(security, 0.0)
            
def my_record_vars(context, data):
    """
    Record variables at the end of each day.
    """
    longs = shorts = 0
    for position in context.portfolio.positions.itervalues():
        if position.amount > 0:
            longs += 1
        elif position.amount < 0:
            shorts += 1
    # Record our variables.
    record(leverage=context.account.leverage, long_count=longs, short_count=shorts)
    try:
        log.info("Today's shorts: "  +", ".join([short_.symbol for short_ in context.shorts]))
        log.info("Today's longs: "  +", ".join([long_.symbol for long_ in context.longs]))
    except:
        pass
There was a runtime error.

Looks cool! im going to clone it to see how it works :P

Edit: Would be nice if you list the changelog between each share.