Research Pipeline & Deployment: Strategy Lifecycle Guide › Continuous Evaluation and Strategy Lifecycle
Quantitative FinanceAll chapters Research Pipeline & Deployment: Strategy Lifecycle Guide › Continuous Evaluation and Strategy LifecycleOne section · Sections in this chapter6 / 9 ↓
Reading tools Definitions & inline links
Choose your expertise level to adjust how many terms are explained. Beginners see more tooltips, experts see fewer to maintain reading flow. Hover over underlined terms for instant definitions.
Article links
Make inline references clickable
Deploying a strategy is not the end of the journey. Markets evolve, and edges decay. Continuous evaluation determines whether a strategy should continue running, needs recalibration, or should be retired.
Detecting Performance Degradation
Statistical tests help distinguish between normal variance and performance decay. The challenge is detecting problems early without overreacting to noise.
The fundamental difficulty in performance evaluation is distinguishing signal from noise. A strategy with a 1.0 Sharpe ratio will have many months where it appears to have a much lower Sharpe. This is not degradation; it is the nature of random returns. True degradation occurs when the underlying edge has diminished, but we observe only the noisy realized returns. Statistical tests help by quantifying how unlikely the observed returns would be if the strategy were performing at its historical level.
!uv pip install scipy
import pandas as pd
import numpy as np
from scipy import stats
from typing import List, Dict
class PerformanceEvaluator:
"""
Evaluate ongoing strategy performance and detect degradation.
"""
def __init__(self, expected_sharpe: float, expected_return: float):
self.expected_sharpe = expected_sharpe
self.expected_return = expected_return
self.returns_history: List[float] = []
self.evaluation_history: List[Dict] = []
def add_returns(self, returns: List[float]):
"""Add new return observations."""
self.returns_history.extend(returns)
def calculate_rolling_metrics(self, window: int = 63) -> Dict:
"""Calculate rolling performance metrics."""
if len(self.returns_history) < window:
return None
recent_returns = self.returns_history[-window:]
mean_return = np.mean(recent_returns)
std_return = np.std(recent_returns)
# Annualize
annual_return = mean_return * 252
annual_vol = std_return * np.sqrt(252)
sharpe = annual_return / annual_vol if annual_vol > 0 else 0
return {
'window_days': window,
'mean_daily_return': mean_return,
'daily_volatility': std_return,
'annualized_return': annual_return,
'annualized_volatility': annual_vol,
'sharpe_ratio': sharpe
}
def test_performance_degradation(self, confidence: float = 0.95) -> Dict:
"""
Test if recent performance is significantly worse than expected.
Uses a t-test to compare recent returns against expected return.
"""
if len(self.returns_history) < 30:
return {'test_result': 'insufficient_data', 'p_value': None}
recent_returns = self.returns_history[-63:] # ~3 months
expected_daily = self.expected_return / 252
# One-sample t-test: is mean return significantly less than expected?
t_stat, p_value = stats.ttest_1samp(recent_returns, expected_daily)
# One-tailed test (only care if performance is worse)
p_value_one_tail = p_value / 2 if t_stat < 0 else 1 - p_value / 2
degraded = p_value_one_tail < (1 - confidence) and t_stat < 0
return {
'test_result': 'degraded' if degraded else 'acceptable',
't_statistic': t_stat,
'p_value': p_value_one_tail,
'confidence_level': confidence,
'recent_return_annualized': np.mean(recent_returns) * 252,
'expected_return_annualized': self.expected_return
}
def calculate_regime_stability(self) -> Dict:
"""
Check if strategy behavior has changed across time periods.
Compares first half vs second half of recent history.
"""
if len(self.returns_history) < 126:
return {'test_result': 'insufficient_data'}
recent = self.returns_history[-126:]
first_half = recent[:63]
second_half = recent[63:]
# Two-sample t-test for means
t_stat_mean, p_value_mean = stats.ttest_ind(first_half, second_half)
# Levene's test for variance equality
levene_stat, p_value_var = stats.levene(first_half, second_half)
regime_shift = p_value_mean < 0.05 or p_value_var < 0.05
return {
'test_result': 'regime_shift_detected' if regime_shift else 'stable',
'mean_shift_p_value': p_value_mean,
'variance_shift_p_value': p_value_var,
'first_half_sharpe': np.mean(first_half) / np.std(first_half) * np.sqrt(252),
'second_half_sharpe': np.mean(second_half) / np.std(second_half) * np.sqrt(252)
}
def generate_evaluation_report(self) -> pd.DataFrame:
"""Generate comprehensive evaluation report."""
rolling = self.calculate_rolling_metrics()
degradation = self.test_performance_degradation()
stability = self.calculate_regime_stability()
report_data = [
('Rolling Sharpe (63d)', f"{rolling['sharpe_ratio']:.2f}" if rolling else 'N/A'),
('Expected Sharpe', f"{self.expected_sharpe:.2f}"),
('Degradation Test', degradation['test_result']),
('Degradation p-value', f"{degradation.get('p_value', 'N/A'):.3f}" if degradation.get('p_value') else 'N/A'),
('Regime Stability', stability['test_result']),
]
return pd.DataFrame(report_data, columns=['Metric', 'Value'])
# Simulate strategy returns with gradual degradation
np.random.seed(321)
evaluator = PerformanceEvaluator(expected_sharpe=1.2, expected_return=0.08)
def calibrated_returns(mean_return, volatility, observations):
"""Create a reproducible sample with the requested realized moments."""
innovations = np.random.normal(size=observations)
standardized = (innovations - innovations.mean()) / innovations.std(ddof=1)
return mean_return + volatility * standardized
# First period: roughly 7.5% annual return at an expected Sharpe near 1.2
good_returns = calibrated_returns(0.0003, 0.004, 100)
# Second period: negative drift and higher volatility after the edge decays
degraded_returns = calibrated_returns(-0.0015, 0.006, 80)
all_returns = list(good_returns) + list(degraded_returns)
evaluator.add_returns(all_returns)rolling = evaluator.calculate_rolling_metrics()
degradation = evaluator.test_performance_degradation()
stability = evaluator.calculate_regime_stability()Strategy Performance Evaluation ============================================================ Rolling Metrics (63-day window): window_days: 63 mean_daily_return: -0.0012 daily_volatility: 0.0057 annualized_return: -0.3112 annualized_volatility: 0.0899 sharpe_ratio: -3.4607 Degradation Test: test_result: degraded t_statistic: -2.1578 p_value: 0.0174 confidence_level: 0.9500 recent_return_annualized: -0.3112 expected_return_annualized: 0.0800 Regime Stability Test: test_result: stable mean_shift_p_value: 0.2141 variance_shift_p_value: 0.5010 first_half_sharpe: -0.0344 second_half_sharpe: -3.4607
The performance evaluation flags statistically significant recent underperformance, but the separate stability test does not detect a significant mean or variance shift at the chosen confidence level. That distinction matters: the strategy has degraded enough to warrant investigation, yet this sample does not establish a structural regime change. In production, the result would justify tighter monitoring and additional evidence before recalibrating or retiring the strategy.
Evaluation Parameters
The performance evaluation system depends on parameters that control the sensitivity and reliability of degradation detection.
The window parameter sets the lookback period, such as 63 days, for calculating rolling metrics. This window determines the granularity of performance assessment. Shorter windows are more sensitive to recent changes but noisier, potentially generating false alarms from normal variance. Longer windows provide more stable estimates but may delay detection of persistent deterioration. A 63-day window, representing approximately one quarter, balances these considerations for most strategies. Strategies with higher expected Sharpe ratios can use shorter windows because their signal-to-noise ratio is better.
The confidence parameter establishes the statistical confidence level, such as 0.95, for the degradation test. This controls the false positive rate: a 95% confidence level means that we expect to incorrectly flag acceptable performance as degraded about 5% of the time. Higher confidence levels reduce false alarms but may delay detection of real issues. The appropriate level depends on the cost of each type of error. If stopping a profitable strategy is very costly, use higher confidence. If continuing a broken strategy is very costly, use lower confidence.
The expected_return parameter gives the baseline annualized return expected from the strategy. This is the null hypothesis for degradation testing: we ask whether observed returns are materially below this level. Setting this parameter requires judgment about what the strategy should achieve. Too optimistic an expectation leads to frequent false alarms; too pessimistic an expectation lets performance degradation go undetected. The best approach is to use the return observed during out-of-sample testing before production, which gives a realistic benchmark free from in-sample optimization bias.
Strategy Lifecycle Management
Strategies have a natural lifecycle. They are born from research, mature through paper trading, live in production, and eventually retire as their edge decays. Managing this lifecycle systematically prevents the common mistake of running strategies long past their useful life.
The lifecycle framework below formalizes this process. Each strategy exists in exactly one state at any time, and transitions between states follow defined rules based on performance metrics. This structure ensures that decision-making about strategies is consistent and documented, avoiding the ad-hoc judgments that often keep underperforming strategies running too long or retire profitable strategies prematurely.
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from typing import Dict, List, Tuple
class StrategyLifecycleState(Enum):
RESEARCH = "research"
PAPER_TRADING = "paper_trading"
PRODUCTION_RAMP = "production_ramp"
PRODUCTION_FULL = "production_full"
UNDER_REVIEW = "under_review"
DEPRECATED = "deprecated"
RETIRED = "retired"
@dataclass
class StrategyLifecycle:
"""Track and manage strategy lifecycle."""
strategy_id: str
state: StrategyLifecycleState
created_date: datetime
state_history: List[Dict] = field(default_factory=list)
# Performance thresholds
production_sharpe_threshold: float = 0.5
deprecation_sharpe_threshold: float = 0.3
review_trigger_drawdown: float = -0.15
# Current metrics
current_sharpe: float = None
current_drawdown: float = None
days_in_current_state: int = 0
def transition_to(self, new_state: StrategyLifecycleState, reason: str):
"""Transition strategy to a new lifecycle state."""
self.state_history.append(
{
"from_state": self.state.value,
"to_state": new_state.value,
"timestamp": datetime.now().isoformat(),
"reason": reason,
}
)
self.state = new_state
self.days_in_current_state = 0
def evaluate_state(self) -> Tuple[bool, str]:
"""
Evaluate if strategy should transition to a different state.
Returns (should_transition, recommended_action).
"""
if self.state == StrategyLifecycleState.PRODUCTION_FULL:
# Check for deprecation triggers
if (
self.current_sharpe is not None
and self.current_sharpe < self.deprecation_sharpe_threshold
):
return True, "Move to UNDER_REVIEW: Sharpe below threshold"
if (
self.current_drawdown is not None
and self.current_drawdown < self.review_trigger_drawdown
):
return True, "Move to UNDER_REVIEW: Drawdown exceeds limit"
elif self.state == StrategyLifecycleState.UNDER_REVIEW:
# Strategy has been under review - decide next step
if self.days_in_current_state > 30:
if (
self.current_sharpe is not None
and self.current_sharpe > self.production_sharpe_threshold
):
return (
True,
"Return to PRODUCTION_FULL: Performance recovered",
)
else:
return (
True,
"Move to DEPRECATED: Performance did not recover",
)
elif self.state == StrategyLifecycleState.DEPRECATED:
# Deprecated strategies should be retired after wind-down
if self.days_in_current_state > 14:
return True, "Move to RETIRED: Wind-down period complete"
return False, "No transition recommended"
def get_lifecycle_summary(self) -> Dict:
"""Get summary of strategy lifecycle."""
return {
"strategy_id": self.strategy_id,
"current_state": self.state.value,
"days_in_state": self.days_in_current_state,
"created_date": self.created_date.isoformat(),
"total_transitions": len(self.state_history),
"current_sharpe": self.current_sharpe,
"current_drawdown": self.current_drawdown,
}
# Demonstrate lifecycle management
lifecycle = StrategyLifecycle(
strategy_id="MOM_001",
state=StrategyLifecycleState.PRODUCTION_FULL,
created_date=datetime(2023, 1, 15),
)
# Simulate strategy performance degradation
lifecycle.current_sharpe = 0.25 # Below threshold
lifecycle.current_drawdown = -0.12
lifecycle.days_in_current_state = 90summary = lifecycle.get_lifecycle_summary()
should_transition, recommendation = lifecycle.evaluate_state()
# Execute the transition
if should_transition:
lifecycle.transition_to(StrategyLifecycleState.UNDER_REVIEW, recommendation)
new_state = lifecycle.state.value
history_len = len(lifecycle.state_history)Strategy Lifecycle Management ============================================================ Current Lifecycle State: strategy_id: MOM_001 current_state: production_full days_in_state: 90 created_date: 2023-01-15T00:00:00 total_transitions: 0 current_sharpe: 0.25 current_drawdown: -0.12 Lifecycle Evaluation: Should Transition: True Recommendation: Move to UNDER_REVIEW: Sharpe below threshold Transitioned to: under_review Transition History: 1 transitions recorded
The lifecycle management system enforces a disciplined process for handling strategy evolution. By automatically transitioning the strategy to "Under Review" based on performance triggers, it ensures that deteriorating strategies are paused and assessed rather than being allowed to decay unnoticed in production. The state history provides an audit trail that documents why each transition occurred, which is valuable for learning from both successes and failures.