212 lines
15 KiB
JSON
212 lines
15 KiB
JSON
{
|
||
"criteria": [
|
||
{
|
||
"name": "Scenario Plausibility",
|
||
"description": "Scenarios are possible given current knowledge, not fantasy. Counterfactuals were realistic alternatives at decision time.",
|
||
"scale": {
|
||
"1": "Implausible scenarios (magic, impossible foreknowledge). Counterfactuals couldn't have been chosen at the time.",
|
||
"3": "Mostly plausible but some unrealistic assumptions. Counterfactuals stretch believability.",
|
||
"5": "All scenarios plausible given what was/is known. Counterfactuals were genuine alternatives available at decision time."
|
||
}
|
||
},
|
||
{
|
||
"name": "Minimal Rewrite Principle (Counterfactuals)",
|
||
"description": "Counterfactuals change as little as possible to isolate causal factor. Not multiple changes bundled together.",
|
||
"scale": {
|
||
"1": "Many factors changed simultaneously. Can't tell which caused different outcome. 'What if X AND Y AND Z...'",
|
||
"3": "Some attempt at isolation but still multiple changes. Primary factor identified but confounded.",
|
||
"5": "Perfect isolation: single factor changed, all else held constant. Causal factor clearly identified."
|
||
}
|
||
},
|
||
{
|
||
"name": "Causal Mechanism Specification",
|
||
"description": "Explains HOW change leads to different outcome. Not just stating result but tracing causal chain.",
|
||
"scale": {
|
||
"1": "No mechanism specified. Just outcome stated ('sales would be higher') without explanation.",
|
||
"3": "Partial mechanism. Some causal steps identified but incomplete chain.",
|
||
"5": "Complete causal chain: initial change → immediate effect → secondary effects → final outcome. Each step explained."
|
||
}
|
||
},
|
||
{
|
||
"name": "Probability Calibration",
|
||
"description": "Scenarios assigned probabilities based on evidence, base rates, analogies. Not all weighted equally.",
|
||
"scale": {
|
||
"1": "No probabilities assigned, or all scenarios treated as equally likely. No base rate consideration.",
|
||
"3": "Rough probabilities assigned but weak justification. Some consideration of likelihood.",
|
||
"5": "Well-calibrated probabilities using base rates, analogies, expert judgment. Sum to 100%. Clear reasoning for each."
|
||
}
|
||
},
|
||
{
|
||
"name": "Pre-Mortem Rigor",
|
||
"description": "For pre-mortems: follows 6-step process, generates novel failure modes specific to context, not just generic risks.",
|
||
"scale": {
|
||
"1": "Generic risk list copied from elsewhere. Hindsight bias ('obvious' failures). No structured process.",
|
||
"3": "Some specific risks but mixed with generic ones. Process partially followed.",
|
||
"5": "Rigorous 6-step process: silent brainstorm, round-robin, voting, mitigations. Context-specific failure modes identified."
|
||
}
|
||
},
|
||
{
|
||
"name": "Action Extraction",
|
||
"description": "Clear extraction of common actions, hedges, options, and decision points from scenarios. Not just stories.",
|
||
"scale": {
|
||
"1": "Scenarios developed but no actions extracted. 'Interesting stories' with no operational implications.",
|
||
"3": "Some actions identified but incomplete. Missing hedges or options.",
|
||
"5": "Comprehensive action plan: common actions (all scenarios), hedges (downside), options (upside), decision triggers clearly specified."
|
||
}
|
||
},
|
||
{
|
||
"name": "Leading Indicator Quality",
|
||
"description": "Indicators are observable, early (6+ months advance), and actionable. Clear thresholds defined.",
|
||
"scale": {
|
||
"1": "No indicators, or lagging indicators (show scenario after it's happened). No thresholds.",
|
||
"3": "Some leading indicators but vague thresholds or not truly early signals.",
|
||
"5": "High-quality leading indicators: observable metrics, 6+ months advance notice, clear thresholds, trigger specific actions."
|
||
}
|
||
},
|
||
{
|
||
"name": "Scenario Diversity",
|
||
"description": "Scenarios are qualitatively different, not just magnitude variations. Cover meaningful range of futures.",
|
||
"scale": {
|
||
"1": "Scenarios differ only in magnitude (10% growth vs 15% vs 20%). Basically same story.",
|
||
"3": "Some qualitative differences but scenarios too similar or narrow range.",
|
||
"5": "Meaningfully different scenarios: qualitative distinctions, broad range captured, distinct strategic implications for each."
|
||
}
|
||
},
|
||
{
|
||
"name": "Bias Avoidance (Hindsight/Confirmation)",
|
||
"description": "Avoids hindsight bias in counterfactuals, confirmation bias in scenario selection, anchoring on current trends.",
|
||
"scale": {
|
||
"1": "Strong hindsight bias ('we should have known'). Only scenarios confirming current view. Anchored on status quo.",
|
||
"3": "Some bias awareness but incomplete mitigation. Mostly avoids obvious biases.",
|
||
"5": "Rigorous bias mitigation: re-inhabits decision context, considers disconfirming scenarios, challenges assumptions, uses base rates."
|
||
}
|
||
},
|
||
{
|
||
"name": "Monitoring and Adaptation Plan",
|
||
"description": "Defined monitoring cadence (quarterly), indicator tracking, scenario probability updates, adaptation triggers.",
|
||
"scale": {
|
||
"1": "No monitoring plan. Set-it-and-forget-it scenarios. No updates planned.",
|
||
"3": "Informal plan to review occasionally. No specific cadence or triggers.",
|
||
"5": "Detailed monitoring: quarterly reviews, indicator dashboard, probability updates, clear adaptation triggers and owner."
|
||
}
|
||
}
|
||
],
|
||
"guidance_by_type": {
|
||
"Strategic Planning (1-3 year horizon)": {
|
||
"target_score": 4.2,
|
||
"key_criteria": ["Scenario Diversity", "Probability Calibration", "Action Extraction"],
|
||
"common_pitfalls": ["Too narrow scenario range", "No hedges against downside", "Monitoring plan missing"],
|
||
"specific_guidance": "Use three-scenario framework (optimistic/baseline/pessimistic) or 2×2 matrix. Assign probabilities (optimistic 15-30%, baseline 40-60%, pessimistic 15-30%). Extract common actions that work across all scenarios, plus hedges for downside. Quarterly monitoring."
|
||
},
|
||
"Pre-Mortem (Project Risk Identification)": {
|
||
"target_score": 4.0,
|
||
"key_criteria": ["Pre-Mortem Rigor", "Action Extraction", "Bias Avoidance"],
|
||
"common_pitfalls": ["Generic risks (not context-specific)", "Hindsight bias ('obvious' failures)", "No mitigations assigned"],
|
||
"specific_guidance": "Follow 6-step process rigorously. Silent brainstorm 5-10 min to prevent groupthink. Generate context-specific failure modes. Vote on top 5-7 risks. Assign mitigation owner and deadline for each."
|
||
},
|
||
"Counterfactual Learning (Post-Decision Analysis)": {
|
||
"target_score": 3.8,
|
||
"key_criteria": ["Minimal Rewrite Principle", "Causal Mechanism Specification", "Bias Avoidance"],
|
||
"common_pitfalls": ["Changing multiple factors (can't isolate cause)", "No causal mechanism (just outcome)", "Hindsight bias ('knew it all along')"],
|
||
"specific_guidance": "Change single factor, hold all else constant. Trace complete causal chain (change → immediate effect → secondary effects → outcome). Re-inhabit decision context to avoid hindsight. Use base rates and analogies to estimate counterfactual probability."
|
||
},
|
||
"Stress Testing (Decision Robustness)": {
|
||
"target_score": 4.0,
|
||
"key_criteria": ["Scenario Diversity", "Causal Mechanism Specification", "Action Extraction"],
|
||
"common_pitfalls": ["Only optimistic/pessimistic (no black swan)", "No mechanism for how extremes occur", "Decision not actually tested"],
|
||
"specific_guidance": "Test decision against optimistic, pessimistic, AND black swan scenarios. Specify HOW extreme outcomes occur. Ask 'Does decision still hold?' for each scenario. Extract hedges to protect against downside extremes."
|
||
},
|
||
"Assumption Reversal (Innovation/Pivots)": {
|
||
"target_score": 3.5,
|
||
"key_criteria": ["Scenario Plausibility", "Action Extraction", "Bias Avoidance"],
|
||
"common_pitfalls": ["Reversed assumptions implausible", "Interesting but no experiments", "Confirmation bias (only reverse convenient assumptions)"],
|
||
"specific_guidance": "Reverse core assumptions ('customers want more features' → 'want fewer'). Test plausibility (could reversal be true?). Design small experiments to test reversal. Challenge assumptions that support current strategy, not just peripheral ones."
|
||
}
|
||
},
|
||
"guidance_by_complexity": {
|
||
"Simple (Routine Decisions, Short-Term)": {
|
||
"target_score": 3.5,
|
||
"focus_areas": ["Scenario Plausibility", "Causal Mechanism Specification", "Action Extraction"],
|
||
"acceptable_shortcuts": ["Informal probabilities", "Two scenarios instead of three", "Simple pre-mortem (no voting)"],
|
||
"specific_guidance": "Quick pre-mortem (30 min) or simple counterfactual analysis. Two scenarios (optimistic/pessimistic). Extract 2-3 key actions. Informal monitoring acceptable."
|
||
},
|
||
"Standard (Strategic Decisions, 1-2 year horizon)": {
|
||
"target_score": 4.0,
|
||
"focus_areas": ["Probability Calibration", "Scenario Diversity", "Leading Indicator Quality", "Monitoring and Adaptation"],
|
||
"acceptable_shortcuts": ["Three scenarios (not full 2×2 matrix)", "Quarterly vs monthly monitoring"],
|
||
"specific_guidance": "Three-scenario framework with probabilities. Extract common actions, hedges, options. Define 5-7 leading indicators. Quarterly scenario reviews and updates. Assign owners for monitoring."
|
||
},
|
||
"Complex (High Stakes, Multi-Year, High Uncertainty)": {
|
||
"target_score": 4.5,
|
||
"focus_areas": ["All criteria", "Rigorous validation", "Comprehensive monitoring"],
|
||
"acceptable_shortcuts": ["None - full rigor required"],
|
||
"specific_guidance": "Full 2×2 scenario matrix or cone of uncertainty. Rigorous probability calibration using base rates and expert judgment. Comprehensive pre-mortem with cross-functional team. Leading indicators with clear thresholds and decision triggers. Monthly monitoring, quarterly deep reviews. All mitigations assigned with owners and deadlines."
|
||
}
|
||
},
|
||
"common_failure_modes": [
|
||
{
|
||
"name": "Implausible Counterfactuals (Fantasy)",
|
||
"symptom": "Counterfactuals require magic, impossible foreknowledge, or weren't real options at decision time. 'What if we had known pandemic was coming?'",
|
||
"detection": "Ask: 'Could a reasonable decision-maker have chosen this alternative given information available then?' If no, implausible.",
|
||
"fix": "Restrict to alternatives actually available at decision time. Use 'what was on the table?' test. Avoid hindsight-dependent counterfactuals."
|
||
},
|
||
{
|
||
"name": "Multiple Changes (Can't Isolate Cause)",
|
||
"symptom": "Counterfactual changes many factors: 'What if we had raised $3M AND launched EU AND hired different CEO...' Can't tell which mattered.",
|
||
"detection": "Count changes. If >1 factor changed, causal isolation violated.",
|
||
"fix": "Minimal rewrite: change ONE factor, hold all else constant. Want to test funding? Change funding only. Want to test geography? Change geography only."
|
||
},
|
||
{
|
||
"name": "No Causal Mechanism",
|
||
"symptom": "Outcome stated without explanation. 'Sales would be 2× higher' but no WHY or HOW.",
|
||
"detection": "Ask 'How does change lead to outcome?' If answer vague or missing, no mechanism.",
|
||
"fix": "Trace causal chain: initial change → immediate effect → secondary effects → final outcome. Each step must be explained with logic or evidence."
|
||
},
|
||
{
|
||
"name": "Scenarios Too Similar",
|
||
"symptom": "Three scenarios differ only in magnitude (10% growth vs 15% vs 20%). Same story, different numbers.",
|
||
"detection": "Read scenarios. Do they describe qualitatively different worlds? If no, too similar.",
|
||
"fix": "Make scenarios qualitatively distinct. Different drivers, different strategic implications. Use 2×2 matrix to force diversity via two independent uncertainties."
|
||
},
|
||
{
|
||
"name": "No Probabilities Assigned",
|
||
"symptom": "All scenarios treated as equally likely, or no probabilities given. Implies 33% each for three scenarios regardless of plausibility.",
|
||
"detection": "Check if probabilities assigned and justified. If missing or all equal, red flag.",
|
||
"fix": "Assign probabilities using base rates, analogies, expert judgment. Baseline typically 40-60%, optimistic/pessimistic 15-30% each. Justify each estimate."
|
||
},
|
||
{
|
||
"name": "Hindsight Bias in Counterfactuals",
|
||
"symptom": "'Obviously we should have done X' - outcome seems inevitable in retrospect. Overconfidence counterfactual would have succeeded.",
|
||
"detection": "Ask: 'Was outcome predictable given information at decision time?' If reasoning depends on information learned after, hindsight bias.",
|
||
"fix": "Re-inhabit decision context: what was known/unknown then? What uncertainty existed? Acknowledge alternative could have failed too. Use base rates to calibrate confidence."
|
||
},
|
||
{
|
||
"name": "Generic Pre-Mortem Risks",
|
||
"symptom": "Pre-mortem lists generic risks ('ran out of money', 'competition', 'tech didn't work') not specific to this project.",
|
||
"detection": "Could these risks apply to any project? If yes, too generic.",
|
||
"fix": "Push for context-specific failure modes. What's unique about THIS project? What specific technical challenges? Which specific competitors? What particular market risks?"
|
||
},
|
||
{
|
||
"name": "Scenarios Without Actions",
|
||
"symptom": "Interesting stories developed but no operational implications. 'So what should we do?' question unanswered.",
|
||
"detection": "Read scenario analysis. Is there action plan with common actions, hedges, options? If no, incomplete.",
|
||
"fix": "Always end with action extraction: (1) Common actions (all scenarios), (2) Hedges (downside protection), (3) Options (upside preparation), (4) Leading indicators (monitoring)."
|
||
},
|
||
{
|
||
"name": "Lagging Indicators (Not Leading)",
|
||
"symptom": "Indicators show scenario after it's happened. 'Revenue collapse' indicates pessimistic scenario, but too late to act.",
|
||
"detection": "Ask: 'Does this indicator give 6+ months advance notice?' If no, it's lagging.",
|
||
"fix": "Find early signals: regulatory votes (before law passed), competitor funding rounds (before product launched), adoption rate trends (before market share shift). Leading indicators are predictive, not reactive."
|
||
},
|
||
{
|
||
"name": "No Monitoring Plan",
|
||
"symptom": "Scenarios developed, actions defined, then filed away. No one tracking which scenario unfolding or updating probabilities.",
|
||
"detection": "Ask: 'Who monitors? How often? What triggers update?' If no answers, no plan.",
|
||
"fix": "Define: (1) Owner responsible for monitoring, (2) Cadence (monthly/quarterly reviews), (3) Indicator dashboard, (4) Decision triggers ('If X crosses threshold Y, then action Z'), (5) Scenario probability update process."
|
||
}
|
||
],
|
||
"minimum_standard": 3.5,
|
||
"target_score": 4.0,
|
||
"excellence_threshold": 4.5
|
||
}
|