class AgentAnalyzer:
@staticmethod
def plot_response_distribution(outcome: Dict):
fig, axes = plt.subplots(2, 2, figsize=(14, 10))
fig.suptitle('Agent Response Evaluation', fontsize=16, fontweight="daring")
responses = outcome['all_responses']
scores = outcome['critic_scores']
uncertainty = outcome['uncertainty']
selected_idx = outcome['selected_index']
ax = axes[0, 0]
score_values = [s.overall_score for s in scores]
bars = ax.bar(vary(len(scores)), score_values, alpha=0.7)
bars[selected_idx].set_color('inexperienced')
bars[selected_idx].set_alpha(1.0)
ax.axhline(np.imply(score_values), shade="crimson", linestyle="--", label=f'Imply: {np.imply(score_values):.3f}')
ax.set_xlabel('Response Index')
ax.set_ylabel('Critic Rating')
ax.set_title('Critic Scores for Every Response')
ax.legend()
ax.grid(True, alpha=0.3)
ax = axes[0, 1]
confidences = [r.confidence for r in responses]
bars = ax.bar(vary(len(responses)), confidences, alpha=0.7, shade="orange")
bars[selected_idx].set_color('inexperienced')
bars[selected_idx].set_alpha(1.0)
ax.axhline(np.imply(confidences), shade="crimson", linestyle="--", label=f'Imply: {np.imply(confidences):.3f}')
ax.set_xlabel('Response Index')
ax.set_ylabel('Confidence')
ax.set_title('Mannequin Confidence per Response')
ax.legend()
ax.grid(True, alpha=0.3)
ax = axes[1, 0]
elements = {
'Accuracy': [s.accuracy_score for s in scores],
'Coherence': [s.coherence_score for s in scores],
'Security': [s.safety_score for s in scores]
}
x = np.arange(len(responses))
width = 0.25
for i, (identify, values) in enumerate(elements.gadgets()):
offset = (i - 1) * width
ax.bar(x + offset, values, width, label=identify, alpha=0.8)
ax.set_xlabel('Response Index')
ax.set_ylabel('Rating')
ax.set_title('Critic Rating Elements')
ax.set_xticks(x)
ax.legend()
ax.grid(True, alpha=0.3, axis="y")
ax = axes[1, 1]
uncertainty_metrics = {
'Entropy': uncertainty.entropy,
'Variance': uncertainty.variance,
'Consistency': uncertainty.consistency_score,
'Epistemic': uncertainty.epistemic_uncertainty,
'Aleatoric': uncertainty.aleatoric_uncertainty
}
bars = ax.barh(listing(uncertainty_metrics.keys()), listing(uncertainty_metrics.values()), alpha=0.7)
ax.set_xlabel('Worth')
ax.set_title(f'Uncertainty Estimates (Danger: {uncertainty.risk_level()})')
ax.grid(True, alpha=0.3, axis="x")
plt.tight_layout()
plt.present()
@staticmethod
def plot_strategy_comparison(agent: CriticAugmentedAgent, immediate: str, ground_truth: Non-compulsory[str] = None):
methods = ["best_score", "most_confident", "most_consistent", "risk_adjusted"]
outcomes = {}
print("Evaluating choice methods...n")
for technique in methods:
print(f"Testing technique: {technique}")
outcome = agent.generate_with_critic(immediate, ground_truth, technique=technique, verbose=False)
outcomes[strategy] = outcome
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
fig.suptitle('Technique Comparability', fontsize=16, fontweight="daring")
ax = axes[0]
selected_scores = [
results[s]['critic_scores'][results[s]['selected_index']].overall_score
for s in methods
]
bars = ax.bar(methods, selected_scores, alpha=0.7, shade="steelblue")
ax.set_ylabel('Critic Rating')
ax.set_title('Chosen Response High quality by Technique')
ax.set_xticklabels(methods, rotation=45, ha="proper")
ax.grid(True, alpha=0.3, axis="y")
ax = axes[1]
for technique in methods:
outcome = outcomes[strategy]
selected_idx = outcome['selected_index']
confidence = outcome['all_responses'][selected_idx].confidence
rating = outcome['critic_scores'][selected_idx].overall_score
ax.scatter(confidence, rating, s=200, alpha=0.6, label=technique)
ax.set_xlabel('Confidence')
ax.set_ylabel('Critic Rating')
ax.set_title('Confidence vs High quality Commerce-off')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.present()
return outcomes
def run_basic_demo():
print("n" + "=" * 80)
print("DEMO 1: Primary Agent with Critic")
print("=" * 80 + "n")
agent = CriticAugmentedAgent(
model_quality=0.8,
risk_tolerance=0.3,
n_samples=5
)
immediate = "What's 15 + 27?"
ground_truth = "42"
outcome = agent.generate_with_critic(
immediate=immediate,
ground_truth=ground_truth,
technique="risk_adjusted",
temperature=0.8
)
print("nš Producing visualizations...")
AgentAnalyzer.plot_response_distribution(outcome)
return outcome
def run_strategy_comparison():
print("n" + "=" * 80)
print("DEMO 2: Technique Comparability")
print("=" * 80 + "n")
agent = CriticAugmentedAgent(
model_quality=0.75,
risk_tolerance=0.5,
n_samples=6
)
immediate = "What's 23 + 19?"
ground_truth = "42"
outcomes = AgentAnalyzer.plot_strategy_comparison(agent, immediate, ground_truth)
return outcomes
def run_uncertainty_analysis():
print("n" + "=" * 80)
print("DEMO 3: Uncertainty Evaluation")
print("=" * 80 + "n")
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
qualities = [0.5, 0.6, 0.7, 0.8, 0.9]
uncertainties = []
consistencies = []
immediate = "What's 30 + 12?"
print("Testing mannequin high quality affect on uncertainty...n")
for high quality in qualities:
agent = CriticAugmentedAgent(model_quality=high quality, n_samples=8)
outcome = agent.generate_with_critic(immediate, verbose=False)
uncertainties.append(outcome['uncertainty'].entropy)
consistencies.append(outcome['uncertainty'].consistency_score)
print(f"High quality: {high quality:.1f} -> Entropy: {outcome['uncertainty'].entropy:.3f}, "
f"Consistency: {outcome['uncertainty'].consistency_score:.3f}")
ax = axes[0]
ax.plot(qualities, uncertainties, 'o-', linewidth=2, markersize=8, label="Entropy")
ax.set_xlabel('Mannequin High quality')
ax.set_ylabel('Entropy')
ax.set_title('Uncertainty vs Mannequin High quality')
ax.grid(True, alpha=0.3)
ax.legend()
ax = axes[1]
ax.plot(qualities, consistencies, 's-', linewidth=2, markersize=8, shade="inexperienced", label="Consistency")
ax.set_xlabel('Mannequin High quality')
ax.set_ylabel('Consistency Rating')
ax.set_title('Self-Consistency vs Mannequin High quality')
ax.grid(True, alpha=0.3)
ax.legend()
plt.tight_layout()
plt.present()
def run_risk_sensitivity_demo():
print("n" + "=" * 80)
print("DEMO 4: Danger Sensitivity Evaluation")
print("=" * 80 + "n")
immediate = "What is eighteen + 24?"
risk_tolerances = [0.1, 0.3, 0.5, 0.7, 0.9]
outcomes = {
'risk_tolerance': [],
'selected_confidence': [],
'selected_score': [],
'uncertainty': []
}
print("Testing completely different threat tolerance ranges...n")
for risk_tol in risk_tolerances:
agent = CriticAugmentedAgent(
model_quality=0.75,
risk_tolerance=risk_tol,
n_samples=6
)
outcome = agent.generate_with_critic(immediate, verbose=False)
selected_idx = outcome['selected_index']
outcomes['risk_tolerance'].append(risk_tol)
outcomes['selected_confidence'].append(
outcome['all_responses'][selected_idx].confidence
)
outcomes['selected_score'].append(
outcome['critic_scores'][selected_idx].overall_score
)
outcomes['uncertainty'].append(outcome['uncertainty'].entropy)
print(f"Danger Tolerance: {risk_tol:.1f} -> "
f"Confidence: {outcomes['selected_confidence'][-1]:.3f}, "
f"Rating: {outcomes['selected_score'][-1]:.3f}")
fig, ax = plt.subplots(1, 1, figsize=(10, 6))
ax.plot(outcomes['risk_tolerance'], outcomes['selected_confidence'], 'o-', linewidth=2, markersize=8, label="Chosen Confidence")
ax.plot(outcomes['risk_tolerance'], outcomes['selected_score'], 's-', linewidth=2, markersize=8, label="Chosen Rating")
ax.set_xlabel('Danger Tolerance')
ax.set_ylabel('Worth')
ax.set_title('Danger Tolerance Affect on Choice')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.present()
def demonstrate_verbalized_uncertainty():
print("n" + "=" * 80)
print("RESEARCH TOPIC: Verbalized Uncertainty")
print("=" * 80 + "n")
print("Idea: Agent not solely estimates uncertainty however explains it.n")
agent = CriticAugmentedAgent(model_quality=0.7, n_samples=5)
immediate = "What's 25 + 17?"
outcome = agent.generate_with_critic(immediate, verbose=False)
uncertainty = outcome['uncertainty']
rationalization = f"""
Uncertainty Evaluation Report:
---------------------------
Danger Degree: {uncertainty.risk_level()}
Detailed Breakdown:
ā¢ Reply Entropy: {uncertainty.entropy:.3f}
ā {'Low' if uncertainty.entropy < 0.5 else 'Medium' if uncertainty.entropy < 1.0 else 'Excessive'} disagreement amongst generated responses
ā¢ Self-Consistency: {uncertainty.consistency_score:.3f}
ā {int(uncertainty.consistency_score * 100)}% of responses agree on the reply
ā¢ Epistemic Uncertainty: {uncertainty.epistemic_uncertainty:.3f}
ā {'Low' if uncertainty.epistemic_uncertainty < 0.3 else 'Medium' if uncertainty.epistemic_uncertainty < 0.6 else 'Excessive'} mannequin uncertainty (information gaps)
ā¢ Aleatoric Uncertainty: {uncertainty.aleatoric_uncertainty:.3f}
ā {'Low' if uncertainty.aleatoric_uncertainty < 0.3 else 'Medium' if uncertainty.aleatoric_uncertainty < 0.6 else 'Excessive'} knowledge uncertainty (inherent randomness)
Suggestion:
"""
if uncertainty.risk_level() == "LOW":
rationalization += "ā Excessive confidence in reply - protected to belief"
elif uncertainty.risk_level() == "MEDIUM":
rationalization += "ā Reasonable confidence - contemplate verification"
else:
rationalization += "ā Low confidence - strongly suggest verification"
print(rationalization)
def demonstrate_self_consistency():
print("n" + "=" * 80)
print("RESEARCH TOPIC: Self-Consistency Reasoning")
print("=" * 80 + "n")
print("Idea: Generate a number of reasoning paths, choose commonest reply.n")
agent = CriticAugmentedAgent(model_quality=0.75, n_samples=7)
immediate = "What's 35 + 7?"
outcome = agent.generate_with_critic(immediate, technique="most_consistent", verbose=False)
estimator = UncertaintyEstimator()
solutions = [estimator._extract_answer(r.content) for r in result['all_responses']]
print("Generated Responses and Solutions:")
print("-" * 80)
for i, (response, reply) in enumerate(zip(outcome['all_responses'], solutions)):
marker = "ā SELECTED" if i == outcome['selected_index'] else ""
print(f"nResponse {i}: {reply} {marker}")
print(f" Confidence: {response.confidence:.3f}")
print(f" Content material: {response.content material[:80]}...")
from collections import Counter
answer_dist = Counter(solutions)
print(f"nnAnswer Distribution:")
print("-" * 80)
for reply, rely in answer_dist.most_common():
proportion = (rely / len(solutions)) * 100
bar = "ā" * int(proportion / 5)
print(f"{reply:>10}: {bar} {rely}/{len(solutions)} ({proportion:.1f}%)")
print(f"nMost Constant Reply: {answer_dist.most_common(1)[0][0]}")
print(f"Consistency Rating: {outcome['uncertainty'].consistency_score:.3f}")
def fundamental():
print("n" + "šÆ" * 40)
print("ADVANCED AGENT WITH INTERNAL CRITIC + UNCERTAINTY ESTIMATION")
print("Tutorial and Demonstrations")
print("šÆ" * 40)
plt.type.use('seaborn-v0_8-darkgrid')
sns.set_palette("husl")
attempt:
result1 = run_basic_demo()
result2 = run_strategy_comparison()
run_uncertainty_analysis()
run_risk_sensitivity_demo()
demonstrate_verbalized_uncertainty()
demonstrate_self_consistency()
print("n" + "=" * 80)
print("ā
ALL DEMONSTRATIONS COMPLETED SUCCESSFULLY")
print("=" * 80)
print("""
Key Takeaways:
1. Inner critics enhance response high quality by way of multi-dimensional analysis
2. Uncertainty estimation allows risk-aware determination making
3. Self-consistency reasoning will increase reliability
4. Completely different choice methods optimize for various targets
5. Verbalized uncertainty helps customers perceive mannequin confidence
Subsequent Steps:
ā¢ Implement with actual LLM APIs (OpenAI, Anthropic, and so on.)
ā¢ Add discovered critic fashions (fine-tuned classifiers)
ā¢ Discover ensemble strategies and meta-learning
ā¢ Combine with retrieval-augmented era (RAG)
ā¢ Deploy in manufacturing with monitoring and suggestions loops
""")
besides Exception as e:
print(f"nā Error throughout demonstration: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
fundamental()