orbiter.eval.ralph

from orbiter.eval.ralph.runner import RalphRunner, RalphResult, ExecuteFn, RePlanFn
from orbiter.eval.ralph.config import (
    StopType,
    ValidationConfig,
    ReflectionConfig,
    StopConditionConfig,
    RalphConfig,
    LoopState,
)
from orbiter.eval.ralph.detectors import (
    StopDecision,
    StopDetector,
    MaxIterationDetector,
    TimeoutDetector,
    CostLimitDetector,
    ConsecutiveFailureDetector,
    ScoreThresholdDetector,
    CompositeDetector,
)

class StopType(StrEnum):
    NONE = "none"
    COMPLETION = "completion"
    MAX_ITERATIONS = "max_iterations"
    TIMEOUT = "timeout"
    MAX_COST = "max_cost"
    MAX_CONSECUTIVE_FAILURES = "max_consecutive_failures"
    SCORE_THRESHOLD = "score_threshold"
    USER_INTERRUPTED = "user_interrupted"
    SYSTEM_ERROR = "system_error"

def is_success(self) -> bool

def is_failure(self) -> bool

ValidationConfig(
    enabled: bool = True,
    scorer_names: tuple[str, ...] = (),
    min_score_threshold: float = 0.5,
    parallel: int = 4,
    timeout: float = 0.0,
)

ReflectionConfig(
    enabled: bool = True,
    level: str = "medium",
    max_history: int = 50,
)

StopConditionConfig(
    max_iterations: int = 10,
    timeout: float = 0.0,
    max_cost: float = 0.0,
    max_consecutive_failures: int = 3,
    score_threshold: float = 0.0,
    enable_user_interrupt: bool = False,
)

RalphConfig(
    validation: ValidationConfig = ValidationConfig(),
    reflection: ReflectionConfig = ReflectionConfig(),
    stop_condition: StopConditionConfig = StopConditionConfig(),
    metadata: dict[str, Any] = {},
)

LoopState()

def elapsed(self) -> float

def success_rate(self) -> float

def latest_score(self) -> dict[str, float]

def best_score(self, metric: str) -> float

def record_score(self, scores: dict[str, float]) -> None

def record_reflection(self, reflection: dict[str, Any]) -> None

def record_success(self, *, tokens: int = 0, cost: float = 0.0) -> None

def record_failure(self, *, cost: float = 0.0) -> None

def to_dict(self) -> dict[str, Any]

StopDecision(
    should_stop: bool,
    stop_type: StopType = StopType.NONE,
    reason: str = "",
    metadata: dict[str, Any] = {},
)

async def check(
    self,
    state: LoopState,
    config: StopConditionConfig,
) -> StopDecision

MaxIterationDetector()

TimeoutDetector()

CostLimitDetector()

ConsecutiveFailureDetector()

ScoreThresholdDetector()

CompositeDetector(detectors: list[StopDetector] | None = None)

def add(self, detector: StopDetector) -> CompositeDetector

async def check(
    self,
    state: LoopState,
    config: StopConditionConfig,
) -> StopDecision

ExecuteFn = Callable[..., Any]  # Async callable: (input: str) -> str
RePlanFn = Callable[..., Any]   # Async callable: (prompt: str) -> str

RalphResult(
    output: str,
    stop_type: StopType,
    reason: str,
    iterations: int,
    scores: dict[str, float],
    state: dict[str, Any],
    reflections: list[dict[str, Any]] = [],
)

RalphRunner(
    execute_fn: ExecuteFn,
    scorers: list[Scorer],
    *,
    config: RalphConfig | None = None,
    reflector: Reflector | None = None,
    replan_fn: RePlanFn | None = None,
)

async def run(self, input: str) -> RalphResult

import asyncio
from orbiter.eval import (
    GeneralReflector,
    OutputCorrectnessScorer,
    OutputLengthScorer,
)
from orbiter.eval.ralph.runner import RalphRunner
from orbiter.eval.ralph.config import (
    RalphConfig,
    ValidationConfig,
    ReflectionConfig,
    StopConditionConfig,
)

iteration_count = 0

async def my_agent(input: str) -> str:
    global iteration_count
    iteration_count += 1
    if iteration_count >= 3:
        return "The capital of France is Paris."
    return "I'm not sure about that."

async def my_judge(prompt: str) -> str:
    return '''{
        "summary": "The answer was incomplete.",
        "key_findings": ["Missing specific answer"],
        "root_causes": ["Insufficient confidence"],
        "insights": ["Need to be more decisive"],
        "suggestions": ["Provide a direct, specific answer"]
    }'''

async def main():
    config = RalphConfig(
        validation=ValidationConfig(min_score_threshold=0.8),
        reflection=ReflectionConfig(enabled=True),
        stop_condition=StopConditionConfig(
            max_iterations=5,
            score_threshold=0.9,
        ),
    )

    runner = RalphRunner(
        execute_fn=my_agent,
        scorers=[
            OutputCorrectnessScorer(keywords=["Paris", "capital"]),
            OutputLengthScorer(min_length=10),
        ],
        config=config,
        reflector=GeneralReflector(judge=my_judge),
    )

    result = await runner.run("What is the capital of France?")

    print(f"Output: {result.output}")
    print(f"Stop type: {result.stop_type}")
    print(f"Iterations: {result.iterations}")
    print(f"Scores: {result.scores}")
    print(f"Success: {result.stop_type.is_success()}")

    # Check reflections
    for ref in result.reflections:
        print(f"  Iteration {ref['iteration']}: {ref['summary']}")

asyncio.run(main())