From 396d7033fa7c6a4e5c889288de4af94b0d4b8b1a Mon Sep 17 00:00:00 2001 From: Ariel Frischer Date: Tue, 18 Feb 2025 20:15:18 -0800 Subject: [PATCH] Show fallback models in config panel (#103) * feat(readme): update experimental fallback handler description for clarity and add recommendation for OPENAI_API_KEY feat(main.py): refactor status display logic into a separate build_status function for better readability and maintainability feat(fallback_handler.py): add debug logging for fallback handler model selection process chore(todo.md): create a new TODO file to track future improvements and tasks * chore(todo.md): remove outdated TODO list file to clean up the repository * feat(main.py): add wrench emoji to FallbackHandler status message for better visibility --- README.md | 2 +- ra_aid/__main__.py | 60 +++++++++++++++++++++++--------------- ra_aid/fallback_handler.py | 10 +++---- 3 files changed, 41 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 53e9ad1..4fb923e 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ More information is available in our [Usage Examples](https://docs.ra-aid.ai/cat - `--hil, -H`: Enable human-in-the-loop mode for interactive assistance during task execution - `--chat`: Enable chat mode with direct human interaction (implies --hil) - `--verbose`: Enable verbose logging output -- `--experimental-fallback-handler`: Enable experimental fallback handler to attempt to fix too calls when they fail 3 times consecutively. +- `--experimental-fallback-handler`: Enable experimental fallback handler to attempt to fix too calls when the same tool fails 3 times consecutively. (OPENAI_API_KEY recommended as openai has the top 5 tool calling models.) See `ra_aid/tool_leaderboard.py` for more info. - `--pretty-logger`: Enables panel markdown formatted logger messages for debugging purposes. - `--temperature`: LLM temperature (0.0-2.0) to control randomness in responses - `--disable-limit-tokens`: Disable token limiting for Anthropic Claude react agents diff --git a/ra_aid/__main__.py b/ra_aid/__main__.py index b986ca6..fa3142f 100644 --- a/ra_aid/__main__.py +++ b/ra_aid/__main__.py @@ -27,6 +27,7 @@ from ra_aid.console.output import cpm from ra_aid.dependencies import check_dependencies from ra_aid.env import validate_environment from ra_aid.exceptions import AgentInterrupt +from ra_aid.fallback_handler import FallbackHandler from ra_aid.llm import initialize_llm from ra_aid.logging_config import get_logger, setup_logging from ra_aid.models_params import DEFAULT_TEMPERATURE, models_params @@ -286,6 +287,40 @@ def is_stage_requested(stage: str) -> bool: return False +def build_status( + args: argparse.Namespace, expert_enabled: bool, web_research_enabled: bool +): + status = Text() + status.append("šŸ¤– ") + status.append(f"{args.provider}/{args.model}") + if args.temperature is not None: + status.append(f" @ T{args.temperature}") + status.append("\n") + + status.append("šŸ¤” ") + if expert_enabled: + status.append(f"{args.expert_provider}/{args.expert_model}") + else: + status.append("Expert: ") + status.append("Disabled", style="italic") + status.append("\n") + + status.append("šŸ” Search: ") + status.append( + "Enabled" if web_research_enabled else "Disabled", + style=None if web_research_enabled else "italic", + ) + + if args.experimental_fallback_handler: + fb_handler = FallbackHandler({}, []) + status.append("\nšŸ”§ FallbackHandler Enabled: ") + msg = ", ".join( + [fb_handler._format_model(m) for m in fb_handler.fallback_tool_models] + ) + status.append(msg) + return status + + def main(): """Main entry point for the ra-aid command line tool.""" args = parse_arguments() @@ -326,30 +361,7 @@ def main(): f"Using default temperature {args.temperature} for model {args.model}" ) - # Display status lines - status = Text() - # Model info - status.append("šŸ¤– ") - status.append(f"{args.provider}/{args.model}") - if args.temperature is not None: - status.append(f" @ T{args.temperature}") - status.append("\n") - - # Expert info - status.append("šŸ¤” ") - if expert_enabled: - status.append(f"{args.expert_provider}/{args.expert_model}") - else: - status.append("Expert: ") - status.append("Disabled", style="italic") - status.append("\n") - - # Search info - status.append("šŸ” Search: ") - status.append( - "Enabled" if web_research_enabled else "Disabled", - style=None if web_research_enabled else "italic", - ) + status = build_status(args, expert_enabled, web_research_enabled) console.print( Panel(status, title="Config", border_style="bright_blue", padding=(0, 1)) diff --git a/ra_aid/fallback_handler.py b/ra_aid/fallback_handler.py index 3f7750c..f951c39 100644 --- a/ra_aid/fallback_handler.py +++ b/ra_aid/fallback_handler.py @@ -51,12 +51,6 @@ class FallbackHandler: self.current_tool_to_bind: None | BaseTool = None self.msg_list: list[BaseMessage] = [] - cpm( - "Fallback models selected: " - + ", ".join([self._format_model(m) for m in self.fallback_tool_models]), - title="Fallback Models", - ) - def _format_model(self, m: dict) -> str: return f"{m.get('model', '')} ({m.get('type', 'prompt')})" @@ -85,6 +79,7 @@ class FallbackHandler: break else: skipped.append(model_name) + final_models = [] for item in supported: if "type" not in item: @@ -99,6 +94,9 @@ class FallbackHandler: "\nSkipped top tool calling models due to missing provider ENV API keys: " + ", ".join(skipped) ) + + logger.debug(f"Fallback Handler: {message}") + return final_models def handle_failure(