Show fallback models in config panel (#103)

* feat(readme): update experimental fallback handler description for clarity and add recommendation for OPENAI_API_KEY
feat(main.py): refactor status display logic into a separate build_status function for better readability and maintainability
feat(fallback_handler.py): add debug logging for fallback handler model selection process
chore(todo.md): create a new TODO file to track future improvements and tasks

* chore(todo.md): remove outdated TODO list file to clean up the repository

* feat(main.py): add wrench emoji to FallbackHandler status message for better visibility

README.md

@@ -175,7 +175,7 @@ More information is available in our [Usage Examples](https://docs.ra-aid.ai/cat
 - `--hil, -H`: Enable human-in-the-loop mode for interactive assistance during task execution
 - `--chat`: Enable chat mode with direct human interaction (implies --hil)
 - `--verbose`: Enable verbose logging output
-- `--experimental-fallback-handler`: Enable experimental fallback handler to attempt to fix too calls when they fail 3 times consecutively.
+- `--experimental-fallback-handler`: Enable experimental fallback handler to attempt to fix too calls when the same tool fails 3 times consecutively. (OPENAI_API_KEY recommended as openai has the top 5 tool calling models.) See `ra_aid/tool_leaderboard.py` for more info.
 - `--pretty-logger`: Enables panel markdown formatted logger messages for debugging purposes.
 - `--temperature`: LLM temperature (0.0-2.0) to control randomness in responses
 - `--disable-limit-tokens`: Disable token limiting for Anthropic Claude react agents

ra_aid/__main__.py

@@ -27,6 +27,7 @@ from ra_aid.console.output import cpm
 from ra_aid.dependencies import check_dependencies
 from ra_aid.env import validate_environment
 from ra_aid.exceptions import AgentInterrupt
+from ra_aid.fallback_handler import FallbackHandler
 from ra_aid.llm import initialize_llm
 from ra_aid.logging_config import get_logger, setup_logging
 from ra_aid.models_params import DEFAULT_TEMPERATURE, models_params
@@ -286,6 +287,40 @@ def is_stage_requested(stage: str) -> bool:
     return False
 
 
+def build_status(
+    args: argparse.Namespace, expert_enabled: bool, web_research_enabled: bool
+):
+    status = Text()
+    status.append("🤖 ")
+    status.append(f"{args.provider}/{args.model}")
+    if args.temperature is not None:
+        status.append(f" @ T{args.temperature}")
+    status.append("\n")
+
+    status.append("🤔 ")
+    if expert_enabled:
+        status.append(f"{args.expert_provider}/{args.expert_model}")
+    else:
+        status.append("Expert: ")
+        status.append("Disabled", style="italic")
+    status.append("\n")
+
+    status.append("🔍 Search: ")
+    status.append(
+        "Enabled" if web_research_enabled else "Disabled",
+        style=None if web_research_enabled else "italic",
+    )
+
+    if args.experimental_fallback_handler:
+        fb_handler = FallbackHandler({}, [])
+        status.append("\n🔧 FallbackHandler Enabled: ")
+        msg = ", ".join(
+            [fb_handler._format_model(m) for m in fb_handler.fallback_tool_models]
+        )
+        status.append(msg)
+    return status
+
+
 def main():
     """Main entry point for the ra-aid command line tool."""
     args = parse_arguments()
@@ -326,30 +361,7 @@ def main():
                 f"Using default temperature {args.temperature} for model {args.model}"
             )
 
-        # Display status lines
+        status = build_status(args, expert_enabled, web_research_enabled)
-        status = Text()
-        # Model info
-        status.append("🤖 ")
-        status.append(f"{args.provider}/{args.model}")
-        if args.temperature is not None:
-            status.append(f" @ T{args.temperature}")
-        status.append("\n")
-
-        # Expert info
-        status.append("🤔 ")
-        if expert_enabled:
-            status.append(f"{args.expert_provider}/{args.expert_model}")
-        else:
-            status.append("Expert: ")
-            status.append("Disabled", style="italic")
-        status.append("\n")
-
-        # Search info
-        status.append("🔍 Search: ")
-        status.append(
-            "Enabled" if web_research_enabled else "Disabled",
-            style=None if web_research_enabled else "italic",
-        )
 
         console.print(
             Panel(status, title="Config", border_style="bright_blue", padding=(0, 1))

ra_aid/fallback_handler.py

@@ -51,12 +51,6 @@ class FallbackHandler:
         self.current_tool_to_bind: None | BaseTool = None
         self.msg_list: list[BaseMessage] = []
 
-        cpm(
-            "Fallback models selected: "
-            + ", ".join([self._format_model(m) for m in self.fallback_tool_models]),
-            title="Fallback Models",
-        )
-
     def _format_model(self, m: dict) -> str:
         return f"{m.get('model', '')} ({m.get('type', 'prompt')})"
 
@@ -85,6 +79,7 @@ class FallbackHandler:
                     break
             else:
                 skipped.append(model_name)
+
         final_models = []
         for item in supported:
             if "type" not in item:
@@ -99,6 +94,9 @@ class FallbackHandler:
                 "\nSkipped top tool calling models due to missing provider ENV API keys: "
                 + ", ".join(skipped)
             )
+
+        logger.debug(f"Fallback Handler: {message}")
+
         return final_models
 
     def handle_failure(