From 53c2afdde3cb0ed950d9e34de75a9c225198d668 Mon Sep 17 00:00:00 2001 From: AI Christianson Date: Thu, 19 Dec 2024 09:18:10 -0500 Subject: [PATCH] Fix env var fallback when base key is given, expert and base provider are different, and expert key is missing. --- CHANGELOG.md | 1 + ra_aid/env.py | 11 ++++--- ra_aid/tools/expert.py | 3 +- tests/ra_aid/test_env.py | 68 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48185de..b8e570b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added monorepo_detected, existing_project_detected, and ui_detected tools so the agent can take specific actions. - Prompt improvements for real-world projects. +- Fix env var fallback when base key is given, expert and base provider are different, and expert key is missing. ## [0.6.3] - 2024-12-18 diff --git a/ra_aid/env.py b/ra_aid/env.py index 585f31f..d2fe997 100644 --- a/ra_aid/env.py +++ b/ra_aid/env.py @@ -56,20 +56,21 @@ def validate_environment(args) -> Tuple[bool, List[str]]: expert_key = f'EXPERT_{config.key_name}' expert_key_missing = not os.environ.get(expert_key) - # Try fallback to base key if providers match - fallback_available = expert_provider == provider and os.environ.get(config.key_name) + # Try fallback to base key for expert provider + fallback_available = os.environ.get(config.key_name) if expert_key_missing and fallback_available: os.environ[expert_key] = os.environ[config.key_name] expert_key_missing = False - + + # Only add to missing list if still missing after fallback attempt if expert_key_missing: expert_missing.append(f'{expert_key} environment variable is not set') - # Special case for openai-compatible expert needing base URL + # Special case for openai-compatible expert needing base URL if expert_provider == "openai-compatible": expert_base = 'EXPERT_OPENAI_API_BASE' base_missing = not os.environ.get(expert_base) - base_fallback = expert_provider == provider and os.environ.get('OPENAI_API_BASE') + base_fallback = os.environ.get('OPENAI_API_BASE') if base_missing and base_fallback: os.environ[expert_base] = os.environ['OPENAI_API_BASE'] diff --git a/ra_aid/tools/expert.py b/ra_aid/tools/expert.py index 44c22f8..4d63c7b 100644 --- a/ra_aid/tools/expert.py +++ b/ra_aid/tools/expert.py @@ -119,7 +119,8 @@ def ask_expert(question: str) -> str: The expert can be extremely useful at logic questions, debugging, and reviewing complex source code, but you must provide all context including source manually. - The query will automatically include any key facts and code snippets from memory, along with any additional context you've provided. + The can see any key facts and code snippets previously noted, along with any additional context you've provided. + But the expert cannot see or reason about anything you have not explicitly provided in this way. Try to phrase your question in a way that it does not expand the scope of our top-level task. diff --git a/tests/ra_aid/test_env.py b/tests/ra_aid/test_env.py index 8b29f72..fe23ba7 100644 --- a/tests/ra_aid/test_env.py +++ b/tests/ra_aid/test_env.py @@ -88,3 +88,71 @@ def test_expert_fallback(clean_env, monkeypatch): assert expert_enabled assert not missing assert os.environ.get('EXPERT_OPENAI_API_KEY') == 'expert-key' + +def test_cross_provider_fallback(clean_env, monkeypatch): + """Test that fallback works even when providers differ""" + args = MockArgs(provider="openai", expert_provider="anthropic") + + # Set base API key for main provider and expert provider + monkeypatch.setenv('OPENAI_API_KEY', 'openai-key') + monkeypatch.setenv('ANTHROPIC_API_KEY', 'anthropic-key') + + # Should enable expert mode with fallback to ANTHROPIC base key + expert_enabled, missing = validate_environment(args) + assert expert_enabled + assert not missing + assert os.environ.get('EXPERT_ANTHROPIC_API_KEY') == 'anthropic-key' + + # Try with openai-compatible expert provider + args = MockArgs(provider="anthropic", expert_provider="openai-compatible") + monkeypatch.setenv('OPENAI_API_KEY', 'openai-key') + monkeypatch.setenv('OPENAI_API_BASE', 'http://test') + + expert_enabled, missing = validate_environment(args) + assert expert_enabled + assert not missing + assert os.environ.get('EXPERT_OPENAI_API_KEY') == 'openai-key' + assert os.environ.get('EXPERT_OPENAI_API_BASE') == 'http://test' + +def test_no_warning_on_fallback(clean_env, monkeypatch): + """Test that no warning is issued when fallback succeeds""" + args = MockArgs(provider="openai", expert_provider="openai") + + # Set only base API key + monkeypatch.setenv('OPENAI_API_KEY', 'test-key') + + # Should enable expert mode with fallback and no warnings + expert_enabled, expert_missing = validate_environment(args) + assert expert_enabled + assert not expert_missing # List should be empty + assert os.environ.get('EXPERT_OPENAI_API_KEY') == 'test-key' + +def test_different_providers_no_expert_key(clean_env, monkeypatch): + """Test behavior when providers differ and only base keys are available""" + args = MockArgs(provider="anthropic", expert_provider="openai") + + # Set only base keys + monkeypatch.setenv('ANTHROPIC_API_KEY', 'anthropic-key') + monkeypatch.setenv('OPENAI_API_KEY', 'openai-key') + + # Should enable expert mode and use base OPENAI key + expert_enabled, missing = validate_environment(args) + assert expert_enabled + assert not missing + assert os.environ.get('EXPERT_OPENAI_API_KEY') == 'openai-key' + +def test_mixed_provider_openai_compatible(clean_env, monkeypatch): + """Test behavior with openai-compatible expert and different main provider""" + args = MockArgs(provider="anthropic", expert_provider="openai-compatible") + + # Set all required keys and URLs + monkeypatch.setenv('ANTHROPIC_API_KEY', 'anthropic-key') + monkeypatch.setenv('OPENAI_API_KEY', 'openai-key') + monkeypatch.setenv('OPENAI_API_BASE', 'http://test') + + # Should enable expert mode and use base openai key and URL + expert_enabled, missing = validate_environment(args) + assert expert_enabled + assert not missing + assert os.environ.get('EXPERT_OPENAI_API_KEY') == 'openai-key' + assert os.environ.get('EXPERT_OPENAI_API_BASE') == 'http://test'