Update client.py

Author: codelion

optillm/plugins/proxy/client.py

@@ -165,6 +165,7 @@ def __init__(self, proxy_client):
     class _Completions:
         def __init__(self, proxy_client):
             self.proxy_client = proxy_client
+            self._system_message_support_cache = {}
 
         def _filter_kwargs(self, kwargs: dict) -> dict:
             """Filter out OptiLLM-specific parameters that shouldn't be sent to providers"""
@@ -175,6 +176,73 @@ def _filter_kwargs(self, kwargs: dict) -> dict:
             }
             return {k: v for k, v in kwargs.items() if k not in optillm_params}
 
+        def _test_system_message_support(self, provider, model: str) -> bool:
+            """Test if a model supports system messages"""
+            cache_key = f"{provider.name}:{model}"
+            
+            if cache_key in self._system_message_support_cache:
+                return self._system_message_support_cache[cache_key]
+            
+            try:
+                test_response = provider.client.chat.completions.create(
+                    model=model,
+                    messages=[
+                        {"role": "system", "content": "test"},
+                        {"role": "user", "content": "hi"}
+                    ],
+                    max_tokens=1,
+                    temperature=0
+                )
+                self._system_message_support_cache[cache_key] = True
+                return True
+            except Exception as e:
+                error_msg = str(e).lower()
+                if any(pattern in error_msg for pattern in [
+                    "developer instruction", "system message", "not enabled", "not supported"
+                ]):
+                    logger.info(f"Provider {provider.name} model {model} does not support system messages")
+                    self._system_message_support_cache[cache_key] = False
+                    return False
+                # Other errors - assume it supports system messages
+                self._system_message_support_cache[cache_key] = True
+                return True
+        
+        def _format_messages_for_provider(self, provider, model: str, messages: list) -> list:
+            """Format messages based on provider's system message support"""
+            # Check if there's a system message
+            has_system = any(msg.get("role") == "system" for msg in messages)
+            
+            if not has_system:
+                return messages
+            
+            # Test system message support
+            supports_system = self._test_system_message_support(provider, model)
+            
+            if supports_system:
+                return messages
+            
+            # Merge system message into first user message
+            formatted_messages = []
+            system_content = None
+            
+            for msg in messages:
+                if msg.get("role") == "system":
+                    system_content = msg.get("content", "")
+                elif msg.get("role") == "user":
+                    if system_content:
+                        # Merge system message with user message
+                        formatted_messages.append({
+                            "role": "user",
+                            "content": f"Instructions: {system_content}\n\nUser: {msg.get('content', '')}"
+                        })
+                        system_content = None
+                    else:
+                        formatted_messages.append(msg)
+                else:
+                    formatted_messages.append(msg)
+            
+            return formatted_messages
+        
         def _make_request_with_timeout(self, provider, request_kwargs):
             """Make a request with timeout handling"""
             # The OpenAI client now supports timeout natively
@@ -232,7 +300,14 @@ def create(self, **kwargs):
                     try:
                         # Map model name if needed and filter out OptiLLM-specific parameters
                         request_kwargs = self._filter_kwargs(kwargs.copy())
-                        request_kwargs['model'] = provider.map_model(model)
+                        mapped_model = provider.map_model(model)
+                        request_kwargs['model'] = mapped_model
+                        
+                        # Format messages based on provider's system message support
+                        if 'messages' in request_kwargs:
+                            request_kwargs['messages'] = self._format_messages_for_provider(
+                                provider, mapped_model, request_kwargs['messages']
+                            )
 
                         # Add timeout to client if supported
                         request_kwargs['timeout'] = self.proxy_client.request_timeout
@@ -279,7 +354,7 @@ def create(self, **kwargs):
                 if self.proxy_client.fallback_client:
                     logger.warning("All proxy providers failed, using fallback client")
                     try:
-                        fallback_kwargs = self._filter_kwargs(kwargs)
+                        fallback_kwargs = self._filter_kwargs(kwargs.copy())
                         fallback_kwargs['timeout'] = self.proxy_client.request_timeout
                         return self.proxy_client.fallback_client.chat.completions.create(**fallback_kwargs)
                     except Exception as e: