improve get available model provider

backend/config/models_config.py

@@ -5,6 +5,7 @@ This file centralizes all configurations related to models and providers used in
 """
 
 # Definition of preferred providers, used in get_available_model_provider.py
+# PREFERRED_PROVIDERS = ["sambanova", "novita"]
 PREFERRED_PROVIDERS = ["fireworks-ai", "sambanova", "novita"]
 
 # Default models to evaluate for evaluation
@@ -18,13 +19,21 @@ DEFAULT_EVALUATION_MODELS = [
     "mistralai/Mistral-Small-24B-Instruct-2501",
 ]
 
+# Modèles alternatifs à utiliser si le modèle par défaut n'est pas disponible
+ALTERNATIVE_BENCHMARK_MODELS = [
+    "meta-llama/Llama-3.3-70B-Instruct",
+    "meta-llama/Llama-3.1-8B-Instruct",
+    "Qwen/Qwen2.5-72B-Instruct",
+    "mistralai/Mistral-Small-24B-Instruct-2501",
+]
+
 # "Qwen/Qwen2.5-72B-Instruct"
 # "meta-llama/Llama-3.1-8B-Instruct"
 # "Qwen/Qwen2.5-32B-Instruct",
 # "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
 
 # Required model for create_bench_config_file.py (only one default model)
-DEFAULT_BENCHMARK_MODEL = "Qwen/Qwen2.5-32B-Instruct"
+DEFAULT_BENCHMARK_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
 
 # Models by roles for benchmark configuration
 # All roles use the default model except chunking

backend/tasks/create_bench_config_file.py

@@ -18,6 +18,8 @@ from config.models_config import (
     DEFAULT_BENCHMARK_MODEL,
     BENCHMARK_MODEL_ROLES,
     DEFAULT_BENCHMARK_TIMEOUT,
+    PREFERRED_PROVIDERS,
+    ALTERNATIVE_BENCHMARK_MODELS,
 )
 
 
@@ -96,14 +98,44 @@ class CreateBenchConfigTask:
             Available provider or None if none found
         """
         self._add_log(f"[INFO] Finding available provider for {model_name}")
+        
+        # Essayer de trouver un provider pour le modèle
         provider = get_available_model_provider(model_name, verbose=True)
         
         if provider:
             self._add_log(f"[INFO] Found provider for {model_name}: {provider}")
             return provider
-        else:
-            self._add_log(f"[WARNING] No available provider found for {model_name}")
-            return None
+        
+        # Si aucun provider n'est trouvé avec la configuration préférée
+        # Essayons de trouver n'importe quel provider disponible en ignorant la préférence
+        from huggingface_hub import model_info
+        from tasks.get_available_model_provider import test_provider
+        
+        self._add_log(f"[WARNING] No preferred provider found for {model_name}, trying all available providers...")
+        
+        try:
+            # Obtenir tous les providers possibles pour ce modèle
+            info = model_info(model_name, expand="inferenceProviderMapping")
+            if hasattr(info, "inference_provider_mapping"):
+                providers = list(info.inference_provider_mapping.keys())
+                
+                # Exclure les providers préférés déjà testés
+                other_providers = [p for p in providers if p not in PREFERRED_PROVIDERS]
+                
+                if other_providers:
+                    self._add_log(f"[INFO] Testing additional providers: {', '.join(other_providers)}")
+                    
+                    # Tester chaque provider
+                    for provider in other_providers:
+                        self._add_log(f"[INFO] Testing provider {provider}")
+                        if test_provider(model_name, provider, verbose=True):
+                            self._add_log(f"[INFO] Found alternative provider for {model_name}: {provider}")
+                            return provider
+        except Exception as e:
+            self._add_log(f"[WARNING] Error while testing additional providers: {str(e)}")
+        
+        self._add_log(f"[WARNING] No available provider found for {model_name}")
+        return None
     
     def generate_base_config(self, hf_org: str, hf_dataset_name: str) -> Dict[str, Any]:
         """
@@ -125,19 +157,46 @@ class CreateBenchConfigTask:
         
         # Get provider for the default model
         provider = self.get_model_provider(DEFAULT_BENCHMARK_MODEL)
+        
+        # Si aucun provider n'est trouvé pour le modèle par défaut, essayer les modèles alternatifs
+        selected_model = DEFAULT_BENCHMARK_MODEL
+        
         if not provider:
-            error_msg = f"Required model not available: {DEFAULT_BENCHMARK_MODEL}. Cannot proceed with benchmark."
+            self._add_log(f"[WARNING] Primary model {DEFAULT_BENCHMARK_MODEL} not available. Trying alternatives...")
+            
+            # Utiliser la liste des modèles alternatifs depuis la configuration
+            for alt_model in ALTERNATIVE_BENCHMARK_MODELS:
+                self._add_log(f"[INFO] Trying alternative model: {alt_model}")
+                alt_provider = self.get_model_provider(alt_model)
+                if alt_provider:
+                    self._add_log(f"[INFO] Found working alternative model: {alt_model} with provider: {alt_provider}")
+                    selected_model = alt_model
+                    provider = alt_provider
+                    break
+        
+        # Si toujours pas de provider, lever une exception
+        if not provider:
+            error_msg = "No model with available provider found. Cannot proceed with benchmark."
             self._add_log(f"[ERROR] {error_msg}")
             raise RuntimeError(error_msg)
             
         # Create model configuration
         model_list = [{
-            "model_name": DEFAULT_BENCHMARK_MODEL,
+            "model_name": selected_model,
             "provider": provider,
             "api_key": "$HF_TOKEN",
             "max_concurrent_requests": 32,
         }]
         
+        # Mettre à jour les roles de modèle si un modèle alternatif est utilisé
+        model_roles = dict(BENCHMARK_MODEL_ROLES)
+        if selected_model != DEFAULT_BENCHMARK_MODEL:
+            for role in model_roles:
+                if role != "chunking":  # Ne pas changer le modèle de chunking
+                    model_roles[role] = [selected_model]
+            
+            self._add_log(f"[INFO] Updated model roles to use {selected_model}")
+        
         # Add minimum delay of 2 seconds for provider_check stage
         self._add_log("[INFO] Finalizing provider check...")
         time.sleep(2)
@@ -156,7 +215,7 @@ class CreateBenchConfigTask:
             },
             "model_list": model_list,
             
-            "model_roles": BENCHMARK_MODEL_ROLES,
+            "model_roles": model_roles,
             "pipeline": {
                 "ingestion": {
                     "source_documents_dir": f"uploaded_files/{self.session_uid}/uploaded_files/",

backend/tasks/get_available_model_provider.py

@@ -49,7 +49,7 @@ def test_provider(model_name: str, provider: str, verbose: bool = False) -> bool
             token=hf_token,
             provider=provider,
             # bill_to=hf_organization,
-            timeout=10  # Increased timeout to allow model loading
+            timeout=3  # Increased timeout to allow model loading
         )
             
         try:
@@ -99,35 +99,77 @@ def get_available_model_provider(model_name, verbose=False):
         # Get HF token from environment
         hf_token = os.environ.get("HF_TOKEN")
         if not hf_token:
+            if verbose:
+                logger.error("HF_TOKEN not defined in environment")
             raise ValueError("HF_TOKEN not defined in environment")
         
         # Get providers for the model and prioritize them
-        info = model_info(model_name, expand="inferenceProviderMapping")
-        if not hasattr(info, "inference_provider_mapping"):
-            if verbose:
-                logger.info(f"No inference providers found for {model_name}")
-            return None
-            
-        providers = list(info.inference_provider_mapping.keys())
-        if not providers:
+        try:
+            info = model_info(model_name, expand="inferenceProviderMapping")
+            if not hasattr(info, "inference_provider_mapping"):
+                if verbose:
+                    logger.info(f"No inference providers found for {model_name}")
+                return None
+                
+            providers = list(info.inference_provider_mapping.keys())
+            if not providers:
+                if verbose:
+                    logger.info(f"Empty list of providers for {model_name}")
+                return None
+        except Exception as e:
             if verbose:
-                logger.info(f"Empty list of providers for {model_name}")
+                logger.error(f"Error retrieving model info for {model_name}: {str(e)}")
             return None
             
         # Prioritize providers
-        providers = prioritize_providers(providers)
+        prioritized_providers = prioritize_providers(providers)
         
         if verbose:
             logger.info(f"Available providers for {model_name}: {', '.join(providers)}")
+            logger.info(f"Prioritized providers: {', '.join(prioritized_providers)}")
         
-        # Test each provider
-        for provider in providers:
-            if test_provider(model_name, provider, verbose):
-                return provider
+        # Test each preferred provider first
+        failed_providers = []
+        for provider in prioritized_providers:
+            if verbose:
+                logger.info(f"Testing provider {provider} for {model_name}")
+            
+            try:
+                if test_provider(model_name, provider, verbose):
+                    if verbose:
+                        logger.info(f"Provider {provider} is available for {model_name}")
+                    return provider
+                else:
+                    failed_providers.append(provider)
+                    if verbose:
+                        logger.warning(f"Provider {provider} test failed for {model_name}")
+            except Exception as e:
+                failed_providers.append(provider)
+                if verbose:
+                    logger.error(f"Exception while testing provider {provider} for {model_name}: {str(e)}")
+                
+        # If all prioritized providers failed, try any remaining providers
+        remaining_providers = [p for p in providers if p not in prioritized_providers and p not in failed_providers]
+        
+        if remaining_providers and verbose:
+            logger.info(f"Trying remaining non-prioritized providers: {', '.join(remaining_providers)}")
+            
+        for provider in remaining_providers:
+            if verbose:
+                logger.info(f"Testing non-prioritized provider {provider} for {model_name}")
+                
+            try:
+                if test_provider(model_name, provider, verbose):
+                    if verbose:
+                        logger.info(f"Non-prioritized provider {provider} is available for {model_name}")
+                    return provider
+            except Exception as e:
+                if verbose:
+                    logger.error(f"Exception while testing non-prioritized provider {provider}: {str(e)}")
                 
         # If we've tried all providers and none worked, log this but don't raise an exception
         if verbose:
-            logger.error(f"No available providers for {model_name}")
+            logger.error(f"No available providers for {model_name}. Tried {len(failed_providers + remaining_providers)} providers.")
         return None
         
     except Exception as e:

frontend/src/components/Benchmark/Display.jsx

@@ -51,18 +51,18 @@ const Display = ({
 
     setIsDownloading(true);
     try {
-      // Requête pour télécharger le dataset
-      const downloadUrl = `${API_CONFIG.BASE_URL}/download-dataset/${sessionId}`;
+      // Requête pour télécharger les questions JSON au lieu du dataset
+      const downloadUrl = `${API_CONFIG.BASE_URL}/download-questions/${sessionId}`;
 
       // Créer un élément a temporaire pour déclencher le téléchargement
       const link = document.createElement("a");
       link.href = downloadUrl;
-      link.setAttribute("download", `yourbench_${sessionId}_dataset.zip`);
+      link.setAttribute("download", `yourbench_${sessionId}_questions.json`);
       document.body.appendChild(link);
       link.click();
       document.body.removeChild(link);
     } catch (error) {
-      console.error("Erreur lors du téléchargement du dataset:", error);
+      console.error("Erreur lors du téléchargement des questions:", error);
       alert("Erreur lors du téléchargement. Veuillez réessayer.");
     } finally {
       setIsDownloading(false);
@@ -85,7 +85,7 @@ const Display = ({
         </Box>
 
         <Box sx={{ display: "flex", gap: 2 }}>
-          <Tooltip title="Download the complete benchmark">
+          <Tooltip title="Download the benchmark questions in JSON format">
             <Button
               variant="outlined"
               color="primary"
@@ -99,7 +99,7 @@ const Display = ({
               onClick={handleDownloadClick}
               disabled={isDownloading || !sessionId}
             >
-              {isDownloading ? "Downloading..." : "Download Benchmark"}
+              {isDownloading ? "Downloading..." : "Download Questions JSON"}
             </Button>
           </Tooltip>