Coverage for credoai/evaluators/utils/utils.py: 91%

1import inspect 

2import os 

3import pkgutil 

4 

5from credoai.evaluators.evaluator import Evaluator 

6 

7 

8def name2evaluator(evaluator_name): 

9 """Converts evaluator name to evaluator class""" 

10 for eval in all_subclasses(Evaluator): 

11 if evaluator_name == eval.__name__: 

12 return eval 

13 raise Exception( 

14 f"<{evaluator_name}> not found in list of Evaluators. Please confirm specified evaluator name is identical to Evaluator class definition." 

15 ) 

16 

17 

18def all_subclasses(cls): 

19 return set(cls.__subclasses__()).union( 

20 [s for c in cls.__subclasses__() for s in all_subclasses(c)] 

21 ) 

22 

23 

24def list_evaluators_exhaustive(): 

25 """List all defined evaluation modules""" 

26 evaluator_path = os.path.dirname(os.path.dirname(__file__)) 

27 return pkgutil.iter_modules([evaluator_path]) 

28 

29 

30def list_evaluators(): 

31 """List subset of all defined evaluators where the module is importable""" 

32 evaluators = list(list_evaluators_exhaustive()) 

33 importer = evaluators[0][0] 

34 # get loadable modules 

35 usable_modules = [] 

36 for evaluator in evaluators: 

37 module = importer.find_module(evaluator[1]) 

38 try: 

39 m = module.load_module() 

40 usable_modules.append((evaluator[1], m)) 

41 except (ModuleNotFoundError, ImportError): 

42 pass 

43 # get evaluators from loadable modules 

44 usable_evaluators = [] 

45 for module in usable_modules: 

46 try: 

47 for m in inspect.getmembers(module[1], inspect.isclass): 

48 if m[1].__module__ == module[0] and isinstance(m[1](), Evaluator): 

49 usable_evaluators.append(m) 

50 except: 

51 pass 

52 return usable_evaluators