Coverage for credoai/lens/utils.py: 28%

1import functools 

2import inspect 

3from typing import Callable, Union 

4 

5import credoai.evaluators 

6from credoai.utils.common import dict_hash 

7 

8 

9def log_command(fun: Callable): 

10 """ 

11 Decorator loggin the full isgnature of a function call. 

12 

13 Parameters 

14 ---------- 

15 fun : Callable 

16 A generic function, specifically used for Lens.add, Lens.delete, Lens.run 

17 

18 """ 

19 

20 @functools.wraps(fun) 

21 def wrapper(self, *args, **kwargs): 

22 tmp = fun(self, *args, **kwargs) 

23 name = fun.__name__ 

24 self.command_list.append(get_command_string(name, args, kwargs)) 

25 return tmp 

26 

27 return wrapper 

28 

29 

30def get_command_string(name: str, arg: dict, kwarg: dict) -> str: 

31 """ 

32 Combines name and function arguments into a signature string. 

33 

34 Parameters 

35 ---------- 

36 name : str 

37 Function's name. 

38 arg : dict 

39 Function's positional arguments. 

40 kwarg : dict 

41 Function's keyword argumwents 

42 

43 Returns 

44 ------- 

45 str 

46 Full function signature,e.g., fun_name(fun_arg1, fun_arg1..., fun_kwarg1...) 

47 """ 

48 

49 arg_parse = [get_arg_info(arg) for arg in arg] 

50 kwarg_parse = [f"{k}={get_arg_info(v)}" for k, v in kwarg.items()] 

51 all_arguments = arg_parse + kwarg_parse 

52 return f"{name}({','.join([x for x in all_arguments if x is not None])})" 

53 

54 

55def get_arg_info(arg: Union[Callable, str, int]) -> str: 

56 """ 

57 Takes a function's arguments and converts them into strings. 

58 

59 Parameters 

60 ---------- 

61 arg : Union[Callable, str, int] 

62 This is quite custom made for usage in Lens(). The positional arguments 

63 can be a call to a class, or int/str. This handles all cases. 

64 

65 Returns 

66 ------- 

67 Union[str,int] 

68 Either a string representing the function signature, or str/int 

69 depending on the argument. 

70 """ 

71 if callable(arg): 

72 # Get only initialization arguments 

73 init_args = { 

74 k: v 

75 for k, v in arg.__dict__.items() 

76 if k in list(inspect.signature(arg.__init__).parameters.keys()) 

77 } 

78 return f"{type(arg).__name__}({get_string_of_arguments_from_kwargs(init_args)})" 

79 elif isinstance(arg, int): 

80 return str(arg) 

81 elif isinstance(arg, str): 

82 return f'"{arg}"' 

83 

84 

85def get_string_of_arguments_from_kwargs(keyarg: dict) -> str: 

86 """ 

87 Transform positional arguments in string. 

88 

89 Parameters 

90 ---------- 

91 keyarg : dict 

92 Function's positional arguments. 

93 

94 Returns 

95 ------- 

96 str 

97 String representing the positional arguments 

98 """ 

99 return ",".join([f"{x[0]}={check_int_str(x[1])}" for x in keyarg.items()]) 

100 

101 

102def check_int_str(x: Union[int, float, str]) -> Union[int, str, float]: 

103 """ 

104 Check what type is the argument and reformats in case it is a string. 

105 """ 

106 if isinstance(x, (int, float)): 

107 return x 

108 elif isinstance(x, str): 

109 return f'"{x}"' 

110 

111 

112def add_metric_keys(prepared_results): 

113 """Adds metric keys to prepared results 

114 

115 Metric keys are used to associated charts, html blobs, and other assets with 

116 specific metrics. They are a hash of most of the metric's attributes, except the value. 

117 So if a metric changes value, the key will stay the same. 

118 

119 Metric keys should be defined after all pertinent information is appended to a metric. 

120 Lens normally handles key association, because it may add additional metadata to a metric 

121 beyond what the assessment creates (e.g., dataset name, model name, etc.) 

122 

123 Parameters 

124 ---------- 

125 prepared_results : DataFrame 

126 output of CredoAssessment.prepare_results() 

127 """ 

128 if prepared_results is None: 

129 return 

130 ignored = ["value", "metadata"] 

131 keys = [ 

132 dict_hash({k: v for k, v in metric_dict.items() if k not in ignored}) 

133 for metric_dict in prepared_results.reset_index().to_dict("records") 

134 ] 

135 prepared_results["metric_key"] = keys