Coverage for credoai/evaluators/evaluator.py: 89%

1from abc import ABC, abstractmethod 

2 

3from connect.evidence import EvidenceContainer 

4from credoai.utils import global_logger 

5from credoai.utils.common import NotRunError, ValidationError 

6 

7 

8class Evaluator(ABC): 

9 """ 

10 Base abstract class for all lens evaluators. 

11 

12 Defines basic functions required from any evaluator object. 

13 

14 This class leverages the special method `__call__` to make artifacts 

15 available in the class enclosure. 

16 

17 .. automethod:: __call__ 

18 .. automethod:: _init_artifacts 

19 .. automethod:: _validate_arguments 

20 .. automethod:: _setup 

21 """ 

22 

23 def __init__(self): 

24 self._results = None 

25 self.artifact_keys = [] 

26 self.logger = global_logger 

27 self.metadata = {} 

28 

29 @property 

30 def name(self): 

31 """The name associated to the Evaluator, equals the class name.""" 

32 return self.__class__.__name__ 

33 

34 @property 

35 def results(self): 

36 """ 

37 Container for all results. 

38 

39 It is expected to be a list of EvidenceContainers. This is enforced in 

40 the associated setter method. 

41 

42 Raises 

43 ------ 

44 NotRunError 

45 It indicates that results are missing, the evaluator was not run. 

46 """ 

47 if self._results is not None: 

48 return self._results 

49 else: 

50 raise NotRunError( 

51 "No results available, please call the method: 'evaluate'." 

52 ) 

53 

54 @results.setter 

55 def results(self, results): 

56 """Requires the results to be list of Evidence Containers""" 

57 if not isinstance(results, list): 

58 raise ValidationError("Results must be a list") 

59 for result in results: 

60 if not isinstance(result, EvidenceContainer): 

61 raise ValidationError("All results must be EvidenceContainers") 

62 self._results = results 

63 

64 @property 

65 @abstractmethod 

66 def required_artifacts(self): 

67 """ 

68 The required artifacts necessary for the functioning of the evaluator 

69 

70 This set contains the :ref:`artifacts<credoai.artifacts>` that Lens can feed to 

71 an evaluator, the accepted values are ``{"model", "assessment_data", "training_data", "data"}``. 

72 

73 The string "data" means that the evaluator can be run on assessment and/or training data 

74 (DataProfiler is an example). Lens will iterate over all the available artifacts internally. 

75 

76 The set can also include the string "sensitive_feature". This is to indicate 

77 that the evaluator depends on sensitive features. Lens will iterate over the available sensitive 

78 features internally. 

79 """ 

80 pass 

81 

82 def __call__(self, **kwargs): 

83 """ 

84 This method is used to pass the model, assessment_data and training_data 

85 artifacts to instantiated evaluator. 

86 

87 The method is called internally by the Lens instance, which only passes the 

88 artifacts specified in the property :meth:`required_artifacts<Evaluator.required_artifacts>`. 

89 

90 After the artifacts are passed, it performs arguments validation and calls :meth:`_setup<Evaluator._setup>` 

91 

92 At the end of these operation, the validated artifacts are available in the evaluator enclosure. 

93 """ 

94 self._init_artifacts(kwargs) 

95 self._validate_arguments() 

96 self._setup() 

97 return self 

98 

99 @abstractmethod 

100 def evaluate(self): 

101 """ 

102 Execute any data/model processing required for the evaluator. 

103 

104 Populates the self.results object. 

105 """ 

106 return self 

107 

108 def get_container_info(self, labels: dict = None, metadata: dict = None): 

109 """ 

110 Expands the base labels and metadata used to populate evidences. 

111 

112 Parameters 

113 ---------- 

114 labels : dict, optional 

115 The default labels can be expanded by the user when defining a new evaluator. 

116 A label is in general any information necessary to identify evidences in the Credo AI Platform, 

117 therefore, by default None. 

118 metadata : dict, optional 

119 Any extra info the user wants to associate to the evidences. Compared 

120 to labels these are not necessary for evidence identification, by default None. 

121 """ 

122 info = self._base_container_info() 

123 if labels: 

124 info["labels"].update(labels) 

125 if metadata: 

126 info["metadata"].update(metadata) 

127 return info 

128 

129 def _base_container_info(self): 

130 """Extract basic info to populate labels and metadata.""" 

131 meta = {**self.metadata, **self._get_artifacts()} 

132 labels = {"evaluator": self.name} 

133 if "dataset_type" in meta: 

134 labels["dataset_type"] = meta["dataset_type"] 

135 return {"labels": labels, "metadata": meta} 

136 

137 def _get_artifacts(self): 

138 """ 

139 Extract artifacts that will be used by the evaluator. 

140 

141 The method also extract name info from the available artifacts. 

142 """ 

143 artifacts = {} 

144 save_keys = { 

145 "model": "model_name", 

146 "data": "dataset_name", 

147 "assessment_data": "assessment_dataset_name", 

148 "training_data": "training_dataset_name", 

149 } 

150 for k in self.artifact_keys: 

151 save_key = save_keys.get(k, k) 

152 try: 

153 artifacts[save_key] = self.__dict__[k].name 

154 except AttributeError: 

155 pass 

156 return artifacts 

157 

158 def _init_artifacts(self, artifacts): 

159 """Adds artifacts to evaluator object 

160 

161 Parameters 

162 ---------- 

163 artifacts : dict 

164 Dictionary of artifacts, e.g. {'model': Model} 

165 """ 

166 self.artifact_keys = list(artifacts.keys()) 

167 self.__dict__.update(artifacts) 

168 

169 @abstractmethod 

170 def _setup(self): 

171 """ 

172 Contains any extra steps necessary to initialize the evaluator 

173 """ 

174 pass 

175 

176 @abstractmethod 

177 def _validate_arguments(self): 

178 """ 

179 Check that basic requirements for the run of an evaluator are met. 

180 """ 

181 pass