Coverage for credoai/evaluators/evaluator.py: 92%
73 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-13 21:56 +0000
1from abc import ABC, abstractmethod
3from connect.evidence import EvidenceContainer
5from credoai import __version__ as version
6from credoai.utils import global_logger
7from credoai.utils.common import NotRunError, ValidationError
9ARTIFACT_LABELS = {
10 "model": "model_name",
11 "data": "dataset_name",
12 "assessment_data": "assessment_dataset_name",
13 "training_data": "training_dataset_name",
14 "sensitive_features": "sensitive_feature",
15}
18class Evaluator(ABC):
19 """
20 Base abstract class for all lens evaluators.
22 Defines basic functions required from any evaluator object.
24 This class leverages the special method `__call__` to make artifacts
25 available in the class enclosure.
27 .. automethod:: __call__
28 .. automethod:: _init_artifacts
29 .. automethod:: _validate_arguments
30 .. automethod:: _setup
31 """
33 def __init__(self):
34 self._results = None
35 self.artifact_keys = []
36 self.logger = global_logger
37 self.metadata = {}
39 @property
40 def name(self):
41 """The name associated to the Evaluator, equals the class name."""
42 return self.__class__.__name__
44 @property
45 def results(self):
46 """
47 Container for all results.
49 It is expected to be a list of EvidenceContainers. This is enforced in
50 the associated setter method.
52 Raises
53 ------
54 NotRunError
55 It indicates that results are missing, the evaluator was not run.
56 """
57 if self._results is not None:
58 return self._results
59 else:
60 raise NotRunError(
61 "No results available, please call the method: 'evaluate'."
62 )
64 @results.setter
65 def results(self, results):
66 """Requires the results to be list of Evidence Containers"""
67 if not isinstance(results, list):
68 raise ValidationError("Results must be a list")
69 for result in results:
70 if not isinstance(result, EvidenceContainer):
71 raise ValidationError("All results must be EvidenceContainers")
72 self._results = results
74 @property
75 @abstractmethod
76 def required_artifacts(self):
77 """
78 The required artifacts necessary for the functioning of the evaluator
80 This set contains the :ref:`artifacts<credoai.artifacts>` that Lens can feed to
81 an evaluator, the accepted values are ``{"model", "assessment_data", "training_data", "data"}``.
83 The string "data" means that the evaluator can be run on assessment and/or training data
84 (DataProfiler is an example). Lens will iterate over all the available artifacts internally.
86 The set can also include the string "sensitive_feature". This is to indicate
87 that the evaluator depends on sensitive features. Lens will iterate over the available sensitive
88 features internally.
89 """
90 pass
92 def __call__(self, **kwargs):
93 """
94 This method is used to pass the model, assessment_data and training_data
95 artifacts to instantiated evaluator.
97 The method is called internally by the Lens instance, which only passes the
98 artifacts specified in the property :meth:`required_artifacts<Evaluator.required_artifacts>`.
100 After the artifacts are passed, it performs arguments validation and calls :meth:`_setup<Evaluator._setup>`
102 At the end of these operation, the validated artifacts are available in the evaluator enclosure.
103 """
104 self._init_artifacts(kwargs)
105 self._validate_arguments()
106 self._setup()
107 return self
109 @abstractmethod
110 def evaluate(self):
111 """
112 Execute any data/model processing required for the evaluator.
114 Populates the self.results object.
115 """
116 return self
118 def get_info(self, labels: dict = None, metadata: dict = None):
119 """
120 Expands the base labels and metadata used to populate evidences.
122 Parameters
123 ----------
124 labels : dict, optional
125 The default labels can be expanded by the user when defining a new evaluator.
126 A label is in general any information necessary to identify evidences in the Credo AI Platform,
127 therefore, by default None.
128 metadata : dict, optional
129 Any extra info the user wants to associate to the evidences. Compared
130 to labels these are not necessary for evidence identification, by default None.
131 """
132 info = self._base_info()
133 if labels:
134 info["labels"].update(labels)
135 if metadata:
136 info["metadata"].update(metadata)
137 return info
139 def _base_info(self):
140 """Extract basic info to populate labels and metadata."""
141 meta = {
142 **self.metadata,
143 **self._get_artifacts(),
144 "source": f"CredoAILens_{version}",
145 }
146 labels = {"evaluator": self.name}
147 # transfer some metadata to labels
148 meta_to_label = ["dataset_type", "sensitive_feature"]
149 for key in meta_to_label:
150 if key in meta:
151 labels[key] = meta[key]
152 return {"labels": labels, "metadata": meta}
154 def _get_artifacts(self):
155 """
156 Extract artifacts that will be used by the evaluator.
158 The method also extracts name info from the available artifacts.
159 """
160 artifacts = {}
161 for k in self.artifact_keys:
162 save_key = ARTIFACT_LABELS.get(k, k)
163 try:
164 artifacts[save_key] = self.__dict__[k].name
165 except AttributeError:
166 pass
167 return artifacts
169 def _init_artifacts(self, artifacts):
170 """Adds artifacts to evaluator object
172 Parameters
173 ----------
174 artifacts : dict
175 Dictionary of artifacts, e.g. {'model': Model}
176 """
177 self.artifact_keys = list(artifacts.keys())
178 self.__dict__.update(artifacts)
180 @abstractmethod
181 def _setup(self):
182 """
183 Contains any extra steps necessary to initialize the evaluator
184 """
185 pass
187 @abstractmethod
188 def _validate_arguments(self):
189 """
190 Check that basic requirements for the run of an evaluator are met.
191 """
192 pass