Coverage for credoai/prism/comparators/comparator.py: 82%

1from abc import ABC, abstractclassmethod 

2 

3 

4# Utility for EvidenceType -> ComparatorType 

5 

6 

7class Comparator(ABC): 

8 """ 

9 Abstract base class for Lens/Prism Comparators 

10 

11 Comparators provide functionality to assess differences in Lens Evaluator results 

12 across datasets, models, etc. Some differences will be numeric while others will be 

13 "change tracking". 

14 

15 Parameters 

16 ---------- 

17 EvidenceContainers: Iterable of EvidenceContainer objects 

18 

19 

20 Different Comparator will exist for each possible EvidenceContainer input. 

21 """ 

22 

23 def __init__(self, EvidenceContainers): 

24 # attributes all comparators will need 

25 self.EvidenceContainers = EvidenceContainers 

26 self.evaluations = set() # to contain all evaluations run, e.g. each metric 

27 self.comparisons = {} # internal container for tracking results of comparisons 

28 self._validate() 

29 self._setup() 

30 

31 @abstractclassmethod 

32 def _setup(self): 

33 """""" 

34 ... 

35 

36 @abstractclassmethod 

37 def _validate(self): 

38 """ 

39 Comparator specific validations, e.g., for metric comparators check 

40 all object passed are of type MetricContainer, etc... 

41 """ 

42 ... 

43 

44 @abstractclassmethod 

45 def compare(self): 

46 """The main function to run the comparison logic""" 

47 ...