Ignore:
Files:
10 added
4 edited

Legend:

Unmodified
Added
Removed
  • Orange/evaluation/scoring.py

    r10367 r10426  
    770770    return ss, df, statc.chisqprob(ss, df) 
    771771 
    772 @deprecated_keywords({"confm": "confusion_matrix"}) 
    773 def sens(confusion_matrix): 
    774     """ 
    775     Return `sensitivity 
     772class CMScore(list): 
     773    """ 
     774    :param test_results: :obj:`~Orange.evaluation.testing.ExperimentResults` 
     775                         or list of :obj:`ConfusionMatrix`. 
     776    :rtype: list of scores, one for each learner.""" 
     777    @deprecated_keywords({"confm": "test_results"}) 
     778    def __init__(self, test_results=None): 
     779        super(CMScore, self).__init__() 
     780 
     781        if test_results is not None: 
     782            self[:] = self.__call__(test_results) 
     783 
     784    def __call__(self, test_results): 
     785        if isinstance(test_results, testing.ExperimentResults): 
     786            test_results = confusion_matrices(test_results, class_index=1) 
     787        if isinstance(test_results, ConfusionMatrix): 
     788            test_results = [test_results] 
     789 
     790        return map(self.compute, test_results) 
     791 
     792 
     793 
     794class Sensitivity(CMScore): 
     795    __doc__ = """Compute `sensitivity 
    776796    <http://en.wikipedia.org/wiki/Sensitivity_and_specificity>`_ (proportion 
    777797    of actual positives which are correctly identified as such). 
    778     """ 
    779     if type(confusion_matrix) == list: 
    780         return [sens(cm) for cm in confusion_matrix] 
    781     else: 
     798    """ + CMScore.__doc__ 
     799    @classmethod 
     800    def compute(self, confusion_matrix): 
    782801        tot = confusion_matrix.TP+confusion_matrix.FN 
    783802        if tot < 1e-6: 
    784803            import warnings 
    785804            warnings.warn("Can't compute sensitivity: one or both classes have no instances") 
    786             return -1 
     805            return None 
    787806 
    788807        return confusion_matrix.TP / tot 
    789808 
    790809 
    791 @deprecated_keywords({"confm": "confusion_matrix"}) 
    792 def recall(confusion_matrix): 
    793     """ 
    794     Return `recall <http://en.wikipedia.org/wiki/Precision_and_recall>`_ 
     810class Recall(Sensitivity): 
     811    __doc__ = """ Compute `recall 
     812    <http://en.wikipedia.org/wiki/Precision_and_recall>`_ 
    795813    (fraction of relevant instances that are retrieved). 
    796     """ 
    797     return sens(confusion_matrix) 
    798  
    799  
    800 @deprecated_keywords({"confm": "confusion_matrix"}) 
    801 def spec(confusion_matrix): 
    802     """ 
    803     Return `specificity 
     814    """ + CMScore.__doc__ 
     815    pass # Recall == Sensitivity 
     816 
     817 
     818class Specificity(CMScore): 
     819    __doc__ = """Compute `specificity 
    804820    <http://en.wikipedia.org/wiki/Sensitivity_and_specificity>`_ 
    805821    (proportion of negatives which are correctly identified). 
    806     """ 
    807     if type(confusion_matrix) == list: 
    808         return [spec(cm) for cm in confusion_matrix] 
    809     else: 
     822    """ + CMScore.__doc__ 
     823    @classmethod 
     824    def compute(self, confusion_matrix): 
    810825        tot = confusion_matrix.FP+confusion_matrix.TN 
    811826        if tot < 1e-6: 
    812827            import warnings 
    813828            warnings.warn("Can't compute specificity: one or both classes have no instances") 
    814             return -1 
     829            return None 
    815830        return confusion_matrix.TN / tot 
    816831 
    817832 
    818 @deprecated_keywords({"confm": "confusion_matrix"}) 
    819 def PPV(confusion_matrix): 
    820     """ 
    821     Return `positive predictive value 
     833class PPV(CMScore): 
     834    __doc__ = """Compute `positive predictive value 
    822835    <http://en.wikipedia.org/wiki/Positive_predictive_value>`_ (proportion of 
    823     subjects with positive test results who are correctly diagnosed).""" 
    824     if type(confusion_matrix) == list: 
    825         return [PPV(cm) for cm in confusion_matrix] 
    826     else: 
     836    subjects with positive test results who are correctly diagnosed). 
     837    """ + CMScore.__doc__ 
     838    @classmethod 
     839    def compute(self, confusion_matrix): 
    827840        tot = confusion_matrix.TP + confusion_matrix.FP 
    828841        if tot < 1e-6: 
    829842            import warnings 
    830843            warnings.warn("Can't compute PPV: one or both classes have no instances") 
    831             return -1 
     844            return None 
    832845        return confusion_matrix.TP/tot 
    833846 
    834847 
    835 @deprecated_keywords({"confm": "confusion_matrix"}) 
    836 def precision(confusion_matrix): 
    837     """ 
    838     Return `precision <http://en.wikipedia.org/wiki/Precision_and_recall>`_ 
     848class Precision(PPV): 
     849    __doc__ = """Compute `precision <http://en.wikipedia.org/wiki/Precision_and_recall>`_ 
    839850    (retrieved instances that are relevant). 
    840     """ 
    841     return PPV(confusion_matrix) 
    842  
    843 @deprecated_keywords({"confm": "confusion_matrix"}) 
    844 def NPV(confusion_matrix): 
    845     """ 
    846     Return `negative predictive value 
     851    """ + CMScore.__doc__ 
     852    pass # Precision == PPV 
     853 
     854 
     855class NPV(CMScore): 
     856    __doc__ = """Compute `negative predictive value 
    847857    <http://en.wikipedia.org/wiki/Negative_predictive_value>`_ (proportion of 
    848858    subjects with a negative test result who are correctly diagnosed). 
    849      """ 
    850     if type(confusion_matrix) == list: 
    851         return [NPV(cm) for cm in confusion_matrix] 
    852     else: 
     859     """ + CMScore.__doc__ 
     860    @classmethod 
     861    def compute(self, confusion_matrix): 
    853862        tot = confusion_matrix.FN + confusion_matrix.TN 
    854863        if tot < 1e-6: 
    855864            import warnings 
    856865            warnings.warn("Can't compute NPV: one or both classes have no instances") 
    857             return -1 
     866            return None 
    858867        return confusion_matrix.TN / tot 
    859868 
    860 @deprecated_keywords({"confm": "confusion_matrix"}) 
    861 def F1(confusion_matrix): 
    862     """ 
    863     Return `F1 score <http://en.wikipedia.org/wiki/F1_score>`_ 
     869 
     870class F1(CMScore): 
     871    __doc__ = """Return `F1 score 
     872    <http://en.wikipedia.org/wiki/F1_score>`_ 
    864873    (harmonic mean of precision and recall). 
    865     """ 
    866     if type(confusion_matrix) == list: 
    867         return [F1(cm) for cm in confusion_matrix] 
    868     else: 
    869         p = precision(confusion_matrix) 
    870         r = recall(confusion_matrix) 
    871         if p + r > 0: 
     874    """ + CMScore.__doc__ 
     875    @classmethod 
     876    def compute(self, confusion_matrix): 
     877        p = Precision.compute(confusion_matrix) 
     878        r = Recall.compute(confusion_matrix) 
     879        if p is not None and r is not None and (p+r) != 0: 
    872880            return 2. * p * r / (p + r) 
    873881        else: 
    874882            import warnings 
    875883            warnings.warn("Can't compute F1: P + R is zero or not defined") 
    876             return -1 
    877  
    878  
    879 @deprecated_keywords({"confm": "confusion_matrix"}) 
    880 def Falpha(confusion_matrix, alpha=1.0): 
    881     """ 
    882     Return the alpha-mean of precision and recall over the given confusion 
     884            return None 
     885 
     886 
     887class Falpha(CMScore): 
     888    __doc__ = """Compute the alpha-mean of precision and recall over the given confusion 
    883889    matrix. 
    884     """ 
    885     if type(confusion_matrix) == list: 
    886         return [Falpha(cm, alpha=alpha) for cm in confusion_matrix] 
    887     else: 
    888         p = precision(confusion_matrix) 
    889         r = recall(confusion_matrix) 
    890         return (1. + alpha) * p * r / (alpha * p + r) 
    891  
    892  
    893 @deprecated_keywords({"confm": "confusion_matrix"}) 
    894 def MCC(confusion_matrix): 
    895     """ 
    896     Return `Matthew correlation coefficient 
     890    """ + CMScore.__doc__ 
     891 
     892    def __init__(self, test_results, alpha=1.): 
     893        self.alpha = alpha 
     894        super(Falpha, self).__init__(test_results) 
     895 
     896    def compute(self, confusion_matrix): 
     897        p = Precision.compute(confusion_matrix) 
     898        r = Recall.compute(confusion_matrix) 
     899        return (1. + self.alpha) * p * r / (self.alpha * p + r) 
     900 
     901 
     902class MCC(CMScore): 
     903    __doc__ = """Compute `Matthew correlation coefficient 
    897904    <http://en.wikipedia.org/wiki/Matthews_correlation_coefficient>`_ 
    898905    (correlation coefficient between the observed and predicted binary 
    899906    classifications). 
    900     """ 
    901     # code by Boris Gorelik 
    902     if type(confusion_matrix) == list: 
    903         return [MCC(cm) for cm in confusion_matrix] 
    904     else: 
    905         truePositive = confusion_matrix.TP 
    906         trueNegative = confusion_matrix.TN 
    907         falsePositive = confusion_matrix.FP 
    908         falseNegative = confusion_matrix.FN 
     907    """ + CMScore.__doc__ 
     908    @classmethod 
     909    def compute(self, cm): 
     910        # code by Boris Gorelik 
     911        TP, TN, FP, FN = cm.TP, cm.TN, cm.FP, cm.FN 
    909912           
    910         try:    
    911             r = (((truePositive * trueNegative) - 
    912                   (falsePositive * falseNegative)) / 
    913                  math.sqrt((truePositive + falsePositive) * 
    914                            (truePositive + falseNegative) *  
    915                            (trueNegative + falsePositive) *  
    916                            (trueNegative + falseNegative))) 
     913        try: 
     914            return (TP*TN - FP*FN) /\ 
     915                 math.sqrt((TP+FP) * (TP+FN) * (TN+ FP) * (TN+FN)) 
    917916        except ZeroDivisionError: 
    918             # Zero difision occurs when there is either no true positives  
     917            # Zero division occurs when there is either no true positives 
    919918            # or no true negatives i.e. the problem contains only one  
    920919            # type of classes. 
    921920            import warnings 
    922921            warnings.warn("Can't compute MCC: TP or TN is zero or not defined") 
    923             r = None 
    924  
    925     return r 
    926922 
    927923 
     
    976972       ret = (prActual - prExpected) / (1.0 - prExpected) 
    977973       return ret 
     974 
     975# Backward compatibility 
     976sens = Sensitivity 
     977spec = Specificity 
     978precision = Precision 
     979recall = Recall 
     980 
     981 
    978982 
    979983@deprecated_keywords({"classIndex": "class_index", 
     
    14681472    """ 
    14691473    Compute the area under ROC curve given a set of experimental results. 
    1470     For multivalued class problems, return the result of 
    1471     :obj:`by_weighted_pairs`. 
    14721474    If testing consisted of multiple folds, each fold is scored and the 
    14731475    average score is returned. If a fold contains only instances with the 
     
    14761478    :param test_results: test results to score 
    14771479    :param ignore_weights: ignore instance weights when calculating score 
    1478     :param method: DEPRECATED, call the appropriate method directly. 
    1479     """ 
     1480    :param multiclass: tells what kind of averaging to perform if the target 
     1481                       class has more than 2 values. 
     1482    """ 
     1483 
     1484    #!Compute AUC for each pair of classes (ignoring instances of all other 
     1485    #!classes) and average the results, weighting them by the number of 
     1486    #!pairs of instances from these two classes (e.g. by the product of 
     1487    #!probabilities of the two classes). AUC computed in this way still 
     1488    #!behaves as the concordance index, e.g., gives the probability that two 
     1489    #!randomly chosen instances from different classes will be correctly 
     1490    #!recognized (if the classifier knows from which two classes the 
     1491    #!instances came). 
     1492    ByWeightedPairs = 0 
     1493 
     1494    #!Similar to ByWeightedPairs, except that the average over class pairs 
     1495    #!is not weighted. This AUC is, like the binary version, independent of 
     1496    #!class distributions, but it is not related to the concordance index 
     1497    #!any more. 
     1498    ByPairs = 1 
     1499 
     1500    #!For each class, it computes AUC for this class against all others (that 
     1501    #!is, treating other classes as one class). The AUCs are then averaged by 
     1502    #!the class probabilities. This is related to the concordance index in 
     1503    #!which we test the classifier's (average) capability of distinguishing 
     1504    #!the instances from a specified class from those that come from other 
     1505    #!classes. 
     1506    #!Unlike the binary AUC, the measure is not independent of class 
     1507    #!distributions. 
     1508    WeightedOneAgainstAll = 2 
     1509 
     1510    #!Similar to weighted_one_against_all, except that the average 
     1511    #!is not weighted. 
     1512    OneAgainstAll = 3 
    14801513 
    14811514    @replace_use_weights 
    1482     def __init__(self, test_results=None, method=0, ignore_weights=False): 
     1515    @deprecated_keywords({"method": "multiclass"}) 
     1516    def __init__(self, test_results=None, multiclass=ByWeightedPairs, ignore_weights=False): 
    14831517 
    14841518        super(AUC, self).__init__() 
    14851519 
    14861520        self.ignore_weights=ignore_weights 
    1487         self.method=method 
     1521        self.method=multiclass 
    14881522 
    14891523        if test_results is not None: 
    1490             self.__call__(test_results) 
     1524            self[:] = self.__call__(test_results) 
    14911525 
    14921526    def __call__(self, test_results): 
     
    14941528            raise ValueError("Cannot compute AUC on a single-class problem") 
    14951529        elif len(test_results.class_values) == 2: 
    1496             self._compute_for_binary_class(test_results) 
    1497         else: 
    1498             self._compute_for_multi_value_class(test_results, self.method) 
    1499  
    1500     @classmethod 
    1501     def by_weighted_pairs(cls, res, ignore_weights=False): 
    1502         """ 
    1503         Compute AUC for each pair of classes (ignoring instances of all other 
    1504         classes) and average the results, weighting them by the number of 
    1505         pairs of instances from these two classes (e.g. by the product of 
    1506         probabilities of the two classes). AUC computed in this way still 
    1507         behaves as the concordance index, e.g., gives the probability that two 
    1508         randomly chosen instances from different classes will be correctly 
    1509         recognized (if the classifier knows from which two classes the 
    1510         instances came). 
    1511         """ 
    1512         auc = AUC(ignore_weights=ignore_weights) 
    1513         auc._compute_for_multi_value_class(res, method=cls.ByWeightedPairs) 
    1514         return auc 
    1515  
    1516     @classmethod 
    1517     def by_pairs(cls, res, ignore_weights=False): 
    1518         """ 
    1519         Similar to by_weighted_pairs, except that the average over class pairs 
    1520         is not weighted. This AUC is, like the binary version, independent of 
    1521         class distributions, but it is not related to the concordance index 
    1522         any more. 
    1523         """ 
    1524         auc = AUC(ignore_weights=ignore_weights) 
    1525         auc._compute_for_multi_value_class(res, method=cls.ByPairs) 
    1526         return auc 
    1527  
    1528     @classmethod 
    1529     def weighted_one_against_all(cls, res, ignore_weights=False): 
    1530         """ 
    1531         For each class, it computes AUC for this class against all others (that 
    1532         is, treating other classes as one class). The AUCs are then averaged by 
    1533         the class probabilities. This is related to the concordance index in 
    1534         which we test the classifier's (average) capability of distinguishing 
    1535         the instances from a specified class from those that come from other 
    1536         classes. 
    1537         Unlike the binary AUC, the measure is not independent of class 
    1538         distributions. 
    1539         """ 
    1540         auc = AUC(ignore_weights=ignore_weights) 
    1541         auc._compute_for_multi_value_class(res, 
    1542             method=cls.WeightedOneAgainstAll) 
    1543         return auc 
    1544  
    1545     @classmethod 
    1546     def one_against_all(cls, res, ignore_weights=False): 
    1547         """ 
    1548         Similar to weighted_one_against_all, except that the average 
    1549         is not weighted. 
    1550         """ 
    1551         auc = AUC(ignore_weights=ignore_weights) 
    1552         auc._compute_for_multi_value_class(res, method=cls.OneAgainstAll) 
    1553         return auc 
    1554  
    1555     @classmethod 
    1556     def single_class(cls, res, class_index=-1, ignore_weights=False): 
    1557         """ 
    1558         Compute AUC where the class with the given class_index is singled 
    1559         out and all other classes are treated as a single class. 
    1560         """ 
    1561         if class_index < 0: 
    1562             if res.base_class >= 0: 
    1563                 class_index = res.base_class 
    1564             else: 
    1565                 class_index = 1 
    1566  
    1567         auc = AUC(ignore_weights=ignore_weights) 
    1568         auc._compute_for_single_class(res, class_index) 
    1569         return auc 
    1570  
    1571     @classmethod 
    1572     def pair(cls, res, class_index1, class_index2, ignore_weights=False): 
    1573         """ 
    1574         Computes AUC between a pair of classes, ignoring instances from all 
    1575         other classes. 
    1576         """ 
    1577         auc = AUC(ignore_weights=ignore_weights) 
    1578         auc._compute_for_pair_of_classes(res, class_index1, class_index2) 
    1579         return auc 
    1580  
    1581     @classmethod 
    1582     def matrix(cls, res, ignore_weights=False): 
    1583         """ 
    1584         Compute a (lower diagonal) matrix with AUCs for all pairs of classes. 
    1585         If there are empty classes, the corresponding elements in the matrix 
    1586         are -1. 
    1587         """ 
    1588         auc = AUC(ignore_weights=ignore_weights) 
    1589         auc._compute_matrix(res) 
    1590         return auc 
     1530            return self._compute_for_binary_class(test_results) 
     1531        else: 
     1532            return self._compute_for_multi_value_class(test_results, self.method) 
    15911533 
    15921534    def _compute_for_binary_class(self, res): 
     
    15981540                (-1, res, res.number_of_iterations)) 
    15991541        else: 
    1600             auc, _ = self._compute_one_class_against_all(res, -1) 
    1601             self[:] = auc 
    1602             return self 
     1542            return self._compute_one_class_against_all(res, -1)[0] 
    16031543 
    16041544    def _compute_for_multi_value_class(self, res, method=0): 
     
    16541594            sum_aucs = [x/usefulClassPairs for x in sum_aucs] 
    16551595 
    1656         self[:] = sum_aucs 
    1657         return self 
     1596        return sum_aucs 
    16581597 
    16591598    # computes the average AUC over folds using "AUCcomputer" (AUC_i or AUC_ij) 
     
    16621601    # over all folds or even this failed; 
    16631602    # in these cases the result is returned immediately 
    1664     @deprecated_keywords({"AUCcomputer": "auc_computer", 
    1665                           "computerArgs": "computer_args"}) 
    16661603    def _compute_for_multiple_folds(self, auc_computer, iterations, 
    16671604                                 computer_args): 
     
    16711608            aucs, foldsUsed = auc_computer(*(ite, ) + computer_args) 
    16721609            if not aucs: 
    1673                 return None 
     1610                import warnings 
     1611                warnings.warn("AUC cannot be computed (all instances belong to the same class).") 
     1612                return 
    16741613            if not foldsUsed: 
     1614                self[:] = aucs 
    16751615                return aucs 
    16761616            subsum_aucs = map(add, subsum_aucs, aucs) 
    1677         self[:] = subsum_aucs 
    1678         return self 
     1617        return subsum_aucs 
    16791618 
    16801619    # Computes AUC 
     
    16841623    def _compute_for_single_class(self, res, class_index): 
    16851624        if res.number_of_iterations > 1: 
    1686             self._compute_for_multiple_folds( 
     1625            return self._compute_for_multiple_folds( 
    16871626                self._compute_one_class_against_all, split_by_iterations(res), 
    16881627                (class_index, res, res.number_of_iterations)) 
    16891628        else: 
    1690             self._compute_one_class_against_all(res, class_index) 
     1629            return self._compute_one_class_against_all(res, class_index) 
    16911630 
    16921631    # Computes AUC for a pair of classes (as if there were no other classes) 
     
    16951634    def _compute_for_pair_of_classes(self, res, class_index1, class_index2): 
    16961635        if res.number_of_iterations > 1: 
    1697             self._compute_for_multiple_folds( 
     1636            return self._compute_for_multiple_folds( 
    16981637                self._compute_one_class_against_another, 
    16991638                split_by_iterations(res), 
    17001639                (class_index1, class_index2, res, res.number_of_iterations)) 
    17011640        else: 
    1702             self._compute_one_class_against_another(res, class_index1, 
     1641            return self._compute_one_class_against_another(res, class_index1, 
    17031642                                                    class_index2) 
    17041643 
     
    17521691        return False, False 
    17531692 
    1754     def _compute_matrix(self, res): 
    1755         numberOfClasses = len(res.class_values) 
    1756         number_of_learners = res.number_of_learners 
    1757         if res.number_of_iterations > 1: 
    1758             iterations, all_ite = split_by_iterations(res), res 
    1759         else: 
    1760             iterations, all_ite = [res], None 
     1693class AUC_for_single_class(AUC): 
     1694    """ 
     1695    Compute AUC where the class with the given class_index is singled 
     1696    out and all other classes are treated as a single class. 
     1697    """ 
     1698    def __init__(self, test_results=None, class_index=-1, ignore_weights=False): 
     1699        if class_index < 0: 
     1700            if test_results and test_results.base_class >= 0: 
     1701                self.class_index = test_results.base_class 
     1702            else: 
     1703                self.class_index = 1 
     1704        else: 
     1705            self.class_index = class_index 
     1706 
     1707        super(AUC_for_single_class, self).__init__(test_results, ignore_weights=ignore_weights) 
     1708 
     1709    def __call__(self, test_results): 
     1710        return self._compute_for_single_class(test_results, self.class_index) 
     1711 
     1712 
     1713class AUC_for_pair_of_classes(AUC): 
     1714    """ 
     1715    Computes AUC between a pair of classes, ignoring instances from all 
     1716    other classes. 
     1717    """ 
     1718    def __init__(self, test_results, class_index1, class_index2, ignore_weights=False): 
     1719        self.class_index1 = class_index1 
     1720        self.class_index2 = class_index2 
     1721 
     1722        super(AUC_for_pair_of_classes, self).__init__(test_results, ignore_weights=ignore_weights) 
     1723 
     1724    def __call__(self, test_results): 
     1725        return self._compute_for_pair_of_classes(test_results, self.class_index1, self.class_index2) 
     1726 
     1727 
     1728class AUC_matrix(AUC): 
     1729    """ 
     1730    Compute a (lower diagonal) matrix with AUCs for all pairs of classes. 
     1731    If there are empty classes, the corresponding elements in the matrix 
     1732    are -1. 
     1733    """ 
     1734 
     1735    def __call__(self, test_results): 
     1736        numberOfClasses = len(test_results.class_values) 
     1737        number_of_learners = test_results.number_of_learners 
     1738        if test_results.number_of_iterations > 1: 
     1739            iterations, all_ite = split_by_iterations(test_results), test_results 
     1740        else: 
     1741            iterations, all_ite = [test_results], None 
    17611742        aucs = [[[] for _ in range(numberOfClasses)] 
    1762                 for _ in range(number_of_learners)] 
     1743        for _ in range(number_of_learners)] 
    17631744        for classIndex1 in range(numberOfClasses): 
    17641745            for classIndex2 in range(classIndex1): 
     
    17661747                    self._compute_one_class_against_another, iterations, 
    17671748                    (classIndex1, classIndex2, all_ite, 
    1768                      res.number_of_iterations)) 
     1749                     test_results.number_of_iterations)) 
    17691750                if pair_aucs: 
    17701751                    for lrn in range(number_of_learners): 
     
    17731754                    for lrn in range(number_of_learners): 
    17741755                        aucs[lrn][classIndex1].append(-1) 
    1775         self[:] = aucs 
    17761756        return aucs 
    17771757 
    17781758#Backward compatibility 
    1779 AUC.ByWeightedPairs = 0 
    1780 AUC.ByPairs = 1 
    1781 AUC.WeightedOneAgainstAll = 2 
    1782 AUC.OneAgainstAll = 3 
    1783  
    17841759@replace_use_weights 
    17851760def AUC_binary(res, ignore_weights=False): 
     
    17951770    return auc 
    17961771 
     1772 
     1773@deprecated_keywords({"AUCcomputer": "auc_computer", 
     1774                      "computerArgs": "computer_args"}) 
    17971775def AUC_iterations(auc_computer, iterations, computer_args): 
    17981776    auc = deprecated_function_name(AUC)() 
     
    18091787def AUC_i(ite, class_index, ignore_weights=False, all_ite=None, 
    18101788          divide_by_if_ite=1.): 
    1811     auc = deprecated_function_name(AUC)() 
     1789    auc = deprecated_function_name(AUC)(ignore_weights=ignore_weights) 
    18121790    result = auc._compute_one_class_against_another(ite, class_index, 
    1813         all_ite=None, divide_by_if_ite=1.) 
     1791        all_ite=all_ite, divide_by_if_ite=divide_by_if_ite) 
    18141792    return result 
    18151793 
     
    18201798    auc = deprecated_function_name(AUC)(ignore_weights=ignore_weights) 
    18211799    result = auc._compute_one_class_against_another( 
    1822         ite, class_index1, class_index2, all_ite=None, divide_by_if_ite=1.) 
     1800        ite, class_index1, class_index2, all_ite=all_ite, divide_by_if_ite=divide_by_if_ite) 
    18231801    return result 
    1824  
    1825  
    1826  
    1827  
    1828 #AUC_binary = replace_use_weights(deprecated_function_name(AUC()._compute_for_binary_class)) 
    1829 #AUC_multi = replace_use_weights(deprecated_function_name(AUC._compute_for_multi_value_class)) 
    1830 #AUC_iterations = replace_use_weights(deprecated_function_name(AUC._compute_for_multiple_folds)) 
    1831 #AUC_x = replace_use_weights(deprecated_function_name(AUC._compute_auc)) 
    1832 #AUC_i = replace_use_weights(deprecated_function_name(AUC._compute_one_class_against_all)) 
    1833 #AUC_ij = replace_use_weights(deprecated_function_name(AUC._compute_one_class_against_another)) 
    18341802 
    18351803AUC_single = replace_use_weights( 
    18361804             deprecated_keywords({"classIndex": "class_index"})( 
    1837              deprecated_function_name(AUC.single_class))) 
     1805             deprecated_function_name(AUC_for_single_class))) 
    18381806AUC_pair = replace_use_weights( 
    18391807           deprecated_keywords({"classIndex1": "class_index1", 
    18401808                                "classIndex2": "class_index2"})( 
    1841            deprecated_function_name(AUC.pair))) 
    1842 AUC_matrix = replace_use_weights(deprecated_function_name(AUC.matrix)) 
     1809           deprecated_function_name(AUC_for_pair_of_classes))) 
     1810AUC_matrix = replace_use_weights(deprecated_function_name(AUC_matrix)) 
    18431811 
    18441812 
  • Orange/testing/unit/tests/test_evaluation_scoring.py

    r10293 r10426  
    4545        ds = data.Table("iris") 
    4646        test_results = testing.cross_validation([self.learner], ds, folds=5) 
    47         auc = scoring.AUC.by_pairs(test_results) 
     47        auc = scoring.AUC(test_results, multiclass=scoring.AUC.ByPairs) 
    4848 
    4949        self.assertEqual(len(auc), 1) 
     
    5252        ds = data.Table("iris") 
    5353        test_results = testing.cross_validation([self.learner], ds, folds=5) 
    54         auc = scoring.AUC.by_weighted_pairs(test_results) 
     54        auc = scoring.AUC(test_results, multiclass=scoring.AUC.ByWeightedPairs) 
    5555 
    5656        self.assertEqual(len(auc), 1) 
     
    5959        ds = data.Table("iris") 
    6060        test_results = testing.cross_validation([self.learner], ds, folds=5) 
    61         auc = scoring.AUC.one_against_all(test_results) 
     61        auc = scoring.AUC(test_results, multiclass=scoring.AUC.OneAgainstAll) 
    6262 
    6363        self.assertEqual(len(auc), 1) 
     
    6666        ds = data.Table("iris") 
    6767        test_results = testing.cross_validation([self.learner], ds, folds=5) 
    68         auc = scoring.AUC.weighted_one_against_all(test_results) 
     68        auc = scoring.AUC(test_results, multiclass=scoring.AUC.WeightedOneAgainstAll) 
    6969 
    7070        self.assertEqual(len(auc), 1) 
     
    7373        ds = data.Table("iris") 
    7474        test_results = testing.cross_validation([self.learner], ds, folds=5) 
    75         auc = scoring.AUC.single_class(test_results) 
    76         self.assertEqual(len(auc), 1) 
    77         auc = scoring.AUC.single_class(test_results, 0) 
    78         self.assertEqual(len(auc), 1) 
    79         auc = scoring.AUC.single_class(test_results, 1) 
    80         self.assertEqual(len(auc), 1) 
    81         auc = scoring.AUC.single_class(test_results, 2) 
     75        auc = scoring.AUC_for_single_class(test_results) 
     76        self.assertEqual(len(auc), 1) 
     77        auc = scoring.AUC_for_single_class(test_results, 0) 
     78        self.assertEqual(len(auc), 1) 
     79        auc = scoring.AUC_for_single_class(test_results, 1) 
     80        self.assertEqual(len(auc), 1) 
     81        auc = scoring.AUC_for_single_class(test_results, 2) 
    8282        self.assertEqual(len(auc), 1) 
    8383 
     
    8585        ds = data.Table("iris") 
    8686        test_results = testing.cross_validation([self.learner], ds, folds=5) 
    87         auc = scoring.AUC.pair(test_results, 0, 1) 
    88         self.assertEqual(len(auc), 1) 
    89         auc = scoring.AUC.pair(test_results, 0, 2) 
    90         self.assertEqual(len(auc), 1) 
    91         auc = scoring.AUC.pair(test_results, 1, 2) 
     87        auc = scoring.AUC_for_pair_of_classes(test_results, 0, 1) 
     88        self.assertEqual(len(auc), 1) 
     89        auc = scoring.AUC_for_pair_of_classes(test_results, 0, 2) 
     90        self.assertEqual(len(auc), 1) 
     91        auc = scoring.AUC_for_pair_of_classes(test_results, 1, 2) 
    9292        self.assertEqual(len(auc), 1) 
    9393 
     
    9595        ds = data.Table("iris") 
    9696        test_results = testing.cross_validation([self.learner], ds, folds=5) 
    97         auc = scoring.AUC.matrix(test_results) 
     97        auc = scoring.AUC_matrix(test_results) 
    9898        self.assertEqual(len(auc), 1) 
    9999        self.assertEqual(len(auc[0]), 3) 
     
    170170        self.assertTrue(hasattr(cm[0], "TP")) 
    171171 
    172  
    173 class TestConfusionMatrix(unittest.TestCase): 
    174     def test_construct_confusion_matrix_from_multiclass(self): 
    175         learner = random_learner 
    176         ds = data.Table("iris") 
    177         pt = testing.proportion_test([learner], ds, times=1) 
    178         cm = scoring.confusion_matrices(pt) 
    179  
    180         self.assertTrue(isinstance(cm[0], list)) 
    181  
    182  
    183     def test_construct_confusion_matrix_from_biclass(self): 
     172class CMScoreTest(object): 
     173    def test_with_test_results_on_biclass(self): 
     174        learner = random_learner 
     175        ds = data.Table("monks-1") 
     176        pt = testing.proportion_test([learner], ds, times=1) 
     177        scores = self.score(pt) 
     178        self.assertIsInstance(scores, list) 
     179 
     180    def test_with_test_results_on_multiclass(self): 
     181        learner = random_learner 
     182        ds = data.Table("iris") 
     183        pt = testing.proportion_test([learner], ds, times=1) 
     184 
     185        scores = self.score(pt) 
     186        self.assertIsInstance(scores, list) 
     187 
     188    def test_with_confusion_matrix_on_biclass(self): 
    184189        learner = random_learner 
    185190        ds = data.Table("monks-1") 
    186191        pt = testing.proportion_test([learner], ds, times=1) 
    187192        cm = scoring.confusion_matrices(pt, class_index=1) 
    188  
    189         self.assertTrue(hasattr(cm[0], "TP")) 
     193        scores = self.score(cm) 
     194        self.assertIsInstance(scores, list) 
     195 
     196    def test_with_confusion_matrix_on_multiclass(self): 
     197        learner = random_learner 
     198        ds = data.Table("iris") 
     199        pt = testing.proportion_test([learner], ds, times=1) 
     200        cm = scoring.confusion_matrices(pt, class_index=1) 
     201        scores = self.score(cm) 
     202        self.assertIsInstance(scores, list) 
     203 
     204class TestSensitivity(CMScoreTest, unittest.TestCase): 
     205    @property 
     206    def score(self): 
     207        return scoring.Sensitivity 
     208 
     209class TestSpecificity(CMScoreTest, unittest.TestCase): 
     210    @property 
     211    def score(self): 
     212        return scoring.Specificity 
     213 
     214class TestPrecision(CMScoreTest, unittest.TestCase): 
     215    @property 
     216    def score(self): 
     217        return scoring.Precision 
     218 
     219class TestRecall(CMScoreTest, unittest.TestCase): 
     220    @property 
     221    def score(self): 
     222        return scoring.Recall 
     223 
     224class TestPPV(CMScoreTest, unittest.TestCase): 
     225    @property 
     226    def score(self): 
     227        return scoring.PPV 
     228 
     229class TestNPV(CMScoreTest, unittest.TestCase): 
     230    @property 
     231    def score(self): 
     232        return scoring.NPV 
     233 
     234class TestF1(CMScoreTest, unittest.TestCase): 
     235    @property 
     236    def score(self): 
     237        return scoring.F1 
     238 
     239class TestFalpha(CMScoreTest, unittest.TestCase): 
     240    @property 
     241    def score(self): 
     242        return scoring.Falpha 
     243 
     244class TestMCC(CMScoreTest, unittest.TestCase): 
     245    @property 
     246    def score(self): 
     247        return scoring.MCC 
    190248 
    191249if __name__ == '__main__': 
  • docs/reference/rst/Orange.evaluation.scoring.rst

    r10343 r10426  
    2727be passed as well. 
    2828 
    29 .. autoclass:: CA 
    30 .. autofunction:: sens 
    31 .. autofunction:: spec 
     29.. autofunction:: CA 
     30.. autofunction:: Sensitivity 
     31.. autofunction:: Specificity 
    3232.. autofunction:: PPV 
    3333.. autofunction:: NPV 
    34 .. autofunction:: precision 
    35 .. autofunction:: recall 
     34.. autofunction:: Precision 
     35.. autofunction:: Recall 
    3636.. autofunction:: F1 
    3737.. autofunction:: Falpha 
     
    4848.. autofunction:: Brier_score 
    4949 
    50 .. autoclass:: AUC 
    51     :members: by_weighted_pairs, by_pairs, 
    52               weighted_one_against_all, one_against_all, single_class, pair, 
    53               matrix 
    54  
     50.. autofunction:: AUC 
     51.. autofunction:: AUC_for_single_class 
     52.. autofunction:: AUC_matrix 
    5553.. autofunction:: AUCWilcoxon 
    5654 
  • setup.py

    r10391 r10424  
    11#!usr/bin/env python 
     2"""Orange: Machine learning and interactive data mining toolbox. 
     3 
     4Orange is a scriptable environment for fast prototyping of new 
     5algorithms and testing schemes. It is a collection of Python packages 
     6that sit over the core library and implement some functionality for 
     7which execution time is not crucial and which is easier done in Python 
     8than in C++. This includes a variety of tasks such as attribute subset, 
     9bagging and boosting, and alike. 
     10 
     11Orange also includes a set of graphical widgets that use methods from 
     12core library and Orange modules. Through visual programming, widgets 
     13can be assembled together into an application by a visual programming 
     14tool called Orange Canvas. 
     15""" 
     16 
     17DOCLINES = __doc__.split("\n") 
    218 
    319import os, sys         
     
    1834from distutils.msvccompiler import MSVCCompiler 
    1935from distutils.unixccompiler import UnixCCompiler 
    20   
     36import subprocess 
     37 
     38CLASSIFIERS = """\ 
     39Development Status :: 4 - Beta 
     40Programming Language :: Python 
     41License :: OSI Approved :: GNU General Public License (GPL) 
     42Operating System :: POSIX 
     43Operating System :: Microsoft :: Windows 
     44Topic :: Scientific/Engineering :: Artificial Intelligence 
     45Topic :: Scientific/Engineering :: Visualization 
     46Intended Audience :: Education 
     47Intended Audience :: Science/Research 
     48""" 
     49 
     50KEYWORDS = """\ 
     51data mining 
     52machine learning 
     53artificial intelligence 
     54""" 
     55 
     56NAME                = 'Orange' 
     57DESCRIPTION         = DOCLINES[0] 
     58LONG_DESCRIPTION    = "\n".join(DOCLINES[2:]) 
     59URL                 = "http://orange.biolab.si" 
     60DOWNLOAD_URL        = "https://bitbucket.org/biolab/orange/downloads" 
     61LICENSE             = 'GNU General Public License (GPL)' 
     62CLASSIFIERS         = filter(None, CLASSIFIERS.split('\n')) 
     63AUTHOR              = "Bioinformatics Laboratory, FRI UL" 
     64AUTHOR_EMAIL        = "orange@fri.uni-lj.si" 
     65KEYWORDS            = filter(None, KEYWORDS.split('\n')) 
     66MAJOR               = 2 
     67MINOR               = 5 
     68MICRO               = 0 
     69ISRELEASED          = False 
     70VERSION             = '%d.%d.%da5' % (MAJOR, MINOR, MICRO) 
     71 
    2172if have_setuptools: 
    2273    setuptools_args = {"zip_safe": False, 
     
    542593                      ) 
    543594     
    544 import fnmatch 
    545 matches = [] 
    546  
    547 #Recursively find '__init__.py's 
    548 for root, dirnames, filenames in os.walk('Orange'):  
    549  
    550   for filename in fnmatch.filter(filenames, '__init__.py'): 
    551       matches.append(os.path.join(root, filename)) 
    552 packages = [os.path.dirname(pkg).replace(os.path.sep, '.') for pkg in matches] 
    553  
    554  
    555 default_version = "2.5a4" 
    556 ############################################ 
    557 # Try to get the hg revision. Do 
    558 #     $ hg parent --template="2.5a3-r{rev} > version 
    559 # before running setup.py (note the example shown does  
    560 # not conform to StrictVersion needed by bdist_msi).  
    561 ############################################ 
    562 if os.path.exists("version"): 
    563     f = open("version") 
    564     version = f.read() 
    565 else: 
    566     version = default_version 
    567      
    568  
    569 setup(cmdclass={"build_ext": pyxtract_build_ext, 
    570                 "install_lib": my_install_lib, 
    571                 "install": my_install}, 
    572       name ="Orange", 
    573       version = version, 
    574       description = "Machine learning and interactive data mining toolbox.", 
    575       author = "Bioinformatics Laboratory, FRI UL", 
    576       author_email = "orange@fri.uni-lj.si", 
    577       url = "http://orange.biolab.si", 
    578       download_url = "https://bitbucket.org/biolab/orange/downloads", 
    579       packages = packages + ["Orange.OrangeCanvas", 
    580                              "Orange.OrangeWidgets", 
    581                              "Orange.OrangeWidgets.Associate", 
    582                              "Orange.OrangeWidgets.Classify", 
    583                              "Orange.OrangeWidgets.Data", 
    584                              "Orange.OrangeWidgets.Evaluate", 
    585                              "Orange.OrangeWidgets.Prototypes", 
    586                              "Orange.OrangeWidgets.Regression", 
    587                              "Orange.OrangeWidgets.Unsupervised", 
    588                              "Orange.OrangeWidgets.Visualize", 
    589                              "Orange.OrangeWidgets.Visualize Qt", 
    590                              "Orange.OrangeWidgets.plot", 
    591                              "Orange.OrangeWidgets.plot.primitives", 
    592                              ], 
    593        
    594       package_data = { 
    595           "Orange" : ["orangerc.cfg", "doc/datasets/*.tab", "doc/datasets/*.csv", "doc/datasets/*.basket", 
    596                       "doc/networks/*.net", "doc/networks/*.tab", 
    597                       "doc/style.css", "doc/widgets/*/*.*", 
    598                       "testing/regression/tests_20/*.tab", 
    599                       "testing/regression/tests_20/*.net", 
    600                       "testing/regression/tests_20/*.basket", 
    601                       "testing/regression/tests_20/*.csv"], 
    602           "Orange.OrangeCanvas": ["icons/*.png", "orngCanvas.pyw", "WidgetTabs.txt"], 
    603           "Orange.OrangeWidgets":["icons/*.png", "icons/backgrounds/*.png", "report/index.html"], 
    604           "Orange.OrangeWidgets.Associate": ["icons/*.png"], 
    605           "Orange.OrangeWidgets.Classify": ["icons/*.png"], 
    606           "Orange.OrangeWidgets.Data": ["icons/*.png"], 
    607           "Orange.OrangeWidgets.Evaluate": ["icons/*.png"], 
    608           "Orange.OrangeWidgets.Prototypes": ["icons/*.png"], 
    609           "Orange.OrangeWidgets.Regression": ["icons/*.png"], 
    610           "Orange.OrangeWidgets.Unsupervised": ["icons/*.png"], 
    611           "Orange.OrangeWidgets.Visualize": ["icons/*.png"], 
    612           "Orange.OrangeWidgets.Visualize Qt": ["icons/*.png"], 
    613           "Orange.OrangeWidgets.plot": ["*.gs", "*.vs"], 
    614           "Orange.OrangeWidgets.plot.primitives": ["*.obj"], 
    615           }, 
    616       ext_modules = [include_ext, orange_ext, orangeom_ext, 
    617                      orangene_ext, corn_ext, statc_ext], 
    618       scripts = ["bin/orange-canvas"], 
    619       license = "GNU General Public License (GPL)", 
    620       keywords = ["data mining", "machine learning", "artificial intelligence"], 
    621       classifiers = ["Development Status :: 4 - Beta", 
    622                      "Programming Language :: Python", 
    623                      "License :: OSI Approved :: GNU General Public License (GPL)", 
    624                      "Operating System :: POSIX", 
    625                      "Operating System :: Microsoft :: Windows", 
    626                      "Topic :: Scientific/Engineering :: Artificial Intelligence", 
    627                      "Topic :: Scientific/Engineering :: Visualization", 
    628                      "Intended Audience :: Education", 
    629                      "Intended Audience :: Science/Research" 
    630                      ], 
    631       long_description="""\ 
    632 Orange data mining library 
    633 ========================== 
    634  
    635 Orange is a scriptable environment for fast prototyping of new 
    636 algorithms and testing schemes. It is a collection of Python packages 
    637 that sit over the core library and implement some functionality for 
    638 which execution time is not crucial and which is easier done in Python 
    639 than in C++. This includes a variety of tasks such as attribute subset, 
    640 bagging and boosting, and alike. 
    641  
    642 Orange also includes a set of graphical widgets that use methods from  
    643 core library and Orange modules. Through visual programming, widgets 
    644 can be assembled together into an application by a visual programming 
    645 tool called Orange Canvas. 
    646 """, 
    647       **setuptools_args) 
    648        
    649  
     595 
     596def get_packages(): 
     597    import fnmatch 
     598    matches = [] 
     599 
     600    #Recursively find '__init__.py's 
     601    for root, dirnames, filenames in os.walk('Orange'): 
     602      # Add packages for Orange 
     603      for filename in fnmatch.filter(filenames, '__init__.py'): 
     604          matches.append(os.path.join(root, filename)) 
     605    return [os.path.dirname(pkg).replace(os.path.sep, '.') for pkg in matches] 
     606 
     607def get_package_data(): 
     608    package_data = { 
     609        "Orange": 
     610            ["orangerc.cfg", "doc/style.css", "doc/widgets/*/*.*"] +\ 
     611             all_with_extension(path="doc/datasets", extensions=("tab", "csv", "basket")) +\ 
     612             all_with_extension(path="doc/networks", extensions=("net", "tab")) +\ 
     613             all_with_extension(path="testing/regression/tests_20", extensions=("net", "tab", "basket", "csv")), 
     614        "Orange.OrangeCanvas": ["icons/*.png", "orngCanvas.pyw", "WidgetTabs.txt"], 
     615        "Orange.OrangeWidgets": ["icons/*.png", "icons/backgrounds/*.png", "report/index.html"], 
     616        "Orange.OrangeWidgets.Associate": ["icons/*.png"], 
     617        "Orange.OrangeWidgets.Classify": ["icons/*.png"], 
     618        "Orange.OrangeWidgets.Data": ["icons/*.png"], 
     619        "Orange.OrangeWidgets.Evaluate": ["icons/*.png"], 
     620        "Orange.OrangeWidgets.Prototypes": ["icons/*.png"], 
     621        "Orange.OrangeWidgets.Regression": ["icons/*.png"], 
     622        "Orange.OrangeWidgets.Unsupervised": ["icons/*.png"], 
     623        "Orange.OrangeWidgets.Visualize": ["icons/*.png"], 
     624        "Orange.OrangeWidgets.Visualize Qt": ["icons/*.png"], 
     625        "Orange.OrangeWidgets.plot": ["*.gs", "*.vs"], 
     626        "Orange.OrangeWidgets.plot.primitives": ["*.obj"], 
     627    } 
     628 
     629    return package_data 
     630 
     631def all_with_extension(path, extensions): 
     632    return [os.path.join(path, "*.%s"%extension) for extension in extensions] 
     633 
     634def hg_revision(): 
     635    # Copied from numpy setup.py and modified to work with hg 
     636    def _minimal_ext_cmd(cmd): 
     637        # construct minimal environment 
     638        env = {} 
     639        for k in ['SYSTEMROOT', 'PATH']: 
     640            v = os.environ.get(k) 
     641            if v is not None: 
     642                env[k] = v 
     643        # LANGUAGE is used on win32 
     644        env['LANGUAGE'] = 'C' 
     645        env['LANG'] = 'C' 
     646        env['LC_ALL'] = 'C' 
     647        out = subprocess.Popen(cmd, stdout = subprocess.PIPE, env=env).communicate()[0] 
     648        return out 
     649 
     650    try: 
     651        out = _minimal_ext_cmd(['hg', 'ide', '-i']) 
     652        HG_REVISION = out.strip().decode('ascii') 
     653    except OSError: 
     654        HG_REVISION = "Unknown" 
     655 
     656    return HG_REVISION 
     657 
     658def write_version_py(filename='Orange/version.py'): 
     659    # Copied from numpy setup.py 
     660    cnt = """ 
     661# THIS FILE IS GENERATED FROM ORANGE SETUP.PY 
     662short_version = '%(version)s' 
     663version = '%(version)s' 
     664full_version = '%(full_version)s' 
     665hg_revision = '%(hg_revision)s' 
     666release = %(isrelease)s 
     667 
     668if not release: 
     669    version = full_version 
     670""" 
     671    FULLVERSION = VERSION 
     672    if os.path.exists('.hg'): 
     673        HG_REVISION = hg_revision() 
     674    elif os.path.exists('Orange/version.py'): 
     675        # must be a source distribution, use existing version file 
     676        from Orange.version import hg_revision as HG_REVISION 
     677    else: 
     678        HG_REVISION = "Unknown" 
     679 
     680    if not ISRELEASED: 
     681        FULLVERSION += '.dev-' + HG_REVISION[:7] 
     682 
     683    a = open(filename, 'w') 
     684    try: 
     685        a.write(cnt % {'version': VERSION, 
     686                       'full_version' : FULLVERSION, 
     687                       'hg_revision' : HG_REVISION, 
     688                       'isrelease': str(ISRELEASED)}) 
     689    finally: 
     690        a.close() 
     691 
     692def setup_package(): 
     693    write_version_py() 
     694 
     695    setup(name =NAME, 
     696          description = DESCRIPTION, 
     697          version = VERSION, 
     698          author = AUTHOR, 
     699          author_email = AUTHOR_EMAIL, 
     700          url = URL, 
     701          download_url = DOWNLOAD_URL, 
     702          classifiers = CLASSIFIERS, 
     703          long_description=LONG_DESCRIPTION, 
     704          license = LICENSE, 
     705          keywords = KEYWORDS, 
     706 
     707          cmdclass={"build_ext": pyxtract_build_ext, 
     708                    "install_lib": my_install_lib, 
     709                    "install": my_install}, 
     710          packages = get_packages(), 
     711          package_data = get_package_data(), 
     712          ext_modules = [include_ext, orange_ext, orangeom_ext, 
     713                         orangene_ext, corn_ext, statc_ext], 
     714          scripts = ["bin/orange-canvas"], 
     715          **setuptools_args) 
     716 
     717if __name__ == '__main__': 
     718    setup_package() 
Note: See TracChangeset for help on using the changeset viewer.