Changeset 10176:789b75ba3f64 in orange


Ignore:
Timestamp:
02/08/12 17:38:27 (2 years ago)
Author:
anze <anze.staric@…>
Branch:
default
rebase_source:
4a564d211eb1a0eedcb3c7db6c8ebab315380284
Message:

Replaced AUC functions with class.

Files:
2 edited

Legend:

Unmodified
Added
Removed
  • Orange/evaluation/scoring.py

    r10063 r10176  
    55import Orange 
    66from Orange import statc, corn 
    7 from Orange.misc import deprecated_keywords 
     7from Orange.misc import deprecated_keywords, deprecated_function_name 
    88from Orange.evaluation import testing 
    99 
     
    13771377        return corn.computeCDT(res, class_index, useweights) 
    13781378 
    1379 ## THIS FUNCTION IS OBSOLETE AND ITS AVERAGING OVER FOLDS IS QUESTIONABLE 
    1380 ## DON'T USE IT 
    1381 def ROCs_from_CDT(cdt, **argkw): 
    1382     """Obsolete, don't use""" 
    1383     if type(cdt) == list: 
    1384         return [ROCs_from_CDT(c) for c in cdt] 
    1385  
    1386     C, D, T = cdt.C, cdt.D, cdt.T 
    1387     N = C+D+T 
    1388     if N < 1e-6: 
    1389         import warnings 
    1390         warnings.warn("Can't compute AUC: one or both classes have no instances") 
    1391         return (-1,)*8 
    1392     if N < 2: 
    1393         import warnings 
    1394         warnings.warn("Can't compute AUC: one or both classes have too few examples") 
    1395  
    1396     som = (C-D)/N 
    1397     c = 0.5*(1+som) 
    1398    
    1399     if (C+D): 
    1400         res = (C/N*100, D/N*100, T/N*100, N, som, (C-D)/(C+D), (C-D)/(N*(N-1)/2), 0.5*(1+som)) 
    1401     else: 
    1402         res = (C/N*100, D/N*100, T/N*100, N, som, -1.0, (C-D)/(N*(N-1)/2), 0.5*(1+som)) 
    1403  
    1404     if argkw.get("print"): 
    1405         print "Concordant  = %5.1f       Somers' D = %1.3f" % (res[0], res[4]) 
    1406         print "Discordant  = %5.1f       Gamma     = %1.3f" % (res[1], res[5]>0 and res[5] or "N/A") 
    1407         print "Tied        = %5.1f       Tau-a     = %1.3f" % (res[2], res[6]) 
    1408         print " %6d pairs             c         = %1.3f"    % (res[3], res[7]) 
    1409  
    1410     return res 
    1411  
    1412 AROC_from_CDT = ROCs_from_CDT  # for backward compatibility, AROC_from_CDT is obsolote 
    1413  
    1414  
    1415  
    1416 # computes AUC using a specified 'cdtComputer' function 
    1417 # It tries to compute AUCs from 'ite' (examples from a single iteration) and, 
    1418 # if C+D+T=0, from 'all_ite' (entire test set). In the former case, the AUCs 
    1419 # are divided by 'divideByIfIte'. Additional flag is returned which is True in 
    1420 # the former case, or False in the latter. 
    1421 @deprecated_keywords({"divideByIfIte": "divide_by_if_ite", 
    1422                       "computerArgs": "computer_args"}) 
    1423 def AUC_x(cdtComputer, ite, all_ite, divide_by_if_ite, computer_args): 
    1424     cdts = cdtComputer(*(ite, ) + computer_args) 
    1425     if not is_CDT_empty(cdts[0]): 
    1426         return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T)/divide_by_if_ite for cdt in cdts], True 
    1427          
    1428     if all_ite: 
    1429         cdts = cdtComputer(*(all_ite, ) + computer_args) 
    1430         if not is_CDT_empty(cdts[0]): 
    1431             return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T) for cdt in cdts], False 
    1432  
    1433     return False, False 
    1434  
    1435      
    1436 # computes AUC between classes i and j as if there we no other classes 
    1437 @deprecated_keywords({"classIndex1": "class_index1", 
    1438                       "classIndex2": "class_index2", 
    1439                       "useWeights": "use_weights", 
    1440                       "divideByIfIte": "divide_by_if_ite"}) 
    1441 def AUC_ij(ite, class_index1, class_index2, use_weights = True, all_ite = None, divide_by_if_ite = 1.0): 
    1442     return AUC_x(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, (class_index1, class_index2, use_weights)) 
    1443  
    1444  
    1445 # computes AUC between class i and the other classes (treating them as the same class) 
    1446 @deprecated_keywords({"classIndex": "class_index", 
    1447                       "useWeights": "use_weights", 
    1448                       "divideByIfIte": "divide_by_if_ite"}) 
    1449 def AUC_i(ite, class_index, use_weights = True, all_ite = None, 
    1450           divide_by_if_ite = 1.0): 
    1451     return AUC_x(corn.computeCDT, ite, all_ite, divide_by_if_ite, (class_index, use_weights)) 
    1452  
    1453  
    1454 # computes the average AUC over folds using a "AUCcomputer" (AUC_i or AUC_ij) 
    1455 # it returns the sum of what is returned by the computer, unless at a certain 
    1456 # fold the computer has to resort to computing over all folds or even this failed; 
    1457 # in these cases the result is returned immediately 
    1458  
    1459 @deprecated_keywords({"AUCcomputer": "auc_computer", 
    1460                       "computerArgs": "computer_args"}) 
    1461 def AUC_iterations(auc_computer, iterations, computer_args): 
    1462     subsum_aucs = [0.] * iterations[0].number_of_learners 
    1463     for ite in iterations: 
    1464         aucs, foldsUsed = auc_computer(*(ite, ) + computer_args) 
    1465         if not aucs: 
    1466             return None 
    1467         if not foldsUsed: 
    1468             return aucs 
    1469         subsum_aucs = map(add, subsum_aucs, aucs) 
    1470     return subsum_aucs 
    1471  
    1472  
    1473 # AUC for binary classification problems 
    1474 @deprecated_keywords({"useWeights": "use_weights"}) 
    1475 def AUC_binary(res, use_weights = True): 
    1476     if res.number_of_iterations > 1: 
    1477         return AUC_iterations(AUC_i, split_by_iterations(res), (-1, use_weights, res, res.number_of_iterations)) 
    1478     else: 
    1479         return AUC_i(res, -1, use_weights)[0] 
    1480  
    1481 # AUC for multiclass problems 
    1482 @deprecated_keywords({"useWeights": "use_weights"}) 
    1483 def AUC_multi(res, use_weights = True, method = 0): 
    1484     numberOfClasses = len(res.class_values) 
    1485      
    1486     if res.number_of_iterations > 1: 
    1487         iterations = split_by_iterations(res) 
    1488         all_ite = res 
    1489     else: 
    1490         iterations = [res] 
    1491         all_ite = None 
    1492      
    1493     # by pairs 
    1494     sum_aucs = [0.] * res.number_of_learners 
    1495     usefulClassPairs = 0. 
    1496  
    1497     if method in [0, 2]: 
    1498         prob = class_probabilities_from_res(res) 
    1499          
    1500     if method <= 1: 
    1501         for classIndex1 in range(numberOfClasses): 
    1502             for classIndex2 in range(classIndex1): 
    1503                 subsum_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, classIndex2, use_weights, all_ite, res.number_of_iterations)) 
     1379 
     1380class AucClass(object): 
     1381    ByWeightedPairs = 0 
     1382    ByPairs = 1 
     1383    WeightedOneAgainstAll = 2 
     1384    OneAgainstAll = 3 
     1385 
     1386    @deprecated_keywords({"useWeights": "use_weights"}) 
     1387    def __call__(self, res, method = 0, use_weights = True): 
     1388        """ Returns the area under ROC curve (AUC) given a set of experimental 
     1389        results. For multivalued class problems, it will compute some sort of 
     1390        average, as specified by the argument method. 
     1391        """ 
     1392        if len(res.class_values) < 2: 
     1393            raise ValueError("Cannot compute AUC on a single-class problem") 
     1394        elif len(res.class_values) == 2: 
     1395            return self.compute_for_binary_class(res, use_weights) 
     1396        else: 
     1397            return self.compute_for_multi_value_class(res, use_weights, method) 
     1398 
     1399    # AUC for binary classification problems 
     1400    @deprecated_keywords({"useWeights": "use_weights"}) 
     1401    def compute_for_binary_class(self, res, use_weights = True): 
     1402        """AUC for binary classification problems""" 
     1403        if res.number_of_iterations > 1: 
     1404            return self.compute_for_multiple_folds( 
     1405                        self.compute_one_class_against_all, 
     1406                        split_by_iterations(res), 
     1407                        (-1, use_weights,res, res.number_of_iterations)) 
     1408        else: 
     1409            return self.compute_one_class_against_all(res, -1, use_weights)[0] 
     1410 
     1411    @deprecated_keywords({"useWeights": "use_weights"}) 
     1412    def compute_for_multi_value_class(self, res, use_weights = True, 
     1413                                      method = 0): 
     1414        """AUC for multiclass classification problems""" 
     1415        numberOfClasses = len(res.class_values) 
     1416 
     1417        if res.number_of_iterations > 1: 
     1418            iterations = split_by_iterations(res) 
     1419            all_ite = res 
     1420        else: 
     1421            iterations = [res] 
     1422            all_ite = None 
     1423 
     1424        # by pairs 
     1425        sum_aucs = [0.] * res.number_of_learners 
     1426        usefulClassPairs = 0. 
     1427 
     1428        if method in [0, 2]: 
     1429            prob = class_probabilities_from_res(res) 
     1430 
     1431        if method <= 1: 
     1432            for classIndex1 in range(numberOfClasses): 
     1433                for classIndex2 in range(classIndex1): 
     1434                    subsum_aucs = self.compute_for_multiple_folds( 
     1435                                             self.compute_one_class_against_another, 
     1436                                             iterations, 
     1437                                             (classIndex1, classIndex2, 
     1438                                             use_weights, all_ite, 
     1439                                             res.number_of_iterations)) 
     1440                    if subsum_aucs: 
     1441                        if method == 0: 
     1442                            p_ij = prob[classIndex1] * prob[classIndex2] 
     1443                            subsum_aucs = [x * p_ij  for x in subsum_aucs] 
     1444                            usefulClassPairs += p_ij 
     1445                        else: 
     1446                            usefulClassPairs += 1 
     1447                        sum_aucs = map(add, sum_aucs, subsum_aucs) 
     1448        else: 
     1449            for classIndex in range(numberOfClasses): 
     1450                subsum_aucs = self.compute_for_multiple_folds( 
     1451                    self.compute_one_class_against_all, 
     1452                    iterations, (classIndex, use_weights, all_ite, 
     1453                                 res.number_of_iterations)) 
    15041454                if subsum_aucs: 
    15051455                    if method == 0: 
    1506                         p_ij = prob[classIndex1] * prob[classIndex2] 
    1507                         subsum_aucs = [x * p_ij  for x in subsum_aucs] 
    1508                         usefulClassPairs += p_ij 
     1456                        p_i = prob[classIndex] 
     1457                        subsum_aucs = [x * p_i  for x in subsum_aucs] 
     1458                        usefulClassPairs += p_i 
    15091459                    else: 
    15101460                        usefulClassPairs += 1 
    15111461                    sum_aucs = map(add, sum_aucs, subsum_aucs) 
    1512     else: 
    1513         for classIndex in range(numberOfClasses): 
    1514             subsum_aucs = AUC_iterations(AUC_i, iterations, (classIndex, use_weights, all_ite, res.number_of_iterations)) 
    1515             if subsum_aucs: 
    1516                 if method == 0: 
    1517                     p_i = prob[classIndex] 
    1518                     subsum_aucs = [x * p_i  for x in subsum_aucs] 
    1519                     usefulClassPairs += p_i 
    1520                 else: 
    1521                     usefulClassPairs += 1 
    1522                 sum_aucs = map(add, sum_aucs, subsum_aucs) 
    1523                      
    1524     if usefulClassPairs > 0: 
    1525         sum_aucs = [x/usefulClassPairs for x in sum_aucs] 
    1526  
    1527     return sum_aucs 
    1528  
    1529 class AucClass(object): 
    1530     pass 
    1531  
    1532  
    1533 def AUC(): 
    1534     pass 
    1535  
    1536 AUC.ByWeightedPairs = 0 
    1537  
    1538 # Computes AUC, possibly for multiple classes (the averaging method can be specified) 
    1539 # Results over folds are averages; if some folds examples from one class only, the folds are merged 
    1540 @deprecated_keywords({"useWeights": "use_weights"}) 
    1541 def AUC(res, method = AUC.ByWeightedPairs, use_weights = True): 
    1542     """ Returns the area under ROC curve (AUC) given a set of experimental 
    1543     results. For multivalued class problems, it will compute some sort of 
    1544     average, as specified by the argument method. 
    1545     """ 
    1546     if len(res.class_values) < 2: 
    1547         raise ValueError("Cannot compute AUC on a single-class problem") 
    1548     elif len(res.class_values) == 2: 
    1549         return AUC_binary(res, use_weights) 
    1550     else: 
    1551         return AUC_multi(res, use_weights, method) 
    1552  
    1553 AUC.ByWeightedPairs = 0 
    1554 AUC.ByPairs = 1 
    1555 AUC.WeightedOneAgainstAll = 2 
    1556 AUC.OneAgainstAll = 3 
    1557  
     1462 
     1463        if usefulClassPairs > 0: 
     1464            sum_aucs = [x/usefulClassPairs for x in sum_aucs] 
     1465 
     1466        return sum_aucs 
     1467 
     1468    # computes the average AUC over folds using a "AUCcomputer" (AUC_i or AUC_ij) 
     1469    # it returns the sum of what is returned by the computer, unless at a certain 
     1470    # fold the computer has to resort to computing over all folds or even this failed; 
     1471    # in these cases the result is returned immediately 
     1472    @deprecated_keywords({"AUCcomputer": "auc_computer", 
     1473                          "computerArgs": "computer_args"}) 
     1474    def compute_for_multiple_folds(self, auc_computer, iterations, 
     1475                                 computer_args): 
     1476        """Compute the average AUC over folds using :obj:`auc_computer`.""" 
     1477        subsum_aucs = [0.] * iterations[0].number_of_learners 
     1478        for ite in iterations: 
     1479            aucs, foldsUsed = auc_computer(*(ite, ) + computer_args) 
     1480            if not aucs: 
     1481                return None 
     1482            if not foldsUsed: 
     1483                return aucs 
     1484            subsum_aucs = map(add, subsum_aucs, aucs) 
     1485        return subsum_aucs 
     1486 
     1487    # computes AUC between class i and the other classes (treating them as the same class) 
     1488    @deprecated_keywords({"classIndex": "class_index", 
     1489                          "useWeights": "use_weights", 
     1490                          "divideByIfIte": "divide_by_if_ite"}) 
     1491    def compute_one_class_against_all(self, ite, class_index, 
     1492            use_weights = True, all_ite = None, divide_by_if_ite = 1.0): 
     1493        """Compute AUC between class i and all the other classes)""" 
     1494        return self.compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite, 
     1495            (class_index, use_weights)) 
     1496 
     1497 
     1498    # computes AUC between classes i and j as if there are no other classes 
     1499    def compute_one_class_against_another(self, ite, class_index1, 
     1500            class_index2, use_weights = True, all_ite = None, 
     1501            divide_by_if_ite = 1.0): 
     1502        """ 
     1503        Compute AUC between classes i and j as if there are no other classes. 
     1504        """ 
     1505        return self.compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, 
     1506            (class_index1, class_index2, use_weights)) 
     1507 
     1508    # computes AUC using a specified 'cdtComputer' function 
     1509    # It tries to compute AUCs from 'ite' (examples from a single iteration) and, 
     1510    # if C+D+T=0, from 'all_ite' (entire test set). In the former case, the AUCs 
     1511    # are divided by 'divideByIfIte'. Additional flag is returned which is True in 
     1512    # the former case, or False in the latter. 
     1513    @deprecated_keywords({"cdt_computer": "cdtComputer", 
     1514                          "divideByIfIte": "divide_by_if_ite", 
     1515                          "computerArgs": "computer_args"}) 
     1516    def compute_auc(self, cdt_computer, ite, all_ite, divide_by_if_ite, 
     1517              computer_args): 
     1518        """ 
     1519        Compute AUC using a :obj:`cdtComputer`. 
     1520        """ 
     1521        cdts = cdt_computer(*(ite, ) + computer_args) 
     1522        if not is_CDT_empty(cdts[0]): 
     1523            return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T)/divide_by_if_ite for cdt in cdts], True 
     1524 
     1525        if all_ite: 
     1526            cdts = cdt_computer(*(all_ite, ) + computer_args) 
     1527            if not is_CDT_empty(cdts[0]): 
     1528                return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T) for cdt in cdts], False 
     1529 
     1530        return False, False 
     1531 
     1532 
     1533AUC = AucClass() 
     1534AUC_binary = deprecated_function_name(AUC.compute_for_binary_class) 
     1535AUC_multi = deprecated_function_name(AUC.compute_for_multi_value_class) 
     1536AUC_iterations = deprecated_function_name(AUC.compute_for_multiple_folds) 
     1537AUC_x = deprecated_function_name(AUC.compute_auc) 
     1538AUC_i = deprecated_function_name(AUC.compute_one_class_against_all) 
     1539AUC_ij = deprecated_function_name(AUC.compute_one_class_against_another) 
    15581540 
    15591541# Computes AUC; in multivalued class problem, AUC is computed as one against all 
  • docs/reference/rst/Orange.evaluation.scoring.rst

    r10127 r10176  
    1818============== 
    1919 
    20 Many scores for evaluation of classification models can be computed solely 
    21 from the confusion matrix constructed manually with the 
    22 :obj:`confusion_matrices` function. If class variable has more than two 
    23 values, the index of the value to calculate the confusion matrix for should 
    24 be passed as well. 
    25  
    2620Calibration scores 
    2721================== 
     22Many scores for evaluation of the classification models measure whether the 
     23model assigns the correct class value to the test instances. Many of these 
     24scores can be computed solely from the confusion matrix constructed manually 
     25with the :obj:`confusion_matrices` function. If class variable has more than 
     26two values, the index of the value to calculate the confusion matrix for should 
     27be passed as well. 
    2828 
    2929.. autosingleton:: CA 
     
    4141.. autofunction:: AP 
    4242.. autofunction:: IS 
     43.. autofunction:: confusion_chi_square 
    4344 
    4445Discriminatory scores 
    4546===================== 
     47Scores that measure how good can the prediction model separate instances with 
     48different classes are called discriminatory scores. 
    4649 
    4750.. autofunction:: Brier_score 
     
    5154    .. attribute:: AUC.ByWeightedPairs (or 0) 
    5255 
    53       Computes AUC for each pair of classes (ignoring instances of all other 
    54       classes) and averages the results, weighting them by the number of 
    55       pairs of instances from these two classes (e.g. by the product of 
    56       probabilities of the two classes). AUC computed in this way still 
    57       behaves as concordance index, e.g., gives the probability that two 
    58       randomly chosen instances from different classes will be correctly 
    59       recognized (this is of course true only if the classifier knows 
    60       from which two classes the instances came). 
    61  
    62    .. attribute:: AUC.ByPairs (or 1) 
    63  
    64       Similar as above, except that the average over class pairs is not 
    65       weighted. This AUC is, like the binary, independent of class 
    66       distributions, but it is not related to concordance index any more. 
    67  
    68    .. attribute:: AUC.WeightedOneAgainstAll (or 2) 
    69  
    70       For each class, it computes AUC for this class against all others (that 
    71       is, treating other classes as one class). The AUCs are then averaged by 
    72       the class probabilities. This is related to concordance index in which 
    73       we test the classifier's (average) capability for distinguishing the 
    74       instances from a specified class from those that come from other classes. 
    75       Unlike the binary AUC, the measure is not independent of class 
    76       distributions. 
    77  
    78    .. attribute:: AUC.OneAgainstAll (or 3) 
    79  
    80       As above, except that the average is not weighted. 
     56        Computes AUC for each pair of classes (ignoring instances of all other 
     57        classes) and averages the results, weighting them by the number of 
     58        pairs of instances from these two classes (e.g. by the product of 
     59        probabilities of the two classes). AUC computed in this way still 
     60        behaves as concordance index, e.g., gives the probability that two 
     61        randomly chosen instances from different classes will be correctly 
     62        recognized (this is of course true only if the classifier knows 
     63        from which two classes the instances came). 
     64 
     65    .. attribute:: AUC.ByPairs (or 1) 
     66 
     67        Similar as above, except that the average over class pairs is not 
     68        weighted. This AUC is, like the binary, independent of class 
     69        distributions, but it is not related to concordance index any more. 
     70 
     71    .. attribute:: AUC.WeightedOneAgainstAll (or 2) 
     72 
     73        For each class, it computes AUC for this class against all others (that 
     74        is, treating other classes as one class). The AUCs are then averaged by 
     75        the class probabilities. This is related to concordance index in which 
     76        we test the classifier's (average) capability for distinguishing the 
     77        instances from a specified class from those that come from other classes. 
     78        Unlike the binary AUC, the measure is not independent of class 
     79        distributions. 
     80 
     81    .. attribute:: AUC.OneAgainstAll (or 3) 
     82 
     83        As above, except that the average is not weighted. 
    8184 
    8285   In case of multiple folds (for instance if the data comes from cross 
Note: See TracChangeset for help on using the changeset viewer.