Changeset 10204:873180a80192 in orange


Ignore:
Timestamp:
02/14/12 13:40:11 (2 years ago)
Author:
anzeh <anze.staric@…>
Branch:
default
Children:
10205:5a51a7f8eae8, 10218:af780bca6b83
Message:

Updated scoring documentation.

Files:
2 edited

Legend:

Unmodified
Added
Removed
  • Orange/evaluation/scoring.py

    r10198 r10204  
    55import Orange 
    66from Orange import statc, corn 
    7 from Orange.misc import deprecated_keywords, deprecated_function_name, deprecation_warning 
     7from Orange.misc import deprecated_keywords, deprecated_function_name, \ 
     8    deprecation_warning, environ 
    89from Orange.evaluation import testing 
    910 
     
    13811382        return corn.computeCDT(res, class_index, useweights) 
    13821383 
     1384# Backward compatibility 
     1385def replace_use_weights(fun): 
     1386    if environ.orange_no_deprecated_members: 
     1387        return fun 
     1388 
     1389    @functools.wraps(fun) 
     1390    def wrapped(*args, **kwargs): 
     1391        use_weights = kwargs.pop("useWeights", None) 
     1392        if use_weights is not None: 
     1393            deprecation_warning("useWeights", "ignore_weights") 
     1394            kwargs["ignore_weights"] = not use_weights 
     1395        return fun(*args, **kwargs) 
     1396    return wrapped 
    13831397 
    13841398class AucClass(object): 
     
    13881402    OneAgainstAll = 3 
    13891403 
    1390     def __call__(self, res, method=0, ignore_weights=False, useWeights=None): 
    1391         """ Returns the area under ROC curve (AUC) given a set of experimental 
    1392         results. For multivalued class problems, it will compute some sort of 
    1393         average, as specified by the argument method. 
    1394         """ 
    1395         if useWeights is not None: 
    1396             deprecation_warning("useWeights", "ignore_weights") 
    1397             ignore_weights = not useWeights 
    1398  
    1399         if len(res.class_values) < 2: 
     1404    @replace_use_weights 
     1405    def __call__(self, test_results, method=0, ignore_weights=False): 
     1406        """ 
     1407        Return the area under ROC curve given a set of experimental results. 
     1408        For multivalued class problems, return the result of :obj:`by_weighted_pairs`. 
     1409 
     1410        :param test_results: test results to score 
     1411        :param ignore_weights: ignore instance weights when calculating score 
     1412        :param method: DEPRECATED, call the appropriate method directly. 
     1413        """ 
     1414        if len(test_results.class_values) < 2: 
    14001415            raise ValueError("Cannot compute AUC on a single-class problem") 
    1401         elif len(res.class_values) == 2: 
    1402             return self.compute_for_binary_class(res, ignore_weights) 
    1403         else: 
    1404             return self.compute_for_multi_value_class(res, ignore_weights, method) 
    1405  
    1406     def compute_for_binary_class(self, res, ignore_weights=False): 
     1416        elif len(test_results.class_values) == 2: 
     1417            return self._compute_for_binary_class(test_results, ignore_weights) 
     1418        else: 
     1419            return self._compute_for_multi_value_class(test_results, ignore_weights, method) 
     1420 
     1421    def by_weighted_pairs(self, res, ignore_weights=False): 
     1422        """ 
     1423        Compute AUC for each pair of classes (ignoring instances of all other 
     1424        classes) and averages the results, weighting them by the number of 
     1425        pairs of instances from these two classes (e.g. by the product of 
     1426        probabilities of the two classes). AUC computed in this way still 
     1427        behaves as concordance index, e.g., gives the probability that two 
     1428        randomly chosen instances from different classes will be correctly 
     1429        recognized (if the classifier knows from which two classes the 
     1430        instances came). 
     1431        """ 
     1432        return self._compute_for_multi_value_class(res, ignore_weights, 
     1433            method=self.ByWeightedPairs) 
     1434 
     1435    def by_pairs(self, res, ignore_weights=False): 
     1436        """ 
     1437        Similar as above, except that the average over class pairs is not 
     1438        weighted. This AUC is, like the binary, independent of class 
     1439        distributions, but it is not related to concordance index any more. 
     1440        """ 
     1441        return self._compute_for_multi_value_class(res, ignore_weights, 
     1442            method=self.ByPairs) 
     1443 
     1444    # Computes AUC; in multivalued class problem, AUC is computed as one against all 
     1445    # Results over folds are averages; if some folds examples from one class only, the folds are merged 
     1446    @replace_use_weights 
     1447    @deprecated_keywords({"classIndex": "class_index"}) 
     1448    def single_class(self, res, class_index=-1, ignore_weights=False): 
     1449        """ 
     1450        Compute AUC where the class with the given class_index is singled 
     1451        out and all other classes are treated as a single class. 
     1452        """ 
     1453        if class_index<0: 
     1454            if res.baseClass>=0: 
     1455                class_index = res.baseClass 
     1456            else: 
     1457                class_index = 1 
     1458 
     1459        if res.number_of_iterations > 1: 
     1460            return AUC_iterations(AUC_i, split_by_iterations(res), 
     1461                (class_index, not ignore_weights, res, res.number_of_iterations)) 
     1462        else: 
     1463            return AUC_i( res, class_index, ignore_weights)[0] 
     1464 
     1465    # Computes AUC for a pair of classes (as if there were no other classes) 
     1466    # Results over folds are averages; if some folds have examples from one class only, the folds are merged 
     1467    def pair(self, res, class_index1, class_index2, ignore_weights=False): 
     1468        """ 
     1469        Computes AUC between a pair of classes, ignoring instances from all 
     1470        other classes. 
     1471        """ 
     1472        if res.number_of_iterations > 1: 
     1473            return AUC_iterations(AUC_ij, split_by_iterations(res), 
     1474                (class_index1, class_index2, not ignore_weights, res, res.number_of_iterations)) 
     1475        else: 
     1476            return AUC_ij(res, class_index1, class_index2, ignore_weights) 
     1477 
     1478    def matrix(self, res, ignore_weights=False): 
     1479        """ 
     1480        Compute a (lower diagonal) matrix with AUCs for all pairs of classes. 
     1481        If there are empty classes, the corresponding elements in the matrix 
     1482        are -1. 
     1483        """ 
     1484        numberOfClasses = len(res.class_values) 
     1485        number_of_learners = res.number_of_learners 
     1486 
     1487        if res.number_of_iterations > 1: 
     1488            iterations, all_ite = split_by_iterations(res), res 
     1489        else: 
     1490            iterations, all_ite = [res], None 
     1491 
     1492        aucs = [[[] for _ in range(numberOfClasses)] for _ in range(number_of_learners)] 
     1493 
     1494        for classIndex1 in range(numberOfClasses): 
     1495            for classIndex2 in range(classIndex1): 
     1496                pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, 
     1497                                                                classIndex2, not ignore_weights, 
     1498                                                                all_ite, res.number_of_iterations)) 
     1499                if pair_aucs: 
     1500                    for lrn in range(number_of_learners): 
     1501                        aucs[lrn][classIndex1].append(pair_aucs[lrn]) 
     1502                else: 
     1503                    for lrn in range(number_of_learners): 
     1504                        aucs[lrn][classIndex1].append(-1) 
     1505        return aucs 
     1506 
     1507    def weighted_one_against_all(self, res, ignore_weights=False): 
     1508        """ 
     1509        For each class, it computes AUC for this class against all others (that 
     1510        is, treating other classes as one class). The AUCs are then averaged by 
     1511        the class probabilities. This is related to concordance index in which 
     1512        we test the classifier's (average) capability for distinguishing the 
     1513        instances from a specified class from those that come from other classes. 
     1514        Unlike the binary AUC, the measure is not independent of class 
     1515        distributions. 
     1516        """ 
     1517        return self._compute_for_multi_value_class(res, ignore_weights, 
     1518            method=self.WeightedOneAgainstAll) 
     1519 
     1520    def one_against_all(self, res, ignore_weights=False): 
     1521        """As above, except that the average is not weighted.""" 
     1522        return self._compute_for_multi_value_class(res, ignore_weights, 
     1523            method=self.OneAgainstAll) 
     1524 
     1525    def _compute_for_binary_class(self, res, ignore_weights=False): 
    14071526        """AUC for binary classification problems""" 
    14081527        if res.number_of_iterations > 1: 
    1409             return self.compute_for_multiple_folds( 
    1410                         self.compute_one_class_against_all, 
     1528            return self._compute_for_multiple_folds( 
     1529                        self._compute_one_class_against_all, 
    14111530                        split_by_iterations(res), 
    14121531                        (-1, not ignore_weights,res, res.number_of_iterations)) 
    14131532        else: 
    1414             return self.compute_one_class_against_all(res, -1, ignore_weights)[0] 
    1415  
    1416     def compute_for_multi_value_class(self, res, ignore_weights=False, 
     1533            return self._compute_one_class_against_all(res, -1, ignore_weights)[0] 
     1534 
     1535    def _compute_for_multi_value_class(self, res, ignore_weights=False, 
    14171536                                      method=0): 
    14181537        """AUC for multiclass classification problems""" 
     
    14371556            for classIndex1 in range(numberOfClasses): 
    14381557                for classIndex2 in range(classIndex1): 
    1439                     subsum_aucs = self.compute_for_multiple_folds( 
    1440                                              self.compute_one_class_against_another, 
     1558                    subsum_aucs = self._compute_for_multiple_folds( 
     1559                                             self._compute_one_class_against_another, 
    14411560                                             iterations, 
    14421561                                             (classIndex1, classIndex2, 
     
    14531572        else: 
    14541573            for classIndex in range(numberOfClasses): 
    1455                 subsum_aucs = self.compute_for_multiple_folds( 
    1456                     self.compute_one_class_against_all, 
     1574                subsum_aucs = self._compute_for_multiple_folds( 
     1575                    self._compute_one_class_against_all, 
    14571576                    iterations, (classIndex, not ignore_weights, all_ite, 
    14581577                                 res.number_of_iterations)) 
     
    14771596    @deprecated_keywords({"AUCcomputer": "auc_computer", 
    14781597                          "computerArgs": "computer_args"}) 
    1479     def compute_for_multiple_folds(self, auc_computer, iterations, 
     1598    def _compute_for_multiple_folds(self, auc_computer, iterations, 
    14801599                                 computer_args): 
    14811600        """Compute the average AUC over folds using :obj:`auc_computer`.""" 
     
    14931612    @deprecated_keywords({"classIndex": "class_index", 
    14941613                          "divideByIfIte": "divide_by_if_ite"}) 
    1495     def compute_one_class_against_all(self, ite, class_index, 
     1614    def _compute_one_class_against_all(self, ite, class_index, 
    14961615                                      ignore_weights=True, all_ite=None, 
    14971616                                      divide_by_if_ite=1.0): 
    14981617        """Compute AUC between class i and all the other classes)""" 
    1499         return self.compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite, 
     1618        return self._compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite, 
    15001619            (class_index, not ignore_weights)) 
    15011620 
    15021621 
    15031622    # computes AUC between classes i and j as if there are no other classes 
    1504     def compute_one_class_against_another(self, ite, class_index1, 
     1623    def _compute_one_class_against_another(self, ite, class_index1, 
    15051624            class_index2, ignore_weights=False, all_ite=None, 
    15061625            divide_by_if_ite=1.0): 
     
    15081627        Compute AUC between classes i and j as if there are no other classes. 
    15091628        """ 
    1510         return self.compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, 
     1629        return self._compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, 
    15111630            (class_index1, class_index2, not ignore_weights)) 
    15121631 
     
    15191638                          "divideByIfIte": "divide_by_if_ite", 
    15201639                          "computerArgs": "computer_args"}) 
    1521     def compute_auc(self, cdt_computer, ite, all_ite, divide_by_if_ite, 
     1640    def _compute_auc(self, cdt_computer, ite, all_ite, divide_by_if_ite, 
    15221641              computer_args): 
    15231642        """ 
    1524         Compute AUC using a :obj:`cdtComputer`. 
     1643        Compute AUC using a :obj:`cdt_computer`. 
    15251644        """ 
    15261645        cdts = cdt_computer(*(ite, ) + computer_args) 
     
    15371656AUC = AucClass() 
    15381657 
    1539 # Backward compatibility 
    1540 def replace_use_weights(fun): 
    1541     @functools.wraps(fun) 
    1542     def wrapped(*args, **kwargs): 
    1543         use_weights = kwargs.pop("useWeights", None) 
    1544         if use_weights is not None: 
    1545             deprecation_warning("useWeights", "ignore_weights") 
    1546             kwargs["ignore_weights"] = not use_weights 
    1547         return fun(*args, **kwargs) 
    1548     return wrapped 
    1549  
    1550  
    1551 AUC_binary = replace_use_weights(deprecated_function_name(AUC.compute_for_binary_class)) 
    1552 AUC_multi = replace_use_weights(deprecated_function_name(AUC.compute_for_multi_value_class)) 
    1553 AUC_iterations = replace_use_weights(deprecated_function_name(AUC.compute_for_multiple_folds)) 
    1554 AUC_x = replace_use_weights(deprecated_function_name(AUC.compute_auc)) 
    1555 AUC_i = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_all)) 
    1556 AUC_ij = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_another)) 
    1557  
    1558 # Computes AUC; in multivalued class problem, AUC is computed as one against all 
    1559 # Results over folds are averages; if some folds examples from one class only, the folds are merged 
    1560 @replace_use_weights 
    1561 @deprecated_keywords({"classIndex": "class_index"}) 
    1562 def AUC_single(res, class_index=-1, ignore_weights=False): 
    1563     """ Computes AUC where the class given classIndex is singled out, and 
    1564     all other classes are treated as a single class. To find how good our 
    1565     classifiers are in distinguishing between vans and other vehicle, call 
    1566     the function like this:: 
    1567      
    1568         Orange.evaluation.scoring.AUC_single(resVeh, \ 
    1569 classIndex = vehicle.domain.classVar.values.index("van")) 
    1570     """ 
    1571     if class_index<0: 
    1572         if res.baseClass>=0: 
    1573             class_index = res.baseClass 
    1574         else: 
    1575             class_index = 1 
    1576  
    1577     if res.number_of_iterations > 1: 
    1578         return AUC_iterations(AUC_i, split_by_iterations(res), 
    1579             (class_index, not ignore_weights, res, res.number_of_iterations)) 
    1580     else: 
    1581         return AUC_i( res, class_index, ignore_weights)[0] 
    1582  
    1583 # Computes AUC for a pair of classes (as if there were no other classes) 
    1584 # Results over folds are averages; if some folds have examples from one class only, the folds are merged 
    1585 @replace_use_weights 
    1586 @deprecated_keywords({"classIndex1": "class_index1", 
    1587                       "classIndex2": "class_index2"}) 
    1588 def AUC_pair(res, class_index1, class_index2, ignore_weights=False): 
    1589     """ Computes AUC between a pair of instances, ignoring instances from all 
    1590     other classes. 
    1591     """ 
    1592     if res.number_of_iterations > 1: 
    1593         return AUC_iterations(AUC_ij, split_by_iterations(res), 
    1594             (class_index1, class_index2, not ignore_weights, res, res.number_of_iterations)) 
    1595     else: 
    1596         return AUC_ij(res, class_index1, class_index2, ignore_weights) 
    1597  
    1598 @replace_use_weights 
    1599 def AUC_matrix(res, ignore_weights=False): 
    1600     """ Computes a (lower diagonal) matrix with AUCs for all pairs of classes. 
    1601     If there are empty classes, the corresponding elements in the matrix 
    1602     are -1. Remember the beautiful(?) code for printing out the confusion 
    1603     matrix? Here it strikes again:: 
    1604      
    1605         classes = vehicle.domain.classVar.values 
    1606         AUCmatrix = Orange.evaluation.scoring.AUC_matrix(resVeh)[0] 
    1607         print "\t"+"\t".join(classes[:-1]) 
    1608         for className, AUCrow in zip(classes[1:], AUCmatrix[1:]): 
    1609             print ("%s" + ("\t%5.3f" * len(AUCrow))) % ((className, ) + tuple(AUCrow)) 
    1610     """ 
    1611     numberOfClasses = len(res.class_values) 
    1612     number_of_learners = res.number_of_learners 
    1613      
    1614     if res.number_of_iterations > 1: 
    1615         iterations, all_ite = split_by_iterations(res), res 
    1616     else: 
    1617         iterations, all_ite = [res], None 
    1618      
    1619     aucs = [[[] for _ in range(numberOfClasses)] for _ in range(number_of_learners)] 
    1620          
    1621     for classIndex1 in range(numberOfClasses): 
    1622         for classIndex2 in range(classIndex1): 
    1623             pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, 
    1624                                                             classIndex2, not ignore_weights, 
    1625                                                             all_ite, res.number_of_iterations)) 
    1626             if pair_aucs: 
    1627                 for lrn in range(number_of_learners): 
    1628                     aucs[lrn][classIndex1].append(pair_aucs[lrn]) 
    1629             else: 
    1630                 for lrn in range(number_of_learners): 
    1631                     aucs[lrn][classIndex1].append(-1) 
    1632     return aucs 
     1658AUC_binary = replace_use_weights(deprecated_function_name(AUC._compute_for_binary_class)) 
     1659AUC_multi = replace_use_weights(deprecated_function_name(AUC._compute_for_multi_value_class)) 
     1660AUC_iterations = replace_use_weights(deprecated_function_name(AUC._compute_for_multiple_folds)) 
     1661AUC_x = replace_use_weights(deprecated_function_name(AUC._compute_auc)) 
     1662AUC_i = replace_use_weights(deprecated_function_name(AUC._compute_one_class_against_all)) 
     1663AUC_ij = replace_use_weights(deprecated_function_name(AUC._compute_one_class_against_another)) 
     1664 
     1665AUC_single = replace_use_weights( 
     1666             deprecated_keywords({"classIndex": "class_index"})( 
     1667             deprecated_function_name(AUC.single_class))) 
     1668AUC_pair = replace_use_weights( 
     1669           deprecated_keywords({"classIndex1": "class_index1", 
     1670                                "classIndex2": "class_index2"})( 
     1671           deprecated_function_name(AUC.pair))) 
     1672AUC_matrix = replace_use_weights(deprecated_function_name(AUC.matrix)) 
    16331673 
    16341674 
  • docs/reference/rst/Orange.evaluation.scoring.rst

    r10202 r10204  
    5252.. autosingleton:: AUC 
    5353.. autoclass:: AucClass 
    54     :members: 
    55  
    56 *AUC.ByWeightedPairs (or 0)* 
    57  
    58     Computes AUC for each pair of classes (ignoring instances of all other 
    59     classes) and averages the results, weighting them by the number of 
    60     pairs of instances from these two classes (e.g. by the product of 
    61     probabilities of the two classes). AUC computed in this way still 
    62     behaves as concordance index, e.g., gives the probability that two 
    63     randomly chosen instances from different classes will be correctly 
    64     recognized (this is of course true only if the classifier knows 
    65     from which two classes the instances came). 
    66  
    67 *AUC.ByPairs (or 1)* 
    68  
    69     Similar as above, except that the average over class pairs is not 
    70     weighted. This AUC is, like the binary, independent of class 
    71     distributions, but it is not related to concordance index any more. 
    72  
    73 *AUC.WeightedOneAgainstAll (or 2)* 
    74  
    75     For each class, it computes AUC for this class against all others (that 
    76     is, treating other classes as one class). The AUCs are then averaged by 
    77     the class probabilities. This is related to concordance index in which 
    78     we test the classifier's (average) capability for distinguishing the 
    79     instances from a specified class from those that come from other classes. 
    80     Unlike the binary AUC, the measure is not independent of class 
    81     distributions. 
    82  
    83 *AUC.OneAgainstAll (or 3)* 
    84  
    85     As above, except that the average is not weighted. 
     54    :members: __call__, by_weighted_pairs, by_pairs, 
     55              weighted_one_against_all, one_against_all, single_class, pair, 
     56              matrix 
    8657 
    8758In case of multiple folds (for instance if the data comes from cross 
     
    12899                 one vs. all:  0.783   0.800   0.500 
    129100 
    130 .. autofunction:: AUC_single 
    131  
    132 .. autofunction:: AUC_pair 
    133  
    134 .. autofunction:: AUC_matrix 
    135  
    136101The remaining functions, which plot the curves and statistically compare 
    137102them, require that the results come from a test with a single iteration, 
Note: See TracChangeset for help on using the changeset viewer.