Changeset 10204:873180a80192 in orange
 Timestamp:
 02/14/12 13:40:11 (2 years ago)
 Branch:
 default
 Children:
 10205:5a51a7f8eae8, 10218:af780bca6b83
 Files:

 2 edited
Legend:
 Unmodified
 Added
 Removed

Orange/evaluation/scoring.py
r10198 r10204 5 5 import Orange 6 6 from Orange import statc, corn 7 from Orange.misc import deprecated_keywords, deprecated_function_name, deprecation_warning 7 from Orange.misc import deprecated_keywords, deprecated_function_name, \ 8 deprecation_warning, environ 8 9 from Orange.evaluation import testing 9 10 … … 1381 1382 return corn.computeCDT(res, class_index, useweights) 1382 1383 1384 # Backward compatibility 1385 def replace_use_weights(fun): 1386 if environ.orange_no_deprecated_members: 1387 return fun 1388 1389 @functools.wraps(fun) 1390 def wrapped(*args, **kwargs): 1391 use_weights = kwargs.pop("useWeights", None) 1392 if use_weights is not None: 1393 deprecation_warning("useWeights", "ignore_weights") 1394 kwargs["ignore_weights"] = not use_weights 1395 return fun(*args, **kwargs) 1396 return wrapped 1383 1397 1384 1398 class AucClass(object): … … 1388 1402 OneAgainstAll = 3 1389 1403 1390 def __call__(self, res, method=0, ignore_weights=False, useWeights=None): 1391 """ Returns the area under ROC curve (AUC) given a set of experimental 1392 results. For multivalued class problems, it will compute some sort of 1393 average, as specified by the argument method. 1394 """ 1395 if useWeights is not None: 1396 deprecation_warning("useWeights", "ignore_weights") 1397 ignore_weights = not useWeights 1398 1399 if len(res.class_values) < 2: 1404 @replace_use_weights 1405 def __call__(self, test_results, method=0, ignore_weights=False): 1406 """ 1407 Return the area under ROC curve given a set of experimental results. 1408 For multivalued class problems, return the result of :obj:`by_weighted_pairs`. 1409 1410 :param test_results: test results to score 1411 :param ignore_weights: ignore instance weights when calculating score 1412 :param method: DEPRECATED, call the appropriate method directly. 1413 """ 1414 if len(test_results.class_values) < 2: 1400 1415 raise ValueError("Cannot compute AUC on a singleclass problem") 1401 elif len(res.class_values) == 2: 1402 return self.compute_for_binary_class(res, ignore_weights) 1403 else: 1404 return self.compute_for_multi_value_class(res, ignore_weights, method) 1405 1406 def compute_for_binary_class(self, res, ignore_weights=False): 1416 elif len(test_results.class_values) == 2: 1417 return self._compute_for_binary_class(test_results, ignore_weights) 1418 else: 1419 return self._compute_for_multi_value_class(test_results, ignore_weights, method) 1420 1421 def by_weighted_pairs(self, res, ignore_weights=False): 1422 """ 1423 Compute AUC for each pair of classes (ignoring instances of all other 1424 classes) and averages the results, weighting them by the number of 1425 pairs of instances from these two classes (e.g. by the product of 1426 probabilities of the two classes). AUC computed in this way still 1427 behaves as concordance index, e.g., gives the probability that two 1428 randomly chosen instances from different classes will be correctly 1429 recognized (if the classifier knows from which two classes the 1430 instances came). 1431 """ 1432 return self._compute_for_multi_value_class(res, ignore_weights, 1433 method=self.ByWeightedPairs) 1434 1435 def by_pairs(self, res, ignore_weights=False): 1436 """ 1437 Similar as above, except that the average over class pairs is not 1438 weighted. This AUC is, like the binary, independent of class 1439 distributions, but it is not related to concordance index any more. 1440 """ 1441 return self._compute_for_multi_value_class(res, ignore_weights, 1442 method=self.ByPairs) 1443 1444 # Computes AUC; in multivalued class problem, AUC is computed as one against all 1445 # Results over folds are averages; if some folds examples from one class only, the folds are merged 1446 @replace_use_weights 1447 @deprecated_keywords({"classIndex": "class_index"}) 1448 def single_class(self, res, class_index=1, ignore_weights=False): 1449 """ 1450 Compute AUC where the class with the given class_index is singled 1451 out and all other classes are treated as a single class. 1452 """ 1453 if class_index<0: 1454 if res.baseClass>=0: 1455 class_index = res.baseClass 1456 else: 1457 class_index = 1 1458 1459 if res.number_of_iterations > 1: 1460 return AUC_iterations(AUC_i, split_by_iterations(res), 1461 (class_index, not ignore_weights, res, res.number_of_iterations)) 1462 else: 1463 return AUC_i( res, class_index, ignore_weights)[0] 1464 1465 # Computes AUC for a pair of classes (as if there were no other classes) 1466 # Results over folds are averages; if some folds have examples from one class only, the folds are merged 1467 def pair(self, res, class_index1, class_index2, ignore_weights=False): 1468 """ 1469 Computes AUC between a pair of classes, ignoring instances from all 1470 other classes. 1471 """ 1472 if res.number_of_iterations > 1: 1473 return AUC_iterations(AUC_ij, split_by_iterations(res), 1474 (class_index1, class_index2, not ignore_weights, res, res.number_of_iterations)) 1475 else: 1476 return AUC_ij(res, class_index1, class_index2, ignore_weights) 1477 1478 def matrix(self, res, ignore_weights=False): 1479 """ 1480 Compute a (lower diagonal) matrix with AUCs for all pairs of classes. 1481 If there are empty classes, the corresponding elements in the matrix 1482 are 1. 1483 """ 1484 numberOfClasses = len(res.class_values) 1485 number_of_learners = res.number_of_learners 1486 1487 if res.number_of_iterations > 1: 1488 iterations, all_ite = split_by_iterations(res), res 1489 else: 1490 iterations, all_ite = [res], None 1491 1492 aucs = [[[] for _ in range(numberOfClasses)] for _ in range(number_of_learners)] 1493 1494 for classIndex1 in range(numberOfClasses): 1495 for classIndex2 in range(classIndex1): 1496 pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, 1497 classIndex2, not ignore_weights, 1498 all_ite, res.number_of_iterations)) 1499 if pair_aucs: 1500 for lrn in range(number_of_learners): 1501 aucs[lrn][classIndex1].append(pair_aucs[lrn]) 1502 else: 1503 for lrn in range(number_of_learners): 1504 aucs[lrn][classIndex1].append(1) 1505 return aucs 1506 1507 def weighted_one_against_all(self, res, ignore_weights=False): 1508 """ 1509 For each class, it computes AUC for this class against all others (that 1510 is, treating other classes as one class). The AUCs are then averaged by 1511 the class probabilities. This is related to concordance index in which 1512 we test the classifier's (average) capability for distinguishing the 1513 instances from a specified class from those that come from other classes. 1514 Unlike the binary AUC, the measure is not independent of class 1515 distributions. 1516 """ 1517 return self._compute_for_multi_value_class(res, ignore_weights, 1518 method=self.WeightedOneAgainstAll) 1519 1520 def one_against_all(self, res, ignore_weights=False): 1521 """As above, except that the average is not weighted.""" 1522 return self._compute_for_multi_value_class(res, ignore_weights, 1523 method=self.OneAgainstAll) 1524 1525 def _compute_for_binary_class(self, res, ignore_weights=False): 1407 1526 """AUC for binary classification problems""" 1408 1527 if res.number_of_iterations > 1: 1409 return self. compute_for_multiple_folds(1410 self. compute_one_class_against_all,1528 return self._compute_for_multiple_folds( 1529 self._compute_one_class_against_all, 1411 1530 split_by_iterations(res), 1412 1531 (1, not ignore_weights,res, res.number_of_iterations)) 1413 1532 else: 1414 return self. compute_one_class_against_all(res, 1, ignore_weights)[0]1415 1416 def compute_for_multi_value_class(self, res, ignore_weights=False,1533 return self._compute_one_class_against_all(res, 1, ignore_weights)[0] 1534 1535 def _compute_for_multi_value_class(self, res, ignore_weights=False, 1417 1536 method=0): 1418 1537 """AUC for multiclass classification problems""" … … 1437 1556 for classIndex1 in range(numberOfClasses): 1438 1557 for classIndex2 in range(classIndex1): 1439 subsum_aucs = self. compute_for_multiple_folds(1440 self. compute_one_class_against_another,1558 subsum_aucs = self._compute_for_multiple_folds( 1559 self._compute_one_class_against_another, 1441 1560 iterations, 1442 1561 (classIndex1, classIndex2, … … 1453 1572 else: 1454 1573 for classIndex in range(numberOfClasses): 1455 subsum_aucs = self. compute_for_multiple_folds(1456 self. compute_one_class_against_all,1574 subsum_aucs = self._compute_for_multiple_folds( 1575 self._compute_one_class_against_all, 1457 1576 iterations, (classIndex, not ignore_weights, all_ite, 1458 1577 res.number_of_iterations)) … … 1477 1596 @deprecated_keywords({"AUCcomputer": "auc_computer", 1478 1597 "computerArgs": "computer_args"}) 1479 def compute_for_multiple_folds(self, auc_computer, iterations,1598 def _compute_for_multiple_folds(self, auc_computer, iterations, 1480 1599 computer_args): 1481 1600 """Compute the average AUC over folds using :obj:`auc_computer`.""" … … 1493 1612 @deprecated_keywords({"classIndex": "class_index", 1494 1613 "divideByIfIte": "divide_by_if_ite"}) 1495 def compute_one_class_against_all(self, ite, class_index,1614 def _compute_one_class_against_all(self, ite, class_index, 1496 1615 ignore_weights=True, all_ite=None, 1497 1616 divide_by_if_ite=1.0): 1498 1617 """Compute AUC between class i and all the other classes)""" 1499 return self. compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite,1618 return self._compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite, 1500 1619 (class_index, not ignore_weights)) 1501 1620 1502 1621 1503 1622 # computes AUC between classes i and j as if there are no other classes 1504 def compute_one_class_against_another(self, ite, class_index1,1623 def _compute_one_class_against_another(self, ite, class_index1, 1505 1624 class_index2, ignore_weights=False, all_ite=None, 1506 1625 divide_by_if_ite=1.0): … … 1508 1627 Compute AUC between classes i and j as if there are no other classes. 1509 1628 """ 1510 return self. compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite,1629 return self._compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, 1511 1630 (class_index1, class_index2, not ignore_weights)) 1512 1631 … … 1519 1638 "divideByIfIte": "divide_by_if_ite", 1520 1639 "computerArgs": "computer_args"}) 1521 def compute_auc(self, cdt_computer, ite, all_ite, divide_by_if_ite,1640 def _compute_auc(self, cdt_computer, ite, all_ite, divide_by_if_ite, 1522 1641 computer_args): 1523 1642 """ 1524 Compute AUC using a :obj:`cdt Computer`.1643 Compute AUC using a :obj:`cdt_computer`. 1525 1644 """ 1526 1645 cdts = cdt_computer(*(ite, ) + computer_args) … … 1537 1656 AUC = AucClass() 1538 1657 1539 # Backward compatibility 1540 def replace_use_weights(fun): 1541 @functools.wraps(fun) 1542 def wrapped(*args, **kwargs): 1543 use_weights = kwargs.pop("useWeights", None) 1544 if use_weights is not None: 1545 deprecation_warning("useWeights", "ignore_weights") 1546 kwargs["ignore_weights"] = not use_weights 1547 return fun(*args, **kwargs) 1548 return wrapped 1549 1550 1551 AUC_binary = replace_use_weights(deprecated_function_name(AUC.compute_for_binary_class)) 1552 AUC_multi = replace_use_weights(deprecated_function_name(AUC.compute_for_multi_value_class)) 1553 AUC_iterations = replace_use_weights(deprecated_function_name(AUC.compute_for_multiple_folds)) 1554 AUC_x = replace_use_weights(deprecated_function_name(AUC.compute_auc)) 1555 AUC_i = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_all)) 1556 AUC_ij = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_another)) 1557 1558 # Computes AUC; in multivalued class problem, AUC is computed as one against all 1559 # Results over folds are averages; if some folds examples from one class only, the folds are merged 1560 @replace_use_weights 1561 @deprecated_keywords({"classIndex": "class_index"}) 1562 def AUC_single(res, class_index=1, ignore_weights=False): 1563 """ Computes AUC where the class given classIndex is singled out, and 1564 all other classes are treated as a single class. To find how good our 1565 classifiers are in distinguishing between vans and other vehicle, call 1566 the function like this:: 1567 1568 Orange.evaluation.scoring.AUC_single(resVeh, \ 1569 classIndex = vehicle.domain.classVar.values.index("van")) 1570 """ 1571 if class_index<0: 1572 if res.baseClass>=0: 1573 class_index = res.baseClass 1574 else: 1575 class_index = 1 1576 1577 if res.number_of_iterations > 1: 1578 return AUC_iterations(AUC_i, split_by_iterations(res), 1579 (class_index, not ignore_weights, res, res.number_of_iterations)) 1580 else: 1581 return AUC_i( res, class_index, ignore_weights)[0] 1582 1583 # Computes AUC for a pair of classes (as if there were no other classes) 1584 # Results over folds are averages; if some folds have examples from one class only, the folds are merged 1585 @replace_use_weights 1586 @deprecated_keywords({"classIndex1": "class_index1", 1587 "classIndex2": "class_index2"}) 1588 def AUC_pair(res, class_index1, class_index2, ignore_weights=False): 1589 """ Computes AUC between a pair of instances, ignoring instances from all 1590 other classes. 1591 """ 1592 if res.number_of_iterations > 1: 1593 return AUC_iterations(AUC_ij, split_by_iterations(res), 1594 (class_index1, class_index2, not ignore_weights, res, res.number_of_iterations)) 1595 else: 1596 return AUC_ij(res, class_index1, class_index2, ignore_weights) 1597 1598 @replace_use_weights 1599 def AUC_matrix(res, ignore_weights=False): 1600 """ Computes a (lower diagonal) matrix with AUCs for all pairs of classes. 1601 If there are empty classes, the corresponding elements in the matrix 1602 are 1. Remember the beautiful(?) code for printing out the confusion 1603 matrix? Here it strikes again:: 1604 1605 classes = vehicle.domain.classVar.values 1606 AUCmatrix = Orange.evaluation.scoring.AUC_matrix(resVeh)[0] 1607 print "\t"+"\t".join(classes[:1]) 1608 for className, AUCrow in zip(classes[1:], AUCmatrix[1:]): 1609 print ("%s" + ("\t%5.3f" * len(AUCrow))) % ((className, ) + tuple(AUCrow)) 1610 """ 1611 numberOfClasses = len(res.class_values) 1612 number_of_learners = res.number_of_learners 1613 1614 if res.number_of_iterations > 1: 1615 iterations, all_ite = split_by_iterations(res), res 1616 else: 1617 iterations, all_ite = [res], None 1618 1619 aucs = [[[] for _ in range(numberOfClasses)] for _ in range(number_of_learners)] 1620 1621 for classIndex1 in range(numberOfClasses): 1622 for classIndex2 in range(classIndex1): 1623 pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, 1624 classIndex2, not ignore_weights, 1625 all_ite, res.number_of_iterations)) 1626 if pair_aucs: 1627 for lrn in range(number_of_learners): 1628 aucs[lrn][classIndex1].append(pair_aucs[lrn]) 1629 else: 1630 for lrn in range(number_of_learners): 1631 aucs[lrn][classIndex1].append(1) 1632 return aucs 1658 AUC_binary = replace_use_weights(deprecated_function_name(AUC._compute_for_binary_class)) 1659 AUC_multi = replace_use_weights(deprecated_function_name(AUC._compute_for_multi_value_class)) 1660 AUC_iterations = replace_use_weights(deprecated_function_name(AUC._compute_for_multiple_folds)) 1661 AUC_x = replace_use_weights(deprecated_function_name(AUC._compute_auc)) 1662 AUC_i = replace_use_weights(deprecated_function_name(AUC._compute_one_class_against_all)) 1663 AUC_ij = replace_use_weights(deprecated_function_name(AUC._compute_one_class_against_another)) 1664 1665 AUC_single = replace_use_weights( 1666 deprecated_keywords({"classIndex": "class_index"})( 1667 deprecated_function_name(AUC.single_class))) 1668 AUC_pair = replace_use_weights( 1669 deprecated_keywords({"classIndex1": "class_index1", 1670 "classIndex2": "class_index2"})( 1671 deprecated_function_name(AUC.pair))) 1672 AUC_matrix = replace_use_weights(deprecated_function_name(AUC.matrix)) 1633 1673 1634 1674 
docs/reference/rst/Orange.evaluation.scoring.rst
r10202 r10204 52 52 .. autosingleton:: AUC 53 53 .. autoclass:: AucClass 54 :members: 55 56 *AUC.ByWeightedPairs (or 0)* 57 58 Computes AUC for each pair of classes (ignoring instances of all other 59 classes) and averages the results, weighting them by the number of 60 pairs of instances from these two classes (e.g. by the product of 61 probabilities of the two classes). AUC computed in this way still 62 behaves as concordance index, e.g., gives the probability that two 63 randomly chosen instances from different classes will be correctly 64 recognized (this is of course true only if the classifier knows 65 from which two classes the instances came). 66 67 *AUC.ByPairs (or 1)* 68 69 Similar as above, except that the average over class pairs is not 70 weighted. This AUC is, like the binary, independent of class 71 distributions, but it is not related to concordance index any more. 72 73 *AUC.WeightedOneAgainstAll (or 2)* 74 75 For each class, it computes AUC for this class against all others (that 76 is, treating other classes as one class). The AUCs are then averaged by 77 the class probabilities. This is related to concordance index in which 78 we test the classifier's (average) capability for distinguishing the 79 instances from a specified class from those that come from other classes. 80 Unlike the binary AUC, the measure is not independent of class 81 distributions. 82 83 *AUC.OneAgainstAll (or 3)* 84 85 As above, except that the average is not weighted. 54 :members: __call__, by_weighted_pairs, by_pairs, 55 weighted_one_against_all, one_against_all, single_class, pair, 56 matrix 86 57 87 58 In case of multiple folds (for instance if the data comes from cross … … 128 99 one vs. all: 0.783 0.800 0.500 129 100 130 .. autofunction:: AUC_single131 132 .. autofunction:: AUC_pair133 134 .. autofunction:: AUC_matrix135 136 101 The remaining functions, which plot the curves and statistically compare 137 102 them, require that the results come from a test with a single iteration,
Note: See TracChangeset
for help on using the changeset viewer.