Changeset 10177:8da5d2e7dfe3 in orange
 Timestamp:
 02/08/12 20:21:35 (2 years ago)
 Branch:
 default
 rebase_source:
 18ed0dd8d7fb04fdf6345f24c90ad93c6f52af3d
 File:

 1 edited
Legend:
 Unmodified
 Added
 Removed

Orange/evaluation/scoring.py
r10176 r10177 1 import operator, math1 import math, functools 2 2 from operator import add 3 3 import numpy … … 5 5 import Orange 6 6 from Orange import statc, corn 7 from Orange.misc import deprecated_keywords, deprecated_function_name 7 from Orange.misc import deprecated_keywords, deprecated_function_name, deprecation_warning 8 8 from Orange.evaluation import testing 9 9 … … 41 41 return [res] 42 42 43 ress = [Orange.evaluation.testing.ExperimentResults(1, res.classifier_names, res.class_values, res.weights, classifiers=res.classifiers, loaded=res.loaded, test_type=res.test_type, labels=res.labels) 43 ress = [Orange.evaluation.testing.ExperimentResults( 44 1, res.classifier_names, res.class_values, 45 res.weights, classifiers=res.classifiers, 46 loaded=res.loaded, test_type=res.test_type, labels=res.labels) 44 47 for i in range(res.number_of_iterations)] 45 48 for te in res.results: … … 359 362 CROSS_VALIDATION = 3 360 363 361 @deprecated_keywords({"reportSE": "report_se"}) 362 def __call__(self, test_results, report_se = False, unweighted=False): 364 @deprecated_keywords({"reportSE": "report_se", 365 "unweighted": "ignore_weights"}) 366 def __call__(self, test_results, report_se = False, ignore_weights=False): 363 367 """Return percentage of matches between predicted and actual class. 364 368 … … 366 370 or :obj:`ConfusionMatrix`. 367 371 :param report_se: include standard error in result. 368 :param unweighted: ignore instance weights.372 :param ignore_weights: ignore instance weights. 369 373 :rtype: list of scores, one for each learner. 370 374 … … 380 384 elif input_type == self.CLASSIFICATION: 381 385 return self.from_classification_results( 382 test_results, report_se, unweighted)386 test_results, report_se, ignore_weights) 383 387 elif input_type == self.CROSS_VALIDATION: 384 388 return self.from_crossvalidation_results( 385 test_results, report_se, unweighted)389 test_results, report_se, ignore_weights) 386 390 387 391 def from_confusion_matrix(self, cm, report_se): … … 398 402 return map(self.from_confusion_matrix, confusion_matrices) # TODO: report_se 399 403 400 def from_classification_results(self, test_results, report_se, unweighted):404 def from_classification_results(self, test_results, report_se, ignore_results): 401 405 CAs = [0.0]*test_results.number_of_learners 402 406 totweight = 0. 403 407 for tex in test_results.results: 404 w = 1. if unweightedelse tex.weight408 w = 1. if ignore_results else tex.weight 405 409 CAs = map(lambda res, cls: res+(cls==tex.actual_class and w), CAs, tex.classes) 406 410 totweight += w … … 413 417 return ca 414 418 415 def from_crossvalidation_results(self, test_results, report_se, unweighted):419 def from_crossvalidation_results(self, test_results, report_se, ignore_weights): 416 420 CAsByFold = [[0.0]*test_results.number_of_iterations for i in range(test_results.number_of_learners)] 417 421 foldN = [0.0]*test_results.number_of_iterations 418 422 419 423 for tex in test_results.results: 420 w = 1. if unweightedelse tex.weight424 w = 1. if ignore_weights else tex.weight 421 425 for lrn in range(test_results.number_of_learners): 422 426 CAsByFold[lrn][tex.iteration_number] += (tex.classes[lrn]==tex.actual_class) and w … … 439 443 CA = CAClass() 440 444 441 @deprecated_keywords({"reportSE": "report_se"}) 442 def AP(res, report_se = False, **argkw): 445 @deprecated_keywords({"reportSE": "report_se", 446 "unweighted": "ignore_weights"}) 447 def AP(res, report_se = False, ignore_weights=False, **argkw): 443 448 """ Computes the average probability assigned to the correct class. """ 444 449 if res.number_of_iterations == 1: 445 450 APs=[0.0]*res.number_of_learners 446 if argkw.get("unweighted", 0)or not res.weights:451 if ignore_weights or not res.weights: 447 452 for tex in res.results: 448 453 APs = map(lambda res, probs: res + probs[tex.actual_class], APs, tex.probabilities) … … 458 463 APsByFold = [[0.0]*res.number_of_learners for i in range(res.number_of_iterations)] 459 464 foldN = [0.0] * res.number_of_iterations 460 if argkw.get("unweighted", 0)or not res.weights:465 if ignore_weights or not res.weights: 461 466 for tex in res.results: 462 467 APsByFold[tex.iteration_number] = map(lambda res, probs: res + probs[tex.actual_class], APsByFold[tex.iteration_number], tex.probabilities) … … 470 475 471 476 472 @deprecated_keywords({"reportSE": "report_se"}) 473 def Brier_score(res, report_se = False, **argkw): 477 @deprecated_keywords({"reportSE": "report_se", 478 "unweighted": "ignore_weights"}) 479 def Brier_score(res, report_se = False, ignore_weights=False, **argkw): 474 480 """ Computes the Brier's score, defined as the average (over test examples) 475 481 of sumx(t(x)p(x))2, where x is a class, t(x) is 1 for the correct class … … 489 495 if res.number_of_iterations == 1: 490 496 MSEs=[0.0]*res.number_of_learners 491 if argkw.get("unweighted", 0)or not res.weights:497 if ignore_weights or not res.weights: 492 498 totweight = 0.0 493 499 for tex in res.results: … … 509 515 foldN = [0.] * res.number_of_iterations 510 516 511 if argkw.get("unweighted", 0)or not res.weights:517 if ignore_weights or not res.weights: 512 518 for tex in res.results: 513 519 BSs[tex.iteration_number] = map(lambda rr, probs: … … 629 635 630 636 @deprecated_keywords({"res": "test_results", 631 "classIndex": "class_index"}) 637 "classIndex": "class_index", 638 "unweighted": "ignore_weights"}) 632 639 def confusion_matrices(test_results, class_index=1, 633 unweighted=False, cutoff=.5):640 ignore_weights=False, cutoff=.5): 634 641 """ 635 642 Return confusion matrices for test_results. … … 638 645 :param class_index: index of class value for which the confusion matrices 639 646 are to be computed. 640 :param unweighted: ignore instance weights.647 :param ignore_weights: ignore instance weights. 641 648 :params cutoff: cutoff for probability 642 649 … … 649 656 if class_index < 1 or numberOfClasses > 2: 650 657 cm = [[[0.0] * numberOfClasses for i in range(numberOfClasses)] for l in range(test_results.number_of_learners)] 651 if unweightedor not test_results.weights:658 if ignore_weights or not test_results.weights: 652 659 for tex in test_results.results: 653 660 trueClass = int(tex.actual_class) … … 671 678 672 679 if cutoff != .5: 673 if unweightedor not test_results.weights:680 if ignore_weights or not test_results.weights: 674 681 for lr in test_results.results: 675 682 isPositive=(lr.actual_class==class_index) … … 682 689 tfpns[i].addTFPosNeg(lr.probabilities[i][class_index]>cutoff, isPositive, lr.weight) 683 690 else: 684 if unweightedor not test_results.weights:691 if ignore_weights or not test_results.weights: 685 692 for lr in test_results.results: 686 693 isPositive=(lr.actual_class==class_index) … … 924 931 return ret 925 932 926 @deprecated_keywords({"classIndex": "class_index"}) 927 def AUCWilcoxon(res, class_index=1, **argkw): 933 @deprecated_keywords({"classIndex": "class_index", 934 "unweighted": "ignore_weights"}) 935 def AUCWilcoxon(res, class_index=1, ignore_weights=False, **argkw): 928 936 """ Computes the area under ROC (AUC) and its standard error using 929 937 Wilcoxon's approach proposed by Hanley and McNeal (1982). If … … 932 940 tuples (aROC, standard error). 933 941 """ 934 useweights = res.weights and not argkw.get("unweighted", 0)942 useweights = res.weights and not ignore_weights 935 943 problists, tots = corn.computeROCCumulative(res, class_index, useweights) 936 944 … … 962 970 963 971 964 @deprecated_keywords({"classIndex": "class_index"}) 965 def compare_2_AUCs(res, lrn1, lrn2, class_index=1, **argkw): 966 return corn.compare2ROCs(res, lrn1, lrn2, class_index, res.weights and not argkw.get("unweighted")) 972 @deprecated_keywords({"classIndex": "class_index", 973 "unweighted": "ignore_weights"}) 974 def compare_2_AUCs(res, lrn1, lrn2, class_index=1, 975 ignore_weights=False, **argkw): 976 return corn.compare2ROCs(res, lrn1, lrn2, class_index, res.weights and not ignore_weights) 967 977 968 978 compare_2_AROCs = compare_2_AUCs # for backward compatibility, compare_2_AROCs is obsolote … … 1348 1358 1349 1359 1350 @deprecated_keywords({"classIndex": "class_index"}) 1351 def compute_CDT(res, class_index=1, **argkw): 1360 @deprecated_keywords({"classIndex": "class_index", 1361 "unweighted": "ignore_weights"}) 1362 def compute_CDT(res, class_index=1, ignore_weights=False, **argkw): 1352 1363 """Obsolete, don't use""" 1353 1364 if class_index<0: … … 1357 1368 class_index = 1 1358 1369 1359 useweights = res.weights and not argkw.get("unweighted", 0)1370 useweights = res.weights and not ignore_weights 1360 1371 weightByClasses = argkw.get("weightByClasses", True) 1361 1372 … … 1384 1395 OneAgainstAll = 3 1385 1396 1386 @deprecated_keywords({"useWeights": "use_weights"}) 1387 def __call__(self, res, method = 0, use_weights = True): 1397 def __call__(self, res, method=0, ignore_weights=False, useWeights=None): 1388 1398 """ Returns the area under ROC curve (AUC) given a set of experimental 1389 1399 results. For multivalued class problems, it will compute some sort of 1390 1400 average, as specified by the argument method. 1391 1401 """ 1402 if useWeights is not None: 1403 deprecation_warning("useWeights", "ignore_weights") 1404 ignore_weights = not useWeights 1405 1392 1406 if len(res.class_values) < 2: 1393 1407 raise ValueError("Cannot compute AUC on a singleclass problem") 1394 1408 elif len(res.class_values) == 2: 1395 return self.compute_for_binary_class(res, use_weights) 1396 else: 1397 return self.compute_for_multi_value_class(res, use_weights, method) 1398 1399 # AUC for binary classification problems 1400 @deprecated_keywords({"useWeights": "use_weights"}) 1401 def compute_for_binary_class(self, res, use_weights = True): 1409 return self.compute_for_binary_class(res, ignore_weights) 1410 else: 1411 return self.compute_for_multi_value_class(res, ignore_weights, method) 1412 1413 def compute_for_binary_class(self, res, ignore_weights=False): 1402 1414 """AUC for binary classification problems""" 1403 1415 if res.number_of_iterations > 1: … … 1405 1417 self.compute_one_class_against_all, 1406 1418 split_by_iterations(res), 1407 (1, use_weights,res, res.number_of_iterations)) 1408 else: 1409 return self.compute_one_class_against_all(res, 1, use_weights)[0] 1410 1411 @deprecated_keywords({"useWeights": "use_weights"}) 1412 def compute_for_multi_value_class(self, res, use_weights = True, 1413 method = 0): 1419 (1, not ignore_weights,res, res.number_of_iterations)) 1420 else: 1421 return self.compute_one_class_against_all(res, 1, ignore_weights)[0] 1422 1423 def compute_for_multi_value_class(self, res, ignore_weights=False, 1424 method=0): 1414 1425 """AUC for multiclass classification problems""" 1415 1426 numberOfClasses = len(res.class_values) … … 1436 1447 iterations, 1437 1448 (classIndex1, classIndex2, 1438 use_weights, all_ite,1449 not ignore_weights, all_ite, 1439 1450 res.number_of_iterations)) 1440 1451 if subsum_aucs: … … 1450 1461 subsum_aucs = self.compute_for_multiple_folds( 1451 1462 self.compute_one_class_against_all, 1452 iterations, (classIndex, use_weights, all_ite,1463 iterations, (classIndex, not ignore_weights, all_ite, 1453 1464 res.number_of_iterations)) 1454 1465 if subsum_aucs: … … 1487 1498 # computes AUC between class i and the other classes (treating them as the same class) 1488 1499 @deprecated_keywords({"classIndex": "class_index", 1489 "useWeights": "use_weights",1490 1500 "divideByIfIte": "divide_by_if_ite"}) 1491 1501 def compute_one_class_against_all(self, ite, class_index, 1492 use_weights = True, all_ite = None, divide_by_if_ite = 1.0): 1502 ignore_weights=True, all_ite=None, 1503 divide_by_if_ite=1.0): 1493 1504 """Compute AUC between class i and all the other classes)""" 1494 1505 return self.compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite, 1495 (class_index, use_weights))1506 (class_index, not ignore_weights)) 1496 1507 1497 1508 1498 1509 # computes AUC between classes i and j as if there are no other classes 1499 1510 def compute_one_class_against_another(self, ite, class_index1, 1500 class_index2, use_weights = True, all_ite =None,1501 divide_by_if_ite =1.0):1511 class_index2, ignore_weights=False, all_ite=None, 1512 divide_by_if_ite=1.0): 1502 1513 """ 1503 1514 Compute AUC between classes i and j as if there are no other classes. 1504 1515 """ 1505 1516 return self.compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, 1506 (class_index1, class_index2, use_weights))1517 (class_index1, class_index2, not ignore_weights)) 1507 1518 1508 1519 # computes AUC using a specified 'cdtComputer' function … … 1530 1541 return False, False 1531 1542 1532 1533 1543 AUC = AucClass() 1534 AUC_binary = deprecated_function_name(AUC.compute_for_binary_class) 1535 AUC_multi = deprecated_function_name(AUC.compute_for_multi_value_class) 1536 AUC_iterations = deprecated_function_name(AUC.compute_for_multiple_folds) 1537 AUC_x = deprecated_function_name(AUC.compute_auc) 1538 AUC_i = deprecated_function_name(AUC.compute_one_class_against_all) 1539 AUC_ij = deprecated_function_name(AUC.compute_one_class_against_another) 1544 1545 # Backward compatibility 1546 def replace_use_weights(fun): 1547 @functools.wraps(fun) 1548 def wrapped(*args, **kwargs): 1549 use_weights = kwargs.pop("useWeights", None) 1550 if use_weights is not None: 1551 deprecation_warning("useWeights", "ignore_weights") 1552 kwargs["ignore_weights"] = not use_weights 1553 return fun(*args, **kwargs) 1554 return wrapped 1555 1556 1557 AUC_binary = replace_use_weights(deprecated_function_name(AUC.compute_for_binary_class)) 1558 AUC_multi = replace_use_weights(deprecated_function_name(AUC.compute_for_multi_value_class)) 1559 AUC_iterations = replace_use_weights(deprecated_function_name(AUC.compute_for_multiple_folds)) 1560 AUC_x = replace_use_weights(deprecated_function_name(AUC.compute_auc)) 1561 AUC_i = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_all)) 1562 AUC_ij = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_another)) 1540 1563 1541 1564 # Computes AUC; in multivalued class problem, AUC is computed as one against all 1542 1565 # Results over folds are averages; if some folds examples from one class only, the folds are merged 1543 @ deprecated_keywords({"classIndex": "class_index",1544 "useWeights": "use_weights"})1545 def AUC_single(res, class_index = 1, use_weights = True):1566 @replace_use_weights 1567 @deprecated_keywords({"classIndex": "class_index"}) 1568 def AUC_single(res, class_index=1, ignore_weights=False): 1546 1569 """ Computes AUC where the class given classIndex is singled out, and 1547 1570 all other classes are treated as a single class. To find how good our … … 1559 1582 1560 1583 if res.number_of_iterations > 1: 1561 return AUC_iterations(AUC_i, split_by_iterations(res), (class_index, use_weights, res, res.number_of_iterations)) 1562 else: 1563 return AUC_i( res, class_index, use_weights)[0] 1584 return AUC_iterations(AUC_i, split_by_iterations(res), 1585 (class_index, not ignore_weights, res, res.number_of_iterations)) 1586 else: 1587 return AUC_i( res, class_index, ignore_weights)[0] 1564 1588 1565 1589 # Computes AUC for a pair of classes (as if there were no other classes) 1566 1590 # Results over folds are averages; if some folds have examples from one class only, the folds are merged 1591 @replace_use_weights 1567 1592 @deprecated_keywords({"classIndex1": "class_index1", 1568 "classIndex2": "class_index2", 1569 "useWeights": "use_weights"}) 1570 def AUC_pair(res, class_index1, class_index2, use_weights = True): 1593 "classIndex2": "class_index2"}) 1594 def AUC_pair(res, class_index1, class_index2, ignore_weights=False): 1571 1595 """ Computes AUC between a pair of instances, ignoring instances from all 1572 1596 other classes. 1573 1597 """ 1574 1598 if res.number_of_iterations > 1: 1575 return AUC_iterations(AUC_ij, split_by_iterations(res), (class_index1, class_index2, use_weights, res, res.number_of_iterations)) 1576 else: 1577 return AUC_ij(res, class_index1, class_index2, use_weights) 1578 1579 1580 # AUC for multiclass problems 1581 @deprecated_keywords({"useWeights": "use_weights"}) 1582 def AUC_matrix(res, use_weights = True): 1599 return AUC_iterations(AUC_ij, split_by_iterations(res), 1600 (class_index1, class_index2, not ignore_weights, res, res.number_of_iterations)) 1601 else: 1602 return AUC_ij(res, class_index1, class_index2, ignore_weights) 1603 1604 @replace_use_weights 1605 def AUC_matrix(res, ignore_weights=False): 1583 1606 """ Computes a (lower diagonal) matrix with AUCs for all pairs of classes. 1584 1607 If there are empty classes, the corresponding elements in the matrix … … 1605 1628 for classIndex1 in range(numberOfClasses): 1606 1629 for classIndex2 in range(classIndex1): 1607 pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, classIndex2, use_weights, all_ite, res.number_of_iterations)) 1630 pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, 1631 classIndex2, not ignore_weights, 1632 all_ite, res.number_of_iterations)) 1608 1633 if pair_aucs: 1609 1634 for lrn in range(number_of_learners): … … 1613 1638 aucs[lrn][classIndex1].append(1) 1614 1639 return aucs 1615 1616 1617 def McNemar(res, **argkw): 1640 1641 1642 @deprecated_keywords({"unweighted": "ignore_weights"}) 1643 def McNemar(res, ignore_weights=False, **argkw): 1618 1644 """ Computes a triangular matrix with McNemar statistics for each pair of 1619 1645 classifiers. The statistics is distributed by chisquare distribution with … … 1625 1651 mcm.append([0.0]*res.number_of_learners) 1626 1652 1627 if not res.weights or argkw.get("unweighted"):1653 if not res.weights or ignore_weights: 1628 1654 for i in res.results: 1629 1655 actual = i.actual_class … … 1662 1688 1663 1689 1664 def McNemar_of_two(res, lrn1, lrn2 ):1690 def McNemar_of_two(res, lrn1, lrn2, ignore_weights=False): 1665 1691 """ McNemar_of_two computes a McNemar statistics for a pair of classifier, 1666 1692 specified by indices learner1 and learner2. 1667 1693 """ 1668 1694 tf = ft = 0.0 1669 if not res.weights or argkw.get("unweighted"):1695 if not res.weights or ignore_weights: 1670 1696 for i in res.results: 1671 1697 actual=i.actual_class
Note: See TracChangeset
for help on using the changeset viewer.