Changeset 10176:789b75ba3f64 in orange
 Timestamp:
 02/08/12 17:38:27 (2 years ago)
 Branch:
 default
 rebase_source:
 4a564d211eb1a0eedcb3c7db6c8ebab315380284
 Files:

 2 edited
Legend:
 Unmodified
 Added
 Removed

Orange/evaluation/scoring.py
r10063 r10176 5 5 import Orange 6 6 from Orange import statc, corn 7 from Orange.misc import deprecated_keywords 7 from Orange.misc import deprecated_keywords, deprecated_function_name 8 8 from Orange.evaluation import testing 9 9 … … 1377 1377 return corn.computeCDT(res, class_index, useweights) 1378 1378 1379 ## THIS FUNCTION IS OBSOLETE AND ITS AVERAGING OVER FOLDS IS QUESTIONABLE 1380 ## DON'T USE IT 1381 def ROCs_from_CDT(cdt, **argkw): 1382 """Obsolete, don't use""" 1383 if type(cdt) == list: 1384 return [ROCs_from_CDT(c) for c in cdt] 1385 1386 C, D, T = cdt.C, cdt.D, cdt.T 1387 N = C+D+T 1388 if N < 1e6: 1389 import warnings 1390 warnings.warn("Can't compute AUC: one or both classes have no instances") 1391 return (1,)*8 1392 if N < 2: 1393 import warnings 1394 warnings.warn("Can't compute AUC: one or both classes have too few examples") 1395 1396 som = (CD)/N 1397 c = 0.5*(1+som) 1398 1399 if (C+D): 1400 res = (C/N*100, D/N*100, T/N*100, N, som, (CD)/(C+D), (CD)/(N*(N1)/2), 0.5*(1+som)) 1401 else: 1402 res = (C/N*100, D/N*100, T/N*100, N, som, 1.0, (CD)/(N*(N1)/2), 0.5*(1+som)) 1403 1404 if argkw.get("print"): 1405 print "Concordant = %5.1f Somers' D = %1.3f" % (res[0], res[4]) 1406 print "Discordant = %5.1f Gamma = %1.3f" % (res[1], res[5]>0 and res[5] or "N/A") 1407 print "Tied = %5.1f Taua = %1.3f" % (res[2], res[6]) 1408 print " %6d pairs c = %1.3f" % (res[3], res[7]) 1409 1410 return res 1411 1412 AROC_from_CDT = ROCs_from_CDT # for backward compatibility, AROC_from_CDT is obsolote 1413 1414 1415 1416 # computes AUC using a specified 'cdtComputer' function 1417 # It tries to compute AUCs from 'ite' (examples from a single iteration) and, 1418 # if C+D+T=0, from 'all_ite' (entire test set). In the former case, the AUCs 1419 # are divided by 'divideByIfIte'. Additional flag is returned which is True in 1420 # the former case, or False in the latter. 1421 @deprecated_keywords({"divideByIfIte": "divide_by_if_ite", 1422 "computerArgs": "computer_args"}) 1423 def AUC_x(cdtComputer, ite, all_ite, divide_by_if_ite, computer_args): 1424 cdts = cdtComputer(*(ite, ) + computer_args) 1425 if not is_CDT_empty(cdts[0]): 1426 return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T)/divide_by_if_ite for cdt in cdts], True 1427 1428 if all_ite: 1429 cdts = cdtComputer(*(all_ite, ) + computer_args) 1430 if not is_CDT_empty(cdts[0]): 1431 return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T) for cdt in cdts], False 1432 1433 return False, False 1434 1435 1436 # computes AUC between classes i and j as if there we no other classes 1437 @deprecated_keywords({"classIndex1": "class_index1", 1438 "classIndex2": "class_index2", 1439 "useWeights": "use_weights", 1440 "divideByIfIte": "divide_by_if_ite"}) 1441 def AUC_ij(ite, class_index1, class_index2, use_weights = True, all_ite = None, divide_by_if_ite = 1.0): 1442 return AUC_x(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, (class_index1, class_index2, use_weights)) 1443 1444 1445 # computes AUC between class i and the other classes (treating them as the same class) 1446 @deprecated_keywords({"classIndex": "class_index", 1447 "useWeights": "use_weights", 1448 "divideByIfIte": "divide_by_if_ite"}) 1449 def AUC_i(ite, class_index, use_weights = True, all_ite = None, 1450 divide_by_if_ite = 1.0): 1451 return AUC_x(corn.computeCDT, ite, all_ite, divide_by_if_ite, (class_index, use_weights)) 1452 1453 1454 # computes the average AUC over folds using a "AUCcomputer" (AUC_i or AUC_ij) 1455 # it returns the sum of what is returned by the computer, unless at a certain 1456 # fold the computer has to resort to computing over all folds or even this failed; 1457 # in these cases the result is returned immediately 1458 1459 @deprecated_keywords({"AUCcomputer": "auc_computer", 1460 "computerArgs": "computer_args"}) 1461 def AUC_iterations(auc_computer, iterations, computer_args): 1462 subsum_aucs = [0.] * iterations[0].number_of_learners 1463 for ite in iterations: 1464 aucs, foldsUsed = auc_computer(*(ite, ) + computer_args) 1465 if not aucs: 1466 return None 1467 if not foldsUsed: 1468 return aucs 1469 subsum_aucs = map(add, subsum_aucs, aucs) 1470 return subsum_aucs 1471 1472 1473 # AUC for binary classification problems 1474 @deprecated_keywords({"useWeights": "use_weights"}) 1475 def AUC_binary(res, use_weights = True): 1476 if res.number_of_iterations > 1: 1477 return AUC_iterations(AUC_i, split_by_iterations(res), (1, use_weights, res, res.number_of_iterations)) 1478 else: 1479 return AUC_i(res, 1, use_weights)[0] 1480 1481 # AUC for multiclass problems 1482 @deprecated_keywords({"useWeights": "use_weights"}) 1483 def AUC_multi(res, use_weights = True, method = 0): 1484 numberOfClasses = len(res.class_values) 1485 1486 if res.number_of_iterations > 1: 1487 iterations = split_by_iterations(res) 1488 all_ite = res 1489 else: 1490 iterations = [res] 1491 all_ite = None 1492 1493 # by pairs 1494 sum_aucs = [0.] * res.number_of_learners 1495 usefulClassPairs = 0. 1496 1497 if method in [0, 2]: 1498 prob = class_probabilities_from_res(res) 1499 1500 if method <= 1: 1501 for classIndex1 in range(numberOfClasses): 1502 for classIndex2 in range(classIndex1): 1503 subsum_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, classIndex2, use_weights, all_ite, res.number_of_iterations)) 1379 1380 class AucClass(object): 1381 ByWeightedPairs = 0 1382 ByPairs = 1 1383 WeightedOneAgainstAll = 2 1384 OneAgainstAll = 3 1385 1386 @deprecated_keywords({"useWeights": "use_weights"}) 1387 def __call__(self, res, method = 0, use_weights = True): 1388 """ Returns the area under ROC curve (AUC) given a set of experimental 1389 results. For multivalued class problems, it will compute some sort of 1390 average, as specified by the argument method. 1391 """ 1392 if len(res.class_values) < 2: 1393 raise ValueError("Cannot compute AUC on a singleclass problem") 1394 elif len(res.class_values) == 2: 1395 return self.compute_for_binary_class(res, use_weights) 1396 else: 1397 return self.compute_for_multi_value_class(res, use_weights, method) 1398 1399 # AUC for binary classification problems 1400 @deprecated_keywords({"useWeights": "use_weights"}) 1401 def compute_for_binary_class(self, res, use_weights = True): 1402 """AUC for binary classification problems""" 1403 if res.number_of_iterations > 1: 1404 return self.compute_for_multiple_folds( 1405 self.compute_one_class_against_all, 1406 split_by_iterations(res), 1407 (1, use_weights,res, res.number_of_iterations)) 1408 else: 1409 return self.compute_one_class_against_all(res, 1, use_weights)[0] 1410 1411 @deprecated_keywords({"useWeights": "use_weights"}) 1412 def compute_for_multi_value_class(self, res, use_weights = True, 1413 method = 0): 1414 """AUC for multiclass classification problems""" 1415 numberOfClasses = len(res.class_values) 1416 1417 if res.number_of_iterations > 1: 1418 iterations = split_by_iterations(res) 1419 all_ite = res 1420 else: 1421 iterations = [res] 1422 all_ite = None 1423 1424 # by pairs 1425 sum_aucs = [0.] * res.number_of_learners 1426 usefulClassPairs = 0. 1427 1428 if method in [0, 2]: 1429 prob = class_probabilities_from_res(res) 1430 1431 if method <= 1: 1432 for classIndex1 in range(numberOfClasses): 1433 for classIndex2 in range(classIndex1): 1434 subsum_aucs = self.compute_for_multiple_folds( 1435 self.compute_one_class_against_another, 1436 iterations, 1437 (classIndex1, classIndex2, 1438 use_weights, all_ite, 1439 res.number_of_iterations)) 1440 if subsum_aucs: 1441 if method == 0: 1442 p_ij = prob[classIndex1] * prob[classIndex2] 1443 subsum_aucs = [x * p_ij for x in subsum_aucs] 1444 usefulClassPairs += p_ij 1445 else: 1446 usefulClassPairs += 1 1447 sum_aucs = map(add, sum_aucs, subsum_aucs) 1448 else: 1449 for classIndex in range(numberOfClasses): 1450 subsum_aucs = self.compute_for_multiple_folds( 1451 self.compute_one_class_against_all, 1452 iterations, (classIndex, use_weights, all_ite, 1453 res.number_of_iterations)) 1504 1454 if subsum_aucs: 1505 1455 if method == 0: 1506 p_i j = prob[classIndex1] * prob[classIndex2]1507 subsum_aucs = [x * p_i jfor x in subsum_aucs]1508 usefulClassPairs += p_i j1456 p_i = prob[classIndex] 1457 subsum_aucs = [x * p_i for x in subsum_aucs] 1458 usefulClassPairs += p_i 1509 1459 else: 1510 1460 usefulClassPairs += 1 1511 1461 sum_aucs = map(add, sum_aucs, subsum_aucs) 1512 else: 1513 for classIndex in range(numberOfClasses): 1514 subsum_aucs = AUC_iterations(AUC_i, iterations, (classIndex, use_weights, all_ite, res.number_of_iterations)) 1515 if subsum_aucs: 1516 if method == 0: 1517 p_i = prob[classIndex] 1518 subsum_aucs = [x * p_i for x in subsum_aucs] 1519 usefulClassPairs += p_i 1520 else: 1521 usefulClassPairs += 1 1522 sum_aucs = map(add, sum_aucs, subsum_aucs) 1523 1524 if usefulClassPairs > 0: 1525 sum_aucs = [x/usefulClassPairs for x in sum_aucs] 1526 1527 return sum_aucs 1528 1529 class AucClass(object): 1530 pass 1531 1532 1533 def AUC(): 1534 pass 1535 1536 AUC.ByWeightedPairs = 0 1537 1538 # Computes AUC, possibly for multiple classes (the averaging method can be specified) 1539 # Results over folds are averages; if some folds examples from one class only, the folds are merged 1540 @deprecated_keywords({"useWeights": "use_weights"}) 1541 def AUC(res, method = AUC.ByWeightedPairs, use_weights = True): 1542 """ Returns the area under ROC curve (AUC) given a set of experimental 1543 results. For multivalued class problems, it will compute some sort of 1544 average, as specified by the argument method. 1545 """ 1546 if len(res.class_values) < 2: 1547 raise ValueError("Cannot compute AUC on a singleclass problem") 1548 elif len(res.class_values) == 2: 1549 return AUC_binary(res, use_weights) 1550 else: 1551 return AUC_multi(res, use_weights, method) 1552 1553 AUC.ByWeightedPairs = 0 1554 AUC.ByPairs = 1 1555 AUC.WeightedOneAgainstAll = 2 1556 AUC.OneAgainstAll = 3 1557 1462 1463 if usefulClassPairs > 0: 1464 sum_aucs = [x/usefulClassPairs for x in sum_aucs] 1465 1466 return sum_aucs 1467 1468 # computes the average AUC over folds using a "AUCcomputer" (AUC_i or AUC_ij) 1469 # it returns the sum of what is returned by the computer, unless at a certain 1470 # fold the computer has to resort to computing over all folds or even this failed; 1471 # in these cases the result is returned immediately 1472 @deprecated_keywords({"AUCcomputer": "auc_computer", 1473 "computerArgs": "computer_args"}) 1474 def compute_for_multiple_folds(self, auc_computer, iterations, 1475 computer_args): 1476 """Compute the average AUC over folds using :obj:`auc_computer`.""" 1477 subsum_aucs = [0.] * iterations[0].number_of_learners 1478 for ite in iterations: 1479 aucs, foldsUsed = auc_computer(*(ite, ) + computer_args) 1480 if not aucs: 1481 return None 1482 if not foldsUsed: 1483 return aucs 1484 subsum_aucs = map(add, subsum_aucs, aucs) 1485 return subsum_aucs 1486 1487 # computes AUC between class i and the other classes (treating them as the same class) 1488 @deprecated_keywords({"classIndex": "class_index", 1489 "useWeights": "use_weights", 1490 "divideByIfIte": "divide_by_if_ite"}) 1491 def compute_one_class_against_all(self, ite, class_index, 1492 use_weights = True, all_ite = None, divide_by_if_ite = 1.0): 1493 """Compute AUC between class i and all the other classes)""" 1494 return self.compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite, 1495 (class_index, use_weights)) 1496 1497 1498 # computes AUC between classes i and j as if there are no other classes 1499 def compute_one_class_against_another(self, ite, class_index1, 1500 class_index2, use_weights = True, all_ite = None, 1501 divide_by_if_ite = 1.0): 1502 """ 1503 Compute AUC between classes i and j as if there are no other classes. 1504 """ 1505 return self.compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, 1506 (class_index1, class_index2, use_weights)) 1507 1508 # computes AUC using a specified 'cdtComputer' function 1509 # It tries to compute AUCs from 'ite' (examples from a single iteration) and, 1510 # if C+D+T=0, from 'all_ite' (entire test set). In the former case, the AUCs 1511 # are divided by 'divideByIfIte'. Additional flag is returned which is True in 1512 # the former case, or False in the latter. 1513 @deprecated_keywords({"cdt_computer": "cdtComputer", 1514 "divideByIfIte": "divide_by_if_ite", 1515 "computerArgs": "computer_args"}) 1516 def compute_auc(self, cdt_computer, ite, all_ite, divide_by_if_ite, 1517 computer_args): 1518 """ 1519 Compute AUC using a :obj:`cdtComputer`. 1520 """ 1521 cdts = cdt_computer(*(ite, ) + computer_args) 1522 if not is_CDT_empty(cdts[0]): 1523 return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T)/divide_by_if_ite for cdt in cdts], True 1524 1525 if all_ite: 1526 cdts = cdt_computer(*(all_ite, ) + computer_args) 1527 if not is_CDT_empty(cdts[0]): 1528 return [(cdt.C+cdt.T/2)/(cdt.C+cdt.D+cdt.T) for cdt in cdts], False 1529 1530 return False, False 1531 1532 1533 AUC = AucClass() 1534 AUC_binary = deprecated_function_name(AUC.compute_for_binary_class) 1535 AUC_multi = deprecated_function_name(AUC.compute_for_multi_value_class) 1536 AUC_iterations = deprecated_function_name(AUC.compute_for_multiple_folds) 1537 AUC_x = deprecated_function_name(AUC.compute_auc) 1538 AUC_i = deprecated_function_name(AUC.compute_one_class_against_all) 1539 AUC_ij = deprecated_function_name(AUC.compute_one_class_against_another) 1558 1540 1559 1541 # Computes AUC; in multivalued class problem, AUC is computed as one against all 
docs/reference/rst/Orange.evaluation.scoring.rst
r10127 r10176 18 18 ============== 19 19 20 Many scores for evaluation of classification models can be computed solely21 from the confusion matrix constructed manually with the22 :obj:`confusion_matrices` function. If class variable has more than two23 values, the index of the value to calculate the confusion matrix for should24 be passed as well.25 26 20 Calibration scores 27 21 ================== 22 Many scores for evaluation of the classification models measure whether the 23 model assigns the correct class value to the test instances. Many of these 24 scores can be computed solely from the confusion matrix constructed manually 25 with the :obj:`confusion_matrices` function. If class variable has more than 26 two values, the index of the value to calculate the confusion matrix for should 27 be passed as well. 28 28 29 29 .. autosingleton:: CA … … 41 41 .. autofunction:: AP 42 42 .. autofunction:: IS 43 .. autofunction:: confusion_chi_square 43 44 44 45 Discriminatory scores 45 46 ===================== 47 Scores that measure how good can the prediction model separate instances with 48 different classes are called discriminatory scores. 46 49 47 50 .. autofunction:: Brier_score … … 51 54 .. attribute:: AUC.ByWeightedPairs (or 0) 52 55 53 Computes AUC for each pair of classes (ignoring instances of all other54 classes) and averages the results, weighting them by the number of55 pairs of instances from these two classes (e.g. by the product of56 probabilities of the two classes). AUC computed in this way still57 behaves as concordance index, e.g., gives the probability that two58 randomly chosen instances from different classes will be correctly59 recognized (this is of course true only if the classifier knows60 from which two classes the instances came).61 62 .. attribute:: AUC.ByPairs (or 1)63 64 Similar as above, except that the average over class pairs is not65 weighted. This AUC is, like the binary, independent of class66 distributions, but it is not related to concordance index any more.67 68 .. attribute:: AUC.WeightedOneAgainstAll (or 2)69 70 For each class, it computes AUC for this class against all others (that71 is, treating other classes as one class). The AUCs are then averaged by72 the class probabilities. This is related to concordance index in which73 we test the classifier's (average) capability for distinguishing the74 instances from a specified class from those that come from other classes.75 Unlike the binary AUC, the measure is not independent of class76 distributions.77 78 .. attribute:: AUC.OneAgainstAll (or 3)79 80 As above, except that the average is not weighted.56 Computes AUC for each pair of classes (ignoring instances of all other 57 classes) and averages the results, weighting them by the number of 58 pairs of instances from these two classes (e.g. by the product of 59 probabilities of the two classes). AUC computed in this way still 60 behaves as concordance index, e.g., gives the probability that two 61 randomly chosen instances from different classes will be correctly 62 recognized (this is of course true only if the classifier knows 63 from which two classes the instances came). 64 65 .. attribute:: AUC.ByPairs (or 1) 66 67 Similar as above, except that the average over class pairs is not 68 weighted. This AUC is, like the binary, independent of class 69 distributions, but it is not related to concordance index any more. 70 71 .. attribute:: AUC.WeightedOneAgainstAll (or 2) 72 73 For each class, it computes AUC for this class against all others (that 74 is, treating other classes as one class). The AUCs are then averaged by 75 the class probabilities. This is related to concordance index in which 76 we test the classifier's (average) capability for distinguishing the 77 instances from a specified class from those that come from other classes. 78 Unlike the binary AUC, the measure is not independent of class 79 distributions. 80 81 .. attribute:: AUC.OneAgainstAll (or 3) 82 83 As above, except that the average is not weighted. 81 84 82 85 In case of multiple folds (for instance if the data comes from cross
Note: See TracChangeset
for help on using the changeset viewer.