Changeset 10177:8da5d2e7dfe3 in orange


Ignore:
Timestamp:
02/08/12 20:21:35 (2 years ago)
Author:
anze <anze.staric@…>
Branch:
default
rebase_source:
18ed0dd8d7fb04fdf6345f24c90ad93c6f52af3d
Message:

Replaced unweighted parameter with ignore_weights.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • Orange/evaluation/scoring.py

    r10176 r10177  
    1 import operator, math 
     1import  math, functools 
    22from operator import add 
    33import numpy 
     
    55import Orange 
    66from Orange import statc, corn 
    7 from Orange.misc import deprecated_keywords, deprecated_function_name 
     7from Orange.misc import deprecated_keywords, deprecated_function_name, deprecation_warning 
    88from Orange.evaluation import testing 
    99 
     
    4141        return [res] 
    4242         
    43     ress = [Orange.evaluation.testing.ExperimentResults(1, res.classifier_names, res.class_values, res.weights, classifiers=res.classifiers, loaded=res.loaded, test_type=res.test_type, labels=res.labels) 
     43    ress = [Orange.evaluation.testing.ExperimentResults( 
     44                1, res.classifier_names, res.class_values, 
     45                res.weights, classifiers=res.classifiers, 
     46                loaded=res.loaded, test_type=res.test_type, labels=res.labels) 
    4447            for i in range(res.number_of_iterations)] 
    4548    for te in res.results: 
     
    359362    CROSS_VALIDATION = 3 
    360363 
    361     @deprecated_keywords({"reportSE": "report_se"}) 
    362     def __call__(self, test_results, report_se = False, unweighted=False): 
     364    @deprecated_keywords({"reportSE": "report_se", 
     365                          "unweighted": "ignore_weights"}) 
     366    def __call__(self, test_results, report_se = False, ignore_weights=False): 
    363367        """Return percentage of matches between predicted and actual class. 
    364368 
     
    366370                             or :obj:`ConfusionMatrix`. 
    367371        :param report_se: include standard error in result. 
    368         :param unweighted: ignore instance weights. 
     372        :param ignore_weights: ignore instance weights. 
    369373        :rtype: list of scores, one for each learner. 
    370374 
     
    380384        elif input_type == self.CLASSIFICATION: 
    381385            return self.from_classification_results( 
    382                                         test_results, report_se, unweighted) 
     386                                        test_results, report_se, ignore_weights) 
    383387        elif input_type == self.CROSS_VALIDATION: 
    384388            return self.from_crossvalidation_results( 
    385                                         test_results, report_se, unweighted) 
     389                                        test_results, report_se, ignore_weights) 
    386390 
    387391    def from_confusion_matrix(self, cm, report_se): 
     
    398402        return map(self.from_confusion_matrix, confusion_matrices) # TODO: report_se 
    399403 
    400     def from_classification_results(self, test_results, report_se, unweighted): 
     404    def from_classification_results(self, test_results, report_se, ignore_results): 
    401405        CAs = [0.0]*test_results.number_of_learners 
    402406        totweight = 0. 
    403407        for tex in test_results.results: 
    404             w = 1. if unweighted else tex.weight 
     408            w = 1. if ignore_results else tex.weight 
    405409            CAs = map(lambda res, cls: res+(cls==tex.actual_class and w), CAs, tex.classes) 
    406410            totweight += w 
     
    413417            return ca 
    414418 
    415     def from_crossvalidation_results(self, test_results, report_se, unweighted): 
     419    def from_crossvalidation_results(self, test_results, report_se, ignore_weights): 
    416420        CAsByFold = [[0.0]*test_results.number_of_iterations for i in range(test_results.number_of_learners)] 
    417421        foldN = [0.0]*test_results.number_of_iterations 
    418422 
    419423        for tex in test_results.results: 
    420             w = 1. if unweighted else tex.weight 
     424            w = 1. if ignore_weights else tex.weight 
    421425            for lrn in range(test_results.number_of_learners): 
    422426                CAsByFold[lrn][tex.iteration_number] += (tex.classes[lrn]==tex.actual_class) and w 
     
    439443CA = CAClass() 
    440444 
    441 @deprecated_keywords({"reportSE": "report_se"}) 
    442 def AP(res, report_se = False, **argkw): 
     445@deprecated_keywords({"reportSE": "report_se", 
     446                      "unweighted": "ignore_weights"}) 
     447def AP(res, report_se = False, ignore_weights=False, **argkw): 
    443448    """ Computes the average probability assigned to the correct class. """ 
    444449    if res.number_of_iterations == 1: 
    445450        APs=[0.0]*res.number_of_learners 
    446         if argkw.get("unweighted", 0) or not res.weights: 
     451        if ignore_weights or not res.weights: 
    447452            for tex in res.results: 
    448453                APs = map(lambda res, probs: res + probs[tex.actual_class], APs, tex.probabilities) 
     
    458463    APsByFold = [[0.0]*res.number_of_learners for i in range(res.number_of_iterations)] 
    459464    foldN = [0.0] * res.number_of_iterations 
    460     if argkw.get("unweighted", 0) or not res.weights: 
     465    if ignore_weights or not res.weights: 
    461466        for tex in res.results: 
    462467            APsByFold[tex.iteration_number] = map(lambda res, probs: res + probs[tex.actual_class], APsByFold[tex.iteration_number], tex.probabilities) 
     
    470475 
    471476 
    472 @deprecated_keywords({"reportSE": "report_se"}) 
    473 def Brier_score(res, report_se = False, **argkw): 
     477@deprecated_keywords({"reportSE": "report_se", 
     478                      "unweighted": "ignore_weights"}) 
     479def Brier_score(res, report_se = False, ignore_weights=False, **argkw): 
    474480    """ Computes the Brier's score, defined as the average (over test examples) 
    475481    of sumx(t(x)-p(x))2, where x is a class, t(x) is 1 for the correct class 
     
    489495    if res.number_of_iterations == 1: 
    490496        MSEs=[0.0]*res.number_of_learners 
    491         if argkw.get("unweighted", 0) or not res.weights: 
     497        if ignore_weights or not res.weights: 
    492498            totweight = 0.0 
    493499            for tex in res.results: 
     
    509515    foldN = [0.] * res.number_of_iterations 
    510516 
    511     if argkw.get("unweighted", 0) or not res.weights: 
     517    if ignore_weights or not res.weights: 
    512518        for tex in res.results: 
    513519            BSs[tex.iteration_number] = map(lambda rr, probs: 
     
    629635 
    630636@deprecated_keywords({"res": "test_results", 
    631                       "classIndex": "class_index"}) 
     637                      "classIndex": "class_index", 
     638                      "unweighted": "ignore_weights"}) 
    632639def confusion_matrices(test_results, class_index=-1, 
    633                        unweighted=False, cutoff=.5): 
     640                       ignore_weights=False, cutoff=.5): 
    634641    """ 
    635642    Return confusion matrices for test_results. 
     
    638645    :param class_index: index of class value for which the confusion matrices 
    639646                        are to be computed. 
    640     :param unweighted: ignore instance weights. 
     647    :param ignore_weights: ignore instance weights. 
    641648    :params cutoff: cutoff for probability 
    642649 
     
    649656        if class_index < -1 or numberOfClasses > 2: 
    650657            cm = [[[0.0] * numberOfClasses for i in range(numberOfClasses)] for l in range(test_results.number_of_learners)] 
    651             if unweighted or not test_results.weights: 
     658            if ignore_weights or not test_results.weights: 
    652659                for tex in test_results.results: 
    653660                    trueClass = int(tex.actual_class) 
     
    671678 
    672679    if cutoff != .5: 
    673         if unweighted or not test_results.weights: 
     680        if ignore_weights or not test_results.weights: 
    674681            for lr in test_results.results: 
    675682                isPositive=(lr.actual_class==class_index) 
     
    682689                    tfpns[i].addTFPosNeg(lr.probabilities[i][class_index]>cutoff, isPositive, lr.weight) 
    683690    else: 
    684         if unweighted or not test_results.weights: 
     691        if ignore_weights or not test_results.weights: 
    685692            for lr in test_results.results: 
    686693                isPositive=(lr.actual_class==class_index) 
     
    924931       return ret 
    925932 
    926 @deprecated_keywords({"classIndex": "class_index"}) 
    927 def AUCWilcoxon(res, class_index=-1, **argkw): 
     933@deprecated_keywords({"classIndex": "class_index", 
     934                      "unweighted": "ignore_weights"}) 
     935def AUCWilcoxon(res, class_index=-1, ignore_weights=False, **argkw): 
    928936    """ Computes the area under ROC (AUC) and its standard error using 
    929937    Wilcoxon's approach proposed by Hanley and McNeal (1982). If  
     
    932940    tuples (aROC, standard error). 
    933941    """ 
    934     useweights = res.weights and not argkw.get("unweighted", 0) 
     942    useweights = res.weights and not ignore_weights 
    935943    problists, tots = corn.computeROCCumulative(res, class_index, useweights) 
    936944 
     
    962970 
    963971 
    964 @deprecated_keywords({"classIndex": "class_index"}) 
    965 def compare_2_AUCs(res, lrn1, lrn2, class_index=-1, **argkw): 
    966     return corn.compare2ROCs(res, lrn1, lrn2, class_index, res.weights and not argkw.get("unweighted")) 
     972@deprecated_keywords({"classIndex": "class_index", 
     973                      "unweighted": "ignore_weights"}) 
     974def compare_2_AUCs(res, lrn1, lrn2, class_index=-1, 
     975                   ignore_weights=False, **argkw): 
     976    return corn.compare2ROCs(res, lrn1, lrn2, class_index, res.weights and not ignore_weights) 
    967977 
    968978compare_2_AROCs = compare_2_AUCs # for backward compatibility, compare_2_AROCs is obsolote 
     
    13481358 
    13491359 
    1350 @deprecated_keywords({"classIndex": "class_index"}) 
    1351 def compute_CDT(res, class_index=-1, **argkw): 
     1360@deprecated_keywords({"classIndex": "class_index", 
     1361                      "unweighted": "ignore_weights"}) 
     1362def compute_CDT(res, class_index=-1, ignore_weights=False, **argkw): 
    13521363    """Obsolete, don't use""" 
    13531364    if class_index<0: 
     
    13571368            class_index = 1 
    13581369             
    1359     useweights = res.weights and not argkw.get("unweighted", 0) 
     1370    useweights = res.weights and not ignore_weights 
    13601371    weightByClasses = argkw.get("weightByClasses", True) 
    13611372 
     
    13841395    OneAgainstAll = 3 
    13851396 
    1386     @deprecated_keywords({"useWeights": "use_weights"}) 
    1387     def __call__(self, res, method = 0, use_weights = True): 
     1397    def __call__(self, res, method=0, ignore_weights=False, useWeights=None): 
    13881398        """ Returns the area under ROC curve (AUC) given a set of experimental 
    13891399        results. For multivalued class problems, it will compute some sort of 
    13901400        average, as specified by the argument method. 
    13911401        """ 
     1402        if useWeights is not None: 
     1403            deprecation_warning("useWeights", "ignore_weights") 
     1404            ignore_weights = not useWeights 
     1405 
    13921406        if len(res.class_values) < 2: 
    13931407            raise ValueError("Cannot compute AUC on a single-class problem") 
    13941408        elif len(res.class_values) == 2: 
    1395             return self.compute_for_binary_class(res, use_weights) 
    1396         else: 
    1397             return self.compute_for_multi_value_class(res, use_weights, method) 
    1398  
    1399     # AUC for binary classification problems 
    1400     @deprecated_keywords({"useWeights": "use_weights"}) 
    1401     def compute_for_binary_class(self, res, use_weights = True): 
     1409            return self.compute_for_binary_class(res, ignore_weights) 
     1410        else: 
     1411            return self.compute_for_multi_value_class(res, ignore_weights, method) 
     1412 
     1413    def compute_for_binary_class(self, res, ignore_weights=False): 
    14021414        """AUC for binary classification problems""" 
    14031415        if res.number_of_iterations > 1: 
     
    14051417                        self.compute_one_class_against_all, 
    14061418                        split_by_iterations(res), 
    1407                         (-1, use_weights,res, res.number_of_iterations)) 
    1408         else: 
    1409             return self.compute_one_class_against_all(res, -1, use_weights)[0] 
    1410  
    1411     @deprecated_keywords({"useWeights": "use_weights"}) 
    1412     def compute_for_multi_value_class(self, res, use_weights = True, 
    1413                                       method = 0): 
     1419                        (-1, not ignore_weights,res, res.number_of_iterations)) 
     1420        else: 
     1421            return self.compute_one_class_against_all(res, -1, ignore_weights)[0] 
     1422 
     1423    def compute_for_multi_value_class(self, res, ignore_weights=False, 
     1424                                      method=0): 
    14141425        """AUC for multiclass classification problems""" 
    14151426        numberOfClasses = len(res.class_values) 
     
    14361447                                             iterations, 
    14371448                                             (classIndex1, classIndex2, 
    1438                                              use_weights, all_ite, 
     1449                                             not ignore_weights, all_ite, 
    14391450                                             res.number_of_iterations)) 
    14401451                    if subsum_aucs: 
     
    14501461                subsum_aucs = self.compute_for_multiple_folds( 
    14511462                    self.compute_one_class_against_all, 
    1452                     iterations, (classIndex, use_weights, all_ite, 
     1463                    iterations, (classIndex, not ignore_weights, all_ite, 
    14531464                                 res.number_of_iterations)) 
    14541465                if subsum_aucs: 
     
    14871498    # computes AUC between class i and the other classes (treating them as the same class) 
    14881499    @deprecated_keywords({"classIndex": "class_index", 
    1489                           "useWeights": "use_weights", 
    14901500                          "divideByIfIte": "divide_by_if_ite"}) 
    14911501    def compute_one_class_against_all(self, ite, class_index, 
    1492             use_weights = True, all_ite = None, divide_by_if_ite = 1.0): 
     1502                                      ignore_weights=True, all_ite=None, 
     1503                                      divide_by_if_ite=1.0): 
    14931504        """Compute AUC between class i and all the other classes)""" 
    14941505        return self.compute_auc(corn.computeCDT, ite, all_ite, divide_by_if_ite, 
    1495             (class_index, use_weights)) 
     1506            (class_index, not ignore_weights)) 
    14961507 
    14971508 
    14981509    # computes AUC between classes i and j as if there are no other classes 
    14991510    def compute_one_class_against_another(self, ite, class_index1, 
    1500             class_index2, use_weights = True, all_ite = None, 
    1501             divide_by_if_ite = 1.0): 
     1511            class_index2, ignore_weights=False, all_ite=None, 
     1512            divide_by_if_ite=1.0): 
    15021513        """ 
    15031514        Compute AUC between classes i and j as if there are no other classes. 
    15041515        """ 
    15051516        return self.compute_auc(corn.computeCDTPair, ite, all_ite, divide_by_if_ite, 
    1506             (class_index1, class_index2, use_weights)) 
     1517            (class_index1, class_index2, not ignore_weights)) 
    15071518 
    15081519    # computes AUC using a specified 'cdtComputer' function 
     
    15301541        return False, False 
    15311542 
    1532  
    15331543AUC = AucClass() 
    1534 AUC_binary = deprecated_function_name(AUC.compute_for_binary_class) 
    1535 AUC_multi = deprecated_function_name(AUC.compute_for_multi_value_class) 
    1536 AUC_iterations = deprecated_function_name(AUC.compute_for_multiple_folds) 
    1537 AUC_x = deprecated_function_name(AUC.compute_auc) 
    1538 AUC_i = deprecated_function_name(AUC.compute_one_class_against_all) 
    1539 AUC_ij = deprecated_function_name(AUC.compute_one_class_against_another) 
     1544 
     1545# Backward compatibility 
     1546def replace_use_weights(fun): 
     1547    @functools.wraps(fun) 
     1548    def wrapped(*args, **kwargs): 
     1549        use_weights = kwargs.pop("useWeights", None) 
     1550        if use_weights is not None: 
     1551            deprecation_warning("useWeights", "ignore_weights") 
     1552            kwargs["ignore_weights"] = not use_weights 
     1553        return fun(*args, **kwargs) 
     1554    return wrapped 
     1555 
     1556 
     1557AUC_binary = replace_use_weights(deprecated_function_name(AUC.compute_for_binary_class)) 
     1558AUC_multi = replace_use_weights(deprecated_function_name(AUC.compute_for_multi_value_class)) 
     1559AUC_iterations = replace_use_weights(deprecated_function_name(AUC.compute_for_multiple_folds)) 
     1560AUC_x = replace_use_weights(deprecated_function_name(AUC.compute_auc)) 
     1561AUC_i = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_all)) 
     1562AUC_ij = replace_use_weights(deprecated_function_name(AUC.compute_one_class_against_another)) 
    15401563 
    15411564# Computes AUC; in multivalued class problem, AUC is computed as one against all 
    15421565# Results over folds are averages; if some folds examples from one class only, the folds are merged 
    1543 @deprecated_keywords({"classIndex": "class_index", 
    1544                       "useWeights": "use_weights"}) 
    1545 def AUC_single(res, class_index = -1, use_weights = True): 
     1566@replace_use_weights 
     1567@deprecated_keywords({"classIndex": "class_index"}) 
     1568def AUC_single(res, class_index=-1, ignore_weights=False): 
    15461569    """ Computes AUC where the class given classIndex is singled out, and 
    15471570    all other classes are treated as a single class. To find how good our 
     
    15591582 
    15601583    if res.number_of_iterations > 1: 
    1561         return AUC_iterations(AUC_i, split_by_iterations(res), (class_index, use_weights, res, res.number_of_iterations)) 
    1562     else: 
    1563         return AUC_i( res, class_index, use_weights)[0] 
     1584        return AUC_iterations(AUC_i, split_by_iterations(res), 
     1585            (class_index, not ignore_weights, res, res.number_of_iterations)) 
     1586    else: 
     1587        return AUC_i( res, class_index, ignore_weights)[0] 
    15641588 
    15651589# Computes AUC for a pair of classes (as if there were no other classes) 
    15661590# Results over folds are averages; if some folds have examples from one class only, the folds are merged 
     1591@replace_use_weights 
    15671592@deprecated_keywords({"classIndex1": "class_index1", 
    1568                       "classIndex2": "class_index2", 
    1569                       "useWeights": "use_weights"}) 
    1570 def AUC_pair(res, class_index1, class_index2, use_weights = True): 
     1593                      "classIndex2": "class_index2"}) 
     1594def AUC_pair(res, class_index1, class_index2, ignore_weights=False): 
    15711595    """ Computes AUC between a pair of instances, ignoring instances from all 
    15721596    other classes. 
    15731597    """ 
    15741598    if res.number_of_iterations > 1: 
    1575         return AUC_iterations(AUC_ij, split_by_iterations(res), (class_index1, class_index2, use_weights, res, res.number_of_iterations)) 
    1576     else: 
    1577         return AUC_ij(res, class_index1, class_index2, use_weights) 
    1578    
    1579  
    1580 # AUC for multiclass problems 
    1581 @deprecated_keywords({"useWeights": "use_weights"}) 
    1582 def AUC_matrix(res, use_weights = True): 
     1599        return AUC_iterations(AUC_ij, split_by_iterations(res), 
     1600            (class_index1, class_index2, not ignore_weights, res, res.number_of_iterations)) 
     1601    else: 
     1602        return AUC_ij(res, class_index1, class_index2, ignore_weights) 
     1603 
     1604@replace_use_weights 
     1605def AUC_matrix(res, ignore_weights=False): 
    15831606    """ Computes a (lower diagonal) matrix with AUCs for all pairs of classes. 
    15841607    If there are empty classes, the corresponding elements in the matrix 
     
    16051628    for classIndex1 in range(numberOfClasses): 
    16061629        for classIndex2 in range(classIndex1): 
    1607             pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, classIndex2, use_weights, all_ite, res.number_of_iterations)) 
     1630            pair_aucs = AUC_iterations(AUC_ij, iterations, (classIndex1, 
     1631                                                            classIndex2, not ignore_weights, 
     1632                                                            all_ite, res.number_of_iterations)) 
    16081633            if pair_aucs: 
    16091634                for lrn in range(number_of_learners): 
     
    16131638                    aucs[lrn][classIndex1].append(-1) 
    16141639    return aucs 
    1615                  
    1616  
    1617 def McNemar(res, **argkw): 
     1640 
     1641 
     1642@deprecated_keywords({"unweighted": "ignore_weights"}) 
     1643def McNemar(res, ignore_weights=False, **argkw): 
    16181644    """ Computes a triangular matrix with McNemar statistics for each pair of 
    16191645    classifiers. The statistics is distributed by chi-square distribution with 
     
    16251651       mcm.append([0.0]*res.number_of_learners) 
    16261652 
    1627     if not res.weights or argkw.get("unweighted"): 
     1653    if not res.weights or ignore_weights: 
    16281654        for i in res.results: 
    16291655            actual = i.actual_class 
     
    16621688 
    16631689 
    1664 def McNemar_of_two(res, lrn1, lrn2): 
     1690def McNemar_of_two(res, lrn1, lrn2, ignore_weights=False): 
    16651691    """ McNemar_of_two computes a McNemar statistics for a pair of classifier, 
    16661692    specified by indices learner1 and learner2. 
    16671693    """ 
    16681694    tf = ft = 0.0 
    1669     if not res.weights or argkw.get("unweighted"): 
     1695    if not res.weights or ignore_weights: 
    16701696        for i in res.results: 
    16711697            actual=i.actual_class 
Note: See TracChangeset for help on using the changeset viewer.