Changeset 9276:c576b05f3a91 in orange


Ignore:
Timestamp:
11/28/11 10:58:26 (2 years ago)
Author:
anze <anze.staric@…>
Branch:
default
Convert:
689bcda688bc2d0f76f1688d0f970265f48506b8
Message:

Refactoring of Orange.evaluation.testing.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • orange/Orange/evaluation/testing.py

    r9224 r9276  
    33import Orange 
    44from Orange.misc import demangle_examples, getobjectname, printVerbose, deprecated_keywords 
    5  
    6  
    7 #### Some private stuff 
    8  
    9  
    105 
    116#### Data structures 
     
    1510    TestedExample stores predictions of different classifiers for a single testing example. 
    1611 
    17     .. attribute:: classes 
    18  
    19         A list of predictions of type Value, one for each classifier. 
    20  
    21     .. attribute:: probabilities 
    22  
    23         A list of probabilities of classes, one for each classifier. 
    24  
    25     .. attribute:: iterationNumber 
    26  
    27         Iteration number (e.g. fold) in which the TestedExample was created/tested. 
    28  
    29     .. attribute:: actualClass 
    30  
    31         The correct class of the example 
    32  
    33     .. attribute:: weight 
    34      
    35         Example's weight. Even if the example set was not weighted, 
    36         this attribute is present and equals 1.0. 
    37  
    38     :param iterationNumber: 
    39     :paramtype iterationNumber: type??? 
    40     :param actualClass: 
    41     :paramtype actualClass: type??? 
    42     :param n: 
    43     :paramtype n: int 
    44     :param weight: 
    45     :paramtype weight: float 
    46  
     12    :var classes: A list of predictions of type Value, one for each classifier. 
     13    :var probabilities: A list of probabilities of classes, one for each classifier. 
     14    :var iterationNumber: Iteration number (e.g. fold) in which the TestedExample was created/tested. 
     15    :var actualClass: The correct class of the example 
     16    :var weight: Example's weight. Even if the example set was not weighted, this attribute is present and equals 1.0. 
    4717    """ 
    4818 
    4919    def __init__(self, iterationNumber=None, actualClass=None, n=0, weight=1.0): 
     20        """ 
     21        :param iterationNumber: 
     22        :param actualClass: 
     23        :param n: 
     24        :param weight: 
     25        """ 
    5026        self.classes = [None]*n 
    5127        self.probabilities = [None]*n 
     
    5329        self.actualClass= actualClass 
    5430        self.weight = weight 
    55      
     31 
    5632    def add_result(self, aclass, aprob): 
    5733        """Appends a new result (class and probability prediction by a single classifier) to the classes and probabilities field.""" 
     
    7349            self.probabilities[i] = list(aprob) 
    7450 
     51    def __repr__(self): 
     52        return str(self.__dict__) 
     53 
    7554class ExperimentResults(object): 
    7655    """ 
     
    7958    circumstances. 
    8059 
    81     .. attribute:: results 
    82  
    83         A list of instances of TestedExample, one for each example in 
    84         the dataset. 
    85  
    86     .. attribute:: classifiers 
    87  
    88         A list of classifiers, one element for each repetition (eg 
    89         fold). Each element is a list of classifiers, one for each 
    90         learner. This field is used only if storing is enabled by 
    91         ``storeClassifiers=1``. 
    92  
    93     .. attribute:: numberOfIterations 
    94  
    95         Number of iterations. This can be the number of folds 
    96         (in cross validation) or the number of repetitions of some 
    97         test. ``TestedExample``'s attribute ``iterationNumber`` should 
    98         be in range ``[0, numberOfIterations-1]``. 
    99  
    100     .. attribute:: numberOfLearners 
    101  
    102         Number of learners. Lengths of lists classes and probabilities 
    103         in each :obj:`TestedExample` should equal ``numberOfLearners``. 
    104  
    105     .. attribute:: loaded 
    106  
    107         If the experimental method supports caching and there are no 
    108         obstacles for caching (such as unknown random seeds), this is a 
    109         list of boolean values. Each element corresponds to a classifier 
    110         and tells whether the experimental results for that classifier 
    111         were computed or loaded from the cache. 
    112  
    113     .. attribute:: weights 
    114  
    115         A flag telling whether the results are weighted. If ``False``, 
    116         weights are still present in ``TestedExamples``, but they are 
    117         all ``1.0``. Clear this flag, if your experimental procedure 
    118         ran on weighted testing examples but you would like to ignore 
    119         the weights in statistics. 
    120  
     60    :var results: A list of instances of TestedExample, one for each example in the dataset. 
     61    :var classifiers: A list of classifiers, one element for each repetition (eg. fold). Each element is a list 
     62      of classifiers, one for each learner. This field is used only if storing is enabled by ``storeClassifiers=1``. 
     63    :var numberOfIterations: Number of iterations. This can be the number of folds (in cross validation) 
     64      or the number of repetitions of some test. ``TestedExample``'s attribute ``iterationNumber`` should 
     65      be in range ``[0, numberOfIterations-1]``. 
     66    :var numberOfLearners: Number of learners. Lengths of lists classes and probabilities in each :obj:`TestedExample` 
     67      should equal ``numberOfLearners``. 
     68    :var loaded: If the experimental method supports caching and there are no obstacles for caching (such as unknown 
     69      random seeds), this is a list of boolean values. Each element corresponds to a classifier and tells whether the 
     70      experimental results for that classifier were computed or loaded from the cache. 
     71    :var weights: A flag telling whether the results are weighted. If ``False``, weights are still present 
     72      in ``TestedExamples``, but they are all ``1.0``. Clear this flag, if your experimental procedure ran on weighted 
     73      testing examples but you would like to ignore the weights in statistics. 
    12174    """ 
    12275    def __init__(self, iterations, classifierNames, classValues=None, weights=None, baseClass=-1, domain=None, **argkw): 
     
    153106                             self.numberOfLearners, 
    154107                             example.getweight(self.weights)) 
    155         pass 
    156108 
    157109    def remove(self, index): 
     
    191143                    self.classifiers[replace] = results.classifiers[i][index] 
    192144 
     145    def __repr__(self): 
     146        return str(self.__dict__) 
    193147#### Experimental procedures 
    194  
    195  
    196 def proportion_test(learners, examples, learnProp, times=10, 
    197                    strat=Orange.core.MakeRandomIndices.StratifiedIfPossible, 
    198                    pps=[], callback=None, **argkw): 
    199     """train-and-test evaluation (train on a subset, test on remaing examples) 
    200  
    201     Splits the data with ``learnProp`` of examples in the learning 
    202     and the rest in the testing set. The test is repeated for a given 
    203     number of times (default 10). Division is stratified by default. The 
    204     Function also accepts keyword arguments for randomization and 
    205     storing classifiers. 
    206  
    207     100 repetitions of the so-called 70:30 test in which 70% of examples 
    208     are used for training and 30% for testing is done by:: 
    209  
    210         res = Orange.evaluation.testing.proportion_test(learners, data, 0.7, 100)  
    211  
    212     Note that Python allows naming the arguments; instead of "100" you 
    213     can use "times=100" to increase the clarity (not so with keyword 
    214     arguments, such as ``storeClassifiers``, ``randseed`` or ``verbose`` 
    215     that must always be given with a name). 
    216  
    217     """ 
    218      
    219     # randomGenerator is set either to what users provided or to orange.RandomGenerator(0) 
    220     # If we left it None or if we set MakeRandomIndices2.randseed, it would give same indices each time it's called 
    221     randomGenerator = argkw.get("indicesrandseed", 0) or argkw.get("randseed", 0) or argkw.get("randomGenerator", 0) 
    222     pick = Orange.core.MakeRandomIndices2(stratified = strat, p0 = learnProp, randomGenerator = randomGenerator) 
    223      
    224     examples, weight = demangle_examples(examples) 
    225     classVar = examples.domain.classVar 
    226     if classVar.varType == Orange.data.Type.Discrete: 
    227         values = list(classVar.values) 
    228         baseValue = classVar.baseValue 
    229     else: 
    230         baseValue = values = None 
    231     testResults = ExperimentResults(times, [l.name for l in learners], values, weight!=0, baseValue) 
    232  
    233     for time in range(times): 
    234         indices = pick(examples) 
    235         learnset = examples.selectref(indices, 0) 
    236         testset = examples.selectref(indices, 1) 
    237         learn_and_test_on_test_data(learners, (learnset, weight), (testset, weight), testResults, time, pps, **argkw) 
    238         if callback: callback() 
    239     return testResults 
    240  
    241  
    242  
    243 def learning_curve_n(learners, examples, folds=10, 
    244                    strat=Orange.core.MakeRandomIndices.StratifiedIfPossible, 
    245                    proportions=Orange.core.frange(0.1), pps=[], **argkw): 
    246     """Construct a learning curve for learners. 
    247  
    248     A simpler interface for the function :obj:`learning_curve`. Instead 
    249     of methods for preparing indices, it simply takes the number of folds 
    250     and a flag telling whether we want a stratified cross-validation or 
    251     not. This function does not return a single :obj:`ExperimentResults` but 
    252     a list of them, one for each proportion. :: 
    253  
    254         prop = [0.2, 0.4, 0.6, 0.8, 1.0] 
    255         res = Orange.evaluation.testing.learning_curve_n(learners, data, folds = 5, proportions = prop) 
    256         for i, p in enumerate(prop): 
    257             print "%5.3f:" % p, 
    258             printResults(res[i]) 
    259  
    260     This function basically prepares a random generator and example selectors 
    261     (``cv`` and ``pick``) and calls :obj:`learning_curve`. 
    262  
    263     """ 
    264  
    265     seed = argkw.get("indicesrandseed", -1) or argkw.get("randseed", -1) 
    266     if seed: 
    267         randomGenerator = Orange.core.RandomGenerator(seed) 
    268     else: 
    269         randomGenerator = argkw.get("randomGenerator", Orange.core.RandomGenerator()) 
    270          
    271     if strat: 
    272         cv=Orange.core.MakeRandomIndicesCV(folds = folds, stratified = strat, randomGenerator = randomGenerator) 
    273         pick=Orange.core.MakeRandomIndices2(stratified = strat, randomGenerator = randomGenerator) 
    274     else: 
    275         cv=Orange.core.RandomIndicesCV(folds = folds, stratified = strat, randomGenerator = randomGenerator) 
    276         pick=Orange.core.RandomIndices2(stratified = strat, randomGenerator = randomGenerator) 
    277     return apply(learning_curve, (learners, examples, cv, pick, proportions, pps), argkw) 
    278  
    279  
    280 def learning_curve(learners, examples, cv=None, pick=None, proportions=Orange.core.frange(0.1), pps=[], **argkw): 
    281     """ 
    282     Computes learning curves using a procedure recommended by Salzberg 
    283     (1997). It first prepares data subsets (folds). For each proportion, 
    284     it performs the cross-validation, but taking only a proportion of 
    285     examples for learning. 
    286  
    287     Arguments ``cv`` and ``pick`` give the methods for preparing 
    288     indices for cross-validation and random selection of learning 
    289     examples. If they are not given, :obj:`orange.MakeRandomIndicesCV` and 
    290     :obj:`orange.MakeRandomIndices2` are used, both will be stratified and the 
    291     cross-validation will be 10-fold. Proportions is a list of proportions 
    292     of learning examples. 
    293  
    294     The function can save time by loading experimental existing data for 
    295     any test that were already conducted and saved. Also, the computed 
    296     results are stored for later use. You can enable this by adding 
    297     a keyword argument ``cache=1``. Another keyword deals with progress 
    298     report. If you add ``verbose=1``, the function will print the proportion 
    299     and the fold number. 
    300  
    301     """ 
    302     verb = argkw.get("verbose", 0) 
    303     cache = argkw.get("cache", 0) 
    304     callback = argkw.get("callback", 0) 
    305  
    306     for pp in pps: 
    307         if pp[0]!="L": 
    308             raise SystemError("cannot preprocess testing examples") 
    309  
    310     if not cv or not pick:     
    311         seed = argkw.get("indicesrandseed", -1) or argkw.get("randseed", -1) 
    312         if seed: 
    313             randomGenerator = Orange.core.RandomGenerator(seed) 
    314         else: 
    315             randomGenerator = argkw.get("randomGenerator", Orange.core.RandomGenerator()) 
    316         if not cv: 
    317             cv = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = randomGenerator) 
    318         if not pick: 
    319             pick = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = randomGenerator) 
    320  
    321     examples, weight = demangle_examples(examples) 
    322     folds = cv(examples) 
    323     ccsum = hex(examples.checksum())[2:] 
    324     ppsp = encode_PP(pps) 
    325     nLrn = len(learners) 
    326  
    327     allResults=[] 
    328     for p in proportions: 
    329         printVerbose("Proportion: %5.3f" % p, verb) 
    330  
    331         if (cv.randseed<0) or (pick.randseed<0): 
    332             cache = 0 
    333         else: 
    334             fnstr = "{learning_curve}_%s_%s_%s_%s%s-%s" % ("%s", p, cv.randseed, pick.randseed, ppsp, ccsum) 
    335             if "*" in fnstr: 
    336                 cache = 0 
    337  
    338         conv = examples.domain.classVar.varType == Orange.data.Type.Discrete and int or float 
    339         testResults = ExperimentResults(cv.folds, [l.name for l in learners], examples.domain.classVar.values.native(), weight!=0, examples.domain.classVar.baseValue) 
    340         testResults.results = [TestedExample(folds[i], conv(examples[i].getclass()), nLrn, examples[i].getweight(weight)) 
    341                                for i in range(len(examples))] 
    342  
    343         if cache and testResults.load_from_files(learners, fnstr): 
    344             printVerbose("  loaded from cache", verb) 
    345         else: 
    346             for fold in range(cv.folds): 
    347                 printVerbose("  fold %d" % fold, verb) 
    348                  
    349                 # learning 
    350                 learnset = examples.selectref(folds, fold, negate=1) 
    351                 learnset = learnset.selectref(pick(learnset, p0=p), 0) 
    352                 if not len(learnset): 
    353                     continue 
    354                  
    355                 for pp in pps: 
    356                     learnset = pp[1](learnset) 
    357  
    358                 classifiers = [None]*nLrn 
    359                 for i in range(nLrn): 
    360                     if not cache or not testResults.loaded[i]: 
    361                         classifiers[i] = learners[i](learnset, weight) 
    362  
    363                 # testing 
    364                 for i in range(len(examples)): 
    365                     if (folds[i]==fold): 
    366                         # This is to prevent cheating: 
    367                         ex = Orange.data.Instance(examples[i]) 
    368                         ex.setclass("?") 
    369                         for cl in range(nLrn): 
    370                             if not cache or not testResults.loaded[cl]: 
    371                                 cls, pro = classifiers[cl](ex, Orange.core.GetBoth) 
    372                                 testResults.results[i].set_result(cl, cls, pro) 
    373                 if callback: callback() 
    374             if cache: 
    375                 testResults.save_to_files(learners, fnstr) 
    376  
    377         allResults.append(testResults) 
    378          
    379     return allResults 
    380  
    381  
    382 def learning_curve_with_test_data(learners, learnset, testset, times=10, 
    383                               proportions=Orange.core.frange(0.1), 
    384                               strat=Orange.core.MakeRandomIndices.StratifiedIfPossible, pps=[], **argkw): 
    385     """ 
    386     This function is suitable for computing a learning curve on datasets, 
    387     where learning and testing examples are split in advance. For each 
    388     proportion of learning examples, it randomly select the requested 
    389     number of learning examples, builds the models and tests them on the 
    390     entire testset. The whole test is repeated for the given number of 
    391     times for each proportion. The result is a list of :obj:`ExperimentResults`, 
    392     one for each proportion. 
    393  
    394     In the following scripts, examples are pre-divided onto training 
    395     and testing set. Learning curves are computed in which 20, 40, 60, 
    396     80 and 100 percents of the examples in the former set are used for 
    397     learning and the latter set is used for testing. Random selection 
    398     of the given proportion of learning set is repeated for five times. 
    399  
    400     .. literalinclude:: code/testing-test.py 
    401         :start-after: Learning curve with pre-separated data 
    402         :end-before: # End 
    403  
    404  
    405     """ 
    406     verb = argkw.get("verbose", 0) 
    407  
    408     learnset, learnweight = demangle_examples(learnset) 
    409     testweight = demangle_examples(testset)[1] 
    410      
    411     randomGenerator = argkw.get("indicesrandseed", 0) or argkw.get("randseed", 0) or argkw.get("randomGenerator", 0) 
    412     pick = Orange.core.MakeRandomIndices2(stratified = strat, randomGenerator = randomGenerator) 
    413     allResults=[] 
    414     for p in proportions: 
    415         printVerbose("Proportion: %5.3f" % p, verb) 
    416         testResults = ExperimentResults(times, [l.name for l in learners], 
    417                                         testset.domain.classVar.values.native(), 
    418                                         testweight!=0, testset.domain.classVar.baseValue) 
    419         testResults.results = [] 
    420          
    421         for t in range(times): 
    422             printVerbose("  repetition %d" % t, verb) 
    423             learn_and_test_on_test_data(learners, (learnset.selectref(pick(learnset, p), 0), learnweight), 
    424                                    testset, testResults, t) 
    425  
    426         allResults.append(testResults) 
    427          
    428     return allResults 
    429  
    430 def learn_and_test_on_test_data(learners, learnset, testset, testResults=None, iterationNumber=0, pps=(), callback=None, **argkw): 
    431     """ 
    432     Perform a test, where learners are learned on one dataset and tested 
    433     on another. 
    434  
    435     :param learners: list of learners to be tested 
    436     :param trainset: a dataset used for training 
    437     :param testset: a dataset used for testing 
    438     :param preprocessors: a list of preprocessors to be used on data. 
    439     :param callback: a function that is be called after each classifier is computed. 
    440     :param store_classifiers: if True, classifiers will be accessible in test_results. 
    441     :param store_examples: if True, examples will be accessible in test_results. 
    442     """ 
    443     storeclassifiers = argkw.get("storeclassifiers", 0) or argkw.get("storeClassifiers", 0) 
    444     storeExamples = argkw.get("storeExamples", 0) 
    445  
    446     learnset, learnweight = demangle_examples(learnset) 
    447     testset, testweight = demangle_examples(testset) 
    448     storeclassifiers = argkw.get("storeclassifiers", 0) or argkw.get("storeClassifiers", 0) 
    449  
    450     learnset, testset = preprocess_data(learnset, testset, pps) 
    451              
    452     classifiers = [] 
    453     for learner in learners: 
    454         classifiers.append(learner(learnset, learnweight)) 
    455         if callback: 
    456             callback() 
    457     for i in range(len(learners)): 
    458         classifiers[i].name = getattr(learners[i], 'name', 'noname') 
    459     testResults = test_on_data(classifiers, (testset, testweight), testResults, iterationNumber, storeExamples) 
    460     if storeclassifiers: 
    461         testResults.classifiers.append(classifiers) 
    462     return testResults 
    463  
    464  
    465 def learn_and_test_on_learn_data(learners, learnset, testResults=None, iterationNumber=0, pps=[], callback=None, **argkw): 
    466     """ 
    467     This function is similar to the above, except that it learns and 
    468     tests on the same data. If first preprocesses the data with ``"B"`` 
    469     preprocessors on the whole data, and afterwards any ``"L"`` or ``"T"`` 
    470     preprocessors on separate datasets. Then it induces the model from 
    471     the learning set and tests it on the testing set. 
    472  
    473     As with :obj:`learn_and_test_on_test_data`, you can pass an already initialized 
    474     :obj:`ExperimentResults` (argument ``results``) and an iteration number to the 
    475     function. In this case, results of the test will be appended with 
    476     the given iteration number. 
    477  
    478     """ 
    479  
    480     storeclassifiers = argkw.get("storeclassifiers", 0) or argkw.get("storeClassifiers", 0) 
    481     storeExamples = argkw.get("storeExamples", 0) 
    482  
    483     learnset, learnweight = demangle_examples(learnset) 
    484  
    485     hasLorT = 0     
    486     for pp in pps: 
    487         if pp[0]=="B": 
    488             learnset = pp[1](learnset) 
    489         else: 
    490             hasLorT = 1 
    491  
    492     if hasLorT: 
    493         testset = Orange.data.Table(learnset) 
    494         for pp in pps: 
    495             if pp[0]=="L": 
    496                 learnset = pp[1](learnset) 
    497             elif pp[0]=="T": 
    498                 testset = pp[1](testset) 
    499             elif pp[0]=="LT": 
    500                 learnset, testset = pp[1](learnset, testset) 
    501     else: 
    502         testset = learnset     
    503  
    504     classifiers = [] 
    505     for learner in learners: 
    506         classifiers.append(learner(learnset, learnweight)) 
    507         if callback: 
    508             callback() 
    509     for i in range(len(learners)): 
    510         classifiers[i].name = getattr(learners[i], "name", "noname") 
    511     testResults = test_on_data(classifiers, (testset, learnweight), testResults, iterationNumber, storeExamples) 
    512     if storeclassifiers: 
    513         testResults.classifiers.append(classifiers) 
    514     return testResults 
    515  
    516  
    517 def test_on_data(classifiers, testset, testResults=None, iterationNumber=0, storeExamples = False, **argkw): 
    518     """ 
    519     This function gets a list of classifiers, not learners like the other 
    520     functions in this module. It classifies each testing example with 
    521     each classifier. You can pass an existing :obj:`ExperimentResults` 
    522     and iteration number, like in :obj:`learnAndTestWithTestData` 
    523     (which actually calls :obj:`testWithTestData`). If you don't, a new 
    524     :obj:`ExperimentResults` will be created. 
    525  
    526     """ 
    527  
    528     testset, testweight = demangle_examples(testset) 
    529  
    530     if not testResults: 
    531         classVar = testset.domain.classVar 
    532         if testset.domain.classVar.varType == Orange.data.Type.Discrete: 
    533             values = classVar.values.native() 
    534             baseValue = classVar.baseValue 
    535         else: 
    536             values = None 
    537             baseValue = -1 
    538         testResults=ExperimentResults(1, [l.name for l in classifiers], values, testweight!=0, baseValue) 
    539  
    540     examples = getattr(testResults, "examples", False) 
    541     if examples and len(examples): 
    542         # We must not modify an example table we do not own, so we clone it the 
    543         # first time we have to add to it 
    544         if not getattr(testResults, "examplesCloned", False): 
    545             testResults.examples = Orange.data.Table(testResults.examples) 
    546             testResults.examplesCloned = True 
    547         testResults.examples.extend(testset) 
    548     else: 
    549         # We do not clone at the first iteration - cloning might never be needed at all... 
    550         testResults.examples = testset 
    551      
    552     conv = testset.domain.classVar.varType == Orange.data.Type.Discrete and int or float 
    553     for ex in testset: 
    554         te = TestedExample(iterationNumber, conv(ex.getclass()), 0, ex.getweight(testweight)) 
    555  
    556         for classifier in classifiers: 
    557             # This is to prevent cheating: 
    558             ex2 = Orange.data.Instance(ex) 
    559             ex2.setclass("?") 
    560             cr = classifier(ex2, Orange.core.GetBoth) 
    561             te.add_result(cr[0], cr[1]) 
    562         testResults.results.append(te) 
    563          
    564     return testResults 
    565  
    566148class Evaluation(object): 
    567149    @deprecated_keywords({"pps": "preprocessors", 
     
    674256 
    675257        # learning 
    676         learn_set, test_set = self.preprocess_data(learn_set, test_set, preprocessors) 
     258        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors) 
    677259        if not learn_set: 
    678260            raise SystemError("no training examples after preprocessing") 
     
    687269 
    688270        return results, classifiers 
     271 
     272 
     273    @deprecated_keywords({"storeExamples": "store_examples", 
     274                          "storeClassifiers": "store_classifiers", 
     275                          "pps": "preprocessors"}) 
     276    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(), 
     277                                     callback=None, store_classifiers=False, store_examples=False): 
     278        """ 
     279        Perform a test where learners are trained and tested on the same data. 
     280 
     281        :param learners: list of learners to be tested 
     282        :param examples: data table on which the learners will be tested 
     283        :param preprocessors: a list of preprocessors to be used on data. 
     284        :param callback: a function that will be called after each fold is computed. 
     285        :param store_classifiers: if True, classifiers will be accessible in test_results. 
     286        :param store_examples: if True, examples will be accessible in test_results. 
     287        """ 
     288 
     289        examples, weight = demangle_examples(examples) 
     290 
     291        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to 
     292        # clone it. 
     293        if not filter(lambda x:x[0]!="B", preprocessors): 
     294            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors) 
     295            test_set = learn_set 
     296        else: 
     297            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors) 
     298 
     299        classifiers = self._train_with_callback(learners, learn_set, weight, callback) 
     300 
     301        test_results = ExperimentResults(1, 
     302                                        classifierNames = [getobjectname(l) for l in learners], 
     303                                        domain=examples.domain, 
     304                                        weights=weight) 
     305        test_results.results = [test_results.create_tested_example(0, example) 
     306                               for i, example in enumerate(examples)] 
     307 
     308        if store_classifiers: 
     309            test_results.classifiers = classifiers 
     310        if store_examples: 
     311            test_results.examples = test_set 
     312 
     313        results = self._test_on_data(classifiers, test_set) 
     314        for example, classifier, result in results: 
     315            test_results.results[example].set_result(classifier, *result) 
     316        return test_results 
     317 
     318    @deprecated_keywords({"storeExamples": "store_examples", 
     319                          "storeClassifiers": "store_classifiers", 
     320                          "pps": "preprocessors"}) 
     321    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(), 
     322                                    callback=None, store_classifiers=False, store_examples=False): 
     323        """ 
     324        Perform a test, where learners are trained on one dataset and tested 
     325        on another. 
     326 
     327        :param learners: list of learners to be tested 
     328        :param learn_set: a dataset used for training 
     329        :param test_set: a dataset used for testing 
     330        :param preprocessors: a list of preprocessors to be used on data. 
     331        :param callback: a function that is be called after each classifier is computed. 
     332        :param store_classifiers: if True, classifiers will be accessible in test_results. 
     333        :param store_examples: if True, examples will be accessible in test_results. 
     334        """ 
     335        learn_set, learn_weight = demangle_examples(learn_set) 
     336        test_set, test_weight = demangle_examples(test_set) 
     337 
     338        test_results = ExperimentResults(1, 
     339                                        classifierNames = [getobjectname(l) for l in learners], 
     340                                        domain=test_set.domain, 
     341                                        weights=test_weight) 
     342        test_results.results = [test_results.create_tested_example(0, example) 
     343                               for i, example in enumerate(test_set)] 
     344 
     345        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback) 
     346 
     347        if store_classifiers: 
     348            test_results.classifiers = classifiers 
     349        if store_examples: 
     350            test_results.examples = test_set 
     351 
     352        for example, classifier, result in results: 
     353            test_results.results[example].set_result(classifier, *result) 
     354        return test_results 
     355 
     356    @deprecated_keywords({"storeExamples": "store_examples", 
     357                          "storeClassifiers": "store_classifiers", 
     358                          "learnProp": "learning_proportion", 
     359                          "strat": "stratification", 
     360                          "pps": "preprocessors", 
     361                          "indicesrandseed": "random_generator", 
     362                          "randseed": "random_generator", 
     363                          "randomGenerator": "random_generator"}) 
     364    def proportion_test(self, learners, examples, learning_proportion, times=10, 
     365                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible, preprocessors=(), random_generator=0, 
     366                   callback=None, store_classifiers=False, store_examples=False): 
     367        """ 
     368        Perform a test, where learners are trained and tested on different data sets. Training and test sets are 
     369        generated by proportionally splitting examples. 
     370 
     371        :param learners: list of learners to be tested 
     372        :param examples: a dataset used for evaluation 
     373        :param learning_proportion: proportion of examples to be used for training 
     374        :param times: number of test repetitions 
     375        :param stratification: use stratification when constructing train and test sets. 
     376        :param preprocessors: a list of preprocessors to be used on data. 
     377        :param callback: a function that is be called after each classifier is computed. 
     378        :param store_classifiers: if True, classifiers will be accessible in test_results. 
     379        :param store_examples: if True, examples will be accessible in test_results. 
     380        """ 
     381        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator) 
     382 
     383        examples, weight = demangle_examples(examples) 
     384 
     385        test_results = ExperimentResults(times, 
     386                                        classifierNames = [getobjectname(l) for l in learners], 
     387                                        domain=examples.domain, 
     388                                        weights=weight) 
     389 
     390        for time in xrange(times): 
     391            indices = pick(examples) 
     392            learn_set = examples.selectref(indices, 0) 
     393            test_set = examples.selectref(indices, 1) 
     394            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors) 
     395 
     396            test_results.results.extend(test_results.create_tested_example(time, example) 
     397                                        for i, example in enumerate(test_set)) 
     398            for example, classifier, result in results: 
     399                test_results.results[example].set_result(classifier, *result) 
     400 
     401            if callback: 
     402                callback() 
     403        return test_results 
     404 
     405    def learning_curve(self, learners, examples, cv=None, pick=None, proportions=Orange.core.frange(0.1), 
     406                       preprocessors=(), random_generator=0, callback=None): 
     407        """ 
     408        Computes learning curves using a procedure recommended by Salzberg 
     409        (1997). It first prepares data subsets (folds). For each proportion, 
     410        it performs the cross-validation, but taking only a proportion of 
     411        examples for learning. 
     412 
     413        Arguments ``cv`` and ``pick`` give the methods for preparing 
     414        indices for cross-validation and random selection of learning 
     415        examples. If they are not given, :obj:`orange.MakeRandomIndicesCV` and 
     416        :obj:`orange.MakeRandomIndices2` are used, both will be stratified and the 
     417        cross-validation will be 10-fold. Proportions is a list of proportions 
     418        of learning examples. 
     419 
     420        The function can save time by loading experimental existing data for 
     421        any test that were already conducted and saved. Also, the computed 
     422        results are stored for later use. You can enable this by adding 
     423        a keyword argument ``cache=1``. Another keyword deals with progress 
     424        report. If you add ``verbose=1``, the function will print the proportion 
     425        and the fold number. 
     426 
     427        """ 
     428        if cv is None: 
     429            cv = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator) 
     430        if pick is None: 
     431            pick = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator) 
     432 
     433        examples, weight = demangle_examples(examples) 
     434        indices = cv(examples) 
     435         
     436        all_results=[] 
     437        for p in proportions: 
     438            def select_proportion_preprocessor(examples): 
     439                return examples.selectref(pick(examples, p0=p), 0) 
     440 
     441            test_results = self.test_with_indices(learners, examples, indices, 
     442                                                  preprocessors=list(preprocessors) + [("L", select_proportion_preprocessor)], 
     443                                                  callback=callback) 
     444            all_results.append(test_results) 
     445        return all_results 
     446 
     447 
     448    def learning_curve_n(self, learners, examples, folds=10, 
     449                       strat=Orange.core.MakeRandomIndices.StratifiedIfPossible, 
     450                       proportions=Orange.core.frange(0.1), pps=[], **argkw): 
     451        """Construct a learning curve for learners. 
     452 
     453        A simpler interface for the function :obj:`learning_curve`. Instead 
     454        of methods for preparing indices, it simply takes the number of folds 
     455        and a flag telling whether we want a stratified cross-validation or 
     456        not. This function does not return a single :obj:`ExperimentResults` but 
     457        a list of them, one for each proportion. :: 
     458 
     459            prop = [0.2, 0.4, 0.6, 0.8, 1.0] 
     460            res = Orange.evaluation.testing.learning_curve_n(learners, data, folds = 5, proportions = prop) 
     461            for i, p in enumerate(prop): 
     462                print "%5.3f:" % p, 
     463                printResults(res[i]) 
     464 
     465        This function basically prepares a random generator and example selectors 
     466        (``cv`` and ``pick``) and calls :obj:`learning_curve`. 
     467 
     468        """ 
     469 
     470        seed = argkw.get("indicesrandseed", -1) or argkw.get("randseed", -1) 
     471        if seed: 
     472            randomGenerator = Orange.core.RandomGenerator(seed) 
     473        else: 
     474            randomGenerator = argkw.get("randomGenerator", Orange.core.RandomGenerator()) 
     475 
     476        if strat: 
     477            cv=Orange.core.MakeRandomIndicesCV(folds = folds, stratified = strat, randomGenerator = randomGenerator) 
     478            pick=Orange.core.MakeRandomIndices2(stratified = strat, randomGenerator = randomGenerator) 
     479        else: 
     480            cv=Orange.core.RandomIndicesCV(folds = folds, stratified = strat, randomGenerator = randomGenerator) 
     481            pick=Orange.core.RandomIndices2(stratified = strat, randomGenerator = randomGenerator) 
     482        return learning_curve(learners, examples, cv, pick, proportions, pps, **argkw) 
     483     
     484    def learning_curve_with_test_data(self, learners, learn_set, test_set, times=10, 
     485                                  proportions=Orange.core.frange(0.1), 
     486                                  stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible, pps=[], random_generator=0): 
     487        """ 
     488        This function is suitable for computing a learning curve on datasets, 
     489        where learning and testing examples are split in advance. For each 
     490        proportion of learning examples, it randomly select the requested 
     491        number of learning examples, builds the models and tests them on the 
     492        entire testset. The whole test is repeated for the given number of 
     493        times for each proportion. The result is a list of :obj:`ExperimentResults`, 
     494        one for each proportion. 
     495 
     496        In the following scripts, examples are pre-divided onto training 
     497        and testing set. Learning curves are computed in which 20, 40, 60, 
     498        80 and 100 percents of the examples in the former set are used for 
     499        learning and the latter set is used for testing. Random selection 
     500        of the given proportion of learning set is repeated for five times. 
     501 
     502        .. literalinclude:: code/testing-test.py 
     503            :start-after: Learning curve with pre-separated data 
     504            :end-before: # End 
     505 
     506 
     507        """ 
     508        learn_set, learn_weight = demangle_examples(learn_set) 
     509        test_set, test_weight = demangle_examples(test_set) 
     510 
     511        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator) 
     512         
     513        all_results=[] 
     514        for p in proportions: 
     515            test_results = ExperimentResults(times, 
     516                                        classifierNames = [getobjectname(l) for l in learners], 
     517                                        domain=test_set.domain, 
     518                                        weights=test_weight) 
     519 
     520            for t in xrange(times): 
     521                test_results.results.extend(test_results.create_tested_example(t, example) 
     522                                            for i, example in enumerate(test_set)) 
     523 
     524                learn_examples = learn_set.selectref(indices(learn_set, p), 0) 
     525                classifiers, results = self._learn_and_test_on_test_data(learners, learn_examples, learn_weight, test_set) 
     526 
     527                for example, classifier, result in results: 
     528                    test_results.results[example].set_result(classifier, *result) 
     529 
     530                test_results.classifiers.append(classifiers) 
     531 
     532            all_results.append(test_results) 
     533        return all_results 
     534 
     535 
     536    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False): 
     537        """ 
     538        Test classifiers on examples 
     539 
     540        :param classifiers: classifiers to test 
     541        :param examples: examples to test on 
     542        :param store_classifiers: if True, classifiers will be accessible in test_results. 
     543        :param store_examples: if True, examples will be accessible in test_results. 
     544        """ 
     545 
     546        examples, weight = demangle_examples(examples) 
     547 
     548        test_results = ExperimentResults(1, 
     549                                        classifierNames = [getobjectname(l) for l in classifiers], 
     550                                        domain=examples.domain, 
     551                                        weights=weight) 
     552        test_results.results = [test_results.create_tested_example(0, example) 
     553                               for i, example in enumerate(examples)] 
     554 
     555        if store_examples: 
     556            test_results.examples = examples 
     557        if store_classifiers: 
     558            test_results.classifiers = classifiers 
     559 
     560        results = self._test_on_data(classifiers, examples) 
     561        for example, classifier, result in results: 
     562            test_results.results[example].set_result(classifier, *result) 
     563        return test_results 
     564 
     565 
     566    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set, 
     567                                     preprocessors=(), callback=None): 
     568        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors) 
     569 
     570        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback) 
     571         
     572        results = self._test_on_data(classifiers, test_set) 
     573        return classifiers, results 
     574 
     575 
     576    def _train_with_callback(self, learners, examples, weight, callback): 
     577        classifiers = [] 
     578        for learner in learners: 
     579            classifier = learner(examples, weight) 
     580            classifier.name = getattr(learner, 'name', 'noname') 
     581            classifiers.append(classifier) 
     582            if callback: 
     583                callback() 
     584        return classifiers 
    689585 
    690586 
     
    706602        return results 
    707603 
    708     def preprocess_data(self, learn_set, test_set, preprocessors): 
     604     
     605    def _preprocess_data(self, learn_set, test_set, preprocessors): 
    709606        """Apply preprocessors to learn and test dataset""" 
    710607        for p_type, preprocessor in preprocessors: 
     
    719616            elif p_type == "LT": 
    720617                (learn_set, test_set) = preprocessor(learn_set, test_set) 
    721  
    722618        return learn_set, test_set 
     619 
    723620 
    724621    def encode_PP(self, pps): 
     
    734631default_evaluation = _default_evaluation = Evaluation() 
    735632 
    736 preprocess_data = _default_evaluation.preprocess_data 
     633cross_validation = _default_evaluation.cross_validation 
     634leave_one_out = _default_evaluation.leave_one_out 
    737635test_with_indices = _default_evaluation.test_with_indices 
    738636one_fold_with_indices = _default_evaluation.one_fold_with_indices 
    739 cross_validation = _default_evaluation.cross_validation 
    740 leave_one_out = _default_evaluation.leave_one_out 
    741  
     637 
     638learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data 
     639learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data 
     640test_on_data = _default_evaluation.test_on_data 
     641 
     642learning_curve = _default_evaluation.learning_curve 
     643learning_curve_n = _default_evaluation.learning_curve_n 
     644learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data 
     645 
     646proportion_test = _default_evaluation.proportion_test 
    742647 
    743648encode_PP = _default_evaluation.encode_PP 
Note: See TracChangeset for help on using the changeset viewer.