source: orange/orange/Orange/evaluation/testing.py @ 9555:0794875e3658

Revision 9555:0794875e3658, 34.7 KB checked in by anze <anze.staric@…>, 2 years ago (diff)

Added deprecated_keywords decorator to learning_curve_n.

Line 
1import itertools
2
3import Orange
4from Orange.misc import demangle_examples, getobjectname, printVerbose, deprecated_keywords, deprecated_members
5
6#### Data structures
7
8TEST_TYPE_SINGLE = 0
9TEST_TYPE_MLC = 1
10
11class TestedExample:
12    """
13    TestedExample stores predictions of different classifiers for a single testing example.
14
15    :var classes: A list of predictions of type Value, one for each classifier.
16    :var probabilities: A list of probabilities of classes, one for each classifier.
17    :var iteration_number: Iteration number (e.g. fold) in which the TestedExample was created/tested.
18    :var actual_class: The correct class of the example
19    :var weight: Example's weight. Even if the example set was not weighted, this attribute is present and equals 1.0.
20    """
21
22    @deprecated_keywords({"iteration_number": "iteration_number",
23                          "actual_class": "actual_class"})
24    def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0):
25        """
26        :param iteration_number:
27        :param actual_class:
28        :param n:
29        :param weight:
30        """
31        self.classes = [None]*n
32        self.probabilities = [None]*n
33        self.iteration_number = iteration_number
34        self.actual_class= actual_class
35        self.weight = weight
36
37    def add_result(self, aclass, aprob):
38        """Appends a new result (class and probability prediction by a single classifier) to the classes and probabilities field."""
39   
40        if type(aclass)==list:
41            self.classes.append(aclass)
42            self.probabilities.append(aprob)
43        elif type(aclass.value)==float:
44            self.classes.append(float(aclass))
45            self.probabilities.append(aprob)
46        else:
47            self.classes.append(int(aclass))
48            self.probabilities.append(list(aprob))
49
50    def set_result(self, i, aclass, aprob):
51        """Sets the result of the i-th classifier to the given values."""
52        if type(aclass)==list:
53            self.classes[i] = aclass
54            self.probabilities[i] = aprob
55        elif type(aclass.value)==float:
56            self.classes[i] = float(aclass)
57            self.probabilities[i] = aprob
58        else:
59            self.classes[i] = int(aclass)
60            self.probabilities[i] = list(aprob)
61
62    def __repr__(self):
63        return str(self.__dict__)
64
65TestedExample = deprecated_members({"iterationNumber": "iteration_number",
66                                    "actualClass": "actual_class"
67                                    })(TestedExample)
68
69class ExperimentResults(object):
70    """
71    ``ExperimentResults`` stores results of one or more repetitions of
72    some test (cross validation, repeated sampling...) under the same
73    circumstances.
74
75    :var results: A list of instances of :obj:`TestedExample`, one for each example in the dataset.
76    :var classifiers: A list of classifiers, one element for each repetition (eg. fold). Each element is a list
77      of classifiers, one for each learner. This field is used only if storing is enabled by ``storeClassifiers=1``.
78    :var number_of_iterations: Number of iterations. This can be the number of folds (in cross validation)
79      or the number of repetitions of some test. :obj:`TestedExample`'s attribute ``iteration_number`` should
80      be in range ``[0, number_of_iterations-1]``.
81    :var number_of_learners: Number of learners. Lengths of lists classes and probabilities in each :obj:`TestedExample`
82      should equal ``number_of_learners``.
83    :var loaded: If the experimental method supports caching and there are no obstacles for caching (such as unknown
84      random seeds), this is a list of boolean values. Each element corresponds to a classifier and tells whether the
85      experimental results for that classifier were computed or loaded from the cache.
86    :var weights: A flag telling whether the results are weighted. If ``False``, weights are still present
87      in :obj:`TestedExample`, but they are all ``1.0``. Clear this flag, if your experimental procedure ran on weighted
88      testing examples but you would like to ignore the weights in statistics.
89    """
90    @deprecated_keywords({"classifierNames": "classifier_names",
91                          "classValues": "class_values",
92                          "baseClass": "base_class",
93                          "numberOfIterations": "number_of_iterations",
94                          "numberOfLearners": "number_of_learners"})
95    def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=-1, domain=None, test_type=TEST_TYPE_SINGLE, **argkw):
96        self.class_values = class_values
97        self.classifier_names = classifier_names
98        self.number_of_iterations = iterations
99        self.number_of_learners = len(classifier_names)
100        self.results = []
101        self.classifiers = []
102        self.loaded = None
103        self.base_class = base_class
104        self.weights = weights
105        self.test_type = test_type
106
107        if domain is not None:
108            self.base_class = self.class_values = None
109            if test_type==TEST_TYPE_SINGLE:
110                if domain.class_var.var_type == Orange.data.Type.Discrete:
111                    self.class_values = list(domain.class_var.values)
112                    self.base_class = domain.class_var.base_value
113                    self.converter = int
114                else:
115                    self.converter = float
116            elif test_type==TEST_TYPE_MLC:
117                self.labels = [var.name for var in domain.class_vars]
118                self.converter = lambda vals: [int(val) if val.variable.var_type == Orange.data.Type.Discrete
119                                               else float(val) for val in vals]
120
121        self.__dict__.update(argkw)
122
123    def load_from_files(self, learners, filename):
124        raise NotImplementedError("This feature is no longer supported.")
125
126    def save_to_files(self, learners, filename):
127        raise NotImplementedError("This feature is no longer supported. Pickle whole class instead.")
128
129    def create_tested_example(self, fold, example):
130        actual = example.getclass() if self.test_type == TEST_TYPE_SINGLE \
131                                  else example.get_classes()
132        return TestedExample(fold,
133                             self.converter(actual),
134                             self.number_of_learners,
135                             example.getweight(self.weights))
136
137    def remove(self, index):
138        """remove one learner from evaluation results"""
139        for r in self.results:
140            del r.classes[index]
141            del r.probabilities[index]
142        del self.classifier_names[index]
143        self.number_of_learners -= 1
144
145    def add(self, results, index, replace=-1):
146        """add evaluation results (for one learner)"""
147        if len(self.results)!=len(results.results):
148            raise SystemError("mismatch in number of test cases")
149        if self.number_of_iterations!=results.number_of_iterations:
150            raise SystemError("mismatch in number of iterations (%d<>%d)" % \
151                  (self.number_of_iterations, results.number_of_iterations))
152        if len(self.classifiers) and len(results.classifiers)==0:
153            raise SystemError("no classifiers in results")
154
155        if replace < 0 or replace >= self.number_of_learners: # results for new learner
156            self.classifier_names.append(results.classifier_names[index])
157            self.number_of_learners += 1
158            for i,r in enumerate(self.results):
159                r.classes.append(results.results[i].classes[index])
160                r.probabilities.append(results.results[i].probabilities[index])
161            if len(self.classifiers):
162                for i in range(self.number_of_iterations):
163                    self.classifiers[i].append(results.classifiers[i][index])
164        else: # replace results of existing learner
165            self.classifier_names[replace] = results.classifier_names[index]
166            for i,r in enumerate(self.results):
167                r.classes[replace] = results.results[i].classes[index]
168                r.probabilities[replace] = results.results[i].probabilities[index]
169            if len(self.classifiers):
170                for i in range(self.number_of_iterations):
171                    self.classifiers[replace] = results.classifiers[i][index]
172
173    def __repr__(self):
174        return str(self.__dict__)
175
176ExperimentResults = deprecated_members({"classValues": "class_values",
177                                        "classifierNames": "classifier_names",
178                                        "baseClass": "base_class",
179                                        "numberOfIterations": "number_of_iterations",
180                                        "numberOfLearners": "number_of_learners"
181})(ExperimentResults)
182
183   
184class Evaluation(object):
185    """Common methods for learner evaluation."""
186
187    @deprecated_keywords({"pps": "preprocessors",
188                          "strat": "stratified",
189                          "randseed": "random_generator",
190                          "indicesrandseed": "random_generator",
191                          "randomGenerator": "random_generator",
192                          "storeClassifiers": "store_classifiers",
193                          "storeExamples": "store_examples"})
194    def cross_validation(self, learners, examples, folds=10,
195            stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible,
196            preprocessors=(), random_generator=0, callback=None,
197            store_classifiers=False, store_examples=False):
198        """Perform cross validation with specified number of folds.
199
200        :param learners: list of learners to be tested
201        :param examples: data table on which the learners will be tested
202        :param folds: number of folds to perform
203        :param stratified: sets, whether indices should be stratified
204        :param preprocessors: a list of preprocessors to be used on data.
205        :param random_generator: random seed or random generator for selection
206               of indices
207        :param callback: a function that will be called after each fold is
208               computed.
209        :param store_classifiers: if True, classifiers will be accessible in
210               test_results.
211        :param store_examples: if True, examples will be accessible in
212               test_results.
213        :return: :obj:`ExperimentResults`
214        """
215        (examples, weight) = demangle_examples(examples)
216
217        indices = Orange.core.MakeRandomIndicesCV(examples, folds,
218            stratified=stratified, random_generator=random_generator)
219
220        return self.test_with_indices(
221            learners=learners,
222            examples=(examples, weight),
223            indices=indices,
224            preprocessors=preprocessors,
225            callback=callback,
226            store_classifiers=store_classifiers,
227            store_examples=store_examples)
228
229
230    @deprecated_keywords({"pps": "preprocessors",
231                          "storeClassifiers": "store_classifiers",
232                          "storeExamples": "store_examples"})
233    def leave_one_out(self, learners, examples, preprocessors=(),
234            callback=None, store_classifiers=False, store_examples=False):
235        """Perform leave-one-out evaluation of learners on a data set.
236
237        :param learners: list of learners to be tested
238        :param examples: data table on which the learners will be tested
239        :param preprocessors: a list of preprocessors to be used on data.
240        :param callback: a function that will be called after each fold is
241               computed.
242        :param store_classifiers: if True, classifiers will be accessible in
243               test_results.
244        :param store_examples: if True, examples will be accessible in
245               test_results.
246        :return: :obj:`ExperimentResults`
247        """
248        return self.test_with_indices(learners, examples, indices=range(len(examples)), preprocessors=preprocessors,
249                                 callback=callback, store_classifiers=store_classifiers, store_examples=store_examples)
250
251    @deprecated_keywords({"storeExamples": "store_examples",
252                          "storeClassifiers": "store_classifiers=True",
253                          "pps":"preprocessors"})
254    def test_with_indices(self, learners, examples, indices, preprocessors=(),
255            callback=None, store_classifiers=False, store_examples=False,
256            **kwargs):
257        """
258        Perform a cross-validation-like test. Examples for each fold are
259        selected based on given indices.
260
261        :param learners: list of learners to be tested
262        :param examples: data table on which the learners will be tested
263        :param indices: a list of integers that defines, which examples will be
264               used for testing in each fold. The number of indices should be
265               equal to the number of examples.
266        :param preprocessors: a list of preprocessors to be used on data.
267        :param callback: a function that will be called after each fold is
268               computed.
269        :param store_classifiers: if True, classifiers will be accessible in test_results.
270        :param store_examples: if True, examples will be accessible in test_results.
271        :return: :obj:`ExperimentResults`
272        """
273        examples, weight = demangle_examples(examples)
274        if not examples:
275            raise ValueError("Test data set with no examples")
276        test_type = self.check_test_type(examples, learners)
277        if "cache" in kwargs:
278            raise ValueError("This feature is no longer supported.")
279
280        niterations = max(indices)+1
281        test_result = ExperimentResults(niterations,
282                                        classifierNames = [getobjectname(l) for l in learners],
283                                        domain=examples.domain,
284                                        weights=weight,
285                                        test_type=test_type)
286
287        test_result.results = [test_result.create_tested_example(indices[i], example)
288                               for i, example in enumerate(examples)]
289
290        if store_examples:
291            test_result.examples = examples
292
293        for fold in xrange(niterations):
294            results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight)
295
296            for example, learner, result in results:
297                test_result.results[example].set_result(learner, *result)
298
299            if store_classifiers:
300                test_result.classifiers.append(classifiers)
301            if callback:
302                callback()
303
304        return test_result
305
306
307    def one_fold_with_indices(self, learners, examples, fold, indices, preprocessors=(), weight=0):
308        """Perform one fold of cross-validation like procedure using provided indices."""
309        learn_set = examples.selectref(indices, fold, negate=1)
310        test_set = examples.selectref(indices, fold, negate=0)
311        if len(learn_set)==0 or len(test_set)==0:
312            return (), ()
313
314        # learning
315        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
316        if not learn_set:
317            raise SystemError("no training examples after preprocessing")
318        if not test_set:
319            raise SystemError("no test examples after preprocessing")
320
321        classifiers = [learner(learn_set, weight) for learner in learners]
322
323        # testing
324        testset_ids = (i for i, _ in enumerate(examples) if indices[i] == fold)
325        results = self._test_on_data(classifiers, test_set, testset_ids)
326
327        return results, classifiers
328
329
330    @deprecated_keywords({"storeExamples": "store_examples",
331                          "storeClassifiers": "store_classifiers",
332                          "pps": "preprocessors"})
333    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(),
334                                     callback=None, store_classifiers=False, store_examples=False):
335        """
336        Perform a test where learners are trained and tested on the same data.
337
338        :param learners: list of learners to be tested
339        :param examples: data table on which the learners will be tested
340        :param preprocessors: a list of preprocessors to be used on data.
341        :param callback: a function that will be called after each fold is computed.
342        :param store_classifiers: if True, classifiers will be accessible in test_results.
343        :param store_examples: if True, examples will be accessible in test_results.
344        :return: :obj:`ExperimentResults`
345        """
346
347        examples, weight = demangle_examples(examples)
348        test_type = self.check_test_type(examples, learners)
349
350        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to
351        # clone it.
352        if not filter(lambda x:x[0]!="B", preprocessors):
353            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors)
354            test_set = learn_set
355        else:
356            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors)
357
358        classifiers = self._train_with_callback(learners, learn_set, weight, callback)
359
360        test_results = ExperimentResults(1,
361                                        classifier_names = [getobjectname(l) for l in learners],
362                                        test_type = test_type,
363                                        domain=examples.domain,
364                                        weights=weight)
365        test_results.results = [test_results.create_tested_example(0, example)
366                               for i, example in enumerate(examples)]
367
368        if store_classifiers:
369            test_results.classifiers = classifiers
370        if store_examples:
371            test_results.examples = test_set
372
373        results = self._test_on_data(classifiers, test_set)
374        for example, classifier, result in results:
375            test_results.results[example].set_result(classifier, *result)
376        return test_results
377
378    @deprecated_keywords({"storeExamples": "store_examples",
379                          "storeClassifiers": "store_classifiers",
380                          "pps": "preprocessors"})
381    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(),
382                                    callback=None, store_classifiers=False, store_examples=False):
383        """
384        Perform a test, where learners are trained on one dataset and tested
385        on another.
386
387        :param learners: list of learners to be tested
388        :param learn_set: a dataset used for training
389        :param test_set: a dataset used for testing
390        :param preprocessors: a list of preprocessors to be used on data.
391        :param callback: a function that is be called after each classifier is computed.
392        :param store_classifiers: if True, classifiers will be accessible in test_results.
393        :param store_examples: if True, examples will be accessible in test_results.
394        :return: :obj:`ExperimentResults`
395        """
396        learn_set, learn_weight = demangle_examples(learn_set)
397        test_set, test_weight = demangle_examples(test_set)
398
399        test_type = self.check_test_type(learn_set, learners)
400        self.check_test_type(test_set, learners)
401       
402        test_results = ExperimentResults(1,
403                                        classifier_names = [getobjectname(l) for l in learners],
404                                        domain=test_set.domain,
405                                        test_type = test_type,
406                                        weights=test_weight)
407        test_results.results = [test_results.create_tested_example(0, example)
408                               for i, example in enumerate(test_set)]
409
410        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback)
411
412        if store_classifiers:
413            test_results.classifiers = classifiers
414        if store_examples:
415            test_results.examples = test_set
416
417        for example, classifier, result in results:
418            test_results.results[example].set_result(classifier, *result)
419        return test_results
420
421    @deprecated_keywords({"storeExamples": "store_examples",
422                          "storeClassifiers": "store_classifiers",
423                          "learnProp": "learning_proportion",
424                          "strat": "stratification",
425                          "pps": "preprocessors",
426                          "indicesrandseed": "random_generator",
427                          "randseed": "random_generator",
428                          "randomGenerator": "random_generator"})
429    def proportion_test(self, learners, examples, learning_proportion, times=10,
430                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible, preprocessors=(), random_generator=0,
431                   callback=None, store_classifiers=False, store_examples=False):
432        """
433        Perform a test, where learners are trained and tested on different data sets. Training and test sets are
434        generated by proportionally splitting examples.
435
436        :param learners: list of learners to be tested
437        :param examples: a dataset used for evaluation
438        :param learning_proportion: proportion of examples to be used for training
439        :param times: number of test repetitions
440        :param stratification: use stratification when constructing train and test sets.
441        :param preprocessors: a list of preprocessors to be used on data.
442        :param callback: a function that is be called after each classifier is computed.
443        :param store_classifiers: if True, classifiers will be accessible in test_results.
444        :param store_examples: if True, examples will be accessible in test_results.
445        :return: :obj:`ExperimentResults`
446        """
447        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator)
448
449        examples, weight = demangle_examples(examples)
450
451        test_type = self.check_test_type(examples, learners)
452       
453        test_results = ExperimentResults(times,
454                                        classifierNames = [getobjectname(l) for l in learners],
455                                        domain=examples.domain,
456                                        test_type = test_type,
457                                        weights=weight)
458        test_results.classifiers = []
459        offset=0
460        for time in xrange(times):
461            indices = pick(examples)
462            learn_set = examples.selectref(indices, 0)
463            test_set = examples.selectref(indices, 1)
464            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors)
465            if store_classifiers:
466                test_results.classifiers.append(classifiers)
467
468            test_results.results.extend(test_results.create_tested_example(time, example)
469                                        for i, example in enumerate(test_set))
470            for example, classifier, result in results:
471                test_results.results[offset+example].set_result(classifier, *result)
472            offset += len(test_set)
473
474            if callback:
475                callback()
476        return test_results
477
478    @deprecated_keywords({"storeExamples": "store_examples",
479                          "storeClassifiers": "store_classifiers",
480                          "learnProp": "learning_proportion",
481                          "strat": "stratification",
482                          "pps": "preprocessors",
483                          "indicesrandseed": "random_generator",
484                          "randseed": "random_generator",
485                          "randomGenerator": "random_generator"})
486    def learning_curve(self, learners, examples, cv_indices=None, proportion_indices=None, proportions=Orange.core.frange(0.1),
487                       preprocessors=(), random_generator=0, callback=None):
488        """
489        Compute a learning curve using multiple cross-validations where
490        models are trained on different portions of the training data.
491
492        :param learners: list of learners to be tested
493        :param examples: a dataset used for evaluation
494        :param cv_indices: indices used for crossvalidation
495        :param proportion_indices: indices for proportion selection
496        :param proportions: proportions of train data to be used
497        :param preprocessors: a list of preprocessors to be used on data.
498        :param callback: a function that is be called after each classifier is computed.
499        :return: list of :obj:`ExperimentResults`
500        """
501        if cv_indices is None:
502            cv_indices = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
503        if proportion_indices is None:
504            proportion_indices = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
505
506        examples, weight = demangle_examples(examples)
507        indices = cv_indices(examples)
508       
509        all_results=[]
510        for p in proportions:
511            def select_proportion_preprocessor(examples):
512                return examples.selectref(proportion_indices(examples, p0=p), 0)
513
514            test_results = self.test_with_indices(learners, examples, indices,
515                                                  preprocessors=list(preprocessors) + [("L", select_proportion_preprocessor)],
516                                                  callback=callback)
517            all_results.append(test_results)
518        return all_results
519
520
521    @deprecated_keywords({"strat": "stratification",
522                          "pps": "preprocessors",
523                          "indicesrandseed": "random_generator",
524                          "randseed": "random_generator",
525                          "randomGenerator": "random_generator"})
526    def learning_curve_n(self, learners, examples, folds=10,
527                       proportions=Orange.core.frange(0.1),
528                       stratification = Orange.core.MakeRandomIndices
529                       .StratifiedIfPossible,
530                       preprocessors=(),
531                       random_generator=0, callback=None):
532        """
533        Compute a learning curve where each cross-validation has given number of folds
534        and models are trained on specified proportion of training data.
535
536        :param learners: list of learners to be tested
537        :param examples: a dataset used for evaluation
538        :param folds: number of folds for cross-validation
539        :param proportions: proportions of train data to be used
540        :param preprocessors: a list of preprocessors to be used on data.
541        :param callback: a function that is called after each classifier is computed.
542        :return: list of :obj:`ExperimentResults`
543        """
544
545        cv=Orange.core.MakeRandomIndicesCV(folds = folds,
546            stratified = stratification, randomGenerator = random_generator)
547        pick=Orange.core.MakeRandomIndices2(stratified = stratification,
548            randomGenerator = random_generator)
549        return learning_curve(learners, examples, cv, pick, proportions,
550            preprocessors, callback=callback)
551   
552    def learning_curve_with_test_data(self, learners, learn_set, test_set,
553            times=10, proportions=Orange.core.frange(0.1),
554            stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
555            preprocessors=(), random_generator=0):
556        """
557        Compute a learning curve given two datasets. Models are learned on
558        proportion of the first dataset and then used to make predictions for
559        the second dataset.
560
561        :param learners: list of learners to be tested
562        :param examples: a dataset used for evaluation
563        :param folds: number of folds for cross-validation
564        :param proportions: proportions of train data to be used
565        :param preprocessors: a list of preprocessors to be used on data.
566        :param callback: a function that is called after each classifier is computed.
567        :return: list of :obj:`ExperimentResults`
568        """
569        learn_set, learn_weight = demangle_examples(learn_set)
570        test_set, test_weight = demangle_examples(test_set)
571        test_type = self.check_test_type(learn_set, learners)
572        self.check_test_type(test_set, learners)
573       
574        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator)
575       
576        all_results=[]
577        for p in proportions:
578            test_results = ExperimentResults(times,
579                                        classifierNames = [getobjectname(l) for l in learners],
580                                        domain=test_set.domain,
581                                        test_type = test_type,
582                                        weights=test_weight)
583            offset = 0
584            for t in xrange(times):
585                test_results.results.extend(test_results.create_tested_example(t, example)
586                                            for i, example in enumerate(test_set))
587
588                learn_examples = learn_set.selectref(indices(learn_set, p), 0)
589                classifiers, results = self._learn_and_test_on_test_data(learners, learn_examples, learn_weight, test_set)
590
591                for example, classifier, result in results:
592                    test_results.results[offset+example].set_result(classifier, *result)
593                offset += len(test_set)
594
595                test_results.classifiers.append(classifiers)
596
597            all_results.append(test_results)
598        return all_results
599
600
601    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False):
602        """
603        Test classifiers on examples
604
605        :param classifiers: classifiers to test
606        :param examples: examples to test on
607        :param store_classifiers: if True, classifiers will be accessible in test_results.
608        :param store_examples: if True, examples will be accessible in test_results.
609        """
610
611        examples, weight = demangle_examples(examples)
612        test_type = self.check_test_type(examples, classifiers)
613
614        test_results = ExperimentResults(1,
615                                        classifierNames = [getobjectname(l) for l in classifiers],
616                                        domain=examples.domain,
617                                        test_type = test_type,
618                                        weights=weight)
619        test_results.results = [test_results.create_tested_example(0, example)
620                               for i, example in enumerate(examples)]
621
622        if store_examples:
623            test_results.examples = examples
624        if store_classifiers:
625            test_results.classifiers = classifiers
626
627        results = self._test_on_data(classifiers, examples)
628        for example, classifier, result in results:
629            test_results.results[example].set_result(classifier, *result)
630        return test_results
631
632
633    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set,
634                                     preprocessors=(), callback=None):
635        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
636
637        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback)
638       
639        results = self._test_on_data(classifiers, test_set)
640        return classifiers, results
641
642
643    def _train_with_callback(self, learners, examples, weight, callback):
644        classifiers = []
645        for learner in learners:
646            classifier = learner(examples, weight)
647            classifier.name = getattr(learner, 'name', 'noname')
648            classifiers.append(classifier)
649            if callback:
650                callback()
651        return classifiers
652
653
654    def _test_on_data(self, classifiers, examples, example_ids=None):
655        results = []
656
657        if example_ids is None:
658            numbered_examples = enumerate(examples)
659        else:
660            numbered_examples = itertools.izip(example_ids, examples)
661
662        for e, example in numbered_examples:
663            for c, classifier in enumerate(classifiers):
664                # Hide actual class to prevent cheating
665                ex2 = Orange.data.Instance(example)
666                if ex2.domain.class_var: ex2.setclass("?")
667                if ex2.domain.class_vars: ex2.set_classes(["?" for cv in ex2.domain.class_vars])
668                result = classifier(ex2, Orange.core.GetBoth)
669                results.append((e, c, result))
670        return results
671
672   
673    def _preprocess_data(self, learn_set, test_set, preprocessors):
674        """Apply preprocessors to learn and test dataset"""
675        for p_type, preprocessor in preprocessors:
676            if p_type == "B":
677                learn_set = preprocessor(learn_set)
678                test_set = preprocessor(test_set)
679        for p_type, preprocessor in preprocessors:
680            if p_type == "L":
681                learn_set = preprocessor(learn_set)
682            elif p_type == "T":
683                test_set = preprocessor(test_set)
684            elif p_type == "LT":
685                (learn_set, test_set) = preprocessor(learn_set, test_set)
686        return learn_set, test_set
687
688
689    def encode_PP(self, pps):
690        pps=""
691        for pp in pps:
692            objname = getobjectname(pp[1], "")
693            if len(objname):
694                pps+="_"+objname
695            else:
696                return "*"
697        return pps
698
699    def check_test_type(self, instances, models):
700        model_is_mlc = [isinstance(m, Orange.multilabel.MultiLabelLearner) or
701                        isinstance(m, Orange.multilabel.MultiLabelClassifier)
702                          for m in models]
703        multi_label = any(model_is_mlc)
704        if multi_label and not all(model_is_mlc):
705            raise ValueError("Test on mixed types of learners (MLC and non-MLC) not possible")
706
707        if multi_label and not instances.domain.class_vars:
708            raise ValueError("Test data with multiple labels (class vars) expected")
709        if not multi_label and not instances.domain.class_var:
710            raise ValueError("Test data set without class attributes")
711
712        return TEST_TYPE_MLC if multi_label else TEST_TYPE_SINGLE
713   
714default_evaluation = _default_evaluation = Evaluation()
715
716cross_validation = _default_evaluation.cross_validation
717leave_one_out = _default_evaluation.leave_one_out
718test_with_indices = _default_evaluation.test_with_indices
719one_fold_with_indices = _default_evaluation.one_fold_with_indices
720
721learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data
722learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data
723test_on_data = _default_evaluation.test_on_data
724
725learning_curve = _default_evaluation.learning_curve
726learning_curve_n = _default_evaluation.learning_curve_n
727learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data
728
729proportion_test = _default_evaluation.proportion_test
730
731encode_PP = _default_evaluation.encode_PP
Note: See TracBrowser for help on using the repository browser.