source: orange/Orange/evaluation/testing.py @ 10234:c24844e20828

Revision 10234:c24844e20828, 35.5 KB checked in by anzeh <anze.staric@…>, 2 years ago (diff)

Fixed a failing test.

Line 
1import itertools
2
3import Orange
4from Orange.misc import demangle_examples, getobjectname, deprecated_keywords, deprecated_members
5
6#### Data structures
7
8TEST_TYPE_SINGLE = 0
9TEST_TYPE_MLC = 1
10TEST_TYPE_MULTITARGET = 2
11
12class TestedExample:
13    """
14    TestedExample stores predictions of different classifiers for a single testing example.
15
16    :var classes: A list of predictions of type Value, one for each classifier.
17    :var probabilities: A list of probabilities of classes, one for each classifier.
18    :var iteration_number: Iteration number (e.g. fold) in which the TestedExample was created/tested.
19    :var actual_class: The correct class of the example
20    :var weight: Example's weight. Even if the example set was not weighted, this attribute is present and equals 1.0.
21    """
22
23    @deprecated_keywords({"iterationNumber": "iteration_number",
24                          "actualClass": "actual_class"})
25    def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0):
26        """
27        :param iteration_number: The iteration number of TestedExample.
28        :param actual_class: The actual class of TestedExample.
29        :param n: The number of learners.
30        :param weight: The weight of the TestedExample.
31        """
32        self.classes = [None]*n
33        self.probabilities = [None]*n
34        self.iteration_number = iteration_number
35        self.actual_class= actual_class
36        self.weight = weight
37
38    def add_result(self, aclass, aprob):
39        """Appends a new result (class and probability prediction by a single classifier) to the classes and probabilities field."""
40   
41        if isinstance(aclass, (list, tuple)):
42            self.classes.append(aclass)
43            self.probabilities.append(aprob)
44        elif type(aclass.value)==float:
45            self.classes.append(float(aclass))
46            self.probabilities.append(aprob)
47        else:
48            self.classes.append(int(aclass))
49            self.probabilities.append(list(aprob))
50
51    def set_result(self, i, aclass, aprob):
52        """Sets the result of the i-th classifier to the given values."""
53        if isinstance(aclass, (list, tuple)):
54            self.classes[i] = aclass
55            self.probabilities[i] = aprob
56        elif type(aclass.value)==float:
57            self.classes[i] = float(aclass)
58            self.probabilities[i] = aprob
59        else:
60            self.classes[i] = int(aclass)
61            self.probabilities[i] = list(aprob)
62
63    def __repr__(self):
64        return str(self.__dict__)
65
66TestedExample = deprecated_members({"iterationNumber": "iteration_number",
67                                    "actualClass": "actual_class"
68                                    })(TestedExample)
69
70class ExperimentResults(object):
71    """
72    ``ExperimentResults`` stores results of one or more repetitions of
73    some test (cross validation, repeated sampling...) under the same
74    circumstances.
75
76    :var results: A list of instances of :obj:`TestedExample`, one for each example in the dataset.
77    :var classifiers: A list of classifiers, one element for each repetition (eg. fold). Each element is a list
78      of classifiers, one for each learner. This field is used only if storing is enabled by ``storeClassifiers=1``.
79    :var number_of_iterations: Number of iterations. This can be the number of folds (in cross validation)
80      or the number of repetitions of some test. :obj:`TestedExample`'s attribute ``iteration_number`` should
81      be in range ``[0, number_of_iterations-1]``.
82    :var number_of_learners: Number of learners. Lengths of lists classes and probabilities in each :obj:`TestedExample`
83      should equal ``number_of_learners``.
84    :var loaded: If the experimental method supports caching and there are no obstacles for caching (such as unknown
85      random seeds), this is a list of boolean values. Each element corresponds to a classifier and tells whether the
86      experimental results for that classifier were computed or loaded from the cache.
87    :var weights: A flag telling whether the results are weighted. If ``False``, weights are still present
88      in :obj:`TestedExample`, but they are all ``1.0``. Clear this flag, if your experimental procedure ran on weighted
89      testing examples but you would like to ignore the weights in statistics.
90    """
91    @deprecated_keywords({"classifierNames": "classifier_names",
92                          "classValues": "class_values",
93                          "baseClass": "base_class",
94                          "numberOfIterations": "number_of_iterations",
95                          "numberOfLearners": "number_of_learners"})
96    def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=-1, domain=None, test_type=TEST_TYPE_SINGLE, labels=None, **argkw):
97        self.class_values = class_values
98        self.classifier_names = classifier_names
99        self.number_of_iterations = iterations
100        self.number_of_learners = len(classifier_names)
101        self.results = []
102        self.classifiers = []
103        self.loaded = None
104        self.base_class = base_class
105        self.weights = weights
106        self.test_type = test_type
107        self.labels = labels
108
109        if domain is not None:
110            self.base_class = self.class_values = None
111            if test_type == TEST_TYPE_SINGLE:
112                if domain.class_var.var_type == Orange.feature.Type.Discrete:
113                    self.class_values = list(domain.class_var.values)
114                    self.base_class = domain.class_var.base_value
115                    self.converter = int
116                else:
117                    self.converter = float
118            elif test_type in (TEST_TYPE_MLC, TEST_TYPE_MULTITARGET):
119                self.labels = [var.name for var in domain.class_vars]
120                self.converter = lambda vals: [int(val) if val.variable.var_type == Orange.feature.Type.Discrete
121                                               else float(val) for val in vals]
122
123        self.__dict__.update(argkw)
124
125    def load_from_files(self, learners, filename):
126        raise NotImplementedError("This feature is no longer supported.")
127
128    def save_to_files(self, learners, filename):
129        raise NotImplementedError("This feature is no longer supported. Pickle whole class instead.")
130
131    def create_tested_example(self, fold, example):
132        actual = example.getclass() if self.test_type == TEST_TYPE_SINGLE \
133                                  else example.get_classes()
134        return TestedExample(fold,
135                             self.converter(actual),
136                             self.number_of_learners,
137                             example.getweight(self.weights))
138
139    def remove(self, index):
140        """remove one learner from evaluation results"""
141        for r in self.results:
142            del r.classes[index]
143            del r.probabilities[index]
144        del self.classifier_names[index]
145        self.number_of_learners -= 1
146
147    def add(self, results, index, replace=-1):
148        """add evaluation results (for one learner)"""
149        if len(self.results)!=len(results.results):
150            raise SystemError("mismatch in number of test cases")
151        if self.number_of_iterations!=results.number_of_iterations:
152            raise SystemError("mismatch in number of iterations (%d<>%d)" % \
153                  (self.number_of_iterations, results.number_of_iterations))
154        if len(self.classifiers) and len(results.classifiers)==0:
155            raise SystemError("no classifiers in results")
156
157        if replace < 0 or replace >= self.number_of_learners: # results for new learner
158            self.classifier_names.append(results.classifier_names[index])
159            self.number_of_learners += 1
160            for i,r in enumerate(self.results):
161                r.classes.append(results.results[i].classes[index])
162                r.probabilities.append(results.results[i].probabilities[index])
163            if len(self.classifiers):
164                for i in range(self.number_of_iterations):
165                    self.classifiers[i].append(results.classifiers[i][index])
166        else: # replace results of existing learner
167            self.classifier_names[replace] = results.classifier_names[index]
168            for i,r in enumerate(self.results):
169                r.classes[replace] = results.results[i].classes[index]
170                r.probabilities[replace] = results.results[i].probabilities[index]
171            if len(self.classifiers):
172                for i in range(self.number_of_iterations):
173                    self.classifiers[replace] = results.classifiers[i][index]
174
175    def __repr__(self):
176        return str(self.__dict__)
177
178ExperimentResults = deprecated_members({"classValues": "class_values",
179                                        "classifierNames": "classifier_names",
180                                        "baseClass": "base_class",
181                                        "numberOfIterations": "number_of_iterations",
182                                        "numberOfLearners": "number_of_learners"
183})(ExperimentResults)
184
185   
186class Evaluation(object):
187    """Common methods for learner evaluation."""
188
189    @deprecated_keywords({"pps": "preprocessors",
190                          "strat": "stratified",
191                          "randseed": "random_generator",
192                          "indicesrandseed": "random_generator",
193                          "randomGenerator": "random_generator",
194                          "storeClassifiers": "store_classifiers",
195                          "storeExamples": "store_examples"})
196    def cross_validation(self, learners, examples, folds=10,
197            stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible,
198            preprocessors=(), random_generator=0, callback=None,
199            store_classifiers=False, store_examples=False):
200        """Perform cross validation with specified number of folds.
201
202        :param learners: list of learners to be tested
203        :param examples: data table on which the learners will be tested
204        :param folds: number of folds to perform
205        :param stratified: sets, whether indices should be stratified
206        :param preprocessors: a list of preprocessors to be used on data.
207        :param random_generator: :obj:`Orange.misc.RandomGenerator` object.
208        :param callback: a function that will be called after each fold is
209               computed.
210        :param store_classifiers: if True, classifiers will be accessible in
211               test_results.
212        :param store_examples: if True, examples will be accessible in
213               test_results.
214        :return: :obj:`ExperimentResults`
215        """
216        (examples, weight) = demangle_examples(examples)
217
218        indices = Orange.core.MakeRandomIndicesCV(examples, folds,
219            stratified=stratified, random_generator=random_generator)
220
221        return self.test_with_indices(
222            learners=learners,
223            examples=(examples, weight),
224            indices=indices,
225            preprocessors=preprocessors,
226            callback=callback,
227            store_classifiers=store_classifiers,
228            store_examples=store_examples)
229
230
231    @deprecated_keywords({"pps": "preprocessors",
232                          "storeClassifiers": "store_classifiers",
233                          "storeExamples": "store_examples"})
234    def leave_one_out(self, learners, examples, preprocessors=(),
235            callback=None, store_classifiers=False, store_examples=False):
236        """Perform leave-one-out evaluation of learners on a data set.
237
238        :param learners: list of learners to be tested
239        :param examples: data table on which the learners will be tested
240        :param preprocessors: a list of preprocessors to be used on data.
241        :param callback: a function that will be called after each fold is
242               computed.
243        :param store_classifiers: if True, classifiers will be accessible in
244               test_results.
245        :param store_examples: if True, examples will be accessible in
246               test_results.
247        :return: :obj:`ExperimentResults`
248        """
249        examples, weight = demangle_examples(examples)
250        return self.test_with_indices(
251            learners, (examples, weight), indices=range(len(examples)),
252            preprocessors=preprocessors, callback=callback,
253            store_classifiers=store_classifiers, store_examples=store_examples)
254
255    @deprecated_keywords({"storeExamples": "store_examples",
256                          "storeClassifiers": "store_classifiers=True",
257                          "pps":"preprocessors"})
258    def test_with_indices(self, learners, examples, indices, preprocessors=(),
259            callback=None, store_classifiers=False, store_examples=False,
260            **kwargs):
261        """
262        Perform a cross-validation-like test. Examples for each fold are
263        selected based on given indices.
264
265        :param learners: list of learners to be tested
266        :param examples: data table on which the learners will be tested
267        :param indices: a list of integers that defines, which examples will be
268               used for testing in each fold. The number of indices should be
269               equal to the number of examples.
270        :param preprocessors: a list of preprocessors to be used on data.
271        :param callback: a function that will be called after each fold is
272               computed.
273        :param store_classifiers: if True, classifiers will be accessible in test_results.
274        :param store_examples: if True, examples will be accessible in test_results.
275        :return: :obj:`ExperimentResults`
276        """
277        examples, weight = demangle_examples(examples)
278        if not examples:
279            raise ValueError("Test data set with no examples")
280        test_type = self.check_test_type(examples, learners)
281        if "cache" in kwargs:
282            raise ValueError("This feature is no longer supported.")
283
284        niterations = max(indices)+1
285        test_result = ExperimentResults(niterations,
286                                        classifier_names = [getobjectname(l) for l in learners],
287                                        domain=examples.domain,
288                                        weights=weight,
289                                        test_type=test_type)
290
291        test_result.results = [test_result.create_tested_example(indices[i], example)
292                               for i, example in enumerate(examples)]
293
294        if store_examples:
295            test_result.examples = examples
296
297        for fold in xrange(niterations):
298            results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight)
299
300            for example, learner, result in results:
301                test_result.results[example].set_result(learner, *result)
302
303            if store_classifiers:
304                test_result.classifiers.append(classifiers)
305            if callback:
306                callback()
307
308        return test_result
309
310
311    def one_fold_with_indices(self, learners, examples, fold, indices, preprocessors=(), weight=0):
312        """Perform one fold of cross-validation like procedure using provided indices."""
313        learn_set = examples.selectref(indices, fold, negate=1)
314        test_set = examples.selectref(indices, fold, negate=0)
315        if len(learn_set)==0 or len(test_set)==0:
316            return (), ()
317
318        # learning
319        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
320        if not learn_set:
321            raise SystemError("no training examples after preprocessing")
322        if not test_set:
323            raise SystemError("no test examples after preprocessing")
324
325        classifiers = [learner(learn_set, weight) for learner in learners]
326
327        # testing
328        testset_ids = (i for i, _ in enumerate(examples) if indices[i] == fold)
329        results = self._test_on_data(classifiers, test_set, testset_ids)
330
331        return results, classifiers
332
333
334    @deprecated_keywords({"storeExamples": "store_examples",
335                          "storeClassifiers": "store_classifiers",
336                          "pps": "preprocessors"})
337    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(),
338                                     callback=None, store_classifiers=False, store_examples=False):
339        """
340        Perform a test where learners are trained and tested on the same data.
341
342        :param learners: list of learners to be tested
343        :param examples: data table on which the learners will be tested
344        :param preprocessors: a list of preprocessors to be used on data.
345        :param callback: a function that will be called after each fold is computed.
346        :param store_classifiers: if True, classifiers will be accessible in test_results.
347        :param store_examples: if True, examples will be accessible in test_results.
348        :return: :obj:`ExperimentResults`
349        """
350
351        examples, weight = demangle_examples(examples)
352        test_type = self.check_test_type(examples, learners)
353
354        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to
355        # clone it.
356        if not filter(lambda x:x[0]!="B", preprocessors):
357            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors)
358            test_set = learn_set
359        else:
360            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors)
361
362        classifiers = self._train_with_callback(learners, learn_set, weight, callback)
363
364        test_results = ExperimentResults(1,
365                                        classifier_names = [getobjectname(l) for l in learners],
366                                        test_type = test_type,
367                                        domain=examples.domain,
368                                        weights=weight)
369        test_results.results = [test_results.create_tested_example(0, example)
370                               for i, example in enumerate(examples)]
371
372        if store_classifiers:
373            test_results.classifiers = classifiers
374        if store_examples:
375            test_results.examples = test_set
376
377        results = self._test_on_data(classifiers, test_set)
378        for example, classifier, result in results:
379            test_results.results[example].set_result(classifier, *result)
380        return test_results
381
382    @deprecated_keywords({"storeExamples": "store_examples",
383                          "storeClassifiers": "store_classifiers",
384                          "pps": "preprocessors"})
385    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(),
386                                    callback=None, store_classifiers=False, store_examples=False):
387        """
388        Perform a test, where learners are trained on one dataset and tested
389        on another.
390
391        :param learners: list of learners to be tested
392        :param learn_set: a dataset used for training
393        :param test_set: a dataset used for testing
394        :param preprocessors: a list of preprocessors to be used on data.
395        :param callback: a function that is be called after each classifier is computed.
396        :param store_classifiers: if True, classifiers will be accessible in test_results.
397        :param store_examples: if True, examples will be accessible in test_results.
398        :return: :obj:`ExperimentResults`
399        """
400        learn_set, learn_weight = demangle_examples(learn_set)
401        test_set, test_weight = demangle_examples(test_set)
402
403        test_type = self.check_test_type(learn_set, learners)
404        self.check_test_type(test_set, learners)
405       
406        test_results = ExperimentResults(1,
407                                        classifier_names = [getobjectname(l) for l in learners],
408                                        domain=test_set.domain,
409                                        test_type = test_type,
410                                        weights=test_weight)
411        test_results.results = [test_results.create_tested_example(0, example)
412                               for i, example in enumerate(test_set)]
413
414        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback)
415
416        if store_classifiers:
417            test_results.classifiers = classifiers
418        if store_examples:
419            test_results.examples = test_set
420
421        for example, classifier, result in results:
422            test_results.results[example].set_result(classifier, *result)
423        return test_results
424
425    @deprecated_keywords({"storeExamples": "store_examples",
426                          "storeClassifiers": "store_classifiers",
427                          "learnProp": "learning_proportion",
428                          "strat": "stratification",
429                          "pps": "preprocessors",
430                          "indicesrandseed": "random_generator",
431                          "randseed": "random_generator",
432                          "randomGenerator": "random_generator"})
433    def proportion_test(self, learners, examples, learning_proportion=.7, times=10,
434                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
435                   preprocessors=(), random_generator=0,
436                   callback=None, store_classifiers=False, store_examples=False):
437        """
438        Perform a test, where learners are trained and tested on different data sets. Training and test sets are
439        generated by proportionally splitting examples.
440
441        :param learners: list of learners to be tested
442        :param examples: a dataset used for evaluation
443        :param learning_proportion: proportion of examples to be used for training
444        :param times: number of test repetitions
445        :param stratification: use stratification when constructing train and test sets.
446        :param preprocessors: a list of preprocessors to be used on data.
447        :param callback: a function that is be called after each classifier is computed.
448        :param store_classifiers: if True, classifiers will be accessible in test_results.
449        :param store_examples: if True, examples will be accessible in test_results.
450        :return: :obj:`ExperimentResults`
451        """
452        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator)
453
454        examples, weight = demangle_examples(examples)
455
456        test_type = self.check_test_type(examples, learners)
457       
458        test_results = ExperimentResults(times,
459                                        classifier_names = [getobjectname(l) for l in learners],
460                                        domain=examples.domain,
461                                        test_type = test_type,
462                                        weights=weight)
463        if store_examples:
464            test_results.examples = []
465        test_results.classifiers = []
466        offset=0
467        for time in xrange(times):
468            indices = pick(examples)
469            learn_set = examples.selectref(indices, 0)
470            test_set = examples.selectref(indices, 1)
471            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors)
472            if store_classifiers:
473                test_results.classifiers.append(classifiers)
474            if store_examples:
475                test_results.examples.append(learn_set)
476
477            test_results.results.extend(test_results.create_tested_example(time, example)
478                                        for i, example in enumerate(test_set))
479            for example, classifier, result in results:
480                test_results.results[offset+example].set_result(classifier, *result)
481            offset += len(test_set)
482
483            if callback:
484                callback()
485        return test_results
486
487    @deprecated_keywords({"storeExamples": "store_examples",
488                          "storeClassifiers": "store_classifiers",
489                          "learnProp": "learning_proportion",
490                          "strat": "stratification",
491                          "pps": "preprocessors",
492                          "indicesrandseed": "random_generator",
493                          "randseed": "random_generator",
494                          "randomGenerator": "random_generator"})
495    def learning_curve(self, learners, examples, cv_indices=None, proportion_indices=None, proportions=Orange.core.frange(0.1),
496                       preprocessors=(), random_generator=0, callback=None):
497        """
498        Compute a learning curve using multiple cross-validations where
499        models are trained on different portions of the training data.
500
501        :param learners: list of learners to be tested
502        :param examples: a dataset used for evaluation
503        :param cv_indices: indices used for crossvalidation
504        :param proportion_indices: indices for proportion selection
505        :param proportions: proportions of train data to be used
506        :param preprocessors: a list of preprocessors to be used on data.
507        :param callback: a function that is be called after each classifier is computed.
508        :return: list of :obj:`ExperimentResults`
509        """
510        if cv_indices is None:
511            cv_indices = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
512        if proportion_indices is None:
513            proportion_indices = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
514
515        examples, weight = demangle_examples(examples)
516        indices = cv_indices(examples)
517       
518        all_results=[]
519        for p in proportions:
520            def select_proportion_preprocessor(examples):
521                return examples.selectref(proportion_indices(examples, p0=p), 0)
522
523            test_results = self.test_with_indices(learners, examples, indices,
524                                                  preprocessors=list(preprocessors) + [("L", select_proportion_preprocessor)],
525                                                  callback=callback)
526            all_results.append(test_results)
527        return all_results
528
529
530    @deprecated_keywords({"strat": "stratification",
531                          "pps": "preprocessors",
532                          "indicesrandseed": "random_generator",
533                          "randseed": "random_generator",
534                          "randomGenerator": "random_generator"})
535    def learning_curve_n(self, learners, examples, folds=10,
536                       proportions=Orange.core.frange(0.1),
537                       stratification = Orange.core.MakeRandomIndices
538                       .StratifiedIfPossible,
539                       preprocessors=(),
540                       random_generator=0, callback=None):
541        """
542        Compute a learning curve where each cross-validation has given number of folds
543        and models are trained on specified proportion of training data.
544
545        :param learners: list of learners to be tested
546        :param examples: a dataset used for evaluation
547        :param folds: number of folds for cross-validation
548        :param proportions: proportions of train data to be used
549        :param preprocessors: a list of preprocessors to be used on data.
550        :param callback: a function that is called after each classifier is computed.
551        :return: list of :obj:`ExperimentResults`
552        """
553
554        cv=Orange.core.MakeRandomIndicesCV(folds = folds,
555            stratified = stratification, randomGenerator = random_generator)
556        pick=Orange.core.MakeRandomIndices2(stratified = stratification,
557            randomGenerator = random_generator)
558        return learning_curve(learners, examples, cv, pick, proportions,
559            preprocessors, callback=callback)
560   
561    def learning_curve_with_test_data(self, learners, learn_set, test_set,
562            times=10, proportions=Orange.core.frange(0.1),
563            stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
564            preprocessors=(), random_generator=0, store_classifiers=False,
565            store_examples=False):
566        """
567        Compute a learning curve given two datasets. Models are learned on
568        proportion of the first dataset and then used to make predictions for
569        the second dataset.
570
571        :param learners: list of learners to be tested
572        :param learn_set: a dataset used for evaluation
573        :param test_set: a dataset used for evaluation
574        :param proportions: proportions of train data to be used
575        :param preprocessors: a list of preprocessors to be used on data.
576        :return: list of :obj:`ExperimentResults`
577        """
578        learn_set, learn_weight = demangle_examples(learn_set)
579        test_set, test_weight = demangle_examples(test_set)
580        test_type = self.check_test_type(learn_set, learners)
581        self.check_test_type(test_set, learners)
582       
583        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator)
584       
585        all_results=[]
586        for p in proportions:
587            test_results = ExperimentResults(times,
588                                        classifier_names = [getobjectname(l) for l in learners],
589                                        domain=test_set.domain,
590                                        test_type = test_type,
591                                        weights=test_weight)
592            offset = 0
593            for t in xrange(times):
594                test_results.results.extend(test_results.create_tested_example(t, example)
595                                            for i, example in enumerate(test_set))
596
597                learn_examples = learn_set.selectref(indices(learn_set, p), 0)
598                classifiers, results = self._learn_and_test_on_test_data\
599                    (learners, learn_examples, learn_weight, test_set,
600                    preprocessors=preprocessors)
601
602                for example, classifier, result in results:
603                    test_results.results[offset+example].set_result(classifier, *result)
604                offset += len(test_set)
605
606                if store_classifiers:
607                    test_results.classifiers.append(classifiers)
608                if store_examples:
609                    test_results.examples = learn_examples
610
611            all_results.append(test_results)
612        return all_results
613
614
615    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False):
616        """
617        Test classifiers on examples
618
619        :param classifiers: classifiers to test
620        :param examples: examples to test on
621        :param store_classifiers: if True, classifiers will be accessible in test_results.
622        :param store_examples: if True, examples will be accessible in test_results.
623        """
624
625        examples, weight = demangle_examples(examples)
626        test_type = self.check_test_type(examples, classifiers)
627
628        test_results = ExperimentResults(1,
629                                        classifier_names = [getobjectname(l) for l in classifiers],
630                                        domain=examples.domain,
631                                        test_type = test_type,
632                                        weights=weight)
633        test_results.results = [test_results.create_tested_example(0, example)
634                               for i, example in enumerate(examples)]
635
636        if store_examples:
637            test_results.examples = examples
638        if store_classifiers:
639            test_results.classifiers = classifiers
640
641        results = self._test_on_data(classifiers, examples)
642        for example, classifier, result in results:
643            test_results.results[example].set_result(classifier, *result)
644        return test_results
645
646
647    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set,
648                                     preprocessors=(), callback=None):
649        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
650
651        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback)
652       
653        results = self._test_on_data(classifiers, test_set)
654        return classifiers, results
655
656
657    def _train_with_callback(self, learners, examples, weight, callback):
658        classifiers = []
659        for learner in learners:
660            classifier = learner(examples, weight)
661            classifier.name = getattr(learner, 'name', 'noname')
662            classifiers.append(classifier)
663            if callback:
664                callback()
665        return classifiers
666
667
668    def _test_on_data(self, classifiers, examples, example_ids=None):
669        results = []
670
671        if example_ids is None:
672            numbered_examples = enumerate(examples)
673        else:
674            numbered_examples = itertools.izip(example_ids, examples)
675
676        for e, example in numbered_examples:
677            for c, classifier in enumerate(classifiers):
678                # Hide actual class to prevent cheating
679                ex2 = Orange.data.Instance(example)
680                if ex2.domain.class_var: ex2.setclass("?")
681                if ex2.domain.class_vars: ex2.set_classes(["?" for _ in ex2
682                .domain.class_vars])
683                result = classifier(ex2, Orange.core.GetBoth)
684                results.append((e, c, result))
685        return results
686
687   
688    def _preprocess_data(self, learn_set, test_set, preprocessors):
689        """Apply preprocessors to learn and test dataset"""
690        for p_type, preprocessor in preprocessors:
691            if p_type == "B":
692                learn_set = preprocessor(learn_set)
693                test_set = preprocessor(test_set)
694        for p_type, preprocessor in preprocessors:
695            if p_type == "L":
696                learn_set = preprocessor(learn_set)
697            elif p_type == "T":
698                test_set = preprocessor(test_set)
699            elif p_type == "LT":
700                (learn_set, test_set) = preprocessor(learn_set, test_set)
701        return learn_set, test_set
702
703
704    def encode_PP(self, pps):
705        for pp in pps:
706            objname = getobjectname(pp[1])
707            if len(objname):
708                pps+="_"+objname
709            else:
710                return "*"
711        return pps
712
713    def check_test_type(self, instances, models):
714        model_is_mlc = [isinstance(m, Orange.multilabel.MultiLabelLearner) or
715                        isinstance(m, Orange.multilabel.MultiLabelClassifier)
716                          for m in models]
717        multi_label = any(model_is_mlc)
718        if multi_label and not all(model_is_mlc):
719            raise ValueError("Test on mixed types of learners (MLC and non-MLC) not possible")
720        multi_target = instances.domain.class_vars and not multi_label
721
722        if (multi_label or multi_target) and not instances.domain.class_vars:
723            raise ValueError("Test data with multiple labels (class vars) expected")
724        if not (multi_label or multi_target or instances.domain.class_var):
725            raise ValueError("Test data set without class attributes")
726
727        return TEST_TYPE_MLC if multi_label else (
728            TEST_TYPE_MULTITARGET if multi_target else TEST_TYPE_SINGLE)
729   
730default_evaluation = _default_evaluation = Evaluation()
731
732cross_validation = _default_evaluation.cross_validation
733leave_one_out = _default_evaluation.leave_one_out
734test_with_indices = _default_evaluation.test_with_indices
735one_fold_with_indices = _default_evaluation.one_fold_with_indices
736
737learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data
738learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data
739test_on_data = _default_evaluation.test_on_data
740
741learning_curve = _default_evaluation.learning_curve
742learning_curve_n = _default_evaluation.learning_curve_n
743learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data
744
745proportion_test = _default_evaluation.proportion_test
746
747encode_PP = _default_evaluation.encode_PP
Note: See TracBrowser for help on using the repository browser.