source: orange/Orange/evaluation/testing.py @ 10846:3cce56b9a0d7

Revision 10846:3cce56b9a0d7, 36.5 KB checked in by Miran@…, 2 years ago (diff)

[BUG] Pickling bug, replaced lambda function in evaluation.testing

Line 
1import itertools
2
3import Orange
4from Orange.utils import demangle_examples, getobjectname
5from Orange.utils import deprecated_keywords, deprecated_members
6
7#### Data structures
8
9TEST_TYPE_SINGLE = 0
10TEST_TYPE_MLC = 1
11TEST_TYPE_MULTITARGET = 2
12
13class TestedExample:
14    """
15    TestedExample stores predictions of different classifiers for a
16    single testing data instance.
17
18    .. attribute:: classes
19
20        A list of predictions of type Value, one for each classifier.
21
22    .. attribute:: probabilities
23
24        A list of probabilities of classes, one for each classifier.
25
26    .. attribute:: iteration_number
27
28        Iteration number (e.g. fold) in which the TestedExample was
29        created/tested.
30
31    .. attribute actual_class
32
33        The correct class of the example
34
35    .. attribute weight
36
37        Instance's weight; 1.0 if data was not weighted
38    """
39
40    @deprecated_keywords({"iterationNumber": "iteration_number",
41                          "actualClass": "actual_class"})
42    def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0):
43        """
44        :param iteration_number: The iteration number of TestedExample.
45        :param actual_class: The actual class of TestedExample.
46        :param n: The number of learners.
47        :param weight: The weight of the TestedExample.
48        """
49        self.classes = [None]*n
50        self.probabilities = [None]*n
51        self.iteration_number = iteration_number
52        self.actual_class= actual_class
53        self.weight = weight
54
55    def add_result(self, aclass, aprob):
56        """Append a new result (class and probability prediction by a single classifier) to the classes and probabilities field."""
57   
58        if isinstance(aclass, (list, tuple)):
59            self.classes.append(aclass)
60            self.probabilities.append(aprob)
61        elif type(aclass.value)==float:
62            self.classes.append(float(aclass))
63            self.probabilities.append(aprob)
64        else:
65            self.classes.append(int(aclass))
66            self.probabilities.append(list(aprob))
67
68    def set_result(self, i, aclass, aprob):
69        """Set the result of the i-th classifier to the given values."""
70        if isinstance(aclass, (list, tuple)):
71            self.classes[i] = aclass
72            self.probabilities[i] = aprob
73        elif type(aclass.value)==float:
74            self.classes[i] = float(aclass)
75            self.probabilities[i] = aprob
76        else:
77            self.classes[i] = int(aclass)
78            self.probabilities[i] = list(aprob)
79
80    def __repr__(self):
81        return str(self.__dict__)
82
83TestedExample = deprecated_members({"iterationNumber": "iteration_number",
84                                    "actualClass": "actual_class"
85                                    })(TestedExample)
86
87def mt_vals(vals):
88    """
89    Substitution for the unpicklable lambda function for multi-target classifiers.
90    """
91    return [int(val) if val.variable.var_type == Orange.feature.Type.Discrete
92                                            else float(val) for val in vals]
93
94class ExperimentResults(object):
95    """
96    ``ExperimentResults`` stores results of one or more repetitions of
97    some test (cross validation, repeated sampling...) under the same
98    circumstances. Instances of this class are constructed by sampling
99    and testing functions from module :obj:`Orange.evaluation.testing`
100    and used by methods in module :obj:`Orange.evaluation.scoring`.
101
102    .. attribute:: results
103
104        A list of instances of :obj:`TestedExample`, one for each
105        example in the dataset.
106
107    .. attribute:: number_of_iterations
108
109        Number of iterations. This can be the number of folds (in
110        cross validation) or the number of repetitions of some
111        test. :obj:`TestedExample`'s attribute ``iteration_number``
112        should be in range ``[0, number_of_iterations-1]``.
113
114    .. attribute:: number_of_learners
115
116        Number of learners. Lengths of lists classes and probabilities
117        in each :obj:`TestedExample` should equal
118        ``number_of_learners``.
119
120    .. attribute:: classifier_names
121
122        Stores the names of the classifiers.
123
124    .. attribute:: classifiers
125
126        A list of classifiers, one element for each iteration of
127        sampling and learning (eg. fold). Each element is a list of
128        classifiers, one for each learner. For instance,
129        ``classifiers[2][4]`` refers to the 3rd repetition, 5th
130        learning algorithm.
131
132        Note that functions from :obj:`~Orange.evaluation.testing`
133        only store classifiers it enabled by setting
134        ``storeClassifiers`` to ``1``.
135
136    ..
137        .. attribute:: loaded
138
139            If the experimental method supports caching and there are no
140            obstacles for caching (such as unknown random seeds), this is a
141            list of boolean values. Each element corresponds to a classifier
142            and tells whether the experimental results for that classifier
143            were computed or loaded from the cache.
144
145    .. attribute:: base_class
146
147       The reference class for measures like AUC.
148
149    .. attribute:: class_values
150
151        The list of class values.
152
153    .. attribute:: weights
154
155        A flag telling whether the results are weighted. If ``False``,
156        weights are still present in :obj:`TestedExample`, but they
157        are all ``1.0``. Clear this flag, if your experimental
158        procedure ran on weighted testing examples but you would like
159        to ignore the weights in statistics.
160    """
161    @deprecated_keywords({"classifierNames": "classifier_names",
162                          "classValues": "class_values",
163                          "baseClass": "base_class",
164                          "numberOfIterations": "number_of_iterations",
165                          "numberOfLearners": "number_of_learners"})
166    def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=-1, domain=None, test_type=TEST_TYPE_SINGLE, labels=None, **argkw):
167        self.class_values = class_values
168        self.classifier_names = classifier_names
169        self.number_of_iterations = iterations
170        self.number_of_learners = len(classifier_names)
171        self.results = []
172        self.classifiers = []
173        self.loaded = None
174        self.base_class = base_class
175        self.weights = weights
176        self.test_type = test_type
177        self.labels = labels
178
179        if domain is not None:
180            self.base_class = self.class_values = None
181            if test_type == TEST_TYPE_SINGLE:
182                if domain.class_var.var_type == Orange.feature.Type.Discrete:
183                    self.class_values = list(domain.class_var.values)
184                    self.base_class = domain.class_var.base_value
185                    self.converter = int
186                else:
187                    self.converter = float
188            elif test_type in (TEST_TYPE_MLC, TEST_TYPE_MULTITARGET):
189                self.labels = [var.name for var in domain.class_vars]
190                self.converter = mt_vals
191
192
193        self.__dict__.update(argkw)
194
195    def load_from_files(self, learners, filename):
196        raise NotImplementedError("This feature is no longer supported.")
197
198    def save_to_files(self, learners, filename):
199        raise NotImplementedError("This feature is no longer supported. Pickle whole class instead.")
200
201    def create_tested_example(self, fold, example):
202        actual = example.getclass() if self.test_type == TEST_TYPE_SINGLE \
203                                  else example.get_classes()
204        return TestedExample(fold,
205                             self.converter(actual),
206                             self.number_of_learners,
207                             example.getweight(self.weights))
208
209    def remove(self, index):
210        """remove one learner from evaluation results"""
211        for r in self.results:
212            del r.classes[index]
213            del r.probabilities[index]
214        del self.classifier_names[index]
215        self.number_of_learners -= 1
216
217    def add(self, results, index, replace=-1):
218        """add evaluation results (for one learner)"""
219        if len(self.results)!=len(results.results):
220            raise SystemError("mismatch in number of test cases")
221        if self.number_of_iterations!=results.number_of_iterations:
222            raise SystemError("mismatch in number of iterations (%d<>%d)" % \
223                  (self.number_of_iterations, results.number_of_iterations))
224        if len(self.classifiers) and len(results.classifiers)==0:
225            raise SystemError("no classifiers in results")
226
227        if replace < 0 or replace >= self.number_of_learners: # results for new learner
228            self.classifier_names.append(results.classifier_names[index])
229            self.number_of_learners += 1
230            for i,r in enumerate(self.results):
231                r.classes.append(results.results[i].classes[index])
232                r.probabilities.append(results.results[i].probabilities[index])
233            if len(self.classifiers):
234                for i in range(self.number_of_iterations):
235                    self.classifiers[i].append(results.classifiers[i][index])
236        else: # replace results of existing learner
237            self.classifier_names[replace] = results.classifier_names[index]
238            for i,r in enumerate(self.results):
239                r.classes[replace] = results.results[i].classes[index]
240                r.probabilities[replace] = results.results[i].probabilities[index]
241            if len(self.classifiers):
242                for i in range(self.number_of_iterations):
243                    self.classifiers[replace] = results.classifiers[i][index]
244
245    def __repr__(self):
246        return str(self.__dict__)
247
248ExperimentResults = deprecated_members({"classValues": "class_values",
249                                        "classifierNames": "classifier_names",
250                                        "baseClass": "base_class",
251                                        "numberOfIterations": "number_of_iterations",
252                                        "numberOfLearners": "number_of_learners"
253})(ExperimentResults)
254
255   
256class Evaluation(object):
257    """Common methods for learner evaluation."""
258
259    @deprecated_keywords({"pps": "preprocessors",
260                          "strat": "stratified",
261                          "randseed": "random_generator",
262                          "indicesrandseed": "random_generator",
263                          "randomGenerator": "random_generator",
264                          "storeClassifiers": "store_classifiers",
265                          "storeExamples": "store_examples"})
266    def cross_validation(self, learners, examples, folds=10,
267            stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible,
268            preprocessors=(), random_generator=0, callback=None,
269            store_classifiers=False, store_examples=False):
270        """Cross validation test with specified number of folds.
271
272        :param learners: list of learning algorithms
273        :param examples: data instances used for training and testing
274        :param folds: number of folds
275        :param stratified: tells whether to stratify the sampling
276        :param preprocessors: a list of preprocessors to be used on data (obsolete)
277        :param random_generator: random seed or generator (see above)
278        :param callback: a function that is called after finishing each fold
279        :param store_classifiers: if ``True``, classifiers are stored in results
280        :param store_examples: if ``True``, examples are stored in results
281        :return: :obj:`ExperimentResults`
282        """
283        (examples, weight) = demangle_examples(examples)
284
285        indices = Orange.core.MakeRandomIndicesCV(examples, folds,
286            stratified=stratified, random_generator=random_generator)
287
288        return self.test_with_indices(
289            learners=learners,
290            examples=(examples, weight),
291            indices=indices,
292            preprocessors=preprocessors,
293            callback=callback,
294            store_classifiers=store_classifiers,
295            store_examples=store_examples)
296
297
298    @deprecated_keywords({"pps": "preprocessors",
299                          "storeClassifiers": "store_classifiers",
300                          "storeExamples": "store_examples"})
301    def leave_one_out(self, learners, examples, preprocessors=(),
302            callback=None, store_classifiers=False, store_examples=False):
303        """Leave-one-out evaluation of learning algorithms.
304
305        :param learners: list of learning algorithms
306        :param examples: data instances used for training and testing
307        :param preprocessors: a list of preprocessors (obsolete)
308        :param callback: a function that is called after finishing each fold
309        :param store_classifiers: if ``True``, classifiers are stored in results
310        :param store_examples: if ``True``, examples are stored in results
311        :return: :obj:`ExperimentResults`
312        """
313        examples, weight = demangle_examples(examples)
314        return self.test_with_indices(
315            learners, (examples, weight), indices=range(len(examples)),
316            preprocessors=preprocessors, callback=callback,
317            store_classifiers=store_classifiers, store_examples=store_examples)
318
319    @deprecated_keywords({"storeExamples": "store_examples",
320                          "storeClassifiers": "store_classifiers=True",
321                          "pps":"preprocessors"})
322    def test_with_indices(self, learners, examples, indices, preprocessors=(),
323            callback=None, store_classifiers=False, store_examples=False):
324        """
325        Perform a cross-validation-like test. Examples for each fold are
326        selected based on given indices.
327
328        :param learners: list of learning algorithms
329        :param examples: data instances used for training and testing
330        :param indices: a list of integer indices that sort examples into folds; each index corresponds to an example from ``examples``
331        :param preprocessors: a list of preprocessors (obsolete)
332        :param callback: a function that is called after each fold
333        :param store_classifiers: if ``True``, classifiers are stored in results
334        :param store_examples: if ``True``, examples are stored in results
335        :return: :obj:`ExperimentResults`
336        """
337        examples, weight = demangle_examples(examples)
338        if not examples:
339            raise ValueError("Test data set with no examples")
340        test_type = self.check_test_type(examples, learners)
341
342        niterations = max(indices)+1
343        test_result = ExperimentResults(niterations,
344                                        classifier_names = [getobjectname(l) for l in learners],
345                                        domain=examples.domain,
346                                        weights=weight,
347                                        test_type=test_type)
348
349        test_result.results = [test_result.create_tested_example(indices[i], example)
350                               for i, example in enumerate(examples)]
351
352        if store_examples:
353            test_result.examples = examples
354
355        for fold in xrange(niterations):
356            results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight)
357
358            for example, learner, result in results:
359                test_result.results[example].set_result(learner, *result)
360
361            if store_classifiers:
362                test_result.classifiers.append(classifiers)
363            if callback:
364                callback()
365
366        return test_result
367
368
369    def one_fold_with_indices(self, learners, examples, fold, indices, preprocessors=(), weight=0):
370        """Similar to :obj:`test_with_indices` except that it performs single fold of cross-validation, given by argument ``fold``."""
371        learn_set = examples.selectref(indices, fold, negate=1)
372        test_set = examples.selectref(indices, fold, negate=0)
373        if len(learn_set)==0 or len(test_set)==0:
374            return (), ()
375
376        # learning
377        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
378        if not learn_set:
379            raise SystemError("no training examples after preprocessing")
380        if not test_set:
381            raise SystemError("no test examples after preprocessing")
382
383        classifiers = [learner(learn_set, weight) for learner in learners]
384
385        # testing
386        testset_ids = (i for i, _ in enumerate(examples) if indices[i] == fold)
387        results = self._test_on_data(classifiers, test_set, testset_ids)
388
389        return results, classifiers
390
391
392    @deprecated_keywords({"storeExamples": "store_examples",
393                          "storeClassifiers": "store_classifiers",
394                          "pps": "preprocessors"})
395    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(),
396                                     callback=None, store_classifiers=False, store_examples=False):
397        """
398        Train learning algorithms and test them on the same data.
399
400        :param learners: list of learning algorithms
401        :param examples: data instances used for training and testing
402        :param preprocessors: a list of preprocessors (obsolete)
403        :param callback: a function that is called after each learning
404        :param store_classifiers: if ``True``, classifiers are stored in results
405        :param store_examples: if ``True``, examples are stored in results
406        :return: :obj:`ExperimentResults`
407        """
408
409        examples, weight = demangle_examples(examples)
410        test_type = self.check_test_type(examples, learners)
411
412        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to
413        # clone it.
414        if not filter(lambda x:x[0]!="B", preprocessors):
415            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors)
416            test_set = learn_set
417        else:
418            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors)
419
420        classifiers = self._train_with_callback(learners, learn_set, weight, callback)
421
422        test_results = ExperimentResults(1,
423                                        classifier_names = [getobjectname(l) for l in learners],
424                                        test_type = test_type,
425                                        domain=examples.domain,
426                                        weights=weight)
427        test_results.results = [test_results.create_tested_example(0, example)
428                               for i, example in enumerate(examples)]
429
430        if store_classifiers:
431            test_results.classifiers = classifiers
432        if store_examples:
433            test_results.examples = test_set
434
435        results = self._test_on_data(classifiers, test_set)
436        for example, classifier, result in results:
437            test_results.results[example].set_result(classifier, *result)
438        return test_results
439
440    @deprecated_keywords({"storeExamples": "store_examples",
441                          "storeClassifiers": "store_classifiers",
442                          "pps": "preprocessors"})
443    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(),
444                                    callback=None, store_classifiers=False, store_examples=False):
445        """
446        Train learning algorithms on one data sets and test them on another.
447
448        :param learners: list of learning algorithms
449        :param learn_set: training instances
450        :param test_set: testing instances
451        :param preprocessors: a list of preprocessors (obsolete)
452        :param callback: a function that is called after each learning
453        :param store_classifiers: if ``True``, classifiers are stored in results
454        :param store_examples: if ``True``, examples are stored in results
455        :return: :obj:`ExperimentResults`
456        """
457        learn_set, learn_weight = demangle_examples(learn_set)
458        test_set, test_weight = demangle_examples(test_set)
459
460        test_type = self.check_test_type(learn_set, learners)
461        self.check_test_type(test_set, learners)
462       
463        test_results = ExperimentResults(1,
464                                        classifier_names = [getobjectname(l) for l in learners],
465                                        domain=test_set.domain,
466                                        test_type = test_type,
467                                        weights=test_weight)
468        test_results.results = [test_results.create_tested_example(0, example)
469                               for i, example in enumerate(test_set)]
470
471        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback)
472
473        if store_classifiers:
474            test_results.classifiers = classifiers
475        if store_examples:
476            test_results.examples = test_set
477
478        for example, classifier, result in results:
479            test_results.results[example].set_result(classifier, *result)
480        return test_results
481
482    @deprecated_keywords({"storeExamples": "store_examples",
483                          "storeClassifiers": "store_classifiers",
484                          "learnProp": "learning_proportion",
485                          "strat": "stratification",
486                          "pps": "preprocessors",
487                          "indicesrandseed": "random_generator",
488                          "randseed": "random_generator",
489                          "randomGenerator": "random_generator"})
490    def proportion_test(self, learners, examples, learning_proportion=.7, times=10,
491                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
492                   preprocessors=(), random_generator=0,
493                   callback=None, store_classifiers=False, store_examples=False):
494        """
495        Iteratively split the data into training and testing set, and train and test the learnign algorithms.
496
497        :param learners: list of learning algorithms
498        :param examples: data instances used for training and testing
499        :param learning_proportion: proportion of data used for training
500        :param times: number of iterations
501        :param stratification: use stratified sampling
502        :param preprocessors: a list of preprocessors (obsolete)
503        :param random_generator: random seed or generator (see above)
504        :param callback: a function that is called after each fold
505        :param store_classifiers: if ``True``, classifiers are stored in results
506        :param store_examples: if ``True``, examples are stored in results
507        :return: :obj:`ExperimentResults`
508        """
509        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator)
510
511        examples, weight = demangle_examples(examples)
512
513        test_type = self.check_test_type(examples, learners)
514       
515        test_results = ExperimentResults(times,
516                                        classifier_names = [getobjectname(l) for l in learners],
517                                        domain=examples.domain,
518                                        test_type = test_type,
519                                        weights=weight)
520        if store_examples:
521            test_results.examples = []
522        test_results.classifiers = []
523        offset=0
524        for time in xrange(times):
525            indices = pick(examples)
526            learn_set = examples.selectref(indices, 0)
527            test_set = examples.selectref(indices, 1)
528            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors)
529            if store_classifiers:
530                test_results.classifiers.append(classifiers)
531            if store_examples:
532                test_results.examples.append(learn_set)
533
534            test_results.results.extend(test_results.create_tested_example(time, example)
535                                        for i, example in enumerate(test_set))
536            for example, classifier, result in results:
537                test_results.results[offset+example].set_result(classifier, *result)
538            offset += len(test_set)
539
540            if callback:
541                callback()
542        return test_results
543
544    @deprecated_keywords({"storeExamples": "store_examples",
545                          "storeClassifiers": "store_classifiers",
546                          "learnProp": "learning_proportion",
547                          "strat": "stratification",
548                          "pps": "preprocessors",
549                          "indicesrandseed": "random_generator",
550                          "randseed": "random_generator",
551                          "randomGenerator": "random_generator"})
552    def learning_curve(self, learners, examples, cv_indices=None, proportion_indices=None, proportions=Orange.core.frange(0.1),
553                       preprocessors=(), random_generator=0, callback=None):
554        """
555        Compute a learning curve using multiple cross-validations where
556        models are trained on different portions of the training data.
557
558        :param learners: list of learning algorithms
559        :param examples: data instances used for training and testing
560        :param cv_indices: indices used for cross validation (leave ``None`` for 10-fold CV)
561        :param proportion_indices: indices for proportion selection (leave ``None`` to let the function construct the folds)
562        :param proportions: list of proportions of data used for training
563        :param preprocessors: a list of preprocessors (obsolete)
564        :param random_generator: random seed or generator (see above)
565        :param callback: a function that is be called after each learning
566        :return: list of :obj:`ExperimentResults`
567        """
568        if cv_indices is None:
569            cv_indices = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
570        if proportion_indices is None:
571            proportion_indices = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
572
573        examples, weight = demangle_examples(examples)
574        indices = cv_indices(examples)
575       
576        all_results=[]
577        for p in proportions:
578            def select_proportion_preprocessor(examples):
579                return examples.selectref(proportion_indices(examples, p0=p), 0)
580
581            test_results = self.test_with_indices(learners, examples, indices,
582                                                  preprocessors=list(preprocessors) + [("L", select_proportion_preprocessor)],
583                                                  callback=callback)
584            all_results.append(test_results)
585        return all_results
586
587
588    @deprecated_keywords({"strat": "stratification",
589                          "pps": "preprocessors",
590                          "indicesrandseed": "random_generator",
591                          "randseed": "random_generator",
592                          "randomGenerator": "random_generator"})
593    def learning_curve_n(self, learners, examples, folds=10,
594                       proportions=Orange.core.frange(0.1),
595                       stratification = Orange.core.MakeRandomIndices
596                       .StratifiedIfPossible,
597                       preprocessors=(),
598                       random_generator=0, callback=None):
599        """
600        Compute a learning curve using multiple cross-validations where
601        models are trained on different portions of the training data.
602        Similar to :obj:`learning_curve` except for simpler arguments.
603
604        :param learners: list of learning algorithms
605        :param examples: data instances used for training and testing
606        :param folds: number of folds for cross-validation
607        :param proportions: list of proportions of data used for training
608        :param stratification: use stratified sampling
609        :param preprocessors: a list of preprocessors (obsolete)
610        :param random_generator: random seed or generator (see above)
611        :param callback: a function that is be called after each learning
612        :return: list of :obj:`ExperimentResults`
613        """
614
615        cv=Orange.core.MakeRandomIndicesCV(folds = folds,
616            stratified = stratification, randomGenerator = random_generator)
617        pick=Orange.core.MakeRandomIndices2(stratified = stratification,
618            randomGenerator = random_generator)
619        return learning_curve(learners, examples, cv, pick, proportions,
620            preprocessors, callback=callback)
621   
622    def learning_curve_with_test_data(self, learners, learn_set, test_set,
623            times=10, proportions=Orange.core.frange(0.1),
624            stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
625            preprocessors=(), random_generator=0, store_classifiers=False,
626            store_examples=False):
627        """
628        Compute a learning curve given two datasets. Models are learned on
629        proportion of the first dataset and then tested on the second.
630
631        :param learners: list of learning algorithms
632        :param learn_set: training data
633        :param test_set: testing data
634        :param times: number of iterations
635        :param straitification: use stratified sampling
636        :param proportions: a list of proportions of training data to be used
637        :param preprocessors: a list of preprocessors (obsolete)
638        :param random_generator: random seed or generator (see above)
639        :param store_classifiers: if ``True``, classifiers are stored in results
640        :param store_examples: if ``True``, examples are stored in results
641        :return: list of :obj:`ExperimentResults`
642        """
643        learn_set, learn_weight = demangle_examples(learn_set)
644        test_set, test_weight = demangle_examples(test_set)
645        test_type = self.check_test_type(learn_set, learners)
646        self.check_test_type(test_set, learners)
647       
648        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator)
649       
650        all_results=[]
651        for p in proportions:
652            test_results = ExperimentResults(times,
653                                        classifier_names = [getobjectname(l) for l in learners],
654                                        domain=test_set.domain,
655                                        test_type = test_type,
656                                        weights=test_weight)
657            offset = 0
658            for t in xrange(times):
659                test_results.results.extend(test_results.create_tested_example(t, example)
660                                            for i, example in enumerate(test_set))
661
662                learn_examples = learn_set.selectref(indices(learn_set, p), 0)
663                classifiers, results = self._learn_and_test_on_test_data\
664                    (learners, learn_examples, learn_weight, test_set,
665                    preprocessors=preprocessors)
666
667                for example, classifier, result in results:
668                    test_results.results[offset+example].set_result(classifier, *result)
669                offset += len(test_set)
670
671                if store_classifiers:
672                    test_results.classifiers.append(classifiers)
673                if store_examples:
674                    test_results.examples = learn_examples
675
676            all_results.append(test_results)
677        return all_results
678
679
680    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False):
681        """
682        Test classifiers on the given data
683
684        :param classifiers: a list of classifiers
685        :param examples: testing data
686        :param store_classifiers: if ``True``, classifiers are stored in results
687        :param store_examples: if ``True``, examples are stored in results
688        """
689
690        examples, weight = demangle_examples(examples)
691        test_type = self.check_test_type(examples, classifiers)
692
693        test_results = ExperimentResults(1,
694                                        classifier_names = [getobjectname(l) for l in classifiers],
695                                        domain=examples.domain,
696                                        test_type = test_type,
697                                        weights=weight)
698        test_results.results = [test_results.create_tested_example(0, example)
699                               for i, example in enumerate(examples)]
700
701        if store_examples:
702            test_results.examples = examples
703        if store_classifiers:
704            test_results.classifiers = classifiers
705
706        results = self._test_on_data(classifiers, examples)
707        for example, classifier, result in results:
708            test_results.results[example].set_result(classifier, *result)
709        return test_results
710
711
712    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set,
713                                     preprocessors=(), callback=None):
714        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
715
716        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback)
717       
718        results = self._test_on_data(classifiers, test_set)
719        return classifiers, results
720
721
722    def _train_with_callback(self, learners, examples, weight, callback):
723        classifiers = []
724        for learner in learners:
725            classifier = learner(examples, weight)
726            classifier.name = getattr(learner, 'name', 'noname')
727            classifiers.append(classifier)
728            if callback:
729                callback()
730        return classifiers
731
732
733    def _test_on_data(self, classifiers, examples, example_ids=None):
734        results = []
735
736        if example_ids is None:
737            numbered_examples = enumerate(examples)
738        else:
739            numbered_examples = itertools.izip(example_ids, examples)
740
741        for e, example in numbered_examples:
742            for c, classifier in enumerate(classifiers):
743                # Hide actual class to prevent cheating
744                ex2 = Orange.data.Instance(example)
745                if ex2.domain.class_var: ex2.setclass("?")
746                if ex2.domain.class_vars: ex2.set_classes(["?" for _ in ex2
747                .domain.class_vars])
748                result = classifier(ex2, Orange.core.GetBoth)
749                results.append((e, c, result))
750        return results
751
752   
753    def _preprocess_data(self, learn_set, test_set, preprocessors):
754        """Apply preprocessors to learn and test dataset (obsolete)"""
755        for p_type, preprocessor in preprocessors:
756            if p_type == "B":
757                learn_set = preprocessor(learn_set)
758                test_set = preprocessor(test_set)
759        for p_type, preprocessor in preprocessors:
760            if p_type == "L":
761                learn_set = preprocessor(learn_set)
762            elif p_type == "T":
763                test_set = preprocessor(test_set)
764            elif p_type == "LT":
765                (learn_set, test_set) = preprocessor(learn_set, test_set)
766        return learn_set, test_set
767
768
769    def encode_PP(self, pps):
770        for pp in pps:
771            objname = getobjectname(pp[1])
772            if len(objname):
773                pps+="_"+objname
774            else:
775                return "*"
776        return pps
777
778    def check_test_type(self, instances, models):
779        model_is_mlc = [isinstance(m, Orange.multilabel.MultiLabelLearner) or
780                        isinstance(m, Orange.multilabel.MultiLabelClassifier)
781                          for m in models]
782        multi_label = any(model_is_mlc)
783        if multi_label and not all(model_is_mlc):
784            raise ValueError("Test on mixed types of learners (MLC and non-MLC) not possible")
785        multi_target = instances.domain.class_vars and not multi_label
786
787        if (multi_label or multi_target) and not instances.domain.class_vars:
788            raise ValueError("Test data with multiple labels (class vars) expected")
789        if not (multi_label or multi_target or instances.domain.class_var):
790            raise ValueError("Test data set without class attributes")
791
792        return TEST_TYPE_MLC if multi_label else (
793            TEST_TYPE_MULTITARGET if multi_target else TEST_TYPE_SINGLE)
794   
795default_evaluation = _default_evaluation = Evaluation()
796
797cross_validation = _default_evaluation.cross_validation
798leave_one_out = _default_evaluation.leave_one_out
799test_with_indices = _default_evaluation.test_with_indices
800one_fold_with_indices = _default_evaluation.one_fold_with_indices
801
802learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data
803learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data
804test_on_data = _default_evaluation.test_on_data
805
806learning_curve = _default_evaluation.learning_curve
807learning_curve_n = _default_evaluation.learning_curve_n
808learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data
809
810proportion_test = _default_evaluation.proportion_test
811
812encode_PP = _default_evaluation.encode_PP
Note: See TracBrowser for help on using the repository browser.