source: orange/Orange/evaluation/testing.py @ 10414:ac84efa944dc

Revision 10414:ac84efa944dc, 36.3 KB checked in by janezd <janez.demsar@…>, 2 years ago (diff)

Half-polished Orange.evaluation documentation

Line 
1import itertools
2
3import Orange
4from Orange.misc import demangle_examples, getobjectname, deprecated_keywords, deprecated_members
5
6#### Data structures
7
8TEST_TYPE_SINGLE = 0
9TEST_TYPE_MLC = 1
10TEST_TYPE_MULTITARGET = 2
11
12class TestedExample:
13    """
14    TestedExample stores predictions of different classifiers for a
15    single testing data instance.
16
17    .. attribute:: classes
18
19        A list of predictions of type Value, one for each classifier.
20
21    .. attribute:: probabilities
22
23        A list of probabilities of classes, one for each classifier.
24
25    .. attribute:: iteration_number
26
27        Iteration number (e.g. fold) in which the TestedExample was
28        created/tested.
29
30    .. attribute actual_class
31
32        The correct class of the example
33
34    .. attribute weight
35
36        Instance's weight; 1.0 if data was not weighted
37    """
38
39    @deprecated_keywords({"iterationNumber": "iteration_number",
40                          "actualClass": "actual_class"})
41    def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0):
42        """
43        :param iteration_number: The iteration number of TestedExample.
44        :param actual_class: The actual class of TestedExample.
45        :param n: The number of learners.
46        :param weight: The weight of the TestedExample.
47        """
48        self.classes = [None]*n
49        self.probabilities = [None]*n
50        self.iteration_number = iteration_number
51        self.actual_class= actual_class
52        self.weight = weight
53
54    def add_result(self, aclass, aprob):
55        """Append a new result (class and probability prediction by a single classifier) to the classes and probabilities field."""
56   
57        if isinstance(aclass, (list, tuple)):
58            self.classes.append(aclass)
59            self.probabilities.append(aprob)
60        elif type(aclass.value)==float:
61            self.classes.append(float(aclass))
62            self.probabilities.append(aprob)
63        else:
64            self.classes.append(int(aclass))
65            self.probabilities.append(list(aprob))
66
67    def set_result(self, i, aclass, aprob):
68        """Set the result of the i-th classifier to the given values."""
69        if isinstance(aclass, (list, tuple)):
70            self.classes[i] = aclass
71            self.probabilities[i] = aprob
72        elif type(aclass.value)==float:
73            self.classes[i] = float(aclass)
74            self.probabilities[i] = aprob
75        else:
76            self.classes[i] = int(aclass)
77            self.probabilities[i] = list(aprob)
78
79    def __repr__(self):
80        return str(self.__dict__)
81
82TestedExample = deprecated_members({"iterationNumber": "iteration_number",
83                                    "actualClass": "actual_class"
84                                    })(TestedExample)
85
86class ExperimentResults(object):
87    """
88    ``ExperimentResults`` stores results of one or more repetitions of
89    some test (cross validation, repeated sampling...) under the same
90    circumstances. Instances of this class are constructed by sampling
91    and testing functions from module :obj:`Orange.evaluation.testing`
92    and used by methods in module :obj:`Orange.evaluation.scoring`.
93
94    .. attribute:: results
95
96        A list of instances of :obj:`TestedExample`, one for each
97        example in the dataset.
98
99    .. attribute:: number_of_iterations
100
101        Number of iterations. This can be the number of folds (in
102        cross validation) or the number of repetitions of some
103        test. :obj:`TestedExample`'s attribute ``iteration_number``
104        should be in range ``[0, number_of_iterations-1]``.
105
106    .. attribute:: number_of_learners
107
108        Number of learners. Lengths of lists classes and probabilities
109        in each :obj:`TestedExample` should equal
110        ``number_of_learners``.
111
112    .. attribute:: classifier_names
113
114        Stores the names of the classifiers.
115
116    .. attribute:: classifiers
117
118        A list of classifiers, one element for each iteration of
119        sampling and learning (eg. fold). Each element is a list of
120        classifiers, one for each learner. For instance,
121        ``classifiers[2][4]`` refers to the 3rd repetition, 5th
122        learning algorithm.
123
124        Note that functions from :obj:`~Orange.evaluation.testing`
125        only store classifiers it enabled by setting
126        ``storeClassifiers`` to ``1``.
127
128    ..
129        .. attribute:: loaded
130
131            If the experimental method supports caching and there are no
132            obstacles for caching (such as unknown random seeds), this is a
133            list of boolean values. Each element corresponds to a classifier
134            and tells whether the experimental results for that classifier
135            were computed or loaded from the cache.
136
137    .. attribute:: base_class
138
139       The reference class for measures like AUC.
140
141    .. attribute:: class_values
142
143        The list of class values.
144
145    .. attribute:: weights
146
147        A flag telling whether the results are weighted. If ``False``,
148        weights are still present in :obj:`TestedExample`, but they
149        are all ``1.0``. Clear this flag, if your experimental
150        procedure ran on weighted testing examples but you would like
151        to ignore the weights in statistics.
152    """
153    @deprecated_keywords({"classifierNames": "classifier_names",
154                          "classValues": "class_values",
155                          "baseClass": "base_class",
156                          "numberOfIterations": "number_of_iterations",
157                          "numberOfLearners": "number_of_learners"})
158    def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=-1, domain=None, test_type=TEST_TYPE_SINGLE, labels=None, **argkw):
159        self.class_values = class_values
160        self.classifier_names = classifier_names
161        self.number_of_iterations = iterations
162        self.number_of_learners = len(classifier_names)
163        self.results = []
164        self.classifiers = []
165        self.loaded = None
166        self.base_class = base_class
167        self.weights = weights
168        self.test_type = test_type
169        self.labels = labels
170
171        if domain is not None:
172            self.base_class = self.class_values = None
173            if test_type == TEST_TYPE_SINGLE:
174                if domain.class_var.var_type == Orange.feature.Type.Discrete:
175                    self.class_values = list(domain.class_var.values)
176                    self.base_class = domain.class_var.base_value
177                    self.converter = int
178                else:
179                    self.converter = float
180            elif test_type in (TEST_TYPE_MLC, TEST_TYPE_MULTITARGET):
181                self.labels = [var.name for var in domain.class_vars]
182                self.converter = lambda vals: [int(val) if val.variable.var_type == Orange.feature.Type.Discrete
183                                               else float(val) for val in vals]
184
185        self.__dict__.update(argkw)
186
187    def load_from_files(self, learners, filename):
188        raise NotImplementedError("This feature is no longer supported.")
189
190    def save_to_files(self, learners, filename):
191        raise NotImplementedError("This feature is no longer supported. Pickle whole class instead.")
192
193    def create_tested_example(self, fold, example):
194        actual = example.getclass() if self.test_type == TEST_TYPE_SINGLE \
195                                  else example.get_classes()
196        return TestedExample(fold,
197                             self.converter(actual),
198                             self.number_of_learners,
199                             example.getweight(self.weights))
200
201    def remove(self, index):
202        """remove one learner from evaluation results"""
203        for r in self.results:
204            del r.classes[index]
205            del r.probabilities[index]
206        del self.classifier_names[index]
207        self.number_of_learners -= 1
208
209    def add(self, results, index, replace=-1):
210        """add evaluation results (for one learner)"""
211        if len(self.results)!=len(results.results):
212            raise SystemError("mismatch in number of test cases")
213        if self.number_of_iterations!=results.number_of_iterations:
214            raise SystemError("mismatch in number of iterations (%d<>%d)" % \
215                  (self.number_of_iterations, results.number_of_iterations))
216        if len(self.classifiers) and len(results.classifiers)==0:
217            raise SystemError("no classifiers in results")
218
219        if replace < 0 or replace >= self.number_of_learners: # results for new learner
220            self.classifier_names.append(results.classifier_names[index])
221            self.number_of_learners += 1
222            for i,r in enumerate(self.results):
223                r.classes.append(results.results[i].classes[index])
224                r.probabilities.append(results.results[i].probabilities[index])
225            if len(self.classifiers):
226                for i in range(self.number_of_iterations):
227                    self.classifiers[i].append(results.classifiers[i][index])
228        else: # replace results of existing learner
229            self.classifier_names[replace] = results.classifier_names[index]
230            for i,r in enumerate(self.results):
231                r.classes[replace] = results.results[i].classes[index]
232                r.probabilities[replace] = results.results[i].probabilities[index]
233            if len(self.classifiers):
234                for i in range(self.number_of_iterations):
235                    self.classifiers[replace] = results.classifiers[i][index]
236
237    def __repr__(self):
238        return str(self.__dict__)
239
240ExperimentResults = deprecated_members({"classValues": "class_values",
241                                        "classifierNames": "classifier_names",
242                                        "baseClass": "base_class",
243                                        "numberOfIterations": "number_of_iterations",
244                                        "numberOfLearners": "number_of_learners"
245})(ExperimentResults)
246
247   
248class Evaluation(object):
249    """Common methods for learner evaluation."""
250
251    @deprecated_keywords({"pps": "preprocessors",
252                          "strat": "stratified",
253                          "randseed": "random_generator",
254                          "indicesrandseed": "random_generator",
255                          "randomGenerator": "random_generator",
256                          "storeClassifiers": "store_classifiers",
257                          "storeExamples": "store_examples"})
258    def cross_validation(self, learners, examples, folds=10,
259            stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible,
260            preprocessors=(), random_generator=0, callback=None,
261            store_classifiers=False, store_examples=False):
262        """Cross validation test with specified number of folds.
263
264        :param learners: list of learning algorithms
265        :param examples: data instances used for training and testing
266        :param folds: number of folds
267        :param stratified: tells whether to stratify the sampling
268        :param preprocessors: a list of preprocessors to be used on data (obsolete)
269        :param random_generator: random seed or generator (see above)
270        :param callback: a function that is called after finishing each fold
271        :param store_classifiers: if ``True``, classifiers are stored in results
272        :param store_examples: if ``True``, examples are stored in results
273        :return: :obj:`ExperimentResults`
274        """
275        (examples, weight) = demangle_examples(examples)
276
277        indices = Orange.core.MakeRandomIndicesCV(examples, folds,
278            stratified=stratified, random_generator=random_generator)
279
280        return self.test_with_indices(
281            learners=learners,
282            examples=(examples, weight),
283            indices=indices,
284            preprocessors=preprocessors,
285            callback=callback,
286            store_classifiers=store_classifiers,
287            store_examples=store_examples)
288
289
290    @deprecated_keywords({"pps": "preprocessors",
291                          "storeClassifiers": "store_classifiers",
292                          "storeExamples": "store_examples"})
293    def leave_one_out(self, learners, examples, preprocessors=(),
294            callback=None, store_classifiers=False, store_examples=False):
295        """Leave-one-out evaluation of learning algorithms.
296
297        :param learners: list of learning algorithms
298        :param examples: data instances used for training and testing
299        :param preprocessors: a list of preprocessors (obsolete)
300        :param callback: a function that is called after finishing each fold
301        :param store_classifiers: if ``True``, classifiers are stored in results
302        :param store_examples: if ``True``, examples are stored in results
303        :return: :obj:`ExperimentResults`
304        """
305        examples, weight = demangle_examples(examples)
306        return self.test_with_indices(
307            learners, (examples, weight), indices=range(len(examples)),
308            preprocessors=preprocessors, callback=callback,
309            store_classifiers=store_classifiers, store_examples=store_examples)
310
311    @deprecated_keywords({"storeExamples": "store_examples",
312                          "storeClassifiers": "store_classifiers=True",
313                          "pps":"preprocessors"})
314    def test_with_indices(self, learners, examples, indices, preprocessors=(),
315            callback=None, store_classifiers=False, store_examples=False):
316        """
317        Perform a cross-validation-like test. Examples for each fold are
318        selected based on given indices.
319
320        :param learners: list of learning algorithms
321        :param examples: data instances used for training and testing
322        :param indices: a list of integer indices that sort examples into folds; each index corresponds to an example from ``examples``
323        :param preprocessors: a list of preprocessors (obsolete)
324        :param callback: a function that is called after each fold
325        :param store_classifiers: if ``True``, classifiers are stored in results
326        :param store_examples: if ``True``, examples are stored in results
327        :return: :obj:`ExperimentResults`
328        """
329        examples, weight = demangle_examples(examples)
330        if not examples:
331            raise ValueError("Test data set with no examples")
332        test_type = self.check_test_type(examples, learners)
333
334        niterations = max(indices)+1
335        test_result = ExperimentResults(niterations,
336                                        classifier_names = [getobjectname(l) for l in learners],
337                                        domain=examples.domain,
338                                        weights=weight,
339                                        test_type=test_type)
340
341        test_result.results = [test_result.create_tested_example(indices[i], example)
342                               for i, example in enumerate(examples)]
343
344        if store_examples:
345            test_result.examples = examples
346
347        for fold in xrange(niterations):
348            results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight)
349
350            for example, learner, result in results:
351                test_result.results[example].set_result(learner, *result)
352
353            if store_classifiers:
354                test_result.classifiers.append(classifiers)
355            if callback:
356                callback()
357
358        return test_result
359
360
361    def one_fold_with_indices(self, learners, examples, fold, indices, preprocessors=(), weight=0):
362        """Similar to :obj:`test_with_indices` except that it performs single fold of cross-validation, given by argument ``fold``."""
363        learn_set = examples.selectref(indices, fold, negate=1)
364        test_set = examples.selectref(indices, fold, negate=0)
365        if len(learn_set)==0 or len(test_set)==0:
366            return (), ()
367
368        # learning
369        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
370        if not learn_set:
371            raise SystemError("no training examples after preprocessing")
372        if not test_set:
373            raise SystemError("no test examples after preprocessing")
374
375        classifiers = [learner(learn_set, weight) for learner in learners]
376
377        # testing
378        testset_ids = (i for i, _ in enumerate(examples) if indices[i] == fold)
379        results = self._test_on_data(classifiers, test_set, testset_ids)
380
381        return results, classifiers
382
383
384    @deprecated_keywords({"storeExamples": "store_examples",
385                          "storeClassifiers": "store_classifiers",
386                          "pps": "preprocessors"})
387    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(),
388                                     callback=None, store_classifiers=False, store_examples=False):
389        """
390        Train learning algorithms and test them on the same data.
391
392        :param learners: list of learning algorithms
393        :param examples: data instances used for training and testing
394        :param preprocessors: a list of preprocessors (obsolete)
395        :param callback: a function that is called after each learning
396        :param store_classifiers: if ``True``, classifiers are stored in results
397        :param store_examples: if ``True``, examples are stored in results
398        :return: :obj:`ExperimentResults`
399        """
400
401        examples, weight = demangle_examples(examples)
402        test_type = self.check_test_type(examples, learners)
403
404        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to
405        # clone it.
406        if not filter(lambda x:x[0]!="B", preprocessors):
407            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors)
408            test_set = learn_set
409        else:
410            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors)
411
412        classifiers = self._train_with_callback(learners, learn_set, weight, callback)
413
414        test_results = ExperimentResults(1,
415                                        classifier_names = [getobjectname(l) for l in learners],
416                                        test_type = test_type,
417                                        domain=examples.domain,
418                                        weights=weight)
419        test_results.results = [test_results.create_tested_example(0, example)
420                               for i, example in enumerate(examples)]
421
422        if store_classifiers:
423            test_results.classifiers = classifiers
424        if store_examples:
425            test_results.examples = test_set
426
427        results = self._test_on_data(classifiers, test_set)
428        for example, classifier, result in results:
429            test_results.results[example].set_result(classifier, *result)
430        return test_results
431
432    @deprecated_keywords({"storeExamples": "store_examples",
433                          "storeClassifiers": "store_classifiers",
434                          "pps": "preprocessors"})
435    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(),
436                                    callback=None, store_classifiers=False, store_examples=False):
437        """
438        Train learning algorithms on one data sets and test them on another.
439
440        :param learners: list of learning algorithms
441        :param learn_set: training instances
442        :param test_set: testing instances
443        :param preprocessors: a list of preprocessors (obsolete)
444        :param callback: a function that is called after each learning
445        :param store_classifiers: if ``True``, classifiers are stored in results
446        :param store_examples: if ``True``, examples are stored in results
447        :return: :obj:`ExperimentResults`
448        """
449        learn_set, learn_weight = demangle_examples(learn_set)
450        test_set, test_weight = demangle_examples(test_set)
451
452        test_type = self.check_test_type(learn_set, learners)
453        self.check_test_type(test_set, learners)
454       
455        test_results = ExperimentResults(1,
456                                        classifier_names = [getobjectname(l) for l in learners],
457                                        domain=test_set.domain,
458                                        test_type = test_type,
459                                        weights=test_weight)
460        test_results.results = [test_results.create_tested_example(0, example)
461                               for i, example in enumerate(test_set)]
462
463        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback)
464
465        if store_classifiers:
466            test_results.classifiers = classifiers
467        if store_examples:
468            test_results.examples = test_set
469
470        for example, classifier, result in results:
471            test_results.results[example].set_result(classifier, *result)
472        return test_results
473
474    @deprecated_keywords({"storeExamples": "store_examples",
475                          "storeClassifiers": "store_classifiers",
476                          "learnProp": "learning_proportion",
477                          "strat": "stratification",
478                          "pps": "preprocessors",
479                          "indicesrandseed": "random_generator",
480                          "randseed": "random_generator",
481                          "randomGenerator": "random_generator"})
482    def proportion_test(self, learners, examples, learning_proportion=.7, times=10,
483                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
484                   preprocessors=(), random_generator=0,
485                   callback=None, store_classifiers=False, store_examples=False):
486        """
487        Iteratively split the data into training and testing set, and train and test the learnign algorithms.
488
489        :param learners: list of learning algorithms
490        :param examples: data instances used for training and testing
491        :param learning_proportion: proportion of data used for training
492        :param times: number of iterations
493        :param stratification: use stratified sampling
494        :param preprocessors: a list of preprocessors (obsolete)
495        :param random_generator: random seed or generator (see above)
496        :param callback: a function that is called after each fold
497        :param store_classifiers: if ``True``, classifiers are stored in results
498        :param store_examples: if ``True``, examples are stored in results
499        :return: :obj:`ExperimentResults`
500        """
501        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator)
502
503        examples, weight = demangle_examples(examples)
504
505        test_type = self.check_test_type(examples, learners)
506       
507        test_results = ExperimentResults(times,
508                                        classifier_names = [getobjectname(l) for l in learners],
509                                        domain=examples.domain,
510                                        test_type = test_type,
511                                        weights=weight)
512        if store_examples:
513            test_results.examples = []
514        test_results.classifiers = []
515        offset=0
516        for time in xrange(times):
517            indices = pick(examples)
518            learn_set = examples.selectref(indices, 0)
519            test_set = examples.selectref(indices, 1)
520            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors)
521            if store_classifiers:
522                test_results.classifiers.append(classifiers)
523            if store_examples:
524                test_results.examples.append(learn_set)
525
526            test_results.results.extend(test_results.create_tested_example(time, example)
527                                        for i, example in enumerate(test_set))
528            for example, classifier, result in results:
529                test_results.results[offset+example].set_result(classifier, *result)
530            offset += len(test_set)
531
532            if callback:
533                callback()
534        return test_results
535
536    @deprecated_keywords({"storeExamples": "store_examples",
537                          "storeClassifiers": "store_classifiers",
538                          "learnProp": "learning_proportion",
539                          "strat": "stratification",
540                          "pps": "preprocessors",
541                          "indicesrandseed": "random_generator",
542                          "randseed": "random_generator",
543                          "randomGenerator": "random_generator"})
544    def learning_curve(self, learners, examples, cv_indices=None, proportion_indices=None, proportions=Orange.core.frange(0.1),
545                       preprocessors=(), random_generator=0, callback=None):
546        """
547        Compute a learning curve using multiple cross-validations where
548        models are trained on different portions of the training data.
549
550        :param learners: list of learning algorithms
551        :param examples: data instances used for training and testing
552        :param cv_indices: indices used for cross validation (leave ``None`` for 10-fold CV)
553        :param proportion_indices: indices for proportion selection (leave ``None`` to let the function construct the folds)
554        :param proportions: list of proportions of data used for training
555        :param preprocessors: a list of preprocessors (obsolete)
556        :param random_generator: random seed or generator (see above)
557        :param callback: a function that is be called after each learning
558        :return: list of :obj:`ExperimentResults`
559        """
560        if cv_indices is None:
561            cv_indices = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
562        if proportion_indices is None:
563            proportion_indices = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
564
565        examples, weight = demangle_examples(examples)
566        indices = cv_indices(examples)
567       
568        all_results=[]
569        for p in proportions:
570            def select_proportion_preprocessor(examples):
571                return examples.selectref(proportion_indices(examples, p0=p), 0)
572
573            test_results = self.test_with_indices(learners, examples, indices,
574                                                  preprocessors=list(preprocessors) + [("L", select_proportion_preprocessor)],
575                                                  callback=callback)
576            all_results.append(test_results)
577        return all_results
578
579
580    @deprecated_keywords({"strat": "stratification",
581                          "pps": "preprocessors",
582                          "indicesrandseed": "random_generator",
583                          "randseed": "random_generator",
584                          "randomGenerator": "random_generator"})
585    def learning_curve_n(self, learners, examples, folds=10,
586                       proportions=Orange.core.frange(0.1),
587                       stratification = Orange.core.MakeRandomIndices
588                       .StratifiedIfPossible,
589                       preprocessors=(),
590                       random_generator=0, callback=None):
591        """
592        Compute a learning curve using multiple cross-validations where
593        models are trained on different portions of the training data.
594        Similar to :obj:`learning_curve` except for simpler arguments.
595
596        :param learners: list of learning algorithms
597        :param examples: data instances used for training and testing
598        :param folds: number of folds for cross-validation
599        :param proportions: list of proportions of data used for training
600        :param stratification: use stratified sampling
601        :param preprocessors: a list of preprocessors (obsolete)
602        :param random_generator: random seed or generator (see above)
603        :param callback: a function that is be called after each learning
604        :return: list of :obj:`ExperimentResults`
605        """
606
607        cv=Orange.core.MakeRandomIndicesCV(folds = folds,
608            stratified = stratification, randomGenerator = random_generator)
609        pick=Orange.core.MakeRandomIndices2(stratified = stratification,
610            randomGenerator = random_generator)
611        return learning_curve(learners, examples, cv, pick, proportions,
612            preprocessors, callback=callback)
613   
614    def learning_curve_with_test_data(self, learners, learn_set, test_set,
615            times=10, proportions=Orange.core.frange(0.1),
616            stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
617            preprocessors=(), random_generator=0, store_classifiers=False,
618            store_examples=False):
619        """
620        Compute a learning curve given two datasets. Models are learned on
621        proportion of the first dataset and then tested on the second.
622
623        :param learners: list of learning algorithms
624        :param learn_set: training data
625        :param test_set: testing data
626        :param times: number of iterations
627        :param straitification: use stratified sampling
628        :param proportions: a list of proportions of training data to be used
629        :param preprocessors: a list of preprocessors (obsolete)
630        :param random_generator: random seed or generator (see above)
631        :param store_classifiers: if ``True``, classifiers are stored in results
632        :param store_examples: if ``True``, examples are stored in results
633        :return: list of :obj:`ExperimentResults`
634        """
635        learn_set, learn_weight = demangle_examples(learn_set)
636        test_set, test_weight = demangle_examples(test_set)
637        test_type = self.check_test_type(learn_set, learners)
638        self.check_test_type(test_set, learners)
639       
640        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator)
641       
642        all_results=[]
643        for p in proportions:
644            test_results = ExperimentResults(times,
645                                        classifier_names = [getobjectname(l) for l in learners],
646                                        domain=test_set.domain,
647                                        test_type = test_type,
648                                        weights=test_weight)
649            offset = 0
650            for t in xrange(times):
651                test_results.results.extend(test_results.create_tested_example(t, example)
652                                            for i, example in enumerate(test_set))
653
654                learn_examples = learn_set.selectref(indices(learn_set, p), 0)
655                classifiers, results = self._learn_and_test_on_test_data\
656                    (learners, learn_examples, learn_weight, test_set,
657                    preprocessors=preprocessors)
658
659                for example, classifier, result in results:
660                    test_results.results[offset+example].set_result(classifier, *result)
661                offset += len(test_set)
662
663                if store_classifiers:
664                    test_results.classifiers.append(classifiers)
665                if store_examples:
666                    test_results.examples = learn_examples
667
668            all_results.append(test_results)
669        return all_results
670
671
672    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False):
673        """
674        Test classifiers on the given data
675
676        :param classifiers: a list of classifiers
677        :param examples: testing data
678        :param store_classifiers: if ``True``, classifiers are stored in results
679        :param store_examples: if ``True``, examples are stored in results
680        """
681
682        examples, weight = demangle_examples(examples)
683        test_type = self.check_test_type(examples, classifiers)
684
685        test_results = ExperimentResults(1,
686                                        classifier_names = [getobjectname(l) for l in classifiers],
687                                        domain=examples.domain,
688                                        test_type = test_type,
689                                        weights=weight)
690        test_results.results = [test_results.create_tested_example(0, example)
691                               for i, example in enumerate(examples)]
692
693        if store_examples:
694            test_results.examples = examples
695        if store_classifiers:
696            test_results.classifiers = classifiers
697
698        results = self._test_on_data(classifiers, examples)
699        for example, classifier, result in results:
700            test_results.results[example].set_result(classifier, *result)
701        return test_results
702
703
704    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set,
705                                     preprocessors=(), callback=None):
706        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
707
708        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback)
709       
710        results = self._test_on_data(classifiers, test_set)
711        return classifiers, results
712
713
714    def _train_with_callback(self, learners, examples, weight, callback):
715        classifiers = []
716        for learner in learners:
717            classifier = learner(examples, weight)
718            classifier.name = getattr(learner, 'name', 'noname')
719            classifiers.append(classifier)
720            if callback:
721                callback()
722        return classifiers
723
724
725    def _test_on_data(self, classifiers, examples, example_ids=None):
726        results = []
727
728        if example_ids is None:
729            numbered_examples = enumerate(examples)
730        else:
731            numbered_examples = itertools.izip(example_ids, examples)
732
733        for e, example in numbered_examples:
734            for c, classifier in enumerate(classifiers):
735                # Hide actual class to prevent cheating
736                ex2 = Orange.data.Instance(example)
737                if ex2.domain.class_var: ex2.setclass("?")
738                if ex2.domain.class_vars: ex2.set_classes(["?" for _ in ex2
739                .domain.class_vars])
740                result = classifier(ex2, Orange.core.GetBoth)
741                results.append((e, c, result))
742        return results
743
744   
745    def _preprocess_data(self, learn_set, test_set, preprocessors):
746        """Apply preprocessors to learn and test dataset (obsolete)"""
747        for p_type, preprocessor in preprocessors:
748            if p_type == "B":
749                learn_set = preprocessor(learn_set)
750                test_set = preprocessor(test_set)
751        for p_type, preprocessor in preprocessors:
752            if p_type == "L":
753                learn_set = preprocessor(learn_set)
754            elif p_type == "T":
755                test_set = preprocessor(test_set)
756            elif p_type == "LT":
757                (learn_set, test_set) = preprocessor(learn_set, test_set)
758        return learn_set, test_set
759
760
761    def encode_PP(self, pps):
762        for pp in pps:
763            objname = getobjectname(pp[1])
764            if len(objname):
765                pps+="_"+objname
766            else:
767                return "*"
768        return pps
769
770    def check_test_type(self, instances, models):
771        model_is_mlc = [isinstance(m, Orange.multilabel.MultiLabelLearner) or
772                        isinstance(m, Orange.multilabel.MultiLabelClassifier)
773                          for m in models]
774        multi_label = any(model_is_mlc)
775        if multi_label and not all(model_is_mlc):
776            raise ValueError("Test on mixed types of learners (MLC and non-MLC) not possible")
777        multi_target = instances.domain.class_vars and not multi_label
778
779        if (multi_label or multi_target) and not instances.domain.class_vars:
780            raise ValueError("Test data with multiple labels (class vars) expected")
781        if not (multi_label or multi_target or instances.domain.class_var):
782            raise ValueError("Test data set without class attributes")
783
784        return TEST_TYPE_MLC if multi_label else (
785            TEST_TYPE_MULTITARGET if multi_target else TEST_TYPE_SINGLE)
786   
787default_evaluation = _default_evaluation = Evaluation()
788
789cross_validation = _default_evaluation.cross_validation
790leave_one_out = _default_evaluation.leave_one_out
791test_with_indices = _default_evaluation.test_with_indices
792one_fold_with_indices = _default_evaluation.one_fold_with_indices
793
794learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data
795learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data
796test_on_data = _default_evaluation.test_on_data
797
798learning_curve = _default_evaluation.learning_curve
799learning_curve_n = _default_evaluation.learning_curve_n
800learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data
801
802proportion_test = _default_evaluation.proportion_test
803
804encode_PP = _default_evaluation.encode_PP
Note: See TracBrowser for help on using the repository browser.