source: orange/Orange/evaluation/testing.py @ 10581:94a831b04ec3

Revision 10581:94a831b04ec3, 36.3 KB checked in by markotoplak, 2 years ago (diff)

Moved some other scripts from misc to utils and Orange imports and canvas not works, although not systematically tested.

Line 
1import itertools
2
3import Orange
4from Orange.utils import demangle_examples, getobjectname
5from Orange.utils import deprecated_keywords, deprecated_members
6
7#### Data structures
8
9TEST_TYPE_SINGLE = 0
10TEST_TYPE_MLC = 1
11TEST_TYPE_MULTITARGET = 2
12
13class TestedExample:
14    """
15    TestedExample stores predictions of different classifiers for a
16    single testing data instance.
17
18    .. attribute:: classes
19
20        A list of predictions of type Value, one for each classifier.
21
22    .. attribute:: probabilities
23
24        A list of probabilities of classes, one for each classifier.
25
26    .. attribute:: iteration_number
27
28        Iteration number (e.g. fold) in which the TestedExample was
29        created/tested.
30
31    .. attribute actual_class
32
33        The correct class of the example
34
35    .. attribute weight
36
37        Instance's weight; 1.0 if data was not weighted
38    """
39
40    @deprecated_keywords({"iterationNumber": "iteration_number",
41                          "actualClass": "actual_class"})
42    def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0):
43        """
44        :param iteration_number: The iteration number of TestedExample.
45        :param actual_class: The actual class of TestedExample.
46        :param n: The number of learners.
47        :param weight: The weight of the TestedExample.
48        """
49        self.classes = [None]*n
50        self.probabilities = [None]*n
51        self.iteration_number = iteration_number
52        self.actual_class= actual_class
53        self.weight = weight
54
55    def add_result(self, aclass, aprob):
56        """Append a new result (class and probability prediction by a single classifier) to the classes and probabilities field."""
57   
58        if isinstance(aclass, (list, tuple)):
59            self.classes.append(aclass)
60            self.probabilities.append(aprob)
61        elif type(aclass.value)==float:
62            self.classes.append(float(aclass))
63            self.probabilities.append(aprob)
64        else:
65            self.classes.append(int(aclass))
66            self.probabilities.append(list(aprob))
67
68    def set_result(self, i, aclass, aprob):
69        """Set the result of the i-th classifier to the given values."""
70        if isinstance(aclass, (list, tuple)):
71            self.classes[i] = aclass
72            self.probabilities[i] = aprob
73        elif type(aclass.value)==float:
74            self.classes[i] = float(aclass)
75            self.probabilities[i] = aprob
76        else:
77            self.classes[i] = int(aclass)
78            self.probabilities[i] = list(aprob)
79
80    def __repr__(self):
81        return str(self.__dict__)
82
83TestedExample = deprecated_members({"iterationNumber": "iteration_number",
84                                    "actualClass": "actual_class"
85                                    })(TestedExample)
86
87class ExperimentResults(object):
88    """
89    ``ExperimentResults`` stores results of one or more repetitions of
90    some test (cross validation, repeated sampling...) under the same
91    circumstances. Instances of this class are constructed by sampling
92    and testing functions from module :obj:`Orange.evaluation.testing`
93    and used by methods in module :obj:`Orange.evaluation.scoring`.
94
95    .. attribute:: results
96
97        A list of instances of :obj:`TestedExample`, one for each
98        example in the dataset.
99
100    .. attribute:: number_of_iterations
101
102        Number of iterations. This can be the number of folds (in
103        cross validation) or the number of repetitions of some
104        test. :obj:`TestedExample`'s attribute ``iteration_number``
105        should be in range ``[0, number_of_iterations-1]``.
106
107    .. attribute:: number_of_learners
108
109        Number of learners. Lengths of lists classes and probabilities
110        in each :obj:`TestedExample` should equal
111        ``number_of_learners``.
112
113    .. attribute:: classifier_names
114
115        Stores the names of the classifiers.
116
117    .. attribute:: classifiers
118
119        A list of classifiers, one element for each iteration of
120        sampling and learning (eg. fold). Each element is a list of
121        classifiers, one for each learner. For instance,
122        ``classifiers[2][4]`` refers to the 3rd repetition, 5th
123        learning algorithm.
124
125        Note that functions from :obj:`~Orange.evaluation.testing`
126        only store classifiers it enabled by setting
127        ``storeClassifiers`` to ``1``.
128
129    ..
130        .. attribute:: loaded
131
132            If the experimental method supports caching and there are no
133            obstacles for caching (such as unknown random seeds), this is a
134            list of boolean values. Each element corresponds to a classifier
135            and tells whether the experimental results for that classifier
136            were computed or loaded from the cache.
137
138    .. attribute:: base_class
139
140       The reference class for measures like AUC.
141
142    .. attribute:: class_values
143
144        The list of class values.
145
146    .. attribute:: weights
147
148        A flag telling whether the results are weighted. If ``False``,
149        weights are still present in :obj:`TestedExample`, but they
150        are all ``1.0``. Clear this flag, if your experimental
151        procedure ran on weighted testing examples but you would like
152        to ignore the weights in statistics.
153    """
154    @deprecated_keywords({"classifierNames": "classifier_names",
155                          "classValues": "class_values",
156                          "baseClass": "base_class",
157                          "numberOfIterations": "number_of_iterations",
158                          "numberOfLearners": "number_of_learners"})
159    def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=-1, domain=None, test_type=TEST_TYPE_SINGLE, labels=None, **argkw):
160        self.class_values = class_values
161        self.classifier_names = classifier_names
162        self.number_of_iterations = iterations
163        self.number_of_learners = len(classifier_names)
164        self.results = []
165        self.classifiers = []
166        self.loaded = None
167        self.base_class = base_class
168        self.weights = weights
169        self.test_type = test_type
170        self.labels = labels
171
172        if domain is not None:
173            self.base_class = self.class_values = None
174            if test_type == TEST_TYPE_SINGLE:
175                if domain.class_var.var_type == Orange.feature.Type.Discrete:
176                    self.class_values = list(domain.class_var.values)
177                    self.base_class = domain.class_var.base_value
178                    self.converter = int
179                else:
180                    self.converter = float
181            elif test_type in (TEST_TYPE_MLC, TEST_TYPE_MULTITARGET):
182                self.labels = [var.name for var in domain.class_vars]
183                self.converter = lambda vals: [int(val) if val.variable.var_type == Orange.feature.Type.Discrete
184                                               else float(val) for val in vals]
185
186        self.__dict__.update(argkw)
187
188    def load_from_files(self, learners, filename):
189        raise NotImplementedError("This feature is no longer supported.")
190
191    def save_to_files(self, learners, filename):
192        raise NotImplementedError("This feature is no longer supported. Pickle whole class instead.")
193
194    def create_tested_example(self, fold, example):
195        actual = example.getclass() if self.test_type == TEST_TYPE_SINGLE \
196                                  else example.get_classes()
197        return TestedExample(fold,
198                             self.converter(actual),
199                             self.number_of_learners,
200                             example.getweight(self.weights))
201
202    def remove(self, index):
203        """remove one learner from evaluation results"""
204        for r in self.results:
205            del r.classes[index]
206            del r.probabilities[index]
207        del self.classifier_names[index]
208        self.number_of_learners -= 1
209
210    def add(self, results, index, replace=-1):
211        """add evaluation results (for one learner)"""
212        if len(self.results)!=len(results.results):
213            raise SystemError("mismatch in number of test cases")
214        if self.number_of_iterations!=results.number_of_iterations:
215            raise SystemError("mismatch in number of iterations (%d<>%d)" % \
216                  (self.number_of_iterations, results.number_of_iterations))
217        if len(self.classifiers) and len(results.classifiers)==0:
218            raise SystemError("no classifiers in results")
219
220        if replace < 0 or replace >= self.number_of_learners: # results for new learner
221            self.classifier_names.append(results.classifier_names[index])
222            self.number_of_learners += 1
223            for i,r in enumerate(self.results):
224                r.classes.append(results.results[i].classes[index])
225                r.probabilities.append(results.results[i].probabilities[index])
226            if len(self.classifiers):
227                for i in range(self.number_of_iterations):
228                    self.classifiers[i].append(results.classifiers[i][index])
229        else: # replace results of existing learner
230            self.classifier_names[replace] = results.classifier_names[index]
231            for i,r in enumerate(self.results):
232                r.classes[replace] = results.results[i].classes[index]
233                r.probabilities[replace] = results.results[i].probabilities[index]
234            if len(self.classifiers):
235                for i in range(self.number_of_iterations):
236                    self.classifiers[replace] = results.classifiers[i][index]
237
238    def __repr__(self):
239        return str(self.__dict__)
240
241ExperimentResults = deprecated_members({"classValues": "class_values",
242                                        "classifierNames": "classifier_names",
243                                        "baseClass": "base_class",
244                                        "numberOfIterations": "number_of_iterations",
245                                        "numberOfLearners": "number_of_learners"
246})(ExperimentResults)
247
248   
249class Evaluation(object):
250    """Common methods for learner evaluation."""
251
252    @deprecated_keywords({"pps": "preprocessors",
253                          "strat": "stratified",
254                          "randseed": "random_generator",
255                          "indicesrandseed": "random_generator",
256                          "randomGenerator": "random_generator",
257                          "storeClassifiers": "store_classifiers",
258                          "storeExamples": "store_examples"})
259    def cross_validation(self, learners, examples, folds=10,
260            stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible,
261            preprocessors=(), random_generator=0, callback=None,
262            store_classifiers=False, store_examples=False):
263        """Cross validation test with specified number of folds.
264
265        :param learners: list of learning algorithms
266        :param examples: data instances used for training and testing
267        :param folds: number of folds
268        :param stratified: tells whether to stratify the sampling
269        :param preprocessors: a list of preprocessors to be used on data (obsolete)
270        :param random_generator: random seed or generator (see above)
271        :param callback: a function that is called after finishing each fold
272        :param store_classifiers: if ``True``, classifiers are stored in results
273        :param store_examples: if ``True``, examples are stored in results
274        :return: :obj:`ExperimentResults`
275        """
276        (examples, weight) = demangle_examples(examples)
277
278        indices = Orange.core.MakeRandomIndicesCV(examples, folds,
279            stratified=stratified, random_generator=random_generator)
280
281        return self.test_with_indices(
282            learners=learners,
283            examples=(examples, weight),
284            indices=indices,
285            preprocessors=preprocessors,
286            callback=callback,
287            store_classifiers=store_classifiers,
288            store_examples=store_examples)
289
290
291    @deprecated_keywords({"pps": "preprocessors",
292                          "storeClassifiers": "store_classifiers",
293                          "storeExamples": "store_examples"})
294    def leave_one_out(self, learners, examples, preprocessors=(),
295            callback=None, store_classifiers=False, store_examples=False):
296        """Leave-one-out evaluation of learning algorithms.
297
298        :param learners: list of learning algorithms
299        :param examples: data instances used for training and testing
300        :param preprocessors: a list of preprocessors (obsolete)
301        :param callback: a function that is called after finishing each fold
302        :param store_classifiers: if ``True``, classifiers are stored in results
303        :param store_examples: if ``True``, examples are stored in results
304        :return: :obj:`ExperimentResults`
305        """
306        examples, weight = demangle_examples(examples)
307        return self.test_with_indices(
308            learners, (examples, weight), indices=range(len(examples)),
309            preprocessors=preprocessors, callback=callback,
310            store_classifiers=store_classifiers, store_examples=store_examples)
311
312    @deprecated_keywords({"storeExamples": "store_examples",
313                          "storeClassifiers": "store_classifiers=True",
314                          "pps":"preprocessors"})
315    def test_with_indices(self, learners, examples, indices, preprocessors=(),
316            callback=None, store_classifiers=False, store_examples=False):
317        """
318        Perform a cross-validation-like test. Examples for each fold are
319        selected based on given indices.
320
321        :param learners: list of learning algorithms
322        :param examples: data instances used for training and testing
323        :param indices: a list of integer indices that sort examples into folds; each index corresponds to an example from ``examples``
324        :param preprocessors: a list of preprocessors (obsolete)
325        :param callback: a function that is called after each fold
326        :param store_classifiers: if ``True``, classifiers are stored in results
327        :param store_examples: if ``True``, examples are stored in results
328        :return: :obj:`ExperimentResults`
329        """
330        examples, weight = demangle_examples(examples)
331        if not examples:
332            raise ValueError("Test data set with no examples")
333        test_type = self.check_test_type(examples, learners)
334
335        niterations = max(indices)+1
336        test_result = ExperimentResults(niterations,
337                                        classifier_names = [getobjectname(l) for l in learners],
338                                        domain=examples.domain,
339                                        weights=weight,
340                                        test_type=test_type)
341
342        test_result.results = [test_result.create_tested_example(indices[i], example)
343                               for i, example in enumerate(examples)]
344
345        if store_examples:
346            test_result.examples = examples
347
348        for fold in xrange(niterations):
349            results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight)
350
351            for example, learner, result in results:
352                test_result.results[example].set_result(learner, *result)
353
354            if store_classifiers:
355                test_result.classifiers.append(classifiers)
356            if callback:
357                callback()
358
359        return test_result
360
361
362    def one_fold_with_indices(self, learners, examples, fold, indices, preprocessors=(), weight=0):
363        """Similar to :obj:`test_with_indices` except that it performs single fold of cross-validation, given by argument ``fold``."""
364        learn_set = examples.selectref(indices, fold, negate=1)
365        test_set = examples.selectref(indices, fold, negate=0)
366        if len(learn_set)==0 or len(test_set)==0:
367            return (), ()
368
369        # learning
370        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
371        if not learn_set:
372            raise SystemError("no training examples after preprocessing")
373        if not test_set:
374            raise SystemError("no test examples after preprocessing")
375
376        classifiers = [learner(learn_set, weight) for learner in learners]
377
378        # testing
379        testset_ids = (i for i, _ in enumerate(examples) if indices[i] == fold)
380        results = self._test_on_data(classifiers, test_set, testset_ids)
381
382        return results, classifiers
383
384
385    @deprecated_keywords({"storeExamples": "store_examples",
386                          "storeClassifiers": "store_classifiers",
387                          "pps": "preprocessors"})
388    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(),
389                                     callback=None, store_classifiers=False, store_examples=False):
390        """
391        Train learning algorithms and test them on the same data.
392
393        :param learners: list of learning algorithms
394        :param examples: data instances used for training and testing
395        :param preprocessors: a list of preprocessors (obsolete)
396        :param callback: a function that is called after each learning
397        :param store_classifiers: if ``True``, classifiers are stored in results
398        :param store_examples: if ``True``, examples are stored in results
399        :return: :obj:`ExperimentResults`
400        """
401
402        examples, weight = demangle_examples(examples)
403        test_type = self.check_test_type(examples, learners)
404
405        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to
406        # clone it.
407        if not filter(lambda x:x[0]!="B", preprocessors):
408            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors)
409            test_set = learn_set
410        else:
411            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors)
412
413        classifiers = self._train_with_callback(learners, learn_set, weight, callback)
414
415        test_results = ExperimentResults(1,
416                                        classifier_names = [getobjectname(l) for l in learners],
417                                        test_type = test_type,
418                                        domain=examples.domain,
419                                        weights=weight)
420        test_results.results = [test_results.create_tested_example(0, example)
421                               for i, example in enumerate(examples)]
422
423        if store_classifiers:
424            test_results.classifiers = classifiers
425        if store_examples:
426            test_results.examples = test_set
427
428        results = self._test_on_data(classifiers, test_set)
429        for example, classifier, result in results:
430            test_results.results[example].set_result(classifier, *result)
431        return test_results
432
433    @deprecated_keywords({"storeExamples": "store_examples",
434                          "storeClassifiers": "store_classifiers",
435                          "pps": "preprocessors"})
436    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(),
437                                    callback=None, store_classifiers=False, store_examples=False):
438        """
439        Train learning algorithms on one data sets and test them on another.
440
441        :param learners: list of learning algorithms
442        :param learn_set: training instances
443        :param test_set: testing instances
444        :param preprocessors: a list of preprocessors (obsolete)
445        :param callback: a function that is called after each learning
446        :param store_classifiers: if ``True``, classifiers are stored in results
447        :param store_examples: if ``True``, examples are stored in results
448        :return: :obj:`ExperimentResults`
449        """
450        learn_set, learn_weight = demangle_examples(learn_set)
451        test_set, test_weight = demangle_examples(test_set)
452
453        test_type = self.check_test_type(learn_set, learners)
454        self.check_test_type(test_set, learners)
455       
456        test_results = ExperimentResults(1,
457                                        classifier_names = [getobjectname(l) for l in learners],
458                                        domain=test_set.domain,
459                                        test_type = test_type,
460                                        weights=test_weight)
461        test_results.results = [test_results.create_tested_example(0, example)
462                               for i, example in enumerate(test_set)]
463
464        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback)
465
466        if store_classifiers:
467            test_results.classifiers = classifiers
468        if store_examples:
469            test_results.examples = test_set
470
471        for example, classifier, result in results:
472            test_results.results[example].set_result(classifier, *result)
473        return test_results
474
475    @deprecated_keywords({"storeExamples": "store_examples",
476                          "storeClassifiers": "store_classifiers",
477                          "learnProp": "learning_proportion",
478                          "strat": "stratification",
479                          "pps": "preprocessors",
480                          "indicesrandseed": "random_generator",
481                          "randseed": "random_generator",
482                          "randomGenerator": "random_generator"})
483    def proportion_test(self, learners, examples, learning_proportion=.7, times=10,
484                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
485                   preprocessors=(), random_generator=0,
486                   callback=None, store_classifiers=False, store_examples=False):
487        """
488        Iteratively split the data into training and testing set, and train and test the learnign algorithms.
489
490        :param learners: list of learning algorithms
491        :param examples: data instances used for training and testing
492        :param learning_proportion: proportion of data used for training
493        :param times: number of iterations
494        :param stratification: use stratified sampling
495        :param preprocessors: a list of preprocessors (obsolete)
496        :param random_generator: random seed or generator (see above)
497        :param callback: a function that is called after each fold
498        :param store_classifiers: if ``True``, classifiers are stored in results
499        :param store_examples: if ``True``, examples are stored in results
500        :return: :obj:`ExperimentResults`
501        """
502        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator)
503
504        examples, weight = demangle_examples(examples)
505
506        test_type = self.check_test_type(examples, learners)
507       
508        test_results = ExperimentResults(times,
509                                        classifier_names = [getobjectname(l) for l in learners],
510                                        domain=examples.domain,
511                                        test_type = test_type,
512                                        weights=weight)
513        if store_examples:
514            test_results.examples = []
515        test_results.classifiers = []
516        offset=0
517        for time in xrange(times):
518            indices = pick(examples)
519            learn_set = examples.selectref(indices, 0)
520            test_set = examples.selectref(indices, 1)
521            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors)
522            if store_classifiers:
523                test_results.classifiers.append(classifiers)
524            if store_examples:
525                test_results.examples.append(learn_set)
526
527            test_results.results.extend(test_results.create_tested_example(time, example)
528                                        for i, example in enumerate(test_set))
529            for example, classifier, result in results:
530                test_results.results[offset+example].set_result(classifier, *result)
531            offset += len(test_set)
532
533            if callback:
534                callback()
535        return test_results
536
537    @deprecated_keywords({"storeExamples": "store_examples",
538                          "storeClassifiers": "store_classifiers",
539                          "learnProp": "learning_proportion",
540                          "strat": "stratification",
541                          "pps": "preprocessors",
542                          "indicesrandseed": "random_generator",
543                          "randseed": "random_generator",
544                          "randomGenerator": "random_generator"})
545    def learning_curve(self, learners, examples, cv_indices=None, proportion_indices=None, proportions=Orange.core.frange(0.1),
546                       preprocessors=(), random_generator=0, callback=None):
547        """
548        Compute a learning curve using multiple cross-validations where
549        models are trained on different portions of the training data.
550
551        :param learners: list of learning algorithms
552        :param examples: data instances used for training and testing
553        :param cv_indices: indices used for cross validation (leave ``None`` for 10-fold CV)
554        :param proportion_indices: indices for proportion selection (leave ``None`` to let the function construct the folds)
555        :param proportions: list of proportions of data used for training
556        :param preprocessors: a list of preprocessors (obsolete)
557        :param random_generator: random seed or generator (see above)
558        :param callback: a function that is be called after each learning
559        :return: list of :obj:`ExperimentResults`
560        """
561        if cv_indices is None:
562            cv_indices = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
563        if proportion_indices is None:
564            proportion_indices = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
565
566        examples, weight = demangle_examples(examples)
567        indices = cv_indices(examples)
568       
569        all_results=[]
570        for p in proportions:
571            def select_proportion_preprocessor(examples):
572                return examples.selectref(proportion_indices(examples, p0=p), 0)
573
574            test_results = self.test_with_indices(learners, examples, indices,
575                                                  preprocessors=list(preprocessors) + [("L", select_proportion_preprocessor)],
576                                                  callback=callback)
577            all_results.append(test_results)
578        return all_results
579
580
581    @deprecated_keywords({"strat": "stratification",
582                          "pps": "preprocessors",
583                          "indicesrandseed": "random_generator",
584                          "randseed": "random_generator",
585                          "randomGenerator": "random_generator"})
586    def learning_curve_n(self, learners, examples, folds=10,
587                       proportions=Orange.core.frange(0.1),
588                       stratification = Orange.core.MakeRandomIndices
589                       .StratifiedIfPossible,
590                       preprocessors=(),
591                       random_generator=0, callback=None):
592        """
593        Compute a learning curve using multiple cross-validations where
594        models are trained on different portions of the training data.
595        Similar to :obj:`learning_curve` except for simpler arguments.
596
597        :param learners: list of learning algorithms
598        :param examples: data instances used for training and testing
599        :param folds: number of folds for cross-validation
600        :param proportions: list of proportions of data used for training
601        :param stratification: use stratified sampling
602        :param preprocessors: a list of preprocessors (obsolete)
603        :param random_generator: random seed or generator (see above)
604        :param callback: a function that is be called after each learning
605        :return: list of :obj:`ExperimentResults`
606        """
607
608        cv=Orange.core.MakeRandomIndicesCV(folds = folds,
609            stratified = stratification, randomGenerator = random_generator)
610        pick=Orange.core.MakeRandomIndices2(stratified = stratification,
611            randomGenerator = random_generator)
612        return learning_curve(learners, examples, cv, pick, proportions,
613            preprocessors, callback=callback)
614   
615    def learning_curve_with_test_data(self, learners, learn_set, test_set,
616            times=10, proportions=Orange.core.frange(0.1),
617            stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
618            preprocessors=(), random_generator=0, store_classifiers=False,
619            store_examples=False):
620        """
621        Compute a learning curve given two datasets. Models are learned on
622        proportion of the first dataset and then tested on the second.
623
624        :param learners: list of learning algorithms
625        :param learn_set: training data
626        :param test_set: testing data
627        :param times: number of iterations
628        :param straitification: use stratified sampling
629        :param proportions: a list of proportions of training data to be used
630        :param preprocessors: a list of preprocessors (obsolete)
631        :param random_generator: random seed or generator (see above)
632        :param store_classifiers: if ``True``, classifiers are stored in results
633        :param store_examples: if ``True``, examples are stored in results
634        :return: list of :obj:`ExperimentResults`
635        """
636        learn_set, learn_weight = demangle_examples(learn_set)
637        test_set, test_weight = demangle_examples(test_set)
638        test_type = self.check_test_type(learn_set, learners)
639        self.check_test_type(test_set, learners)
640       
641        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator)
642       
643        all_results=[]
644        for p in proportions:
645            test_results = ExperimentResults(times,
646                                        classifier_names = [getobjectname(l) for l in learners],
647                                        domain=test_set.domain,
648                                        test_type = test_type,
649                                        weights=test_weight)
650            offset = 0
651            for t in xrange(times):
652                test_results.results.extend(test_results.create_tested_example(t, example)
653                                            for i, example in enumerate(test_set))
654
655                learn_examples = learn_set.selectref(indices(learn_set, p), 0)
656                classifiers, results = self._learn_and_test_on_test_data\
657                    (learners, learn_examples, learn_weight, test_set,
658                    preprocessors=preprocessors)
659
660                for example, classifier, result in results:
661                    test_results.results[offset+example].set_result(classifier, *result)
662                offset += len(test_set)
663
664                if store_classifiers:
665                    test_results.classifiers.append(classifiers)
666                if store_examples:
667                    test_results.examples = learn_examples
668
669            all_results.append(test_results)
670        return all_results
671
672
673    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False):
674        """
675        Test classifiers on the given data
676
677        :param classifiers: a list of classifiers
678        :param examples: testing data
679        :param store_classifiers: if ``True``, classifiers are stored in results
680        :param store_examples: if ``True``, examples are stored in results
681        """
682
683        examples, weight = demangle_examples(examples)
684        test_type = self.check_test_type(examples, classifiers)
685
686        test_results = ExperimentResults(1,
687                                        classifier_names = [getobjectname(l) for l in classifiers],
688                                        domain=examples.domain,
689                                        test_type = test_type,
690                                        weights=weight)
691        test_results.results = [test_results.create_tested_example(0, example)
692                               for i, example in enumerate(examples)]
693
694        if store_examples:
695            test_results.examples = examples
696        if store_classifiers:
697            test_results.classifiers = classifiers
698
699        results = self._test_on_data(classifiers, examples)
700        for example, classifier, result in results:
701            test_results.results[example].set_result(classifier, *result)
702        return test_results
703
704
705    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set,
706                                     preprocessors=(), callback=None):
707        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
708
709        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback)
710       
711        results = self._test_on_data(classifiers, test_set)
712        return classifiers, results
713
714
715    def _train_with_callback(self, learners, examples, weight, callback):
716        classifiers = []
717        for learner in learners:
718            classifier = learner(examples, weight)
719            classifier.name = getattr(learner, 'name', 'noname')
720            classifiers.append(classifier)
721            if callback:
722                callback()
723        return classifiers
724
725
726    def _test_on_data(self, classifiers, examples, example_ids=None):
727        results = []
728
729        if example_ids is None:
730            numbered_examples = enumerate(examples)
731        else:
732            numbered_examples = itertools.izip(example_ids, examples)
733
734        for e, example in numbered_examples:
735            for c, classifier in enumerate(classifiers):
736                # Hide actual class to prevent cheating
737                ex2 = Orange.data.Instance(example)
738                if ex2.domain.class_var: ex2.setclass("?")
739                if ex2.domain.class_vars: ex2.set_classes(["?" for _ in ex2
740                .domain.class_vars])
741                result = classifier(ex2, Orange.core.GetBoth)
742                results.append((e, c, result))
743        return results
744
745   
746    def _preprocess_data(self, learn_set, test_set, preprocessors):
747        """Apply preprocessors to learn and test dataset (obsolete)"""
748        for p_type, preprocessor in preprocessors:
749            if p_type == "B":
750                learn_set = preprocessor(learn_set)
751                test_set = preprocessor(test_set)
752        for p_type, preprocessor in preprocessors:
753            if p_type == "L":
754                learn_set = preprocessor(learn_set)
755            elif p_type == "T":
756                test_set = preprocessor(test_set)
757            elif p_type == "LT":
758                (learn_set, test_set) = preprocessor(learn_set, test_set)
759        return learn_set, test_set
760
761
762    def encode_PP(self, pps):
763        for pp in pps:
764            objname = getobjectname(pp[1])
765            if len(objname):
766                pps+="_"+objname
767            else:
768                return "*"
769        return pps
770
771    def check_test_type(self, instances, models):
772        model_is_mlc = [isinstance(m, Orange.multilabel.MultiLabelLearner) or
773                        isinstance(m, Orange.multilabel.MultiLabelClassifier)
774                          for m in models]
775        multi_label = any(model_is_mlc)
776        if multi_label and not all(model_is_mlc):
777            raise ValueError("Test on mixed types of learners (MLC and non-MLC) not possible")
778        multi_target = instances.domain.class_vars and not multi_label
779
780        if (multi_label or multi_target) and not instances.domain.class_vars:
781            raise ValueError("Test data with multiple labels (class vars) expected")
782        if not (multi_label or multi_target or instances.domain.class_var):
783            raise ValueError("Test data set without class attributes")
784
785        return TEST_TYPE_MLC if multi_label else (
786            TEST_TYPE_MULTITARGET if multi_target else TEST_TYPE_SINGLE)
787   
788default_evaluation = _default_evaluation = Evaluation()
789
790cross_validation = _default_evaluation.cross_validation
791leave_one_out = _default_evaluation.leave_one_out
792test_with_indices = _default_evaluation.test_with_indices
793one_fold_with_indices = _default_evaluation.one_fold_with_indices
794
795learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data
796learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data
797test_on_data = _default_evaluation.test_on_data
798
799learning_curve = _default_evaluation.learning_curve
800learning_curve_n = _default_evaluation.learning_curve_n
801learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data
802
803proportion_test = _default_evaluation.proportion_test
804
805encode_PP = _default_evaluation.encode_PP
Note: See TracBrowser for help on using the repository browser.