source: orange/Orange/evaluation/testing.py @ 11039:4fb098cb546d

Revision 11039:4fb098cb546d, 36.5 KB checked in by Lan Zagar <lan.zagar@…>, 17 months ago (diff)

Corrected order of preprocessors.

Line 
1import itertools
2
3import Orange
4from Orange.utils import demangle_examples, getobjectname
5from Orange.utils import deprecated_keywords, deprecated_members
6
7#### Data structures
8
9TEST_TYPE_SINGLE = 0
10TEST_TYPE_MLC = 1
11TEST_TYPE_MULTITARGET = 2
12
13class TestedExample:
14    """
15    TestedExample stores predictions of different classifiers for a
16    single testing data instance.
17
18    .. attribute:: classes
19
20        A list of predictions of type Value, one for each classifier.
21
22    .. attribute:: probabilities
23
24        A list of probabilities of classes, one for each classifier.
25
26    .. attribute:: iteration_number
27
28        Iteration number (e.g. fold) in which the TestedExample was
29        created/tested.
30
31    .. attribute actual_class
32
33        The correct class of the example
34
35    .. attribute weight
36
37        Instance's weight; 1.0 if data was not weighted
38    """
39
40    @deprecated_keywords({"iterationNumber": "iteration_number",
41                          "actualClass": "actual_class"})
42    def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0):
43        """
44        :param iteration_number: The iteration number of TestedExample.
45        :param actual_class: The actual class of TestedExample.
46        :param n: The number of learners.
47        :param weight: The weight of the TestedExample.
48        """
49        self.classes = [None]*n
50        self.probabilities = [None]*n
51        self.iteration_number = iteration_number
52        self.actual_class= actual_class
53        self.weight = weight
54
55    def add_result(self, aclass, aprob):
56        """Append a new result (class and probability prediction by a single classifier) to the classes and probabilities field."""
57   
58        if isinstance(aclass, (list, tuple)):
59            self.classes.append(aclass)
60            self.probabilities.append(aprob)
61        elif type(aclass.value)==float:
62            self.classes.append(float(aclass))
63            self.probabilities.append(aprob)
64        else:
65            self.classes.append(int(aclass))
66            self.probabilities.append(aprob)
67
68    def set_result(self, i, aclass, aprob):
69        """Set the result of the i-th classifier to the given values."""
70        if isinstance(aclass, (list, tuple)):
71            self.classes[i] = aclass
72            self.probabilities[i] = aprob
73        elif type(aclass.value)==float:
74            self.classes[i] = float(aclass)
75            self.probabilities[i] = aprob
76        else:
77            self.classes[i] = int(aclass)
78            self.probabilities[i] = aprob
79
80    def __repr__(self):
81        return str(self.__dict__)
82
83TestedExample = deprecated_members({"iterationNumber": "iteration_number",
84                                    "actualClass": "actual_class"
85                                    })(TestedExample)
86
87def mt_vals(vals):
88    """
89    Substitution for the unpicklable lambda function for multi-target classifiers.
90    """
91    return [val if val.is_DK() else int(val) if val.variable.var_type == Orange.feature.Type.Discrete
92                                            else float(val) for val in vals]
93
94class ExperimentResults(object):
95    """
96    ``ExperimentResults`` stores results of one or more repetitions of
97    some test (cross validation, repeated sampling...) under the same
98    circumstances. Instances of this class are constructed by sampling
99    and testing functions from module :obj:`Orange.evaluation.testing`
100    and used by methods in module :obj:`Orange.evaluation.scoring`.
101
102    .. attribute:: results
103
104        A list of instances of :obj:`TestedExample`, one for each
105        example in the dataset.
106
107    .. attribute:: number_of_iterations
108
109        Number of iterations. This can be the number of folds (in
110        cross validation) or the number of repetitions of some
111        test. :obj:`TestedExample`'s attribute ``iteration_number``
112        should be in range ``[0, number_of_iterations-1]``.
113
114    .. attribute:: number_of_learners
115
116        Number of learners. Lengths of lists classes and probabilities
117        in each :obj:`TestedExample` should equal
118        ``number_of_learners``.
119
120    .. attribute:: classifier_names
121
122        Stores the names of the classifiers.
123
124    .. attribute:: classifiers
125
126        A list of classifiers, one element for each iteration of
127        sampling and learning (eg. fold). Each element is a list of
128        classifiers, one for each learner. For instance,
129        ``classifiers[2][4]`` refers to the 3rd repetition, 5th
130        learning algorithm.
131
132        Note that functions from :obj:`~Orange.evaluation.testing`
133        only store classifiers it enabled by setting
134        ``storeClassifiers`` to ``1``.
135
136    ..
137        .. attribute:: loaded
138
139            If the experimental method supports caching and there are no
140            obstacles for caching (such as unknown random seeds), this is a
141            list of boolean values. Each element corresponds to a classifier
142            and tells whether the experimental results for that classifier
143            were computed or loaded from the cache.
144
145    .. attribute:: base_class
146
147       The reference class for measures like AUC.
148
149    .. attribute:: class_values
150
151        The list of class values.
152
153    .. attribute:: weights
154
155        A flag telling whether the results are weighted. If ``False``,
156        weights are still present in :obj:`TestedExample`, but they
157        are all ``1.0``. Clear this flag, if your experimental
158        procedure ran on weighted testing examples but you would like
159        to ignore the weights in statistics.
160    """
161    @deprecated_keywords({"classifierNames": "classifier_names",
162                          "classValues": "class_values",
163                          "baseClass": "base_class",
164                          "numberOfIterations": "number_of_iterations",
165                          "numberOfLearners": "number_of_learners"})
166    def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=-1, domain=None, test_type=TEST_TYPE_SINGLE, labels=None, **argkw):
167        self.class_values = class_values
168        self.classifier_names = classifier_names
169        self.number_of_iterations = iterations
170        self.number_of_learners = len(classifier_names)
171        self.results = []
172        self.classifiers = []
173        self.loaded = None
174        self.base_class = base_class
175        self.weights = weights
176        self.test_type = test_type
177        self.labels = labels
178
179        if domain is not None:
180            self.base_class = self.class_values = None
181            if test_type == TEST_TYPE_SINGLE:
182                if domain.class_var.var_type == Orange.feature.Type.Discrete:
183                    self.class_values = list(domain.class_var.values)
184                    self.base_class = domain.class_var.base_value
185                    self.converter = int
186                else:
187                    self.converter = float
188            elif test_type in (TEST_TYPE_MLC, TEST_TYPE_MULTITARGET):
189                self.class_values = [list(cv.values) if cv.var_type == cv.Discrete else None for cv in domain.class_vars]
190                self.labels = [var.name for var in domain.class_vars]
191                self.converter = mt_vals
192
193
194        self.__dict__.update(argkw)
195
196    def load_from_files(self, learners, filename):
197        raise NotImplementedError("This feature is no longer supported.")
198
199    def save_to_files(self, learners, filename):
200        raise NotImplementedError("This feature is no longer supported. Pickle whole class instead.")
201
202    def create_tested_example(self, fold, example):
203        actual = example.getclass() if self.test_type == TEST_TYPE_SINGLE \
204                                  else example.get_classes()
205        return TestedExample(fold,
206                             self.converter(actual),
207                             self.number_of_learners,
208                             example.getweight(self.weights))
209
210    def remove(self, index):
211        """remove one learner from evaluation results"""
212        for r in self.results:
213            del r.classes[index]
214            del r.probabilities[index]
215        del self.classifier_names[index]
216        self.number_of_learners -= 1
217
218    def add(self, results, index, replace=-1):
219        """add evaluation results (for one learner)"""
220        if len(self.results)!=len(results.results):
221            raise SystemError("mismatch in number of test cases")
222        if self.number_of_iterations!=results.number_of_iterations:
223            raise SystemError("mismatch in number of iterations (%d<>%d)" % \
224                  (self.number_of_iterations, results.number_of_iterations))
225        if len(self.classifiers) and len(results.classifiers)==0:
226            raise SystemError("no classifiers in results")
227
228        if replace < 0 or replace >= self.number_of_learners: # results for new learner
229            self.classifier_names.append(results.classifier_names[index])
230            self.number_of_learners += 1
231            for i,r in enumerate(self.results):
232                r.classes.append(results.results[i].classes[index])
233                r.probabilities.append(results.results[i].probabilities[index])
234            if len(self.classifiers):
235                for i in range(self.number_of_iterations):
236                    self.classifiers[i].append(results.classifiers[i][index])
237        else: # replace results of existing learner
238            self.classifier_names[replace] = results.classifier_names[index]
239            for i,r in enumerate(self.results):
240                r.classes[replace] = results.results[i].classes[index]
241                r.probabilities[replace] = results.results[i].probabilities[index]
242            if len(self.classifiers):
243                for i in range(self.number_of_iterations):
244                    self.classifiers[replace] = results.classifiers[i][index]
245
246    def __repr__(self):
247        return str(self.__dict__)
248
249ExperimentResults = deprecated_members({"classValues": "class_values",
250                                        "classifierNames": "classifier_names",
251                                        "baseClass": "base_class",
252                                        "numberOfIterations": "number_of_iterations",
253                                        "numberOfLearners": "number_of_learners"
254})(ExperimentResults)
255
256   
257class Evaluation(object):
258    """Common methods for learner evaluation."""
259
260    @deprecated_keywords({"pps": "preprocessors",
261                          "strat": "stratified",
262                          "randseed": "random_generator",
263                          "indicesrandseed": "random_generator",
264                          "randomGenerator": "random_generator",
265                          "storeClassifiers": "store_classifiers",
266                          "storeExamples": "store_examples"})
267    def cross_validation(self, learners, examples, folds=10,
268            stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible,
269            preprocessors=(), random_generator=0, callback=None,
270            store_classifiers=False, store_examples=False):
271        """Cross validation test with specified number of folds.
272
273        :param learners: list of learning algorithms
274        :param examples: data instances used for training and testing
275        :param folds: number of folds
276        :param stratified: tells whether to stratify the sampling
277        :param preprocessors: a list of preprocessors to be used on data (obsolete)
278        :param random_generator: random seed or generator (see above)
279        :param callback: a function that is called after finishing each fold
280        :param store_classifiers: if ``True``, classifiers are stored in results
281        :param store_examples: if ``True``, examples are stored in results
282        :return: :obj:`ExperimentResults`
283        """
284        (examples, weight) = demangle_examples(examples)
285
286        indices = Orange.core.MakeRandomIndicesCV(examples, folds,
287            stratified=stratified, random_generator=random_generator)
288
289        return self.test_with_indices(
290            learners=learners,
291            examples=(examples, weight),
292            indices=indices,
293            preprocessors=preprocessors,
294            callback=callback,
295            store_classifiers=store_classifiers,
296            store_examples=store_examples)
297
298
299    @deprecated_keywords({"pps": "preprocessors",
300                          "storeClassifiers": "store_classifiers",
301                          "storeExamples": "store_examples"})
302    def leave_one_out(self, learners, examples, preprocessors=(),
303            callback=None, store_classifiers=False, store_examples=False):
304        """Leave-one-out evaluation of learning algorithms.
305
306        :param learners: list of learning algorithms
307        :param examples: data instances used for training and testing
308        :param preprocessors: a list of preprocessors (obsolete)
309        :param callback: a function that is called after finishing each fold
310        :param store_classifiers: if ``True``, classifiers are stored in results
311        :param store_examples: if ``True``, examples are stored in results
312        :return: :obj:`ExperimentResults`
313        """
314        examples, weight = demangle_examples(examples)
315        return self.test_with_indices(
316            learners, (examples, weight), indices=range(len(examples)),
317            preprocessors=preprocessors, callback=callback,
318            store_classifiers=store_classifiers, store_examples=store_examples)
319
320    @deprecated_keywords({"storeExamples": "store_examples",
321                          "storeClassifiers": "store_classifiers=True",
322                          "pps":"preprocessors"})
323    def test_with_indices(self, learners, examples, indices, preprocessors=(),
324            callback=None, store_classifiers=False, store_examples=False):
325        """
326        Perform a cross-validation-like test. Examples for each fold are
327        selected based on given indices.
328
329        :param learners: list of learning algorithms
330        :param examples: data instances used for training and testing
331        :param indices: a list of integer indices that sort examples into folds; each index corresponds to an example from ``examples``
332        :param preprocessors: a list of preprocessors (obsolete)
333        :param callback: a function that is called after each fold
334        :param store_classifiers: if ``True``, classifiers are stored in results
335        :param store_examples: if ``True``, examples are stored in results
336        :return: :obj:`ExperimentResults`
337        """
338        examples, weight = demangle_examples(examples)
339        if not examples:
340            raise ValueError("Test data set with no examples")
341        test_type = self.check_test_type(examples, learners)
342
343        niterations = max(indices)+1
344        test_result = ExperimentResults(niterations,
345                                        classifier_names = [getobjectname(l) for l in learners],
346                                        domain=examples.domain,
347                                        weights=weight,
348                                        test_type=test_type)
349
350        test_result.results = [test_result.create_tested_example(indices[i], example)
351                               for i, example in enumerate(examples)]
352
353        if store_examples:
354            test_result.examples = examples
355
356        for fold in xrange(niterations):
357            results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight)
358
359            for example, learner, result in results:
360                test_result.results[example].set_result(learner, *result)
361
362            if store_classifiers:
363                test_result.classifiers.append(classifiers)
364            if callback:
365                callback()
366
367        return test_result
368
369
370    def one_fold_with_indices(self, learners, examples, fold, indices, preprocessors=(), weight=0):
371        """Similar to :obj:`test_with_indices` except that it performs single fold of cross-validation, given by argument ``fold``."""
372        learn_set = examples.selectref(indices, fold, negate=1)
373        test_set = examples.selectref(indices, fold, negate=0)
374        if len(learn_set)==0 or len(test_set)==0:
375            return (), ()
376
377        # learning
378        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
379        if not learn_set:
380            raise SystemError("no training examples after preprocessing")
381        if not test_set:
382            raise SystemError("no test examples after preprocessing")
383
384        classifiers = [learner(learn_set, weight) for learner in learners]
385
386        # testing
387        testset_ids = (i for i, _ in enumerate(examples) if indices[i] == fold)
388        results = self._test_on_data(classifiers, test_set, testset_ids)
389
390        return results, classifiers
391
392
393    @deprecated_keywords({"storeExamples": "store_examples",
394                          "storeClassifiers": "store_classifiers",
395                          "pps": "preprocessors"})
396    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(),
397                                     callback=None, store_classifiers=False, store_examples=False):
398        """
399        Train learning algorithms and test them on the same data.
400
401        :param learners: list of learning algorithms
402        :param examples: data instances used for training and testing
403        :param preprocessors: a list of preprocessors (obsolete)
404        :param callback: a function that is called after each learning
405        :param store_classifiers: if ``True``, classifiers are stored in results
406        :param store_examples: if ``True``, examples are stored in results
407        :return: :obj:`ExperimentResults`
408        """
409
410        examples, weight = demangle_examples(examples)
411        test_type = self.check_test_type(examples, learners)
412
413        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to
414        # clone it.
415        if not filter(lambda x:x[0]!="B", preprocessors):
416            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors)
417            test_set = learn_set
418        else:
419            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors)
420
421        classifiers = self._train_with_callback(learners, learn_set, weight, callback)
422
423        test_results = ExperimentResults(1,
424                                        classifier_names = [getobjectname(l) for l in learners],
425                                        test_type = test_type,
426                                        domain=examples.domain,
427                                        weights=weight)
428        test_results.results = [test_results.create_tested_example(0, example)
429                               for i, example in enumerate(examples)]
430
431        if store_classifiers:
432            test_results.classifiers = classifiers
433        if store_examples:
434            test_results.examples = test_set
435
436        results = self._test_on_data(classifiers, test_set)
437        for example, classifier, result in results:
438            test_results.results[example].set_result(classifier, *result)
439        return test_results
440
441    @deprecated_keywords({"storeExamples": "store_examples",
442                          "storeClassifiers": "store_classifiers",
443                          "pps": "preprocessors"})
444    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(),
445                                    callback=None, store_classifiers=False, store_examples=False):
446        """
447        Train learning algorithms on one data sets and test them on another.
448
449        :param learners: list of learning algorithms
450        :param learn_set: training instances
451        :param test_set: testing instances
452        :param preprocessors: a list of preprocessors (obsolete)
453        :param callback: a function that is called after each learning
454        :param store_classifiers: if ``True``, classifiers are stored in results
455        :param store_examples: if ``True``, examples are stored in results
456        :return: :obj:`ExperimentResults`
457        """
458        learn_set, learn_weight = demangle_examples(learn_set)
459        test_set, test_weight = demangle_examples(test_set)
460
461        test_type = self.check_test_type(learn_set, learners)
462        self.check_test_type(test_set, learners)
463       
464        test_results = ExperimentResults(1,
465                                        classifier_names = [getobjectname(l) for l in learners],
466                                        domain=test_set.domain,
467                                        test_type = test_type,
468                                        weights=test_weight)
469        test_results.results = [test_results.create_tested_example(0, example)
470                               for i, example in enumerate(test_set)]
471
472        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback)
473
474        if store_classifiers:
475            test_results.classifiers = classifiers
476        if store_examples:
477            test_results.examples = test_set
478
479        for example, classifier, result in results:
480            test_results.results[example].set_result(classifier, *result)
481        return test_results
482
483    @deprecated_keywords({"storeExamples": "store_examples",
484                          "storeClassifiers": "store_classifiers",
485                          "learnProp": "learning_proportion",
486                          "strat": "stratification",
487                          "pps": "preprocessors",
488                          "indicesrandseed": "random_generator",
489                          "randseed": "random_generator",
490                          "randomGenerator": "random_generator"})
491    def proportion_test(self, learners, examples, learning_proportion=.7, times=10,
492                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
493                   preprocessors=(), random_generator=0,
494                   callback=None, store_classifiers=False, store_examples=False):
495        """
496        Iteratively split the data into training and testing set, and train and test the learnign algorithms.
497
498        :param learners: list of learning algorithms
499        :param examples: data instances used for training and testing
500        :param learning_proportion: proportion of data used for training
501        :param times: number of iterations
502        :param stratification: use stratified sampling
503        :param preprocessors: a list of preprocessors (obsolete)
504        :param random_generator: random seed or generator (see above)
505        :param callback: a function that is called after each fold
506        :param store_classifiers: if ``True``, classifiers are stored in results
507        :param store_examples: if ``True``, examples are stored in results
508        :return: :obj:`ExperimentResults`
509        """
510        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator)
511
512        examples, weight = demangle_examples(examples)
513
514        test_type = self.check_test_type(examples, learners)
515       
516        test_results = ExperimentResults(times,
517                                        classifier_names = [getobjectname(l) for l in learners],
518                                        domain=examples.domain,
519                                        test_type = test_type,
520                                        weights=weight)
521        if store_examples:
522            test_results.examples = []
523        test_results.classifiers = []
524        offset=0
525        for time in xrange(times):
526            indices = pick(examples)
527            learn_set = examples.selectref(indices, 0)
528            test_set = examples.selectref(indices, 1)
529            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors)
530            if store_classifiers:
531                test_results.classifiers.append(classifiers)
532            if store_examples:
533                test_results.examples.append(learn_set)
534
535            test_results.results.extend(test_results.create_tested_example(time, example)
536                                        for i, example in enumerate(test_set))
537            for example, classifier, result in results:
538                test_results.results[offset+example].set_result(classifier, *result)
539            offset += len(test_set)
540
541            if callback:
542                callback()
543        return test_results
544
545    @deprecated_keywords({"storeExamples": "store_examples",
546                          "storeClassifiers": "store_classifiers",
547                          "learnProp": "learning_proportion",
548                          "strat": "stratification",
549                          "pps": "preprocessors",
550                          "indicesrandseed": "random_generator",
551                          "randseed": "random_generator",
552                          "randomGenerator": "random_generator"})
553    def learning_curve(self, learners, examples, cv_indices=None, proportion_indices=None, proportions=Orange.core.frange(0.1),
554                       preprocessors=(), random_generator=0, callback=None):
555        """
556        Compute a learning curve using multiple cross-validations where
557        models are trained on different portions of the training data.
558
559        :param learners: list of learning algorithms
560        :param examples: data instances used for training and testing
561        :param cv_indices: indices used for cross validation (leave ``None`` for 10-fold CV)
562        :param proportion_indices: indices for proportion selection (leave ``None`` to let the function construct the folds)
563        :param proportions: list of proportions of data used for training
564        :param preprocessors: a list of preprocessors (obsolete)
565        :param random_generator: random seed or generator (see above)
566        :param callback: a function that is be called after each learning
567        :return: list of :obj:`ExperimentResults`
568        """
569        if cv_indices is None:
570            cv_indices = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
571        if proportion_indices is None:
572            proportion_indices = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
573
574        examples, weight = demangle_examples(examples)
575        indices = cv_indices(examples)
576       
577        all_results=[]
578        for p in proportions:
579            def select_proportion_preprocessor(examples):
580                return examples.selectref(proportion_indices(examples, p0=p), 0)
581
582            test_results = self.test_with_indices(learners, examples, indices,
583                preprocessors=[("L", select_proportion_preprocessor)] +
584                list(preprocessors), callback=callback)
585            all_results.append(test_results)
586        return all_results
587
588
589    @deprecated_keywords({"strat": "stratification",
590                          "pps": "preprocessors",
591                          "indicesrandseed": "random_generator",
592                          "randseed": "random_generator",
593                          "randomGenerator": "random_generator"})
594    def learning_curve_n(self, learners, examples, folds=10,
595                       proportions=Orange.core.frange(0.1),
596                       stratification = Orange.core.MakeRandomIndices
597                       .StratifiedIfPossible,
598                       preprocessors=(),
599                       random_generator=0, callback=None):
600        """
601        Compute a learning curve using multiple cross-validations where
602        models are trained on different portions of the training data.
603        Similar to :obj:`learning_curve` except for simpler arguments.
604
605        :param learners: list of learning algorithms
606        :param examples: data instances used for training and testing
607        :param folds: number of folds for cross-validation
608        :param proportions: list of proportions of data used for training
609        :param stratification: use stratified sampling
610        :param preprocessors: a list of preprocessors (obsolete)
611        :param random_generator: random seed or generator (see above)
612        :param callback: a function that is be called after each learning
613        :return: list of :obj:`ExperimentResults`
614        """
615
616        cv=Orange.core.MakeRandomIndicesCV(folds = folds,
617            stratified = stratification, randomGenerator = random_generator)
618        pick=Orange.core.MakeRandomIndices2(stratified = stratification,
619            randomGenerator = random_generator)
620        return learning_curve(learners, examples, cv, pick, proportions,
621            preprocessors, callback=callback)
622   
623    def learning_curve_with_test_data(self, learners, learn_set, test_set,
624            times=10, proportions=Orange.core.frange(0.1),
625            stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
626            preprocessors=(), random_generator=0, store_classifiers=False,
627            store_examples=False):
628        """
629        Compute a learning curve given two datasets. Models are learned on
630        proportion of the first dataset and then tested on the second.
631
632        :param learners: list of learning algorithms
633        :param learn_set: training data
634        :param test_set: testing data
635        :param times: number of iterations
636        :param straitification: use stratified sampling
637        :param proportions: a list of proportions of training data to be used
638        :param preprocessors: a list of preprocessors (obsolete)
639        :param random_generator: random seed or generator (see above)
640        :param store_classifiers: if ``True``, classifiers are stored in results
641        :param store_examples: if ``True``, examples are stored in results
642        :return: list of :obj:`ExperimentResults`
643        """
644        learn_set, learn_weight = demangle_examples(learn_set)
645        test_set, test_weight = demangle_examples(test_set)
646        test_type = self.check_test_type(learn_set, learners)
647        self.check_test_type(test_set, learners)
648       
649        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator)
650       
651        all_results=[]
652        for p in proportions:
653            test_results = ExperimentResults(times,
654                                        classifier_names = [getobjectname(l) for l in learners],
655                                        domain=test_set.domain,
656                                        test_type = test_type,
657                                        weights=test_weight)
658            offset = 0
659            for t in xrange(times):
660                test_results.results.extend(test_results.create_tested_example(t, example)
661                                            for i, example in enumerate(test_set))
662
663                learn_examples = learn_set.selectref(indices(learn_set, p), 0)
664                classifiers, results = self._learn_and_test_on_test_data\
665                    (learners, learn_examples, learn_weight, test_set,
666                    preprocessors=preprocessors)
667
668                for example, classifier, result in results:
669                    test_results.results[offset+example].set_result(classifier, *result)
670                offset += len(test_set)
671
672                if store_classifiers:
673                    test_results.classifiers.append(classifiers)
674                if store_examples:
675                    test_results.examples = learn_examples
676
677            all_results.append(test_results)
678        return all_results
679
680
681    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False):
682        """
683        Test classifiers on the given data
684
685        :param classifiers: a list of classifiers
686        :param examples: testing data
687        :param store_classifiers: if ``True``, classifiers are stored in results
688        :param store_examples: if ``True``, examples are stored in results
689        """
690
691        examples, weight = demangle_examples(examples)
692        test_type = self.check_test_type(examples, classifiers)
693
694        test_results = ExperimentResults(1,
695                                        classifier_names = [getobjectname(l) for l in classifiers],
696                                        domain=examples.domain,
697                                        test_type = test_type,
698                                        weights=weight)
699        test_results.results = [test_results.create_tested_example(0, example)
700                               for i, example in enumerate(examples)]
701
702        if store_examples:
703            test_results.examples = examples
704        if store_classifiers:
705            test_results.classifiers = classifiers
706
707        results = self._test_on_data(classifiers, examples)
708        for example, classifier, result in results:
709            test_results.results[example].set_result(classifier, *result)
710        return test_results
711
712
713    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set,
714                                     preprocessors=(), callback=None):
715        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
716
717        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback)
718       
719        results = self._test_on_data(classifiers, test_set)
720        return classifiers, results
721
722
723    def _train_with_callback(self, learners, examples, weight, callback):
724        classifiers = []
725        for learner in learners:
726            classifier = learner(examples, weight)
727            classifier.name = getattr(learner, 'name', 'noname')
728            classifiers.append(classifier)
729            if callback:
730                callback()
731        return classifiers
732
733
734    def _test_on_data(self, classifiers, examples, example_ids=None):
735        results = []
736
737        if example_ids is None:
738            numbered_examples = enumerate(examples)
739        else:
740            numbered_examples = itertools.izip(example_ids, examples)
741
742        for e, example in numbered_examples:
743            for c, classifier in enumerate(classifiers):
744                # Hide actual class to prevent cheating
745                ex2 = Orange.data.Instance(example)
746                if ex2.domain.class_var: ex2.setclass("?")
747                if ex2.domain.class_vars: ex2.set_classes(["?" for _ in ex2
748                .domain.class_vars])
749                result = classifier(ex2, Orange.core.GetBoth)
750                results.append((e, c, result))
751        return results
752
753   
754    def _preprocess_data(self, learn_set, test_set, preprocessors):
755        """Apply preprocessors to learn and test dataset (obsolete)"""
756        for p_type, preprocessor in preprocessors:
757            if p_type == "B":
758                learn_set = preprocessor(learn_set)
759                test_set = preprocessor(test_set)
760        for p_type, preprocessor in preprocessors:
761            if p_type == "L":
762                learn_set = preprocessor(learn_set)
763            elif p_type == "T":
764                test_set = preprocessor(test_set)
765            elif p_type == "LT":
766                (learn_set, test_set) = preprocessor(learn_set, test_set)
767        return learn_set, test_set
768
769
770    def encode_PP(self, pps):
771        for pp in pps:
772            objname = getobjectname(pp[1])
773            if len(objname):
774                pps+="_"+objname
775            else:
776                return "*"
777        return pps
778
779    def check_test_type(self, instances, models):
780        model_is_mlc = [isinstance(m, Orange.multilabel.MultiLabelLearner) or
781                        isinstance(m, Orange.multilabel.MultiLabelClassifier)
782                          for m in models]
783        multi_label = any(model_is_mlc)
784        if multi_label and not all(model_is_mlc):
785            raise ValueError("Test on mixed types of learners (MLC and non-MLC) not possible")
786        multi_target = instances.domain.class_vars and not multi_label
787
788        if (multi_label or multi_target) and not instances.domain.class_vars:
789            raise ValueError("Test data with multiple labels (class vars) expected")
790        if not (multi_label or multi_target or instances.domain.class_var):
791            raise ValueError("Test data set without class attributes")
792
793        return TEST_TYPE_MLC if multi_label else (
794            TEST_TYPE_MULTITARGET if multi_target else TEST_TYPE_SINGLE)
795   
796default_evaluation = _default_evaluation = Evaluation()
797
798cross_validation = _default_evaluation.cross_validation
799leave_one_out = _default_evaluation.leave_one_out
800test_with_indices = _default_evaluation.test_with_indices
801one_fold_with_indices = _default_evaluation.one_fold_with_indices
802
803learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data
804learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data
805test_on_data = _default_evaluation.test_on_data
806
807learning_curve = _default_evaluation.learning_curve
808learning_curve_n = _default_evaluation.learning_curve_n
809learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data
810
811proportion_test = _default_evaluation.proportion_test
812
813encode_PP = _default_evaluation.encode_PP
Note: See TracBrowser for help on using the repository browser.