source: orange/Orange/evaluation/testing.py @ 10022:61f650b5c37e

Revision 10022:61f650b5c37e, 35.0 KB checked in by Lan Zagar <lan.zagar@…>, 2 years ago (diff)

Added two multitarget helpers for using singletarget scoring methods.

Line 
1import itertools
2
3import Orange
4from Orange.misc import demangle_examples, getobjectname, deprecated_keywords, deprecated_members
5
6#### Data structures
7
8TEST_TYPE_SINGLE = 0
9TEST_TYPE_MLC = 1
10TEST_TYPE_MULTITARGET = 2
11
12class TestedExample:
13    """
14    TestedExample stores predictions of different classifiers for a single testing example.
15
16    :var classes: A list of predictions of type Value, one for each classifier.
17    :var probabilities: A list of probabilities of classes, one for each classifier.
18    :var iteration_number: Iteration number (e.g. fold) in which the TestedExample was created/tested.
19    :var actual_class: The correct class of the example
20    :var weight: Example's weight. Even if the example set was not weighted, this attribute is present and equals 1.0.
21    """
22
23    @deprecated_keywords({"iterationNumber": "iteration_number",
24                          "actualClass": "actual_class"})
25    def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0):
26        """
27        :param iteration_number: The iteration number of TestedExample.
28        :param actual_class: The actual class of TestedExample.
29        :param n: The number of learners.
30        :param weight: The weight of the TestedExample.
31        """
32        self.classes = [None]*n
33        self.probabilities = [None]*n
34        self.iteration_number = iteration_number
35        self.actual_class= actual_class
36        self.weight = weight
37
38    def add_result(self, aclass, aprob):
39        """Appends a new result (class and probability prediction by a single classifier) to the classes and probabilities field."""
40   
41        if isinstance(aclass, (list, tuple)):
42            self.classes.append(aclass)
43            self.probabilities.append(aprob)
44        elif type(aclass.value)==float:
45            self.classes.append(float(aclass))
46            self.probabilities.append(aprob)
47        else:
48            self.classes.append(int(aclass))
49            self.probabilities.append(list(aprob))
50
51    def set_result(self, i, aclass, aprob):
52        """Sets the result of the i-th classifier to the given values."""
53        if isinstance(aclass, (list, tuple)):
54            self.classes[i] = aclass
55            self.probabilities[i] = aprob
56        elif type(aclass.value)==float:
57            self.classes[i] = float(aclass)
58            self.probabilities[i] = aprob
59        else:
60            self.classes[i] = int(aclass)
61            self.probabilities[i] = list(aprob)
62
63    def __repr__(self):
64        return str(self.__dict__)
65
66TestedExample = deprecated_members({"iterationNumber": "iteration_number",
67                                    "actualClass": "actual_class"
68                                    })(TestedExample)
69
70class ExperimentResults(object):
71    """
72    ``ExperimentResults`` stores results of one or more repetitions of
73    some test (cross validation, repeated sampling...) under the same
74    circumstances.
75
76    :var results: A list of instances of :obj:`TestedExample`, one for each example in the dataset.
77    :var classifiers: A list of classifiers, one element for each repetition (eg. fold). Each element is a list
78      of classifiers, one for each learner. This field is used only if storing is enabled by ``storeClassifiers=1``.
79    :var number_of_iterations: Number of iterations. This can be the number of folds (in cross validation)
80      or the number of repetitions of some test. :obj:`TestedExample`'s attribute ``iteration_number`` should
81      be in range ``[0, number_of_iterations-1]``.
82    :var number_of_learners: Number of learners. Lengths of lists classes and probabilities in each :obj:`TestedExample`
83      should equal ``number_of_learners``.
84    :var loaded: If the experimental method supports caching and there are no obstacles for caching (such as unknown
85      random seeds), this is a list of boolean values. Each element corresponds to a classifier and tells whether the
86      experimental results for that classifier were computed or loaded from the cache.
87    :var weights: A flag telling whether the results are weighted. If ``False``, weights are still present
88      in :obj:`TestedExample`, but they are all ``1.0``. Clear this flag, if your experimental procedure ran on weighted
89      testing examples but you would like to ignore the weights in statistics.
90    """
91    @deprecated_keywords({"classifierNames": "classifier_names",
92                          "classValues": "class_values",
93                          "baseClass": "base_class",
94                          "numberOfIterations": "number_of_iterations",
95                          "numberOfLearners": "number_of_learners"})
96    def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=-1, domain=None, test_type=TEST_TYPE_SINGLE, labels=None, **argkw):
97        self.class_values = class_values
98        self.classifier_names = classifier_names
99        self.number_of_iterations = iterations
100        self.number_of_learners = len(classifier_names)
101        self.results = []
102        self.classifiers = []
103        self.loaded = None
104        self.base_class = base_class
105        self.weights = weights
106        self.test_type = test_type
107        self.labels = labels
108
109        if domain is not None:
110            self.base_class = self.class_values = None
111            if test_type == TEST_TYPE_SINGLE:
112                if domain.class_var.var_type == Orange.feature.Type.Discrete:
113                    self.class_values = list(domain.class_var.values)
114                    self.base_class = domain.class_var.base_value
115                    self.converter = int
116                else:
117                    self.converter = float
118            elif test_type in (TEST_TYPE_MLC, TEST_TYPE_MULTITARGET):
119                self.labels = [var.name for var in domain.class_vars]
120                self.converter = lambda vals: [int(val) if val.variable.var_type == Orange.feature.Type.Discrete
121                                               else float(val) for val in vals]
122
123        self.__dict__.update(argkw)
124
125    def load_from_files(self, learners, filename):
126        raise NotImplementedError("This feature is no longer supported.")
127
128    def save_to_files(self, learners, filename):
129        raise NotImplementedError("This feature is no longer supported. Pickle whole class instead.")
130
131    def create_tested_example(self, fold, example):
132        actual = example.getclass() if self.test_type == TEST_TYPE_SINGLE \
133                                  else example.get_classes()
134        return TestedExample(fold,
135                             self.converter(actual),
136                             self.number_of_learners,
137                             example.getweight(self.weights))
138
139    def remove(self, index):
140        """remove one learner from evaluation results"""
141        for r in self.results:
142            del r.classes[index]
143            del r.probabilities[index]
144        del self.classifier_names[index]
145        self.number_of_learners -= 1
146
147    def add(self, results, index, replace=-1):
148        """add evaluation results (for one learner)"""
149        if len(self.results)!=len(results.results):
150            raise SystemError("mismatch in number of test cases")
151        if self.number_of_iterations!=results.number_of_iterations:
152            raise SystemError("mismatch in number of iterations (%d<>%d)" % \
153                  (self.number_of_iterations, results.number_of_iterations))
154        if len(self.classifiers) and len(results.classifiers)==0:
155            raise SystemError("no classifiers in results")
156
157        if replace < 0 or replace >= self.number_of_learners: # results for new learner
158            self.classifier_names.append(results.classifier_names[index])
159            self.number_of_learners += 1
160            for i,r in enumerate(self.results):
161                r.classes.append(results.results[i].classes[index])
162                r.probabilities.append(results.results[i].probabilities[index])
163            if len(self.classifiers):
164                for i in range(self.number_of_iterations):
165                    self.classifiers[i].append(results.classifiers[i][index])
166        else: # replace results of existing learner
167            self.classifier_names[replace] = results.classifier_names[index]
168            for i,r in enumerate(self.results):
169                r.classes[replace] = results.results[i].classes[index]
170                r.probabilities[replace] = results.results[i].probabilities[index]
171            if len(self.classifiers):
172                for i in range(self.number_of_iterations):
173                    self.classifiers[replace] = results.classifiers[i][index]
174
175    def __repr__(self):
176        return str(self.__dict__)
177
178ExperimentResults = deprecated_members({"classValues": "class_values",
179                                        "classifierNames": "classifier_names",
180                                        "baseClass": "base_class",
181                                        "numberOfIterations": "number_of_iterations",
182                                        "numberOfLearners": "number_of_learners"
183})(ExperimentResults)
184
185   
186class Evaluation(object):
187    """Common methods for learner evaluation."""
188
189    @deprecated_keywords({"pps": "preprocessors",
190                          "strat": "stratified",
191                          "randseed": "random_generator",
192                          "indicesrandseed": "random_generator",
193                          "randomGenerator": "random_generator",
194                          "storeClassifiers": "store_classifiers",
195                          "storeExamples": "store_examples"})
196    def cross_validation(self, learners, examples, folds=10,
197            stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible,
198            preprocessors=(), random_generator=0, callback=None,
199            store_classifiers=False, store_examples=False):
200        """Perform cross validation with specified number of folds.
201
202        :param learners: list of learners to be tested
203        :param examples: data table on which the learners will be tested
204        :param folds: number of folds to perform
205        :param stratified: sets, whether indices should be stratified
206        :param preprocessors: a list of preprocessors to be used on data.
207        :param random_generator: :obj:`Orange.misc.RandomGenerator` object.
208        :param callback: a function that will be called after each fold is
209               computed.
210        :param store_classifiers: if True, classifiers will be accessible in
211               test_results.
212        :param store_examples: if True, examples will be accessible in
213               test_results.
214        :return: :obj:`ExperimentResults`
215        """
216        (examples, weight) = demangle_examples(examples)
217
218        indices = Orange.core.MakeRandomIndicesCV(examples, folds,
219            stratified=stratified, random_generator=random_generator)
220
221        return self.test_with_indices(
222            learners=learners,
223            examples=(examples, weight),
224            indices=indices,
225            preprocessors=preprocessors,
226            callback=callback,
227            store_classifiers=store_classifiers,
228            store_examples=store_examples)
229
230
231    @deprecated_keywords({"pps": "preprocessors",
232                          "storeClassifiers": "store_classifiers",
233                          "storeExamples": "store_examples"})
234    def leave_one_out(self, learners, examples, preprocessors=(),
235            callback=None, store_classifiers=False, store_examples=False):
236        """Perform leave-one-out evaluation of learners on a data set.
237
238        :param learners: list of learners to be tested
239        :param examples: data table on which the learners will be tested
240        :param preprocessors: a list of preprocessors to be used on data.
241        :param callback: a function that will be called after each fold is
242               computed.
243        :param store_classifiers: if True, classifiers will be accessible in
244               test_results.
245        :param store_examples: if True, examples will be accessible in
246               test_results.
247        :return: :obj:`ExperimentResults`
248        """
249        return self.test_with_indices(learners, examples, indices=range(len(examples)), preprocessors=preprocessors,
250                                 callback=callback, store_classifiers=store_classifiers, store_examples=store_examples)
251
252    @deprecated_keywords({"storeExamples": "store_examples",
253                          "storeClassifiers": "store_classifiers=True",
254                          "pps":"preprocessors"})
255    def test_with_indices(self, learners, examples, indices, preprocessors=(),
256            callback=None, store_classifiers=False, store_examples=False,
257            **kwargs):
258        """
259        Perform a cross-validation-like test. Examples for each fold are
260        selected based on given indices.
261
262        :param learners: list of learners to be tested
263        :param examples: data table on which the learners will be tested
264        :param indices: a list of integers that defines, which examples will be
265               used for testing in each fold. The number of indices should be
266               equal to the number of examples.
267        :param preprocessors: a list of preprocessors to be used on data.
268        :param callback: a function that will be called after each fold is
269               computed.
270        :param store_classifiers: if True, classifiers will be accessible in test_results.
271        :param store_examples: if True, examples will be accessible in test_results.
272        :return: :obj:`ExperimentResults`
273        """
274        examples, weight = demangle_examples(examples)
275        if not examples:
276            raise ValueError("Test data set with no examples")
277        test_type = self.check_test_type(examples, learners)
278        if "cache" in kwargs:
279            raise ValueError("This feature is no longer supported.")
280
281        niterations = max(indices)+1
282        test_result = ExperimentResults(niterations,
283                                        classifierNames = [getobjectname(l) for l in learners],
284                                        domain=examples.domain,
285                                        weights=weight,
286                                        test_type=test_type)
287
288        test_result.results = [test_result.create_tested_example(indices[i], example)
289                               for i, example in enumerate(examples)]
290
291        if store_examples:
292            test_result.examples = examples
293
294        for fold in xrange(niterations):
295            results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight)
296
297            for example, learner, result in results:
298                test_result.results[example].set_result(learner, *result)
299
300            if store_classifiers:
301                test_result.classifiers.append(classifiers)
302            if callback:
303                callback()
304
305        return test_result
306
307
308    def one_fold_with_indices(self, learners, examples, fold, indices, preprocessors=(), weight=0):
309        """Perform one fold of cross-validation like procedure using provided indices."""
310        learn_set = examples.selectref(indices, fold, negate=1)
311        test_set = examples.selectref(indices, fold, negate=0)
312        if len(learn_set)==0 or len(test_set)==0:
313            return (), ()
314
315        # learning
316        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
317        if not learn_set:
318            raise SystemError("no training examples after preprocessing")
319        if not test_set:
320            raise SystemError("no test examples after preprocessing")
321
322        classifiers = [learner(learn_set, weight) for learner in learners]
323
324        # testing
325        testset_ids = (i for i, _ in enumerate(examples) if indices[i] == fold)
326        results = self._test_on_data(classifiers, test_set, testset_ids)
327
328        return results, classifiers
329
330
331    @deprecated_keywords({"storeExamples": "store_examples",
332                          "storeClassifiers": "store_classifiers",
333                          "pps": "preprocessors"})
334    def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(),
335                                     callback=None, store_classifiers=False, store_examples=False):
336        """
337        Perform a test where learners are trained and tested on the same data.
338
339        :param learners: list of learners to be tested
340        :param examples: data table on which the learners will be tested
341        :param preprocessors: a list of preprocessors to be used on data.
342        :param callback: a function that will be called after each fold is computed.
343        :param store_classifiers: if True, classifiers will be accessible in test_results.
344        :param store_examples: if True, examples will be accessible in test_results.
345        :return: :obj:`ExperimentResults`
346        """
347
348        examples, weight = demangle_examples(examples)
349        test_type = self.check_test_type(examples, learners)
350
351        # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to
352        # clone it.
353        if not filter(lambda x:x[0]!="B", preprocessors):
354            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors)
355            test_set = learn_set
356        else:
357            learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors)
358
359        classifiers = self._train_with_callback(learners, learn_set, weight, callback)
360
361        test_results = ExperimentResults(1,
362                                        classifier_names = [getobjectname(l) for l in learners],
363                                        test_type = test_type,
364                                        domain=examples.domain,
365                                        weights=weight)
366        test_results.results = [test_results.create_tested_example(0, example)
367                               for i, example in enumerate(examples)]
368
369        if store_classifiers:
370            test_results.classifiers = classifiers
371        if store_examples:
372            test_results.examples = test_set
373
374        results = self._test_on_data(classifiers, test_set)
375        for example, classifier, result in results:
376            test_results.results[example].set_result(classifier, *result)
377        return test_results
378
379    @deprecated_keywords({"storeExamples": "store_examples",
380                          "storeClassifiers": "store_classifiers",
381                          "pps": "preprocessors"})
382    def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(),
383                                    callback=None, store_classifiers=False, store_examples=False):
384        """
385        Perform a test, where learners are trained on one dataset and tested
386        on another.
387
388        :param learners: list of learners to be tested
389        :param learn_set: a dataset used for training
390        :param test_set: a dataset used for testing
391        :param preprocessors: a list of preprocessors to be used on data.
392        :param callback: a function that is be called after each classifier is computed.
393        :param store_classifiers: if True, classifiers will be accessible in test_results.
394        :param store_examples: if True, examples will be accessible in test_results.
395        :return: :obj:`ExperimentResults`
396        """
397        learn_set, learn_weight = demangle_examples(learn_set)
398        test_set, test_weight = demangle_examples(test_set)
399
400        test_type = self.check_test_type(learn_set, learners)
401        self.check_test_type(test_set, learners)
402       
403        test_results = ExperimentResults(1,
404                                        classifier_names = [getobjectname(l) for l in learners],
405                                        domain=test_set.domain,
406                                        test_type = test_type,
407                                        weights=test_weight)
408        test_results.results = [test_results.create_tested_example(0, example)
409                               for i, example in enumerate(test_set)]
410
411        classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback)
412
413        if store_classifiers:
414            test_results.classifiers = classifiers
415        if store_examples:
416            test_results.examples = test_set
417
418        for example, classifier, result in results:
419            test_results.results[example].set_result(classifier, *result)
420        return test_results
421
422    @deprecated_keywords({"storeExamples": "store_examples",
423                          "storeClassifiers": "store_classifiers",
424                          "learnProp": "learning_proportion",
425                          "strat": "stratification",
426                          "pps": "preprocessors",
427                          "indicesrandseed": "random_generator",
428                          "randseed": "random_generator",
429                          "randomGenerator": "random_generator"})
430    def proportion_test(self, learners, examples, learning_proportion, times=10,
431                   stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible, preprocessors=(), random_generator=0,
432                   callback=None, store_classifiers=False, store_examples=False):
433        """
434        Perform a test, where learners are trained and tested on different data sets. Training and test sets are
435        generated by proportionally splitting examples.
436
437        :param learners: list of learners to be tested
438        :param examples: a dataset used for evaluation
439        :param learning_proportion: proportion of examples to be used for training
440        :param times: number of test repetitions
441        :param stratification: use stratification when constructing train and test sets.
442        :param preprocessors: a list of preprocessors to be used on data.
443        :param callback: a function that is be called after each classifier is computed.
444        :param store_classifiers: if True, classifiers will be accessible in test_results.
445        :param store_examples: if True, examples will be accessible in test_results.
446        :return: :obj:`ExperimentResults`
447        """
448        pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator)
449
450        examples, weight = demangle_examples(examples)
451
452        test_type = self.check_test_type(examples, learners)
453       
454        test_results = ExperimentResults(times,
455                                        classifierNames = [getobjectname(l) for l in learners],
456                                        domain=examples.domain,
457                                        test_type = test_type,
458                                        weights=weight)
459        test_results.classifiers = []
460        offset=0
461        for time in xrange(times):
462            indices = pick(examples)
463            learn_set = examples.selectref(indices, 0)
464            test_set = examples.selectref(indices, 1)
465            classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors)
466            if store_classifiers:
467                test_results.classifiers.append(classifiers)
468
469            test_results.results.extend(test_results.create_tested_example(time, example)
470                                        for i, example in enumerate(test_set))
471            for example, classifier, result in results:
472                test_results.results[offset+example].set_result(classifier, *result)
473            offset += len(test_set)
474
475            if callback:
476                callback()
477        return test_results
478
479    @deprecated_keywords({"storeExamples": "store_examples",
480                          "storeClassifiers": "store_classifiers",
481                          "learnProp": "learning_proportion",
482                          "strat": "stratification",
483                          "pps": "preprocessors",
484                          "indicesrandseed": "random_generator",
485                          "randseed": "random_generator",
486                          "randomGenerator": "random_generator"})
487    def learning_curve(self, learners, examples, cv_indices=None, proportion_indices=None, proportions=Orange.core.frange(0.1),
488                       preprocessors=(), random_generator=0, callback=None):
489        """
490        Compute a learning curve using multiple cross-validations where
491        models are trained on different portions of the training data.
492
493        :param learners: list of learners to be tested
494        :param examples: a dataset used for evaluation
495        :param cv_indices: indices used for crossvalidation
496        :param proportion_indices: indices for proportion selection
497        :param proportions: proportions of train data to be used
498        :param preprocessors: a list of preprocessors to be used on data.
499        :param callback: a function that is be called after each classifier is computed.
500        :return: list of :obj:`ExperimentResults`
501        """
502        if cv_indices is None:
503            cv_indices = Orange.core.MakeRandomIndicesCV(folds=10, stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
504        if proportion_indices is None:
505            proportion_indices = Orange.core.MakeRandomIndices2(stratified=Orange.core.MakeRandomIndices.StratifiedIfPossible, randomGenerator = random_generator)
506
507        examples, weight = demangle_examples(examples)
508        indices = cv_indices(examples)
509       
510        all_results=[]
511        for p in proportions:
512            def select_proportion_preprocessor(examples):
513                return examples.selectref(proportion_indices(examples, p0=p), 0)
514
515            test_results = self.test_with_indices(learners, examples, indices,
516                                                  preprocessors=list(preprocessors) + [("L", select_proportion_preprocessor)],
517                                                  callback=callback)
518            all_results.append(test_results)
519        return all_results
520
521
522    @deprecated_keywords({"strat": "stratification",
523                          "pps": "preprocessors",
524                          "indicesrandseed": "random_generator",
525                          "randseed": "random_generator",
526                          "randomGenerator": "random_generator"})
527    def learning_curve_n(self, learners, examples, folds=10,
528                       proportions=Orange.core.frange(0.1),
529                       stratification = Orange.core.MakeRandomIndices
530                       .StratifiedIfPossible,
531                       preprocessors=(),
532                       random_generator=0, callback=None):
533        """
534        Compute a learning curve where each cross-validation has given number of folds
535        and models are trained on specified proportion of training data.
536
537        :param learners: list of learners to be tested
538        :param examples: a dataset used for evaluation
539        :param folds: number of folds for cross-validation
540        :param proportions: proportions of train data to be used
541        :param preprocessors: a list of preprocessors to be used on data.
542        :param callback: a function that is called after each classifier is computed.
543        :return: list of :obj:`ExperimentResults`
544        """
545
546        cv=Orange.core.MakeRandomIndicesCV(folds = folds,
547            stratified = stratification, randomGenerator = random_generator)
548        pick=Orange.core.MakeRandomIndices2(stratified = stratification,
549            randomGenerator = random_generator)
550        return learning_curve(learners, examples, cv, pick, proportions,
551            preprocessors, callback=callback)
552   
553    def learning_curve_with_test_data(self, learners, learn_set, test_set,
554            times=10, proportions=Orange.core.frange(0.1),
555            stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible,
556            preprocessors=(), random_generator=0):
557        """
558        Compute a learning curve given two datasets. Models are learned on
559        proportion of the first dataset and then used to make predictions for
560        the second dataset.
561
562        :param learners: list of learners to be tested
563        :param examples: a dataset used for evaluation
564        :param folds: number of folds for cross-validation
565        :param proportions: proportions of train data to be used
566        :param preprocessors: a list of preprocessors to be used on data.
567        :param callback: a function that is called after each classifier is computed.
568        :return: list of :obj:`ExperimentResults`
569        """
570        learn_set, learn_weight = demangle_examples(learn_set)
571        test_set, test_weight = demangle_examples(test_set)
572        test_type = self.check_test_type(learn_set, learners)
573        self.check_test_type(test_set, learners)
574       
575        indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator)
576       
577        all_results=[]
578        for p in proportions:
579            test_results = ExperimentResults(times,
580                                        classifierNames = [getobjectname(l) for l in learners],
581                                        domain=test_set.domain,
582                                        test_type = test_type,
583                                        weights=test_weight)
584            offset = 0
585            for t in xrange(times):
586                test_results.results.extend(test_results.create_tested_example(t, example)
587                                            for i, example in enumerate(test_set))
588
589                learn_examples = learn_set.selectref(indices(learn_set, p), 0)
590                classifiers, results = self._learn_and_test_on_test_data(learners, learn_examples, learn_weight, test_set)
591
592                for example, classifier, result in results:
593                    test_results.results[offset+example].set_result(classifier, *result)
594                offset += len(test_set)
595
596                test_results.classifiers.append(classifiers)
597
598            all_results.append(test_results)
599        return all_results
600
601
602    def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False):
603        """
604        Test classifiers on examples
605
606        :param classifiers: classifiers to test
607        :param examples: examples to test on
608        :param store_classifiers: if True, classifiers will be accessible in test_results.
609        :param store_examples: if True, examples will be accessible in test_results.
610        """
611
612        examples, weight = demangle_examples(examples)
613        test_type = self.check_test_type(examples, classifiers)
614
615        test_results = ExperimentResults(1,
616                                        classifierNames = [getobjectname(l) for l in classifiers],
617                                        domain=examples.domain,
618                                        test_type = test_type,
619                                        weights=weight)
620        test_results.results = [test_results.create_tested_example(0, example)
621                               for i, example in enumerate(examples)]
622
623        if store_examples:
624            test_results.examples = examples
625        if store_classifiers:
626            test_results.classifiers = classifiers
627
628        results = self._test_on_data(classifiers, examples)
629        for example, classifier, result in results:
630            test_results.results[example].set_result(classifier, *result)
631        return test_results
632
633
634    def _learn_and_test_on_test_data(self, learners, learn_set, learn_weight, test_set,
635                                     preprocessors=(), callback=None):
636        learn_set, test_set = self._preprocess_data(learn_set, test_set, preprocessors)
637
638        classifiers = self._train_with_callback(learners, learn_set, learn_weight, callback)
639       
640        results = self._test_on_data(classifiers, test_set)
641        return classifiers, results
642
643
644    def _train_with_callback(self, learners, examples, weight, callback):
645        classifiers = []
646        for learner in learners:
647            classifier = learner(examples, weight)
648            classifier.name = getattr(learner, 'name', 'noname')
649            classifiers.append(classifier)
650            if callback:
651                callback()
652        return classifiers
653
654
655    def _test_on_data(self, classifiers, examples, example_ids=None):
656        results = []
657
658        if example_ids is None:
659            numbered_examples = enumerate(examples)
660        else:
661            numbered_examples = itertools.izip(example_ids, examples)
662
663        for e, example in numbered_examples:
664            for c, classifier in enumerate(classifiers):
665                # Hide actual class to prevent cheating
666                ex2 = Orange.data.Instance(example)
667                if ex2.domain.class_var: ex2.setclass("?")
668                if ex2.domain.class_vars: ex2.set_classes(["?" for cv in ex2.domain.class_vars])
669                result = classifier(ex2, Orange.core.GetBoth)
670                results.append((e, c, result))
671        return results
672
673   
674    def _preprocess_data(self, learn_set, test_set, preprocessors):
675        """Apply preprocessors to learn and test dataset"""
676        for p_type, preprocessor in preprocessors:
677            if p_type == "B":
678                learn_set = preprocessor(learn_set)
679                test_set = preprocessor(test_set)
680        for p_type, preprocessor in preprocessors:
681            if p_type == "L":
682                learn_set = preprocessor(learn_set)
683            elif p_type == "T":
684                test_set = preprocessor(test_set)
685            elif p_type == "LT":
686                (learn_set, test_set) = preprocessor(learn_set, test_set)
687        return learn_set, test_set
688
689
690    def encode_PP(self, pps):
691        pps=""
692        for pp in pps:
693            objname = getobjectname(pp[1], "")
694            if len(objname):
695                pps+="_"+objname
696            else:
697                return "*"
698        return pps
699
700    def check_test_type(self, instances, models):
701        model_is_mlc = [isinstance(m, Orange.multilabel.MultiLabelLearner) or
702                        isinstance(m, Orange.multilabel.MultiLabelClassifier)
703                          for m in models]
704        multi_label = any(model_is_mlc)
705        if multi_label and not all(model_is_mlc):
706            raise ValueError("Test on mixed types of learners (MLC and non-MLC) not possible")
707        multi_target = instances.domain.class_vars and not multi_label
708
709        if (multi_label or multi_target) and not instances.domain.class_vars:
710            raise ValueError("Test data with multiple labels (class vars) expected")
711        if not (multi_label or multi_target or instances.domain.class_var):
712            raise ValueError("Test data set without class attributes")
713
714        return TEST_TYPE_MLC if multi_label else (
715            TEST_TYPE_MULTITARGET if multi_target else TEST_TYPE_SINGLE)
716   
717default_evaluation = _default_evaluation = Evaluation()
718
719cross_validation = _default_evaluation.cross_validation
720leave_one_out = _default_evaluation.leave_one_out
721test_with_indices = _default_evaluation.test_with_indices
722one_fold_with_indices = _default_evaluation.one_fold_with_indices
723
724learn_and_test_on_learn_data = _default_evaluation.learn_and_test_on_learn_data
725learn_and_test_on_test_data = _default_evaluation.learn_and_test_on_test_data
726test_on_data = _default_evaluation.test_on_data
727
728learning_curve = _default_evaluation.learning_curve
729learning_curve_n = _default_evaluation.learning_curve_n
730learning_curve_with_test_data = _default_evaluation.learning_curve_with_test_data
731
732proportion_test = _default_evaluation.proportion_test
733
734encode_PP = _default_evaluation.encode_PP
Note: See TracBrowser for help on using the repository browser.