source: orange/Orange/testing/unit/tests/test_evaluation.py @ 9936:08300d990d26

Revision 9936:08300d990d26, 16.4 KB checked in by markotoplak, 2 years ago (diff)

moved new_meta_ide from feature to feature.Descriptor

Line 
1import itertools, operator, unittest
2from collections import Counter
3
4import Orange
5
6example_no = Orange.feature.Descriptor.new_meta_id()
7
8class DummyLearner(Orange.classification.majority.MajorityLearner):
9    def __init__(self, id=None):
10        self.id=id
11        self.data = []
12        self.classifiers = []
13        self.classifier_no = 0
14        super(DummyLearner, self).__init__()
15
16    def __call__(self, dataset, weight=0):
17        self.data.append(dataset)
18
19        cl = super(DummyLearner, self).__call__(dataset, weight)
20        cl = DummyClassifier(cl, self.id, self.classifier_no)
21        self.classifier_no += 1
22        self.classifiers.append(cl)
23        return cl
24
25class DummyClassifier(object):
26    name="DummyClassifier"
27    def __init__(self, base_class, learner_id, classifier_no):
28        self.base_class = base_class
29        self.classifier_no = classifier_no
30        self.learner_id = learner_id
31        self.data = []
32        self.predictions = []
33
34    def __call__(self, example, options=None):
35        value, probability = self.base_class.__call__(example, options)
36        p = [self.learner_id, self.classifier_no, int(example[example_no])]
37        self.data.append(example)
38        self.predictions.append(p)
39
40        return value, p
41
42class DummyPreprocessor(object):
43    def __init__(self, meta_id):
44        self.meta_id = meta_id
45
46    def __call__(self, *datasets):
47        new_datasets = []
48        for dataset in datasets:
49            new_data = Orange.data.Table(dataset)
50            for example in new_data:
51                example[self.meta_id] = 1.
52            new_datasets.append(new_data)
53
54        return new_data if len(datasets) == 1 else new_datasets
55
56class BrokenPreprocessor(object):
57    def __call__(self, *args):
58        return []
59
60class TestEvaluation(unittest.TestCase):
61    def setUp(self):
62        self.example_no = example_no
63        self.learner = DummyLearner()
64        self.examples = Orange.data.Table("iris")
65        for i, inst in enumerate(self.examples):
66            inst[self.example_no] = i
67
68        self.preprocessed_with_both = Orange.feature.Descriptor.new_meta_id()
69        self.preprocessed_with_learn = Orange.feature.Descriptor.new_meta_id()
70        self.preprocessed_with_test = Orange.feature.Descriptor.new_meta_id()
71        self.preprocessed_with_learn_test = Orange.feature.Descriptor.new_meta_id()
72        self.preprocessors = (("B", DummyPreprocessor(self.preprocessed_with_both)),
73                              ("L", DummyPreprocessor(self.preprocessed_with_learn)),
74                              ("T", DummyPreprocessor(self.preprocessed_with_test)),
75                              ("LT", DummyPreprocessor(self.preprocessed_with_learn_test)))
76
77        self.folds = 3
78        examples_in_fold = len(self.examples) // self.folds
79        self.indices = [i // examples_in_fold for i in range(len(self.examples))]
80        self.callback_calls = 0
81        reload(Orange.evaluation.testing)
82        self.evaluation = Orange.evaluation.testing
83
84    def test_with_indices(self):
85        learners = [DummyLearner(id=0), DummyLearner(id=1), DummyLearner(id=2)]
86        self.callback_calls = 0
87
88        test_results = self.evaluation.test_with_indices(learners, (self.examples, 0), self.indices, callback=self.callback)
89
90        for l, learner in enumerate(learners):
91            predicted_results = [prediction for classifier in learner.classifiers for prediction in classifier.predictions]
92            returned_results = [r.probabilities[l] for r in test_results.results]
93            self.assertItemsEqual(returned_results, predicted_results)
94
95            # Each example should be used for training (n-1)x, where n is the number of folds
96            example_cnt = 0
97            for fold, data in enumerate(learner.data):
98                for example in data:
99                    # Classifier should be trained on examples where fold is different from current fold
100                    self.assertNotEqual(self.indices[int(example[self.example_no])], fold)
101                    example_cnt += 1
102            self.assertEqual(example_cnt, len(self.examples) * (self.folds-1))
103
104            # Each example should be used for testing only once
105            example_cnt = 0
106            for fold, classifier in enumerate(learner.classifiers):
107                for example in classifier.data:
108                    # Classifier should perform classification on examples with same fold number
109                    self.assertEqual(self.indices[int(example[self.example_no])], fold)
110                    example_cnt += 1
111            self.assertEqual(example_cnt, len(self.examples))
112
113        # Callback function should be called once for every fold.
114        self.assertEqual(self.callback_calls, self.folds)
115
116    def callback(self):
117        self.callback_calls += 1
118
119    def test_with_indices_can_store_examples_and_classifiers(self):
120        test_results = self.evaluation.test_with_indices([self.learner], self.examples, self.indices,
121                                                        store_examples=True, store_classifiers=True)
122        self.assertGreater(len(test_results.examples), 0)
123        self.assertGreater(len(test_results.classifiers), 0)
124        classifiers = map(operator.itemgetter(0), test_results.classifiers)
125        self.assertItemsEqual(self.learner.classifiers, classifiers)
126
127    def test_with_indices_uses_preprocessors(self):
128        self.evaluation.test_with_indices([self.learner], self.examples, self.indices,
129                                           preprocessors=self.preprocessors)
130        self.assertPreprocessedCorrectly()
131
132    def test_with_indices_handles_errors(self):
133        learner = DummyLearner()
134        # No data should raise a Value error
135        with self.assertRaises(ValueError):
136            self.evaluation.test_with_indices([learner], [], self.indices)
137
138        # If one fold is not represented in indices, cross validation should still execute
139        self.evaluation.test_with_indices([learner], self.examples, [2] + [1]*(len(self.examples)-1))
140
141        # If preprocessors is "broken" (returns no data), error should be  raised
142        with self.assertRaises(SystemError):
143            self.evaluation.test_with_indices([learner], self.examples, self.indices, preprocessors=(("L", BrokenPreprocessor()),))
144        with self.assertRaises(SystemError):
145            self.evaluation.test_with_indices([learner], self.examples, self.indices, preprocessors=(("T", BrokenPreprocessor()),))
146
147
148    def test_cross_validation(self):
149        learners = [DummyLearner(id=0), DummyLearner(id=1), DummyLearner(id=2)]
150        self.callback_calls = 0
151        folds = 10
152        test_results = self.evaluation.cross_validation(learners, (self.examples, 0), folds=folds, callback=self.callback)
153
154        for l, learner in enumerate(learners):
155            predicted_results = [prediction for classifier in learner.classifiers for prediction in classifier.predictions]
156            returned_results = [r.probabilities[l] for r in test_results.results]
157            self.assertItemsEqual(returned_results, predicted_results)
158
159            # Each example should be used for training (n-1)x, where n is the number of folds
160            example_cnt = 0
161            for fold, data in enumerate(learner.data):
162                for example in data:
163                    example_cnt += 1
164            self.assertEqual(example_cnt, len(self.examples) * (folds-1))
165
166            # Each example should be used for testing only once
167            example_cnt = 0
168            for fold, classifier in enumerate(learner.classifiers):
169                for example in classifier.data:
170                    example_cnt += 1
171            self.assertEqual(example_cnt, len(self.examples))
172
173    def test_leave_one_out(self):
174        learners = [DummyLearner(id=0), DummyLearner(id=1), DummyLearner(id=2)]
175        self.callback_calls = 0
176       
177        test_results = self.evaluation.leave_one_out(learners, self.examples)
178        for l, learner in enumerate(learners):
179            predicted_results = [prediction for classifier in learner.classifiers for prediction in classifier.predictions]
180            returned_results = [r.probabilities[l] for r in test_results.results]
181            self.assertItemsEqual(returned_results, predicted_results)
182
183            # Each example should be used for training (n-1)x, where n is the number of folds
184            example_cnt = 0
185            for fold, data in enumerate(learner.data):
186                for example in data:
187                    example_cnt += 1
188            self.assertEqual(example_cnt, len(self.examples) * (len(self.examples)-1))
189
190            # Each example should be used for testing only once
191            example_cnt = 0
192            for fold, classifier in enumerate(learner.classifiers):
193                for example in classifier.data:
194                    example_cnt += 1
195            self.assertEqual(example_cnt, len(self.examples))
196
197    def test_on_data(self):
198        pass
199
200    def test_on_data_can_store_examples_and_classifiers(self):
201        learner = DummyLearner()
202        classifier = learner(self.examples)
203        # Passing store_examples = True should make examples accessible
204        test_results = self.evaluation.test_on_data([classifier], self.examples,
205                                                    store_examples=True, store_classifiers=True)
206        self.assertGreater(len(test_results.examples), 0)
207        self.assertGreater(len(test_results.classifiers), 0)
208        self.assertItemsEqual(learner.classifiers, test_results.classifiers)
209
210    def test_learn_and_test_on_learn_data(self):
211        self.callback_calls = 0
212        learner = DummyLearner()
213
214        test_results = self.evaluation.learn_and_test_on_learn_data([learner], self.examples, callback=self.callback,
215                                                    store_examples=True, store_classifiers=True)
216
217        self.assertEqual(self.callback_calls, 1)
218
219
220
221    def test_learn_and_test_on_learn_data_with_preprocessors(self):
222        self.learner = DummyLearner()
223        test_results = self.evaluation.learn_and_test_on_learn_data([self.learner], self.examples,
224                                                    preprocessors=self.preprocessors,
225                                                    callback=self.callback, store_examples=True, store_classifiers=True)
226        self.assertPreprocessedCorrectly()
227
228    def test_learn_and_test_on_learn_data_can_store_examples_and_classifiers(self):
229        learner = DummyLearner()
230       
231        test_results = self.evaluation.learn_and_test_on_learn_data([learner], self.examples,
232                                                    store_examples=True, store_classifiers=True)
233        self.assertGreater(len(test_results.examples), 0)
234        self.assertGreater(len(test_results.classifiers), 0)
235        self.assertItemsEqual(learner.classifiers, test_results.classifiers)
236
237    def test_learn_and_test_on_test_data(self):
238        self.callback_calls = 0
239        learner = DummyLearner()
240
241        test_results = self.evaluation.learn_and_test_on_test_data([learner], self.examples, self.examples,
242                                                    callback=self.callback, store_examples=True, store_classifiers=True)
243        self.assertEqual(self.callback_calls, 1)
244
245    def test_learn_and_test_on_test_data_with_preprocessors(self):
246        self.learner = DummyLearner()
247        test_results = self.evaluation.learn_and_test_on_test_data([self.learner], self.examples, self.examples,
248                                                    preprocessors=self.preprocessors,
249                                                    callback=self.callback, store_examples=True, store_classifiers=True)
250        self.assertPreprocessedCorrectly()
251
252    def test_learn_and_test_on_test_data_can_store_examples_and_classifiers(self):
253        learner = DummyLearner()
254
255        test_results = self.evaluation.learn_and_test_on_test_data([learner], self.examples, self.examples,
256                                                    store_examples=True, store_classifiers=True)
257        self.assertGreater(len(test_results.examples), 0)
258        self.assertGreater(len(test_results.classifiers), 0)
259        self.assertItemsEqual(learner.classifiers, test_results.classifiers)
260
261    def test_learning_curve_with_test_data(self):
262        learner = DummyLearner()
263        times=10
264        proportions=Orange.core.frange(0.1)
265        test_results = self.evaluation.learning_curve_with_test_data([learner], self.examples, self.examples,
266                                                                              times=times, proportions=proportions)
267        # We expect the method to return a list of test_results, one instance for each proportion. Each
268        # instance should have "times" folds.
269        self.assertEqual(len(test_results), len(proportions))
270        for test_result in test_results:
271            self.assertEqual(test_result.numberOfIterations, times)
272            self.assertEqual(len(test_result.results), times*len(self.examples))
273
274
275
276    def test_learning_curve_with_test_data_can_store_examples_and_classifiers(self):
277        learner = DummyLearner()
278
279        test_results = self.evaluation.learn_and_test_on_test_data([learner], self.examples, self.examples,
280                                                                            store_examples=True, store_classifiers=True)
281        self.assertGreater(len(test_results.examples), 0)
282        self.assertGreater(len(test_results.classifiers), 0)
283        self.assertItemsEqual(learner.classifiers, test_results.classifiers)
284
285
286    def test_proportion_test(self):
287        self.callback_calls = 0
288        times = 10
289        learner = DummyLearner()
290
291        test_results = self.evaluation.proportion_test([learner], self.examples, learning_proportion=.7, times=times,
292                                                                callback = self.callback)
293
294        self.assertEqual(self.callback_calls, times)
295
296    def test_learning_curve(self):
297        self.callback_calls = 0
298        times = 10
299        proportions=Orange.core.frange(0.1)
300        folds=10
301        learner = DummyLearner()
302
303        test_results = self.evaluation.learning_curve([learner], self.examples,
304                                                                callback = self.callback)
305
306        # Ensure that each iteration is learned on correct proportion of training examples
307        for proportion, data in zip((p for p in proportions for _ in range(10)), learner.data):
308            actual_examples = len(data)
309            expected_examples = len(self.examples)*proportion*(folds-1)/folds
310            self.assertTrue(abs(actual_examples - expected_examples) <= 1)
311
312        # Ensure results are not lost
313        predicted_results = [prediction for classifier in learner.classifiers for prediction in classifier.predictions]
314        returned_results = [r.probabilities[0] for tr in test_results for r in tr.results]
315        self.assertItemsEqual(returned_results, predicted_results)
316       
317        self.assertEqual(self.callback_calls, folds*len(proportions))
318
319    #TODO: LearningCurveN tests
320
321    def assertPreprocessedCorrectly(self):
322        # Original examples should be left intact
323        for example in self.examples:
324            self.assertFalse(example.has_meta(self.preprocessed_with_both))
325            self.assertFalse(example.has_meta(self.preprocessed_with_learn))
326            self.assertFalse(example.has_meta(self.preprocessed_with_test))
327            self.assertFalse(example.has_meta(self.preprocessed_with_learn_test))
328        for fold, data in enumerate(self.learner.data):
329            for example in data:
330                # Preprocessors both, learn and learntest should be applied to learn data.
331                self.assertTrue(example.has_meta(self.preprocessed_with_both))
332                self.assertTrue(example.has_meta(self.preprocessed_with_learn))
333                self.assertTrue(example.has_meta(self.preprocessed_with_learn_test))
334                # Preprocessor test should not be applied to learn data.
335                self.assertFalse(example.has_meta(self.preprocessed_with_test))
336        for fold, classifier in enumerate(self.learner.classifiers):
337            for example in classifier.data:
338                # Preprocessors both, test and learntest should be applied to test data.
339                self.assertTrue(example.has_meta(self.preprocessed_with_both))
340                self.assertTrue(example.has_meta(self.preprocessed_with_test))
341                self.assertTrue(example.has_meta(self.preprocessed_with_learn_test))
342                # Preprocessor learn should not be applied to test data.
343                self.assertFalse(example.has_meta(self.preprocessed_with_learn))
344
345if __name__ == '__main__':
346    unittest.main()
Note: See TracBrowser for help on using the repository browser.