source: orange-reliability/docs/rst/code/reliability-long.py @ 2:e96a42ae4a8f

Revision 2:e96a42ae4a8f, 2.0 KB checked in by Matija Polajnar <matija.polajnar@…>, 2 years ago (diff)

Move reliability regression tests to add-on.

Line 
1# Description: Reliability estimation
2# Category:    evaluation
3# Uses:        prostate
4# Referenced:  Orange.evaluation.reliability
5# Classes:     Orange.evaluation.reliability.Learner
6
7import Orange
8Orange.evaluation.reliability.select_with_repeat.random_generator = None
9Orange.evaluation.reliability.select_with_repeat.randseed = 42
10
11import Orange
12prostate = Orange.data.Table("prostate.tab")
13
14knn = Orange.classification.knn.kNNLearner()
15reliability = Orange.evaluation.reliability.Learner(knn)
16
17res = Orange.evaluation.testing.cross_validation([reliability], prostate)
18
19reliability_res = Orange.evaluation.reliability.get_pearson_r(res)
20
21print
22print "Estimate               r       p"
23for estimate in reliability_res:
24    print "%-20s %7.3f %7.3f" % (Orange.evaluation.reliability.METHOD_NAME[estimate[3]],
25                                 estimate[0], estimate[1])
26
27reliability = Orange.evaluation.reliability.Learner(knn, estimators=[Orange.evaluation.reliability.SensitivityAnalysis()])
28
29res = Orange.evaluation.testing.cross_validation([reliability], prostate)
30
31reliability_res = Orange.evaluation.reliability.get_pearson_r(res)
32
33print
34print "Estimate               r       p"
35for estimate in reliability_res:
36    print "%-20s %7.3f %7.3f" % (Orange.evaluation.reliability.METHOD_NAME[estimate[3]],
37                                 estimate[0], estimate[1])
38
39indices = Orange.data.sample.SubsetIndices2(prostate, p0=0.7)
40train = prostate.select(indices, 0)
41test = prostate.select(indices, 1)
42
43reliability = Orange.evaluation.reliability.Learner(knn, icv=True)
44res = Orange.evaluation.testing.learn_and_test_on_test_data([reliability], train, test)
45
46print
47print "Method used in internal cross-validation: ", Orange.evaluation.reliability.METHOD_NAME[res.results[0].probabilities[0].reliability_estimate[0].method]
48
49top5 = sorted((abs(result.probabilities[0].reliability_estimate[0].estimate), id) for id, result in enumerate(res.results))[:5]
50for estimate, id in top5:
51    print "%7.3f %i" % (estimate, id)
Note: See TracBrowser for help on using the repository browser.