source: orange/docs/reference/rst/code/statExamples.py @ 10429:23daf949f7b2

Revision 10429:23daf949f7b2, 5.7 KB checked in by anzeh <anze.staric@…>, 2 years ago (diff)

Fixed some backward compatibility issues.

Line 
1# Description: Demostrates the use of classification scores
2# Category:    evaluation
3# Uses:        voting.tab
4# Referenced:  orngStat.htm
5
6import Orange
7
8learners = [ Orange.classification.bayes.NaiveLearner(name = "bayes"),
9             Orange.classification.tree.TreeLearner(name="tree"),
10             Orange.classification.majority.MajorityLearner(name="majrty")]
11
12voting = Orange.data.Table("voting")
13res = Orange.evaluation.testing.cross_validation(learners, voting)
14
15vehicle = Orange.data.Table("vehicle")
16resVeh = Orange.evaluation.testing.cross_validation(learners, vehicle)
17
18import Orange.evaluation.scoring
19
20CAs = Orange.evaluation.scoring.CA(res)
21APs = Orange.evaluation.scoring.AP(res)
22Briers = Orange.evaluation.scoring.Brier_score(res)
23ISs = Orange.evaluation.scoring.IS(res)
24
25print
26print "method\tCA\tAP\tBrier\tIS"
27for l in range(len(learners)):
28    print "%s\t%5.3f\t%5.3f\t%5.3f\t%6.3f" % (learners[l].name, CAs[l], APs[l], Briers[l], ISs[l])
29
30
31CAs = Orange.evaluation.scoring.CA(res, reportSE=True)
32APs = Orange.evaluation.scoring.AP(res, reportSE=True)
33Briers = Orange.evaluation.scoring.Brier_score(res, reportSE=True)
34ISs = Orange.evaluation.scoring.IS(res, reportSE=True)
35
36print
37print "method\tCA\tAP\tBrier\tIS"
38for l in range(len(learners)):
39    print "%s\t%5.3f+-%5.3f\t%5.3f+-%5.3f\t%5.3f+-%5.3f\t%6.3f+-%5.3f" % ((learners[l].name, ) + CAs[l] + APs[l] + Briers[l] + ISs[l])
40
41
42print
43cm = Orange.evaluation.scoring.confusion_matrices(res)[0]
44print "Confusion matrix for naive Bayes:"
45print "TP: %i, FP: %i, FN: %s, TN: %i" % (cm.TP, cm.FP, cm.FN, cm.TN)
46
47print
48cm = Orange.evaluation.scoring.confusion_matrices(res, cutoff=0.2)[0]
49print "Confusion matrix for naive Bayes:"
50print "TP: %i, FP: %i, FN: %s, TN: %i" % (cm.TP, cm.FP, cm.FN, cm.TN)
51
52print
53cm = Orange.evaluation.scoring.confusion_matrices(resVeh, vehicle.domain.class_var.values.index("van"))[0]
54print "Confusion matrix for naive Bayes for 'van':"
55print "TP: %i, FP: %i, FN: %s, TN: %i" % (cm.TP, cm.FP, cm.FN, cm.TN)
56
57print
58cm = Orange.evaluation.scoring.confusion_matrices(resVeh, vehicle.domain.class_var.values.index("opel"))[0]
59print "Confusion matrix for naive Bayes for 'opel':"
60print "TP: %i, FP: %i, FN: %s, TN: %i" % (cm.TP, cm.FP, cm.FN, cm.TN)
61
62print
63cm = Orange.evaluation.scoring.confusion_matrices(resVeh)[0]
64classes = vehicle.domain.class_var.values
65print "\t"+"\t".join(classes)
66for className, classConfusions in zip(classes, cm):
67    print ("%s" + ("\t%i" * len(classes))) % ((className, ) + tuple(classConfusions))
68
69cm = Orange.evaluation.scoring.confusion_matrices(res)
70sensitivities = Orange.evaluation.scoring.sens(cm)
71specificities = Orange.evaluation.scoring.spec(cm)
72print
73print "Sensitivity and specificity for 'voting'"
74print "method\tsens\tspec"
75for l in range(len(learners)):
76    print "%s\t%5.3f\t%5.3f" % (learners[l].name, sensitivities[l], specificities[l])
77
78cm = Orange.evaluation.scoring.confusion_matrices(resVeh, vehicle.domain.class_var.values.index("van"))
79sensitivities = Orange.evaluation.scoring.sens(cm)
80specificities = Orange.evaluation.scoring.spec(cm)
81print
82print "Sensitivity and specificity for 'vehicle=van'"
83print "method\tsens\tspec"
84for l in range(len(learners)):
85    print "%s\t%5.3f\t%5.3f" % (learners[l].name, sensitivities[l], specificities[l])
86
87print
88print "AUC (voting)"
89
90AUCs = Orange.evaluation.scoring.AUC(res)
91for l in range(len(learners)):
92    print "%10s: %5.3f" % (learners[l].name, AUCs[l])
93
94
95print
96print "AUC for vehicle using weighted single-out method"
97print "bayes\ttree\tmajority"
98AUCs = Orange.evaluation.scoring.AUC(resVeh, Orange.evaluation.scoring.AUC.WeightedOneAgainstAll)
99print "%5.3f\t%5.3f\t%5.3f" % tuple(AUCs)
100
101print
102print "AUC for vehicle, using different methods"
103methods = ["by pairs, weighted", "by pairs", "one vs. all, weighted", "one vs. all"]
104print " " *25 + \tbayes\ttree\tmajority"
105for i in range(4):
106    AUCs = Orange.evaluation.scoring.AUC(resVeh, i)
107    print "%25s: \t%5.3f\t%5.3f\t%5.3f" % ((methods[i], ) + tuple(AUCs))
108
109
110classes = vehicle.domain.class_var.values
111classDist = Orange.statistics.distribution.Distribution(vehicle.domain.class_var, vehicle)
112
113print
114print "AUC for detecting class 'van' in 'vehicle'"
115AUCs = Orange.evaluation.scoring.AUC_single(resVeh, classIndex = vehicle.domain.class_var.values.index("van"))
116print "%5.3f\t%5.3f\t%5.3f" % tuple(AUCs)
117
118print
119print "AUCs for detecting various classes in 'vehicle'"
120for c,s in enumerate(classes):
121    print "%s (%5.3f) vs others: \t%5.3f\t%5.3f\t%5.3f" % ((s, classDist[c] ) + tuple(Orange.evaluation.scoring.AUC_single(resVeh, c)))
122
123print
124classes = vehicle.domain.class_var.values
125AUCmatrix = Orange.evaluation.scoring.AUC_matrix(resVeh)[0]
126print "\t"+"\t".join(classes[:-1])
127for className, AUCrow in zip(classes[1:], AUCmatrix[1:]):
128    print ("%s" + ("\t%5.3f" * len(AUCrow))) % ((className, ) + tuple(AUCrow))
129
130print
131print "AUCs for detecting various pairs of classes in 'vehicle'"
132for c1, s1 in enumerate(classes):
133    for c2 in range(c1):
134        print "%s vs %s: \t%5.3f\t%5.3f\t%5.3f" % ((s1, classes[c2]) + tuple(Orange.evaluation.scoring.AUC_pair(resVeh, c1, c2)))
135
136
137ri2 = Orange.data.sample.SubsetIndices2(voting, 0.6)
138train = voting.selectref(ri2, 0)
139test = voting.selectref(ri2, 1)
140res1 = Orange.evaluation.testing.learn_and_test_on_test_data(learners, train, test)
141
142print
143print "AUC and SE for voting"
144AUCs = Orange.evaluation.scoring.AUCWilcoxon(res1)
145for li, lrn in enumerate(learners):
146    print "%s: %5.3f+-%5.3f" % (lrn.name, AUCs[li][0], AUCs[li][1])
147
148print
149print "Difference between naive Bayes and tree: %5.3f+-%5.3f" % tuple(Orange.evaluation.scoring.compare_2_AUCs(res1, 0, 1)[2])
150
151print
152print "ROC (first 20 points) for bayes on 'voting'"
153ROC_bayes = Orange.evaluation.scoring.compute_ROC(res1)[0]
154for t in ROC_bayes[:20]:
155    print "%5.3f\t%5.3f" % t
Note: See TracBrowser for help on using the repository browser.