Changeset 9503:e70b002076e2 in orange
 Timestamp:
 12/29/11 12:15:58 (2 years ago)
 Branch:
 default
 Convert:
 29b9fdcdeb48a621f38352499b1492537ca51534
 File:

 1 edited
Legend:
 Unmodified
 Added
 Removed

orange/Orange/evaluation/testing.py
r9502 r9503 2 2 3 3 import Orange 4 from Orange.misc import demangle_examples, getobjectname, printVerbose, deprecated_keywords 4 from Orange.misc import demangle_examples, getobjectname, printVerbose, deprecated_keywords, deprecated_members 5 5 6 6 #### Data structures 7 8 TEST_TYPE_SINGLE = 0 9 TEST_TYPE_MLC = 1 7 10 8 11 class TestedExample: … … 17 20 """ 18 21 19 def __init__(self, iterationNumber=None, actualClass=None, n=0, weight=1.0): 20 """ 21 :param iterationNumber: 22 :param actualClass: 22 @deprecated_keywords({"iterationNumber": "iteration_number", 23 "actualClass": "actual_class"}) 24 def __init__(self, iteration_number=None, actual_class=None, n=0, weight=1.0): 25 """ 26 :param iteration_number: 27 :param actual_class: 23 28 :param n: 24 29 :param weight: … … 26 31 self.classes = [None]*n 27 32 self.probabilities = [None]*n 28 self.iterationNumber = iteration Number29 self.actualClass= actual Class33 self.iterationNumber = iteration_number 34 self.actualClass= actual_class 30 35 self.weight = weight 31 36 … … 33 38 """Appends a new result (class and probability prediction by a single classifier) to the classes and probabilities field.""" 34 39 35 if type(aclass.value)==float: 40 if type(aclass)==list: 41 self.classes.append(aclass) 42 self.probabilities.append(aprob) 43 elif type(aclass.value)==float: 36 44 self.classes.append(float(aclass)) 37 45 self.probabilities.append(aprob) … … 42 50 def set_result(self, i, aclass, aprob): 43 51 """Sets the result of the ith classifier to the given values.""" 44 if type(aclass.value)==float: 52 if type(aclass)==list: 53 self.classes[i] = aclass 54 self.probabilities[i] = aprob 55 elif type(aclass.value)==float: 45 56 self.classes[i] = float(aclass) 46 57 self.probabilities[i] = aprob … … 61 72 :var classifiers: A list of classifiers, one element for each repetition (eg. fold). Each element is a list 62 73 of classifiers, one for each learner. This field is used only if storing is enabled by ``storeClassifiers=1``. 63 :var number OfIterations: Number of iterations. This can be the number of folds (in cross validation)74 :var number_of_iterations: Number of iterations. This can be the number of folds (in cross validation) 64 75 or the number of repetitions of some test. ``TestedExample``'s attribute ``iterationNumber`` should 65 be in range ``[0, number OfIterations1]``.66 :var number OfLearners: Number of learners. Lengths of lists classes and probabilities in each :obj:`TestedExample`67 should equal ``number OfLearners``.76 be in range ``[0, number_of_iterations1]``. 77 :var number_of_learners: Number of learners. Lengths of lists classes and probabilities in each :obj:`TestedExample` 78 should equal ``number_of_learners``. 68 79 :var loaded: If the experimental method supports caching and there are no obstacles for caching (such as unknown 69 80 random seeds), this is a list of boolean values. Each element corresponds to a classifier and tells whether the … … 73 84 testing examples but you would like to ignore the weights in statistics. 74 85 """ 75 def __init__(self, iterations, classifierNames, classValues=None, weights=None, baseClass=1, domain=None, **argkw): 76 self.classValues = classValues 77 self.classifierNames = classifierNames 78 self.numberOfIterations = iterations 79 self.numberOfLearners = len(classifierNames) 86 @deprecated_keywords({"classifierNames": "classifier_names", 87 "classValues": "class_values", 88 "baseClass": "base_class", 89 "numberOfIterations": "number_of_iterations", 90 "numberOfLearners": "number_of_learners"}) 91 def __init__(self, iterations, classifier_names, class_values=None, weights=None, base_class=1, domain=None, test_type=TEST_TYPE_SINGLE, **argkw): 92 self.class_values = class_values 93 self.classifier_names = classifier_names 94 self.number_of_iterations = iterations 95 self.number_of_learners = len(classifier_names) 80 96 self.results = [] 81 97 self.classifiers = [] 82 98 self.loaded = None 83 self.base Class = baseClass99 self.base_class = base_class 84 100 self.weights = weights 101 self.test_type = test_type 85 102 86 103 if domain is not None: 87 if domain.classVar.varType == Orange.data.Type.Discrete: 88 self.classValues = list(domain.classVar.values) 89 self.baseClass = domain.classVar.base_value 90 self.converter = int 91 else: 92 self.baseClass = self.classValues = None 93 self.converter = float 104 self.base_class = self.class_values = None 105 if test_type==TEST_TYPE_SINGLE: 106 if domain.class_var.var_type == Orange.data.Type.Discrete: 107 self.class_values = list(domain.class_var.values) 108 self.base_class = domain.class_var.base_value 109 self.converter = int 110 else: 111 self.converter = float 112 elif test_type==TEST_TYPE_MLC: 113 self.converter = lambda vals: [int(val) if val.variable.var_type == Orange.data.Type.Discrete 114 else float(val) for val in vals] 94 115 95 116 self.__dict__.update(argkw) … … 102 123 103 124 def create_tested_example(self, fold, example): 125 actual = [example.getclass, example.get_classes][self.test_type]() 104 126 return TestedExample(fold, 105 self.converter( example.getclass()),106 self.number OfLearners,127 self.converter(actual), 128 self.number_of_learners, 107 129 example.getweight(self.weights)) 108 130 … … 112 134 del r.classes[index] 113 135 del r.probabilities[index] 114 del self.classifier Names[index]115 self.number OfLearners = 1136 del self.classifier_names[index] 137 self.number_of_learners = 1 116 138 117 139 def add(self, results, index, replace=1): … … 119 141 if len(self.results)!=len(results.results): 120 142 raise SystemError("mismatch in number of test cases") 121 if self.number OfIterations!=results.numberOfIterations:143 if self.number_of_iterations!=results.number_of_iterations: 122 144 raise SystemError("mismatch in number of iterations (%d<>%d)" % \ 123 (self.number OfIterations, results.numberOfIterations))145 (self.number_of_iterations, results.number_of_iterations)) 124 146 if len(self.classifiers) and len(results.classifiers)==0: 125 147 raise SystemError("no classifiers in results") 126 148 127 if replace < 0 or replace >= self.number OfLearners: # results for new learner128 self.classifier Names.append(results.classifierNames[index])129 self.number OfLearners += 1149 if replace < 0 or replace >= self.number_of_learners: # results for new learner 150 self.classifier_names.append(results.classifier_names[index]) 151 self.number_of_learners += 1 130 152 for i,r in enumerate(self.results): 131 153 r.classes.append(results.results[i].classes[index]) 132 154 r.probabilities.append(results.results[i].probabilities[index]) 133 155 if len(self.classifiers): 134 for i in range(self.number OfIterations):156 for i in range(self.number_of_iterations): 135 157 self.classifiers[i].append(results.classifiers[i][index]) 136 158 else: # replace results of existing learner 137 self.classifier Names[replace] = results.classifierNames[index]159 self.classifier_names[replace] = results.classifier_names[index] 138 160 for i,r in enumerate(self.results): 139 161 r.classes[replace] = results.results[i].classes[index] 140 162 r.probabilities[replace] = results.results[i].probabilities[index] 141 163 if len(self.classifiers): 142 for i in range(self.number OfIterations):164 for i in range(self.number_of_iterations): 143 165 self.classifiers[replace] = results.classifiers[i][index] 144 166 145 167 def __repr__(self): 146 168 return str(self.__dict__) 169 170 171 ExperimentResults = deprecated_members({"classValues": "class_values", 172 "classifierNames": "classifier_names", 173 "baseClass": "base_class", 174 "numberOfIterations": "number_of_iterations", 175 "numberOfLearners": "number_of_learners" 176 })(ExperimentResults) 177 147 178 #### Experimental procedures 148 179 class Evaluation(object): … … 192 223 return self.test_with_indices(learners, examples, indices=range(len(examples)), preprocessors=preprocessors, 193 224 callback=callback, store_classifiers=store_classifiers, store_examples=store_examples) 225 226 def check_test_type(self, instances, learners): 227 learner_is_mlc = [isinstance(l, Orange.multilabel.MultiLabelLearner) 228 for l in learners] 229 multi_label = any(learner_is_mlc) 230 if multi_label and not all(learner_is_mlc): 231 raise ValueError("Test on mixed types of learners (MLC and nonMLC) not possible") 232 233 if multi_label and not instances.domain.class_vars: 234 raise ValueError("Test data with multiple labels (class vars) expected") 235 if not multi_label and not instances.domain.class_var: 236 raise ValueError("Test data set without class attributes") 237 238 return [TEST_TYPE_SINGLE, TEST_TYPE_MLC][multi_label] 194 239 195 240 … … 216 261 if not examples: 217 262 raise ValueError("Test data set with no examples") 218 if not examples.domain.classVar: 219 raise ValueError("Test data set without class attribute") 263 test_type = self.check_test_type(examples, learners) 220 264 if "cache" in kwargs: 221 265 raise ValueError("This feature is no longer supported.") 222 223 266 224 267 niterations = max(indices)+1 … … 226 269 classifierNames = [getobjectname(l) for l in learners], 227 270 domain=examples.domain, 228 weights=weight) 271 weights=weight, 272 test_type=test_type) 229 273 230 274 test_result.results = [test_result.create_tested_example(indices[i], example) … … 602 646 # Hide actual class to prevent cheating 603 647 ex2 = Orange.data.Instance(example) 604 ex2.setclass("?") 648 if ex2.domain.class_var: ex2.setclass("?") 649 if ex2.domain.class_vars: ex2.set_classes(["?" for cv in ex2.domain.class_vars]) 605 650 result = classifier(ex2, Orange.core.GetBoth) 606 651 results.append((e, c, result))
Note: See TracChangeset
for help on using the changeset viewer.