source: orange/source/orange/liblinear_interface.cpp @ 11610:33695add91c0

Revision 11610:33695add91c0, 12.3 KB checked in by Ales Erjavec <ales.erjavec@…>, 10 months ago (diff)

Cleanup of TLinearLearner/Classifier.

The training examples are sorted before training so the labels in
LIBLINEAR model match the order of class_var.values.

TLinearClassifier no longer has the 'examples' member, has a changed
constructor and (un)pickle signature.

Line 
1/*
2    This file is part of Orange.
3
4    Copyright 1996-2011 Faculty of Computer and Information Science, University of Ljubljana
5    Contact: janez.demsar@fri.uni-lj.si
6
7    Orange is free software: you can redistribute it and/or modify
8    it under the terms of the GNU General Public License as published by
9    the Free Software Foundation, either version 3 of the License, or
10    (at your option) any later version.
11
12    Orange is distributed in the hope that it will be useful,
13    but WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15    GNU General Public License for more details.
16
17    You should have received a copy of the GNU General Public License
18    along with Orange.  If not, see <http://www.gnu.org/licenses/>.
19*/
20
21#include <iostream>
22#include <sstream>
23
24#include <math.h>
25#include <stdio.h>
26#include <stdlib.h>
27#include <string.h>
28#include <stdarg.h>
29#include <assert.h>
30
31#include "liblinear_interface.ppp"
32
33#define Malloc(type,n) (type *) malloc((n)*sizeof(type))
34
35// Defined in linear.cpp. If a new solver is added this should be updated.
36
37static const char *solver_type_table[]=
38{
39    "L2R_LR", "L2R_L2LOSS_SVC_DUAL", "L2R_L2LOSS_SVC", "L2R_L1LOSS_SVC_DUAL", "MCSVM_CS",
40    "L1R_L2LOSS_SVC", "L1R_LR", "L2R_LR_DUAL", NULL
41};
42
43/*
44 *The folowing load save functions are used for orange pickling
45 */
46
47/*
48 * Save the model to an std::ostream. This is a modified `save_model` function
49 * from `linear.cpp` in LIBLINEAR package.
50 */
51int linear_save_model_alt(ostream &stream, struct model *model_)
52{
53    int i;
54    int nr_feature=model_->nr_feature;
55    int n;
56    const parameter& param = model_->param;
57
58    if(model_->bias>=0)
59        n=nr_feature+1;
60    else
61        n=nr_feature;
62
63    int nr_classifier;
64    if(model_->nr_class==2 && model_->param.solver_type != MCSVM_CS)
65        nr_classifier=1;
66    else
67        nr_classifier=model_->nr_class;
68
69    stream.precision(17);
70
71    stream << "solver_type " << solver_type_table[param.solver_type] << endl;
72    stream << "nr_class " << model_->nr_class << endl;
73    stream << "label";
74    for(i=0; i<model_->nr_class; i++)
75        stream << " " << model_->label[i];
76    stream << endl;
77
78    stream << "nr_feature " << nr_feature << endl;
79
80    stream << "bias " << model_->bias << endl;
81
82    stream << "w" << endl;
83    for(i=0; i<n; i++)
84    {
85        int j;
86        for(j=0; j<nr_classifier; j++)
87            stream << model_->w[i*nr_classifier+j] << " ";
88        stream << endl;
89    }
90
91    if (stream.good())
92        return 0;
93    else
94        return -1;
95}
96
97/*
98 * Save linear model into a std::string.
99 */
100int linear_save_model_alt(string &buffer, struct model *model_)
101{
102    std::ostringstream strstream;
103    int ret = linear_save_model_alt(strstream, model_);
104    buffer = strstream.rdbuf()->str();
105    return ret;
106}
107
108/*
109 * Load a linear model from std::istream. This is a modified `load_model`
110 * function from `linear.cpp` in LIBLINEAR package.
111 */
112struct model *linear_load_model_alt(istream &stream)
113{
114    int i;
115    int nr_feature;
116    int n;
117    int nr_class;
118    double bias;
119    model *model_ = Malloc(model,1);
120    parameter& param = model_->param;
121
122    model_->label = NULL;
123
124    char cmd[81];
125    stream.width(80);
126    while(stream.good())
127    {
128        stream >> cmd;
129        if(strcmp(cmd, "solver_type")==0)
130        {
131            stream >> cmd;
132            int i;
133            for(i=0;solver_type_table[i];i++)
134            {
135                if(strcmp(solver_type_table[i],cmd)==0)
136                {
137                    param.solver_type=i;
138                    break;
139                }
140            }
141            if(solver_type_table[i] == NULL)
142            {
143                fprintf(stderr,"unknown solver type.\n");
144                free(model_->label);
145                free(model_);
146                return NULL;
147            }
148        }
149        else if(strcmp(cmd,"nr_class")==0)
150        {
151            stream >> nr_class;
152            model_->nr_class=nr_class;
153        }
154        else if(strcmp(cmd,"nr_feature")==0)
155        {
156            stream >> nr_feature;
157            model_->nr_feature=nr_feature;
158        }
159        else if(strcmp(cmd,"bias")==0)
160        {
161            stream >> bias;
162            model_->bias=bias;
163        }
164        else if(strcmp(cmd,"w")==0)
165        {
166            break;
167        }
168        else if(strcmp(cmd,"label")==0)
169        {
170            int nr_class = model_->nr_class;
171            model_->label = Malloc(int, nr_class);
172            for(int i=0;i<nr_class;i++)
173                stream >> model_->label[i];
174        }
175        else
176        {
177            fprintf(stderr,"unknown text in model file: [%s]\n",cmd);
178            free(model_->label);
179            free(model_);
180            return NULL;
181        }
182    }
183
184    nr_feature=model_->nr_feature;
185    if(model_->bias>=0)
186        n=nr_feature+1;
187    else
188        n=nr_feature;
189
190    int nr_classifier;
191    if(nr_class==2 && param.solver_type != MCSVM_CS)
192        nr_classifier = 1;
193    else
194        nr_classifier = nr_class;
195
196    model_->w=Malloc(double, n*nr_classifier);
197    for(i=0; i<n; i++)
198    {
199        int j;
200        for(j=0; j<nr_classifier; j++)
201            stream >> model_->w[i*nr_classifier+j];
202    }
203    if (stream.fail())
204        return NULL;
205    else
206        return model_;
207}
208
209/*
210 * Load a linear model from a std:string.
211 */
212struct model *linear_load_model_alt(string &buffer)
213{
214    std::istringstream str_stream(buffer);
215    str_stream.exceptions(ios::failbit | ios::badbit);
216    return linear_load_model_alt(str_stream);
217}
218
219struct NodeSort{
220    bool operator () (const feature_node &lhs, const feature_node &rhs){
221        return lhs.index < rhs.index;
222    }
223};
224
225int countFeatures(const TExample &ex, bool includeMeta, bool includeRegular){
226    int count = 1;
227    if (includeRegular)
228        for (TExample::iterator i=ex.begin(); i!=ex.end(); i++)
229            if ((i->varType==TValue::INTVAR || i->varType==TValue::FLOATVAR) && i->isRegular() && i!=&ex.getClass())
230                count++;
231    if (includeMeta)
232        for (TMetaValues::const_iterator i=ex.meta.begin(); i!=ex.meta.end(); i++)
233            if ((i->second.varType==TValue::INTVAR || i->second.varType==TValue::FLOATVAR) && i->second.isRegular())
234                count++;
235    return count;
236}
237
238feature_node *feature_nodeFromExample(const TExample &ex, double bias){
239    int n_nodes = countFeatures(ex, false, true);
240
241    if (bias >= 0.0)
242        n_nodes++;
243
244    feature_node *nodes = new feature_node[n_nodes];
245    feature_node *ptr = nodes;
246
247    int index = 1;
248
249    for (TExample::iterator i=ex.begin(); i!=ex.end(); i++)
250        if (i!=&ex.getClass()){
251            if ((i->varType==TValue::INTVAR || (i->varType==TValue::FLOATVAR && (*i==*i))) && i->isRegular()){
252                if (i->varType==TValue::INTVAR)
253                    ptr->value = (int) *i;
254                else
255                    ptr->value = (float) *i;
256                ptr->index = index;
257                ptr++;
258            }
259            index++;
260        }
261
262    if (bias >= 0.0)
263    {
264        ptr->value = bias;
265        ptr->index = index;
266        ptr++;
267    }
268
269    ptr->index = -1;
270    return nodes;
271}
272
273problem *problemFromExamples(PExampleGenerator examples, double bias){
274    problem *prob = new problem;
275    prob->l = examples->numberOfExamples();
276    prob->n = examples->domain->attributes->size();
277
278    if (bias >= 0)
279        prob->n++;
280
281    prob->x = new feature_node* [prob->l];
282    prob->y = new int [prob->l];
283    prob->bias = bias;
284    feature_node **ptrX = prob->x;
285    int *ptrY = prob->y;
286    PEITERATE(iter, examples){
287        *ptrX = feature_nodeFromExample(*iter, bias);
288        *ptrY = (int) (*iter).getClass();
289        ptrX++;
290        ptrY++;
291    }
292    return prob;
293}
294
295void destroy_problem(problem *prob){
296    for (int i = 0; i < prob->l; i++)
297        delete[] prob->x[i];
298    delete[] prob->x;
299    delete[] prob->y;
300}
301
302static void dont_print_string(const char *s){}
303
304
305/*
306 * Extract feature weights from a LIBLINEAR model.
307 * The number of class values must be provided.
308 */
309
310TFloatListList * extract_feature_weights(model * model, int nr_class_values) {
311    /* Number of liblinear classifiers.
312     *
313     * NOTE: If some class values do not have any data instances in
314     * the training set they are not present in the liblinear model
315     * so this number might be different than nr_class_values.
316     */
317    int nr_classifier = model->nr_class;
318    if (model->nr_class == 2 && model->param.solver_type != MCSVM_CS) {
319        // model contains a single weight vector
320        nr_classifier = 1;
321    }
322
323    // Number of weight vectors to return.
324    int nr_orange_weights = nr_class_values;
325    if (nr_class_values == 2 && model->param.solver_type != MCSVM_CS) {
326        nr_orange_weights = 1;
327    }
328
329    assert(nr_orange_weights >= nr_classifier);
330
331    int nr_feature = model->nr_feature;
332
333    if (model->bias >= 0.0){
334        nr_feature++;
335    }
336
337    int* labels = new int[model->nr_class];
338    get_labels(model, labels);
339
340    // Initialize the weight matrix (nr_orange_weights x nr_features).
341    TFloatListList * weights = mlnew TFloatListList(nr_orange_weights);
342    for (int i = 0; i < nr_orange_weights; i++){
343        weights->at(i) = mlnew TFloatList(nr_feature, 0.0f);
344    }
345
346    if (nr_classifier > 1) {
347        /*
348         * NOTE: If some class was missing from the training data set
349         * (had no instances) its weight vector will be left initialized
350         * to 0
351         */
352        for (int i = 0; i < nr_classifier; i++) {
353            for (int j = 0; j < nr_feature; j++) {
354                weights->at(labels[i])->at(j) = model->w[j * nr_classifier + i];
355            }
356        }
357    } else {
358        for (int j = 0; j < nr_feature; j++) {
359            if (nr_orange_weights > 1) {
360                /* There were more than 2 orange class values. This means
361                 * there were no instances for one or more classed in the
362                 * training data set. We cannot simply leave the 'negative'
363                 * class vector as zero because we would lose information
364                 * which class was used (i.e. we could not make a proper
365                 * negative classification using the weights).
366                 */
367                weights->at(labels[0])->at(j) = model->w[j];
368                weights->at(labels[1])->at(j) = - model->w[j];
369            } else {
370                weights->at(0)->at(j) = model->w[j];
371            }
372        }
373    }
374
375    delete[] labels;
376
377    return weights;
378}
379
380
381TLinearLearner::TLinearLearner(){
382    solver_type = L2R_LR;
383    eps = 0.01f;
384    C = 1;
385    bias = -1.0;
386    set_print_string_function(&dont_print_string);
387}
388
389
390PClassifier TLinearLearner::operator()(PExampleGenerator examples, const int &weight){
391    parameter *param = new parameter;
392    param->solver_type = solver_type;
393    param->eps = eps;
394    param->C = C;
395    param->nr_weight = 0;
396    param->weight_label = NULL;
397    param->weight = NULL;
398
399    PDomain domain = examples->domain;
400
401    if (!domain->classVar)
402        raiseError("classVar expected");
403
404    if (domain->classVar->varType != TValue::INTVAR)
405        raiseError("Discrete class expected");
406
407    // Shallow copy of examples.
408    PExampleTable train_data = mlnew TExampleTable(examples, /* owns= */ false);
409
410    /*
411     * Sort the training instances by class.
412     * This is necessary because LIBLINEAR's class/label/weight order
413     * is defined by the order of labels encountered in the training
414     * data. By sorting we make sure it matches classVar.values order.
415     */
416    vector<int> sort_column(domain->variables->size() - 1);
417    train_data->sort(sort_column);
418
419    problem *prob = problemFromExamples(train_data, bias);
420
421    const char * error_msg = check_parameter(prob, param);
422    if (error_msg){
423        delete param;
424        destroy_problem(prob);
425        raiseError("LIBLINEAR error: %s", error_msg);
426    }
427    /* The solvers in liblinear use rand() function.
428     * To make the results reproducible we set the seed from the data table's
429     * crc.
430     */
431    srand(train_data->checkSum(false));
432
433    model *model = train(prob, param);
434    destroy_problem(prob);
435
436    return PClassifier(mlnew TLinearClassifier(domain, model));
437}
438
439
440/*
441 * Construct a TLinearClassifer given a domain and a trained LIBLINEAR
442 * constructed model.
443 */
444
445TLinearClassifier::TLinearClassifier(PDomain domain, struct model * model) : TClassifierFD(domain) {
446    linmodel = model;
447    bias = model->bias;
448    dbias = model->bias;
449
450    computesProbabilities = check_probability_model(linmodel) != 0;
451
452    weights = extract_feature_weights(model, get_nr_values());
453}
454
455
456TLinearClassifier::~TLinearClassifier() {
457    if (linmodel)
458        free_and_destroy_model(&linmodel);
459}
460
461/* Return the number of discrete class values, or raise an error
462 * if the class_var is not discrete.
463 */
464int TLinearClassifier::get_nr_values()
465{
466    int nr_values = 0;
467    TEnumVariable * enum_var = NULL;
468    enum_var = dynamic_cast<TEnumVariable*>(classVar.getUnwrappedPtr());
469    if (enum_var) {
470        nr_values = enum_var->noOfValues();
471    }
472    else {
473        raiseError("Discrete class expected.");
474    }
475    return nr_values;
476}
477
478PDistribution TLinearClassifier::classDistribution(const TExample &example){
479    TExample new_example(domain, example);
480    int numClass = get_nr_class(linmodel);
481
482    feature_node *x = feature_nodeFromExample(new_example, bias);
483
484    int *labels = new int [numClass];
485    get_labels(linmodel, labels);
486
487    double *prob_est = new double [numClass];
488    predict_probability(linmodel, x, prob_est);
489
490    PDistribution dist = TDistribution::create(classVar);
491    for (int i=0; i<numClass; i++)
492        dist->setint(labels[i], prob_est[i]);
493
494    delete[] x;
495    delete[] labels;
496    delete[] prob_est;
497    return dist;
498}
499
500TValue TLinearClassifier::operator () (const TExample &example){
501    TExample new_example(domain, example);
502    int numClass = get_nr_class(linmodel);
503
504    feature_node *x = feature_nodeFromExample(new_example, bias);
505
506    int predict_label = predict(linmodel, x);
507    delete[] x;
508    return TValue(predict_label);
509}
Note: See TracBrowser for help on using the repository browser.