lightning/examples/plot_l2_solvers.py at master · scikit-learn-contrib/lightning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
"""
=====================
L2 solver comparison
=====================

This example compares different solvers with L2 regularization.
"""
print(__doc__)

import sys
import time

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_classification
from sklearn.datasets import fetch_20newsgroups_vectorized

from lightning.classification import SVRGClassifier
from lightning.classification import SDCAClassifier
from lightning.classification import CDClassifier
from lightning.classification import AdaGradClassifier
from lightning.classification import SAGAClassifier, SAGClassifier

from lightning.impl.adagrad_fast import _proj_elastic_all

class Callback(object):

    def __init__(self, X, y):
        self.X = X
        self.y = y
        self.obj = []
        self.times = []
        self.start_time = time.time()
        self.test_time = 0

    def __call__(self, clf, t=None):
        test_time = time.time()

        if hasattr(clf, "_finalize_coef"):
            clf._finalize_coef()

        if t is not None:
            _proj_elastic_all(clf.eta, t, clf.g_sum_[0], clf.g_norms_[0],
                              alpha1=0, alpha2=clf.alpha, delta=0,
                              w=clf.coef_[0])


        y_pred = clf.decision_function(self.X).ravel()
        loss = (np.maximum(1 - self.y * y_pred, 0) ** 2).mean()
        coef = clf.coef_.ravel()
        regul = 0.5 * clf.alpha * np.dot(coef, coef)
        self.obj.append(loss + regul)
        self.test_time += time.time() - test_time
        self.times.append(time.time() -  self.start_time - self.test_time)

try:
    dataset = sys.argv[1]
except:
    dataset = "synthetic"

if dataset == "news20":
    bunch = fetch_20newsgroups_vectorized(subset="all")
    X = bunch.data
    y = bunch.target
    y[y >= 1] = 1
    alpha = 1e-4
    eta_svrg = 1e-1
    eta_adagrad = 1
    xlim = (0, 20)

else:
    X, y = make_classification(n_samples=10000,
                               n_features=100,
                               n_classes=2,
                               random_state=0)
    alpha = 1e-2
    eta_svrg = 1e-3
    eta_adagrad = 1e-2
    xlim = [0, 2]

y = y * 2 - 1

# make sure the method does not stop prematurely, we want to see
# the full convergence path
tol = 1e-24

clf1 = SVRGClassifier(loss="squared_hinge", alpha=alpha, eta=eta_svrg,
                      n_inner=1.0, max_iter=100, random_state=0, tol=1e-24)
clf2 = SDCAClassifier(loss="squared_hinge", alpha=alpha,
                      max_iter=100, n_calls=X.shape[0]/2, random_state=0, tol=tol)
clf3 = CDClassifier(loss="squared_hinge", alpha=alpha, C=1.0/X.shape[0],
                    max_iter=50, n_calls=X.shape[1]/3, random_state=0, tol=tol)
clf4 = AdaGradClassifier(loss="squared_hinge", alpha=alpha, eta=eta_adagrad,
                    n_iter=100, n_calls=X.shape[0]/2, random_state=0)
clf5 = SAGAClassifier(loss="squared_hinge", alpha=alpha,
                    max_iter=100, random_state=0, tol=tol)
clf6 = SAGClassifier(loss="squared_hinge", alpha=alpha,
                    max_iter=100, random_state=0, tol=tol)

plt.figure()

data = {}
for clf, name in ((clf1, "SVRG"),
                  (clf2, "SDCA"),
                  (clf3, "PCD"),
                  (clf4, "AdaGrad"),
                  (clf5, "SAGA"),
                  (clf6, "SAG")
                  ):
    print(name)
    cb = Callback(X, y)
    clf.callback = cb

    if name == "PCD" and hasattr(X, "tocsc"):
        clf.fit(X.tocsc(), y)
    else:
        clf.fit(X, y)
    data[name] = (cb.times, np.array(cb.obj))

# get best value
fmin = min([np.min(a[1]) for a in data.values()])
for name in data:
    plt.plot(data[name][0], data[name][1] - fmin, label=name, lw=3)

plt.xlim(xlim)
plt.yscale('log')
plt.xlabel("CPU time")
plt.ylabel("Objective value minus optimum")
plt.legend()
plt.grid()

plt.show()