Try some levenberg marquardt stuff

_code/pset_07/cpp/levenberg_marquardt.cpp

@@ -48,7 +48,7 @@ int main() {
 
     double a = 1;
     double b = 1;
-    double lambda = 1e-3;
+    double lambda = 10;
     constexpr bool vary_lambda = false;
     double error = (values - sin(a + b * samples)).matrix().squaredNorm();
     double new_error;
@@ -58,20 +58,24 @@ int main() {
     Eigen::Vector2d step;
     std::cout << "error: " << error << '\n';
 
-    for (uint32_t i = 0; i < 100; ++i) {
+    for (uint32_t i = 0; i < 1000; ++i) {
+        // compute gradient
         tmp = (values - sin(a + b * samples)) * cos(a + b * samples);
         grad[0] = -2 * tmp.sum();
         grad[1] = -2 * (samples * tmp).sum();
+
+        // compute levenberg marquardt matrix (based on hessian)
         tmp = values * sin(a + b * samples) + cos(2 * (a + b * samples));
-        mat(0, 0) = tmp.sum() * (1 + lambda);
+        mat(0, 0) = tmp.sum() + lambda;
         mat(0, 1) = (samples * tmp).sum();
         mat(1, 0) = mat(0, 1);
-        mat(1, 1) = (samples * samples * tmp).sum() * (1 + lambda);
+        mat(1, 1) = (samples * samples * tmp).sum() + lambda;
+
+        // compute the step
+        step = -mat.inverse() * grad; // Levenberg-Marquardt
+        //step = -lambda * grad;      // gradient descent
 
-        // Levenber-Marquardt
-        //step = -mat.inverse() * grad;
-        // gradient descent
-        step = -lambda * grad;
+        // apply the step
         a += step[0];
         b += step[1];
         new_error = (values - sin(a + b * samples)).matrix().squaredNorm();
@@ -85,8 +89,9 @@ int main() {
         }
         error = new_error;
 
-        std::cout << "error: " << error << '\n';
+        //std::cout << "error: " << error << '\n';
     }
 
+    std::cout << "error: " << error << '\n';
     std::cout << "a = " << a << ", b = " << b << '\n';
 }