diff --git a/internal/ceres/problem_impl.cc b/internal/ceres/problem_impl.cc
index ae87fcb..09eed9e 100644
--- a/internal/ceres/problem_impl.cc
+++ b/internal/ceres/problem_impl.cc
@@ -638,6 +638,9 @@
     for (int i = 0; i < variable_parameter_blocks.size(); ++i) {
       variable_parameter_blocks[i]->SetVarying();
     }
+
+    program_->SetParameterBlockStatePtrsToUserStatePtrs();
+    program_->SetParameterOffsetsAndIndex();
     return false;
   }
 
@@ -696,6 +699,8 @@
     }
   }
 
+  program_->SetParameterBlockStatePtrsToUserStatePtrs();
+  program_->SetParameterOffsetsAndIndex();
   return status;
 }