diff --git a/internal/ceres/block_sparse_matrix.cc b/internal/ceres/block_sparse_matrix.cc
index 9195e9e..214daef 100644
--- a/internal/ceres/block_sparse_matrix.cc
+++ b/internal/ceres/block_sparse_matrix.cc
@@ -81,6 +81,7 @@
   VLOG(2) << "Allocating values array with "
           << num_nonzeros_ * sizeof(double) << " bytes.";  // NOLINT
   values_.reset(new double[num_nonzeros_]);
+  max_num_nonzeros_ = num_nonzeros_;
   CHECK_NOTNULL(values_.get());
 }
 
@@ -299,9 +300,12 @@
     }
   }
 
-  double* new_values = new double[num_nonzeros_];
-  std::copy(values_.get(), values_.get() + old_num_nonzeros, new_values);
-  values_.reset(new_values);
+  if (num_nonzeros_ > max_num_nonzeros_) {
+    double* new_values = new double[num_nonzeros_];
+    std::copy(values_.get(), values_.get() + old_num_nonzeros, new_values);
+    values_.reset(new_values);
+    max_num_nonzeros_ = num_nonzeros_;
+  }
 
   std::copy(m.values(),
             m.values() + m.num_nonzeros(),
diff --git a/internal/ceres/block_sparse_matrix.h b/internal/ceres/block_sparse_matrix.h
index 1b91a75..17491bf 100644
--- a/internal/ceres/block_sparse_matrix.h
+++ b/internal/ceres/block_sparse_matrix.h
@@ -131,6 +131,7 @@
   int num_rows_;
   int num_cols_;
   int num_nonzeros_;
+  int max_num_nonzeros_;
   scoped_array<double> values_;
   scoped_ptr<CompressedRowBlockStructure> block_structure_;
   CERES_DISALLOW_COPY_AND_ASSIGN(BlockSparseMatrix);