diff --git a/internal/ceres/schur_jacobi_preconditioner.cc b/internal/ceres/schur_jacobi_preconditioner.cc
index 338df71..5b87e9f 100644
--- a/internal/ceres/schur_jacobi_preconditioner.cc
+++ b/internal/ceres/schur_jacobi_preconditioner.cc
@@ -33,7 +33,7 @@
 #include <utility>
 #include <vector>
 #include "Eigen/Dense"
-#include "ceres/block_random_access_sparse_matrix.h"
+#include "ceres/block_random_access_diagonal_matrix.h"
 #include "ceres/block_sparse_matrix.h"
 #include "ceres/collections_port.h"
 #include "ceres/detect_structure.h"
@@ -57,16 +57,11 @@
       << "SCHUR_JACOBI preconditioner.";
 
   block_size_.resize(num_blocks);
-  set<pair<int, int> > block_pairs;
-
-  int num_block_diagonal_entries = 0;
   for (int i = 0; i < num_blocks; ++i) {
     block_size_[i] = bs.cols[i + options_.elimination_groups[0]].size;
-    block_pairs.insert(make_pair(i, i));
-    num_block_diagonal_entries += block_size_[i] * block_size_[i];
   }
 
-  m_.reset(new BlockRandomAccessSparseMatrix(block_size_, block_pairs));
+  m_.reset(new BlockRandomAccessDiagonalMatrix(block_size_));
   InitEliminator(bs);
 }
 
@@ -118,7 +113,7 @@
   CHECK_NOTNULL(y);
 
   const double* lhs_values =
-      down_cast<BlockRandomAccessSparseMatrix*>(m_.get())->matrix()->values();
+      down_cast<BlockRandomAccessDiagonalMatrix*>(m_.get())->matrix()->values();
 
   // This loop can be easily multi-threaded with OpenMP if need be.
   for (int i = 0; i < block_size_.size(); ++i) {