diff --git a/internal/ceres/CMakeLists.txt b/internal/ceres/CMakeLists.txt
index eee8abf..02a1731 100644
--- a/internal/ceres/CMakeLists.txt
+++ b/internal/ceres/CMakeLists.txt
@@ -36,6 +36,7 @@
     block_jacobian_writer.cc
     block_random_access_crs_matrix.cc
     block_random_access_dense_matrix.cc
+    block_random_access_diagonal_matrix.cc
     block_random_access_matrix.cc
     block_random_access_sparse_matrix.cc
     block_sparse_matrix.cc
@@ -205,6 +206,7 @@
   CERES_TEST(autodiff_local_parameterization)
   CERES_TEST(block_random_access_crs_matrix)
   CERES_TEST(block_random_access_dense_matrix)
+  CERES_TEST(block_random_access_diagonal_matrix)
   CERES_TEST(block_random_access_sparse_matrix)
   CERES_TEST(block_sparse_matrix)
   CERES_TEST(c_api)
diff --git a/internal/ceres/block_random_access_diagonal_matrix.cc b/internal/ceres/block_random_access_diagonal_matrix.cc
new file mode 100644
index 0000000..d8bf4ef
--- /dev/null
+++ b/internal/ceres/block_random_access_diagonal_matrix.cc
@@ -0,0 +1,120 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2013 Google Inc. All rights reserved.
+// http://code.google.com/p/ceres-solver/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: sameeragarwal@google.com (Sameer Agarwal)
+
+#include "ceres/block_random_access_diagonal_matrix.h"
+
+#include <algorithm>
+#include <set>
+#include <utility>
+#include <vector>
+#include "ceres/internal/port.h"
+#include "ceres/internal/scoped_ptr.h"
+#include "ceres/triplet_sparse_matrix.h"
+#include "ceres/types.h"
+#include "ceres/stl_util.h"
+#include "glog/logging.h"
+
+namespace ceres {
+namespace internal {
+
+BlockRandomAccessDiagonalMatrix::BlockRandomAccessDiagonalMatrix(
+    const vector<int>& blocks)
+    : blocks_(blocks) {
+  // Build the row/column layout vector and count the number of scalar
+  // rows/columns.
+  int num_cols = 0;
+  int num_nonzeros = 0;
+  vector<int> col_layout;
+  for (int i = 0; i < blocks_.size(); ++i) {
+    col_layout.push_back(num_cols);
+    num_cols += blocks_[i];
+    num_nonzeros += blocks_[i] * blocks_[i];
+  }
+
+  VLOG(1) << "Matrix Size [" << num_cols
+          << "," << num_cols
+          << "] " << num_nonzeros;
+
+  tsm_.reset(new TripletSparseMatrix(num_cols, num_cols, num_nonzeros));
+  tsm_->set_num_nonzeros(num_nonzeros);
+  int* rows = tsm_->mutable_rows();
+  int* cols = tsm_->mutable_cols();
+  double* values = tsm_->mutable_values();
+
+  int pos = 0;
+  for (int i = 0; i < blocks_.size(); ++i) {
+    const int block_size = blocks_[i];
+    layout_.push_back(new CellInfo(values + pos));
+    const int block_begin = col_layout[i];
+    for (int r = 0; r < block_size; ++r) {
+      for (int c = 0; c < block_size; ++c, ++pos) {
+        rows[pos] = block_begin + r;
+        cols[pos] = block_begin + c;
+      }
+    }
+  }
+}
+
+// Assume that the user does not hold any locks on any cell blocks
+// when they are calling SetZero.
+BlockRandomAccessDiagonalMatrix::~BlockRandomAccessDiagonalMatrix() {
+  STLDeleteContainerPointers(layout_.begin(), layout_.end());
+}
+
+CellInfo* BlockRandomAccessDiagonalMatrix::GetCell(int row_block_id,
+                                                   int col_block_id,
+                                                   int* row,
+                                                   int* col,
+                                                   int* row_stride,
+                                                   int* col_stride) {
+  if (row_block_id != col_block_id) {
+    return NULL;
+  }
+  const int stride = blocks_[row_block_id];
+
+  // Each cell is stored contiguously as its own little dense matrix.
+  *row = 0;
+  *col = 0;
+  *row_stride = stride;
+  *col_stride = stride;
+  return layout_[row_block_id];
+}
+
+// Assume that the user does not hold any locks on any cell blocks
+// when they are calling SetZero.
+void BlockRandomAccessDiagonalMatrix::SetZero() {
+  if (tsm_->num_nonzeros()) {
+    VectorRef(tsm_->mutable_values(),
+              tsm_->num_nonzeros()).setZero();
+  }
+}
+
+}  // namespace internal
+}  // namespace ceres
diff --git a/internal/ceres/block_random_access_diagonal_matrix.h b/internal/ceres/block_random_access_diagonal_matrix.h
new file mode 100644
index 0000000..6b3cff2
--- /dev/null
+++ b/internal/ceres/block_random_access_diagonal_matrix.h
@@ -0,0 +1,96 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2013 Google Inc. All rights reserved.
+// http://code.google.com/p/ceres-solver/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: sameeragarwal@google.com (Sameer Agarwal)
+
+#ifndef CERES_INTERNAL_BLOCK_RANDOM_ACCESS_DIAGONAL_MATRIX_H_
+#define CERES_INTERNAL_BLOCK_RANDOM_ACCESS_DIAGONAL_MATRIX_H_
+
+#include <set>
+#include <vector>
+#include <utility>
+#include "ceres/mutex.h"
+#include "ceres/block_random_access_matrix.h"
+#include "ceres/collections_port.h"
+#include "ceres/triplet_sparse_matrix.h"
+#include "ceres/integral_types.h"
+#include "ceres/internal/macros.h"
+#include "ceres/internal/port.h"
+#include "ceres/internal/scoped_ptr.h"
+#include "ceres/types.h"
+
+namespace ceres {
+namespace internal {
+
+// A thread safe block diagonal matrix implementation of
+// BlockRandomAccessMatrix.
+class BlockRandomAccessDiagonalMatrix : public BlockRandomAccessMatrix {
+ public:
+  // blocks is an array of block sizes.
+  BlockRandomAccessDiagonalMatrix(const vector<int>& blocks);
+
+  // The destructor is not thread safe. It assumes that no one is
+  // modifying any cells when the matrix is being destroyed.
+  virtual ~BlockRandomAccessDiagonalMatrix();
+
+  // BlockRandomAccessMatrix Interface.
+  virtual CellInfo* GetCell(int row_block_id,
+                            int col_block_id,
+                            int* row,
+                            int* col,
+                            int* row_stride,
+                            int* col_stride);
+
+  // This is not a thread safe method, it assumes that no cell is
+  // locked.
+  virtual void SetZero();
+
+  // Since the matrix is square, num_rows() == num_cols().
+  virtual int num_rows() const { return tsm_->num_rows(); }
+  virtual int num_cols() const { return tsm_->num_cols(); }
+
+  // Access to the underlying matrix object.
+  const TripletSparseMatrix* matrix() const { return tsm_.get(); }
+  TripletSparseMatrix* mutable_matrix() { return tsm_.get(); }
+
+ private:
+  // row/column block sizes.
+  const vector<int> blocks_;
+  vector<CellInfo*> layout_;
+
+  // The underlying matrix object which actually stores the cells.
+  scoped_ptr<TripletSparseMatrix> tsm_;
+
+  friend class BlockRandomAccessDiagonalMatrixTest;
+  CERES_DISALLOW_COPY_AND_ASSIGN(BlockRandomAccessDiagonalMatrix);
+};
+
+}  // namespace internal
+}  // namespace ceres
+
+#endif  // CERES_INTERNAL_BLOCK_RANDOM_ACCESS_DIAGONAL_MATRIX_H_
diff --git a/internal/ceres/block_random_access_diagonal_matrix_test.cc b/internal/ceres/block_random_access_diagonal_matrix_test.cc
new file mode 100644
index 0000000..e19268b
--- /dev/null
+++ b/internal/ceres/block_random_access_diagonal_matrix_test.cc
@@ -0,0 +1,116 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2013 Google Inc. All rights reserved.
+// http://code.google.com/p/ceres-solver/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: sameeragarwal@google.com (Sameer Agarwal)
+
+#include <limits>
+#include <vector>
+
+#include "ceres/block_random_access_diagonal_matrix.h"
+#include "ceres/internal/eigen.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+namespace ceres {
+namespace internal {
+
+TEST(BlockRandomAccessDiagonalMatrix, GetCell) {
+  vector<int> blocks;
+  blocks.push_back(3);
+  blocks.push_back(4);
+  blocks.push_back(5);
+  const int num_rows = 3 + 4 + 5;
+  const int num_nonzeros =  3 * 3 + 4 * 4 + 5 * 5;
+
+  BlockRandomAccessDiagonalMatrix m(blocks);
+  EXPECT_EQ(m.num_rows(), num_rows);
+  EXPECT_EQ(m.num_cols(), num_rows);
+
+  for (int i = 0; i < blocks.size(); ++i) {
+    const int row_block_id = i;
+    int col_block_id;
+    int row;
+    int col;
+    int row_stride;
+    int col_stride;
+
+    for (int j = 0; j < blocks.size(); ++j) {
+      col_block_id = j;
+      CellInfo* cell =  m.GetCell(row_block_id, col_block_id,
+                                  &row, &col,
+                                  &row_stride, &col_stride);
+      // Off diagonal entries are not present.
+      if (i != j) {
+        EXPECT_TRUE(cell == NULL);
+        continue;
+      }
+
+      EXPECT_TRUE(cell != NULL);
+      EXPECT_EQ(row, 0);
+      EXPECT_EQ(col, 0);
+      EXPECT_EQ(row_stride, blocks[row_block_id]);
+      EXPECT_EQ(col_stride, blocks[col_block_id]);
+
+      // Write into the block
+      MatrixRef(cell->values, row_stride, col_stride).block(
+          row, col, blocks[row_block_id], blocks[col_block_id]) =
+          (row_block_id + 1) * (col_block_id +1) *
+          Matrix::Ones(blocks[row_block_id], blocks[col_block_id]);
+    }
+  }
+
+  const TripletSparseMatrix* tsm = m.matrix();
+  EXPECT_EQ(tsm->num_nonzeros(), num_nonzeros);
+  EXPECT_EQ(tsm->max_num_nonzeros(), num_nonzeros);
+
+  Matrix dense;
+  tsm->ToDenseMatrix(&dense);
+
+  double kTolerance = 1e-14;
+
+  // (0,0)
+  EXPECT_NEAR((dense.block(0, 0, 3, 3) - Matrix::Ones(3, 3)).norm(),
+              0.0,
+              kTolerance);
+
+  // (1,1)
+  EXPECT_NEAR((dense.block(3, 3, 4, 4) - 2 * 2 * Matrix::Ones(4, 4)).norm(),
+              0.0,
+              kTolerance);
+
+  // (1,1)
+  EXPECT_NEAR((dense.block(7, 7, 5, 5) - 3 * 3 * Matrix::Ones(5, 5)).norm(),
+              0.0,
+              kTolerance);
+
+  // There is nothing else in the matrix besides these four blocks.
+  EXPECT_NEAR(dense.norm(), sqrt(9.0 + 16. * 16. + 81.0 * 25.), kTolerance);
+}
+
+}  // namespace internal
+}  // namespace ceres
diff --git a/internal/ceres/block_random_access_sparse_matrix.h b/internal/ceres/block_random_access_sparse_matrix.h
index a6b5f39..27b1029 100644
--- a/internal/ceres/block_random_access_sparse_matrix.h
+++ b/internal/ceres/block_random_access_sparse_matrix.h
@@ -47,7 +47,7 @@
 namespace ceres {
 namespace internal {
 
-// A threaf safe square block sparse implementation of
+// A thread safe square block sparse implementation of
 // BlockRandomAccessMatrix. Internally a TripletSparseMatrix is used
 // for doing the actual storage. This class augments this matrix with
 // an unordered_map that allows random read/write access.
diff --git a/internal/ceres/schur_jacobi_preconditioner.cc b/internal/ceres/schur_jacobi_preconditioner.cc
index 338df71..5b87e9f 100644
--- a/internal/ceres/schur_jacobi_preconditioner.cc
+++ b/internal/ceres/schur_jacobi_preconditioner.cc
@@ -33,7 +33,7 @@
 #include <utility>
 #include <vector>
 #include "Eigen/Dense"
-#include "ceres/block_random_access_sparse_matrix.h"
+#include "ceres/block_random_access_diagonal_matrix.h"
 #include "ceres/block_sparse_matrix.h"
 #include "ceres/collections_port.h"
 #include "ceres/detect_structure.h"
@@ -57,16 +57,11 @@
       << "SCHUR_JACOBI preconditioner.";
 
   block_size_.resize(num_blocks);
-  set<pair<int, int> > block_pairs;
-
-  int num_block_diagonal_entries = 0;
   for (int i = 0; i < num_blocks; ++i) {
     block_size_[i] = bs.cols[i + options_.elimination_groups[0]].size;
-    block_pairs.insert(make_pair(i, i));
-    num_block_diagonal_entries += block_size_[i] * block_size_[i];
   }
 
-  m_.reset(new BlockRandomAccessSparseMatrix(block_size_, block_pairs));
+  m_.reset(new BlockRandomAccessDiagonalMatrix(block_size_));
   InitEliminator(bs);
 }
 
@@ -118,7 +113,7 @@
   CHECK_NOTNULL(y);
 
   const double* lhs_values =
-      down_cast<BlockRandomAccessSparseMatrix*>(m_.get())->matrix()->values();
+      down_cast<BlockRandomAccessDiagonalMatrix*>(m_.get())->matrix()->values();
 
   // This loop can be easily multi-threaded with OpenMP if need be.
   for (int i = 0; i < block_size_.size(); ++i) {
diff --git a/internal/ceres/schur_jacobi_preconditioner.h b/internal/ceres/schur_jacobi_preconditioner.h
index f6e7b0d..aecb015 100644
--- a/internal/ceres/schur_jacobi_preconditioner.h
+++ b/internal/ceres/schur_jacobi_preconditioner.h
@@ -49,7 +49,7 @@
 namespace ceres {
 namespace internal {
 
-class BlockRandomAccessSparseMatrix;
+class BlockRandomAccessDiagonalMatrix;
 class BlockSparseMatrix;
 struct CompressedRowBlockStructure;
 class SchurEliminatorBase;
@@ -100,7 +100,7 @@
   scoped_ptr<SchurEliminatorBase> eliminator_;
 
   // Preconditioner matrix.
-  scoped_ptr<BlockRandomAccessSparseMatrix> m_;
+  scoped_ptr<BlockRandomAccessDiagonalMatrix> m_;
   CERES_DISALLOW_COPY_AND_ASSIGN(SchurJacobiPreconditioner);
 };