diff --git a/internal/ceres/CMakeLists.txt b/internal/ceres/CMakeLists.txt
index 45df9be..b1a7d52 100644
--- a/internal/ceres/CMakeLists.txt
+++ b/internal/ceres/CMakeLists.txt
@@ -33,6 +33,7 @@
     block_evaluate_preparer.cc
     block_jacobi_preconditioner.cc
     block_jacobian_writer.cc
+    block_random_access_crs_matrix.cc
     block_random_access_dense_matrix.cc
     block_random_access_matrix.cc
     block_random_access_sparse_matrix.cc
@@ -244,6 +245,7 @@
   CERES_TEST(autodiff_cost_function)
   CERES_TEST(autodiff_local_parameterization)
   CERES_TEST(blas)
+  CERES_TEST(block_random_access_crs_matrix)
   CERES_TEST(block_random_access_dense_matrix)
   CERES_TEST(block_random_access_sparse_matrix)
   CERES_TEST(block_sparse_matrix)
diff --git a/internal/ceres/block_random_access_crs_matrix.cc b/internal/ceres/block_random_access_crs_matrix.cc
new file mode 100644
index 0000000..5b008e2
--- /dev/null
+++ b/internal/ceres/block_random_access_crs_matrix.cc
@@ -0,0 +1,170 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2013 Google Inc. All rights reserved.
+// http://code.google.com/p/ceres-solver/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: sameeragarwal@google.com (Sameer Agarwal)
+
+#include "ceres/block_random_access_crs_matrix.h"
+
+#include <algorithm>
+#include <set>
+#include <utility>
+#include <vector>
+#include "ceres/compressed_row_sparse_matrix.h"
+#include "ceres/internal/port.h"
+#include "ceres/internal/scoped_ptr.h"
+#include "ceres/mutex.h"
+#include "ceres/triplet_sparse_matrix.h"
+#include "ceres/types.h"
+#include "glog/logging.h"
+
+namespace ceres {
+namespace internal {
+
+BlockRandomAccessCRSMatrix::BlockRandomAccessCRSMatrix(
+    const vector<int>& blocks,
+    const set<pair<int, int> >& block_pairs)
+    : kMaxRowBlocks(10 * 1000 * 1000),
+      blocks_(blocks) {
+  CHECK_LT(blocks.size(), kMaxRowBlocks);
+
+  col_layout_.resize(blocks_.size(), 0);
+  row_strides_.resize(blocks_.size(), 0);
+
+  // Build the row/column layout vector and count the number of scalar
+  // rows/columns.
+  int num_cols = 0;
+  for (int i = 0; i < blocks_.size(); ++i) {
+    col_layout_[i] = num_cols;
+    num_cols += blocks_[i];
+  }
+
+  // Walk the sparsity pattern and count the number of non-zeros.
+  int num_nonzeros = 0;
+  for (set<pair<int, int> >::const_iterator it = block_pairs.begin();
+       it != block_pairs.end();
+       ++it) {
+    const int row_block_size = blocks_[it->first];
+    const int col_block_size = blocks_[it->second];
+    num_nonzeros += row_block_size * col_block_size;
+  }
+
+  VLOG(2) << "Matrix Size [" << num_cols
+          << "," << num_cols
+          << "] " << num_nonzeros;
+
+  crsm_.reset(new CompressedRowSparseMatrix(num_cols, num_cols, num_nonzeros));
+  int* rows = crsm_->mutable_rows();
+  int* cols = crsm_->mutable_cols();
+  double* values = crsm_->mutable_values();
+
+  // Iterate over the sparsity pattern and fill the scalar sparsity
+  // pattern of the underlying compressed sparse row matrix. Along the
+  // way also fill out the Layout object which will allow random
+  // access into the CRS Matrix.
+  set<pair<int, int> >::const_iterator it = block_pairs.begin();
+  vector<int> col_blocks;
+  int row_pos = 0;
+  rows[0] = 0;
+  while (it != block_pairs.end()) {
+    // Add entries to layout_ for all the blocks for this row.
+    col_blocks.clear();
+    const int row_block_id = it->first;
+    const int row_block_size = blocks_[row_block_id];
+    int num_cols = 0;
+    while ((it != block_pairs.end()) && (it->first == row_block_id)) {
+      layout_[IntPairToLong(it->first, it->second)] =
+          new CellInfo(values + num_cols);
+      col_blocks.push_back(it->second);
+      num_cols += blocks_[it->second];
+      ++it;
+    };
+
+    // Count the number of non-zeros in the row block.
+    for (int j = 0; j < row_block_size; ++j) {
+      rows[row_pos + j + 1] = rows[row_pos + j] + num_cols;
+    }
+
+    // Fill out the sparsity pattern for each row.
+    int col_pos = 0;
+    for (int j = 0; j < col_blocks.size(); ++j) {
+      const int col_block_id = col_blocks[j];
+      const int col_block_size = blocks_[col_block_id];
+      for (int r = 0; r < row_block_size; ++r) {
+        const int column_block_begin = rows[row_pos + r] + col_pos;
+        for (int c = 0; c < col_block_size; ++c) {
+          cols[column_block_begin + c] = col_layout_[col_block_id] + c;
+        }
+      }
+      col_pos += col_block_size;
+    }
+
+    row_pos += row_block_size;
+    values += row_block_size * num_cols;
+    row_strides_[row_block_id] = num_cols;
+  }
+}
+
+// Assume that the user does not hold any locks on any cell blocks
+// when they are calling SetZero.
+BlockRandomAccessCRSMatrix::~BlockRandomAccessCRSMatrix() {
+  // TODO(sameeragarwal) this should be rationalized going forward and
+  // perhaps moved into BlockRandomAccessMatrix.
+  for (LayoutType::iterator it = layout_.begin();
+       it != layout_.end();
+       ++it) {
+    delete it->second;
+  }
+}
+
+CellInfo* BlockRandomAccessCRSMatrix::GetCell(int row_block_id,
+                                              int col_block_id,
+                                              int* row,
+                                              int* col,
+                                              int* row_stride,
+                                              int* col_stride) {
+  const LayoutType::iterator it  =
+      layout_.find(IntPairToLong(row_block_id, col_block_id));
+  if (it == layout_.end()) {
+    return NULL;
+  }
+
+  *row = 0;
+  *col = 0;
+  *row_stride = blocks_[row_block_id];
+  *col_stride = row_strides_[row_block_id];
+  return it->second;
+}
+
+// Assume that the user does not hold any locks on any cell blocks
+// when they are calling SetZero.
+void BlockRandomAccessCRSMatrix::SetZero() {
+  crsm_->SetZero();
+}
+
+}  // namespace internal
+}  // namespace ceres
diff --git a/internal/ceres/block_random_access_crs_matrix.h b/internal/ceres/block_random_access_crs_matrix.h
new file mode 100644
index 0000000..11a203b
--- /dev/null
+++ b/internal/ceres/block_random_access_crs_matrix.h
@@ -0,0 +1,108 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2013 Google Inc. All rights reserved.
+// http://code.google.com/p/ceres-solver/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: sameeragarwal@google.com (Sameer Agarwal)
+
+#ifndef CERES_INTERNAL_BLOCK_RANDOM_ACCESS_CRS_MATRIX_H_
+#define CERES_INTERNAL_BLOCK_RANDOM_ACCESS_CRS_MATRIX_H_
+
+#include <set>
+#include <vector>
+#include <utility>
+#include "ceres/mutex.h"
+#include "ceres/block_random_access_matrix.h"
+#include "ceres/compressed_row_sparse_matrix.h"
+#include "ceres/collections_port.h"
+#include "ceres/integral_types.h"
+#include "ceres/internal/macros.h"
+#include "ceres/internal/port.h"
+#include "ceres/internal/scoped_ptr.h"
+#include "ceres/types.h"
+
+namespace ceres {
+namespace internal {
+
+// A square BlockRandomAccessMatrix where the underlying storage is a
+// compressed row sparse matrix. The matrix need not be symmetric.
+class BlockRandomAccessCRSMatrix : public BlockRandomAccessMatrix {
+ public:
+  // blocks is an array of block sizes. block_pairs is a set of
+  // <row_block_id, col_block_id> pairs to identify the non-zero cells
+  // of this matrix.
+  BlockRandomAccessCRSMatrix(const vector<int>& blocks,
+                             const set<pair<int, int> >& block_pairs);
+
+  // The destructor is not thread safe. It assumes that no one is
+  // modifying any cells when the matrix is being destroyed.
+  virtual ~BlockRandomAccessCRSMatrix();
+
+  // BlockRandomAccessMatrix Interface.
+  virtual CellInfo* GetCell(int row_block_id,
+                            int col_block_id,
+                            int* row,
+                            int* col,
+                            int* row_stride,
+                            int* col_stride);
+
+  // This is not a thread safe method, it assumes that no cell is
+  // locked.
+  virtual void SetZero();
+
+  // Since the matrix is square, num_rows() == num_cols().
+  virtual int num_rows() const { return crsm_->num_rows(); }
+  virtual int num_cols() const { return crsm_->num_cols(); }
+
+    // Access to the underlying matrix object.
+  const CompressedRowSparseMatrix* matrix() const { return crsm_.get(); }
+  CompressedRowSparseMatrix* mutable_matrix() { return crsm_.get(); }
+
+ private:
+  int64 IntPairToLong(int a, int b) {
+    return a * kMaxRowBlocks + b;
+  }
+
+  const int64 kMaxRowBlocks;
+  // row/column block sizes.
+  const vector<int> blocks_;
+  vector<int> col_layout_;
+  vector<int> row_strides_;
+
+  // A mapping from <row_block_id, col_block_id> to the position in
+  // the values array of tsm_ where the block is stored.
+  typedef HashMap<long int, CellInfo* > LayoutType;
+  LayoutType layout_;
+
+  scoped_ptr<CompressedRowSparseMatrix> crsm_;
+  friend class BlockRandomAccessCRSMatrixTest;
+  CERES_DISALLOW_COPY_AND_ASSIGN(BlockRandomAccessCRSMatrix);
+};
+
+}  // namespace internal
+}  // namespace ceres
+
+#endif  // CERES_INTERNAL_BLOCK_RANDOM_ACCESS_CRS_MATRIX_H_
diff --git a/internal/ceres/block_random_access_crs_matrix_test.cc b/internal/ceres/block_random_access_crs_matrix_test.cc
new file mode 100644
index 0000000..1266c4f
--- /dev/null
+++ b/internal/ceres/block_random_access_crs_matrix_test.cc
@@ -0,0 +1,148 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2013 Google Inc. All rights reserved.
+// http://code.google.com/p/ceres-solver/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: sameeragarwal@google.com (Sameer Agarwal)
+
+#include <limits>
+#include <vector>
+#include "ceres/block_random_access_crs_matrix.h"
+#include "ceres/internal/eigen.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+namespace ceres {
+namespace internal {
+
+TEST(BlockRandomAccessCRSMatrix, GetCell) {
+  vector<int> blocks;
+  blocks.push_back(3);
+  blocks.push_back(4);
+  blocks.push_back(5);
+  const int num_rows = 3 + 4 + 5;
+
+  set< pair<int, int> > block_pairs;
+  int num_nonzeros = 0;
+  block_pairs.insert(make_pair(0, 0));
+  num_nonzeros += blocks[0] * blocks[0];
+
+  block_pairs.insert(make_pair(1, 1));
+  num_nonzeros += blocks[1] * blocks[1];
+
+  block_pairs.insert(make_pair(1, 2));
+  num_nonzeros += blocks[1] * blocks[2];
+
+  block_pairs.insert(make_pair(2, 0));
+  num_nonzeros += blocks[2] * blocks[0];
+
+  BlockRandomAccessCRSMatrix m(blocks, block_pairs);
+  EXPECT_EQ(m.num_rows(), num_rows);
+  EXPECT_EQ(m.num_cols(), num_rows);
+
+  for (set<pair<int, int> >::const_iterator it = block_pairs.begin();
+       it != block_pairs.end();
+       ++it) {
+    const int row_block_id = it->first;
+    const int col_block_id = it->second;
+    int row;
+    int col;
+    int row_stride;
+    int col_stride;
+    CellInfo* cell =  m.GetCell(row_block_id, col_block_id,
+                                &row, &col,
+                                &row_stride, &col_stride);
+    EXPECT_TRUE(cell != NULL);
+    EXPECT_EQ(row, 0);
+    EXPECT_EQ(col, 0);
+
+    // Write into the block.
+    MatrixRef(cell->values, row_stride, col_stride).block(
+        row, col, blocks[row_block_id], blocks[col_block_id]) =
+        (row_block_id + 1) * (col_block_id +1) *
+        Matrix::Ones(blocks[row_block_id], blocks[col_block_id]);
+  }
+
+  const CompressedRowSparseMatrix* crsm = m.matrix();
+  EXPECT_EQ(crsm->num_nonzeros(), num_nonzeros);
+
+  Matrix dense;
+  crsm->ToDenseMatrix(&dense);
+
+  double kTolerance = 1e-14;
+
+  // (0,0)
+  EXPECT_NEAR((dense.block(0, 0, 3, 3) - Matrix::Ones(3, 3)).norm(),
+              0.0,
+              kTolerance);
+  // (1,1)
+  EXPECT_NEAR((dense.block(3, 3, 4, 4) - 2 * 2 * Matrix::Ones(4, 4)).norm(),
+              0.0,
+              kTolerance);
+  // (1,2)
+  EXPECT_NEAR((dense.block(3, 3 + 4, 4, 5) - 2 * 3 * Matrix::Ones(4, 5)).norm(),
+              0.0,
+              kTolerance);
+  // (2,0)
+  EXPECT_NEAR((dense.block(3 + 4, 0, 5, 3) - 3 * 1 * Matrix::Ones(5, 3)).norm(),
+              0.0,
+              kTolerance);
+
+  // There is nothing else in the matrix besides these four blocks.
+  EXPECT_NEAR(dense.norm(), sqrt(9. + 16. * 16. + 36. * 20. + 9. * 15.),
+              kTolerance);
+}
+
+// IntPairToLong is private, thus this fixture is needed to access and
+// test it.
+class BlockRandomAccessCRSMatrixTest : public ::testing::Test {
+ public:
+  virtual void SetUp() {
+    vector<int> blocks;
+    blocks.push_back(1);
+    set< pair<int, int> > block_pairs;
+    block_pairs.insert(make_pair(0, 0));
+    m_.reset(new BlockRandomAccessCRSMatrix(blocks, block_pairs));
+  }
+
+  void CheckIntPair(int a, int b) {
+    int64 value = m_->IntPairToLong(a, b);
+    EXPECT_GT(value, 0) << "Overflow a = " << a << " b = " << b;
+    EXPECT_GT(value, a) << "Overflow a = " << a << " b = " << b;
+    EXPECT_GT(value, b) << "Overflow a = " << a << " b = " << b;
+  }
+
+ private:
+  scoped_ptr<BlockRandomAccessCRSMatrix> m_;
+};
+
+TEST_F(BlockRandomAccessCRSMatrixTest, IntPairToLongOverflow) {
+  CheckIntPair(numeric_limits<int>::max(), numeric_limits<int>::max());
+}
+
+
+}  // namespace internal
+}  // namespace ceres
diff --git a/internal/ceres/block_random_access_sparse_matrix.h b/internal/ceres/block_random_access_sparse_matrix.h
index 48a0043..a6b5f39 100644
--- a/internal/ceres/block_random_access_sparse_matrix.h
+++ b/internal/ceres/block_random_access_sparse_matrix.h
@@ -74,7 +74,6 @@
   // This is not a thread safe method, it assumes that no cell is
   // locked.
   virtual void SetZero();
-  virtual bool IsThreadSafe() const { return true; }
 
   // Since the matrix is square, num_rows() == num_cols().
   virtual int num_rows() const { return tsm_->num_rows(); }