diff --git a/internal/ceres/block_random_access_diagonal_matrix.cc b/internal/ceres/block_random_access_diagonal_matrix.cc
index 357fc31..643dbf1 100644
--- a/internal/ceres/block_random_access_diagonal_matrix.cc
+++ b/internal/ceres/block_random_access_diagonal_matrix.cc
@@ -52,10 +52,10 @@
     : context_(context), num_threads_(num_threads) {
   m_ = CompressedRowSparseMatrix::CreateBlockDiagonalMatrix(nullptr, blocks);
   double* values = m_->mutable_values();
-  layout_.reserve(blocks.size());
-  for (auto& block : blocks) {
-    layout_.emplace_back(std::make_unique<CellInfo>(values));
-    values += block.size * block.size;
+  layout_ = std::make_unique<CellInfo[]>(blocks.size());
+  for (int i = 0; i < blocks.size(); ++i) {
+    layout_[i].values = values;
+    values += blocks[i].size * blocks[i].size;
   }
 }
 
@@ -77,7 +77,7 @@
   *col = 0;
   *row_stride = stride;
   *col_stride = stride;
-  return layout_[row_block_id].get();
+  return &layout_[row_block_id];
 }
 
 // Assume that the user does not hold any locks on any cell blocks
@@ -91,9 +91,9 @@
   auto& blocks = m_->row_blocks();
   const int num_blocks = blocks.size();
   ParallelFor(context_, 0, num_blocks, num_threads_, [this, blocks](int i) {
-    auto* cell_info = layout_[i].get();
+    auto& cell_info = layout_[i];
     auto& block = blocks[i];
-    MatrixRef b(cell_info->values, block.size, block.size);
+    MatrixRef b(cell_info.values, block.size, block.size);
     b = b.selfadjointView<Eigen::Upper>().llt().solve(
         Matrix::Identity(block.size, block.size));
   });
@@ -107,9 +107,9 @@
   const int num_blocks = blocks.size();
   ParallelFor(
       context_, 0, num_blocks, num_threads_, [this, blocks, x, y](int i) {
-        auto* cell_info = layout_[i].get();
+        auto& cell_info = layout_[i];
         auto& block = blocks[i];
-        ConstMatrixRef b(cell_info->values, block.size, block.size);
+        ConstMatrixRef b(cell_info.values, block.size, block.size);
         VectorRef(y + block.position, block.size).noalias() +=
             b * ConstVectorRef(x + block.position, block.size);
       });
diff --git a/internal/ceres/block_random_access_diagonal_matrix.h b/internal/ceres/block_random_access_diagonal_matrix.h
index e5f50b0..9671f3e 100644
--- a/internal/ceres/block_random_access_diagonal_matrix.h
+++ b/internal/ceres/block_random_access_diagonal_matrix.h
@@ -33,7 +33,6 @@
 
 #include <memory>
 #include <utility>
-#include <vector>
 
 #include "ceres/block_random_access_matrix.h"
 #include "ceres/block_structure.h"
@@ -85,7 +84,7 @@
   ContextImpl* context_ = nullptr;
   const int num_threads_ = 1;
   std::unique_ptr<CompressedRowSparseMatrix> m_;
-  std::vector<std::unique_ptr<CellInfo>> layout_;
+  std::unique_ptr<CellInfo[]> layout_;
 };
 
 }  // namespace ceres::internal
diff --git a/internal/ceres/block_random_access_sparse_matrix.cc b/internal/ceres/block_random_access_sparse_matrix.cc
index 271544e..b9f8b36 100644
--- a/internal/ceres/block_random_access_sparse_matrix.cc
+++ b/internal/ceres/block_random_access_sparse_matrix.cc
@@ -91,8 +91,7 @@
     for (auto& c : cells) {
       const int col_block_id = c.block_id;
       double* const data = values + c.position;
-      layout_[IntPairToInt64(row_block_id, col_block_id)] =
-          std::make_unique<CellInfo>(data);
+      layout_.emplace(IntPairToInt64(row_block_id, col_block_id), data);
     }
   }
 }
@@ -113,7 +112,7 @@
   *col = 0;
   *row_stride = blocks_[row_block_id].size;
   *col_stride = blocks_[col_block_id].size;
-  return it->second.get();
+  return &it->second;
 }
 
 // Assume that the user does not hold any locks on any cell blocks
diff --git a/internal/ceres/block_random_access_sparse_matrix.h b/internal/ceres/block_random_access_sparse_matrix.h
index c509a01..5121832 100644
--- a/internal/ceres/block_random_access_sparse_matrix.h
+++ b/internal/ceres/block_random_access_sparse_matrix.h
@@ -113,8 +113,8 @@
   const int num_threads_ = 1;
 
   // A mapping from <row_block_id, col_block_id> to the position in
-  // the values array of tsm_ where the block is stored.
-  using LayoutType = std::unordered_map<int64_t, std::unique_ptr<CellInfo>>;
+  // the values array of bsm_ where the block is stored.
+  using LayoutType = std::unordered_map<std::int64_t, CellInfo>;
   LayoutType layout_;
 
   // The underlying matrix object which actually stores the cells.