Detect large Jacobians and return failure instead of crashing.

Detect when the number of non-zeros overflows when constructing
BlockSparseMatrix and CompressedRowSparseMatrix and return
with an error message instead of crashing.

Change-Id: I45e102f7c0519eef441ce0586b7adf96e4a954a9

internal/ceres/block_jacobian_writer.cc

diff --git a/internal/ceres/block_jacobian_writer.cc b/internal/ceres/block_jacobian_writer.cc
index b009e96..adeaf41 100644
--- a/internal/ceres/block_jacobian_writer.cc
+++ b/internal/ceres/block_jacobian_writer.cc
@@ -63,7 +63,7 @@
 //
 // TODO(keir): Consider if we should use a boolean for each parameter block
 // instead of num_eliminate_blocks.
-void BuildJacobianLayout(const Program& program,
+bool BuildJacobianLayout(const Program& program,
                          int num_eliminate_blocks,
                          std::vector<int*>* jacobian_layout,
                          std::vector<int>* jacobian_layout_storage) {
@@ -73,8 +73,8 @@
   // Iterate over all the active residual blocks and determine how many E blocks
   // are there. This will determine where the F blocks start in the jacobian
   // matrix. Also compute the number of jacobian blocks.
-  int f_block_pos = 0;
-  int num_jacobian_blocks = 0;
+  unsigned int f_block_pos = 0;
+  unsigned int num_jacobian_blocks = 0;
   for (auto* residual_block : residual_blocks) {
     const int num_residuals = residual_block->NumResiduals();
     const int num_parameter_blocks = residual_block->NumParameterBlocks();
@@ -90,6 +90,11 @@
         }
       }
     }
+    if (num_jacobian_blocks > std::numeric_limits<int>::max()) {
+      LOG(ERROR) << "Overlow error. Too many blocks in the jacobian matrix : "
+                 << num_jacobian_blocks;
+      return false;
+    }
   }
 
   // We now know that the E blocks are laid out starting at zero, and the F
@@ -145,12 +150,18 @@
         jacobian_pos[k] = e_block_pos;
         e_block_pos += jacobian_block_size;
       } else {
-        jacobian_pos[k] = f_block_pos;
+        jacobian_pos[k] = static_cast<int>(f_block_pos);
         f_block_pos += jacobian_block_size;
+        if (f_block_pos > std::numeric_limits<int>::max()) {
+          LOG(ERROR)
+              << "Overlow error. Too many entries in the Jacobian matrix.";
+          return false;
+        }
       }
     }
     jacobian_pos += active_parameter_blocks.size();
   }
+  return true;
 }
 
 }  // namespace
@@ -161,10 +172,10 @@
   CHECK_GE(options.num_eliminate_blocks, 0)
       << "num_eliminate_blocks must be greater than 0.";
 
-  BuildJacobianLayout(*program,
-                      options.num_eliminate_blocks,
-                      &jacobian_layout_,
-                      &jacobian_layout_storage_);
+  jacobian_layout_is_valid_ = BuildJacobianLayout(*program,
+                                                  options.num_eliminate_blocks,
+                                                  &jacobian_layout_,
+                                                  &jacobian_layout_storage_);
 }
 
 // Create evaluate prepareres that point directly into the final jacobian. This
@@ -183,6 +194,12 @@
 }
 
 std::unique_ptr<SparseMatrix> BlockJacobianWriter::CreateJacobian() const {
+  if (!jacobian_layout_is_valid_) {
+    LOG(ERROR) << "Unable to create Jacobian matrix. Too many entries in the "
+                  "Jacobian matrix.";
+    return nullptr;
+  }
+
   auto* bs = new CompressedRowBlockStructure;
 
   const std::vector<ParameterBlock*>& parameter_blocks =