commit 58ad13c05daf619f77f2d7ad943c62a2c905bc33
author Sameer Agarwal <sameeragarwal@google.com> Tue Jun 05 17:46:53 2012 -0700
committer Sameer Agarwal <sameeragarwal@google.com> Tue Jun 05 18:06:00 2012 -0700
tree f0f1a4a1192e6ca93354c61bd6b87502800387ad
parent 8e2420edb17b89ab5f5a0f7056ec7fdcc0c18943

BlockRandomAccessSparseMatrix::IntPairToLong suffers from integer
overflow.

Even though the return value of this function is a long int, the
computation happens with three ints, which causes an overflow before
the upgrade happens.

The fix is to upgrade the constant used int his computation to be a
long int, which causes the computation to be done in longs instead of
ints.

A test has been added to verify that the fix works.

Change-Id: Ibb0aef877125bb37ca28754cb07b8e1627fd1d5a

internal/ceres/block_random_access_sparse_matrix.h

diff --git a/internal/ceres/block_random_access_sparse_matrix.h b/internal/ceres/block_random_access_sparse_matrix.h
index 27e30a7..48a0043 100644
--- a/internal/ceres/block_random_access_sparse_matrix.h
+++ b/internal/ceres/block_random_access_sparse_matrix.h
@@ -38,6 +38,7 @@
 #include "ceres/block_random_access_matrix.h"
 #include "ceres/collections_port.h"
 #include "ceres/triplet_sparse_matrix.h"
+#include "ceres/integral_types.h"
 #include "ceres/internal/macros.h"
 #include "ceres/internal/port.h"
 #include "ceres/internal/scoped_ptr.h"
@@ -84,11 +85,11 @@
   TripletSparseMatrix* mutable_matrix() { return tsm_.get(); }
 
  private:
-  long int IntPairToLong(int a, int b) {
+  int64 IntPairToLong(int a, int b) {
     return a * kMaxRowBlocks + b;
   }
 
-  const int kMaxRowBlocks;
+  const int64 kMaxRowBlocks;
   // row/column block sizes.
   const vector<int> blocks_;
 
@@ -100,6 +101,7 @@
   // The underlying matrix object which actually stores the cells.
   scoped_ptr<TripletSparseMatrix> tsm_;
 
+  friend class BlockRandomAccessSparseMatrixTest;
   CERES_DISALLOW_COPY_AND_ASSIGN(BlockRandomAccessSparseMatrix);
 };