diff --git a/internal/ceres/problem_impl.h b/internal/ceres/problem_impl.h
index 75bdc2b..e846c03 100644
--- a/internal/ceres/problem_impl.h
+++ b/internal/ceres/problem_impl.h
@@ -45,6 +45,7 @@
 #include "ceres/internal/macros.h"
 #include "ceres/internal/port.h"
 #include "ceres/internal/scoped_ptr.h"
+#include "ceres/collections_port.h"
 #include "ceres/problem.h"
 #include "ceres/types.h"
 
@@ -63,6 +64,7 @@
 class ProblemImpl {
  public:
   typedef map<double*, ParameterBlock*> ParameterMap;
+  typedef HashSet<ResidualBlock*> ResidualBlockSet;
 
   ProblemImpl();
   explicit ProblemImpl(const Problem::Options& options);
@@ -160,9 +162,15 @@
   Program* mutable_program() { return program_.get(); }
 
   const ParameterMap& parameter_map() const { return parameter_block_map_; }
+  const ResidualBlockSet& residual_block_set() const {
+    CHECK(options_.enable_fast_removal)
+        << "Fast removal not enabled, residual_block_set is not maintained.";
+    return residual_block_set_;
+  }
 
  private:
   ParameterBlock* InternalAddParameterBlock(double* values, int size);
+  void InternalRemoveResidualBlock(ResidualBlock* residual_block);
 
   bool InternalEvaluate(Program* program,
                         double* cost,
@@ -184,6 +192,9 @@
   // The mapping from user pointers to parameter blocks.
   map<double*, ParameterBlock*> parameter_block_map_;
 
+  // Iff enable_fast_removal is enabled, contains the current residual blocks.
+  ResidualBlockSet residual_block_set_;
+
   // The actual parameter and residual blocks.
   internal::scoped_ptr<internal::Program> program_;