diff --git a/internal/ceres/CMakeLists.txt b/internal/ceres/CMakeLists.txt
index a8813a6..9329162 100644
--- a/internal/ceres/CMakeLists.txt
+++ b/internal/ceres/CMakeLists.txt
@@ -511,9 +511,6 @@
 endmacro()
 
 if (BUILD_BENCHMARKS)
-  add_executable(autodiff_cost_function_benchmark autodiff_cost_function_benchmark.cc)
-  add_dependencies_to_benchmark(autodiff_cost_function_benchmark)
-
   add_executable(small_blas_gemv_benchmark small_blas_gemv_benchmark.cc)
   add_dependencies_to_benchmark(small_blas_gemv_benchmark)
 
@@ -525,9 +522,10 @@
 
   add_executable(schur_eliminator_benchmark schur_eliminator_benchmark.cc)
   add_dependencies_to_benchmark(schur_eliminator_benchmark)
+
+  add_subdirectory(autodiff_benchmarks)
 endif (BUILD_BENCHMARKS)
 
 if(CODE_GENERATION)
   add_subdirectory(codegen)
 endif()
-
diff --git a/internal/ceres/autodiff_benchmarks/CMakeLists.txt b/internal/ceres/autodiff_benchmarks/CMakeLists.txt
new file mode 100644
index 0000000..e42e584
--- /dev/null
+++ b/internal/ceres/autodiff_benchmarks/CMakeLists.txt
@@ -0,0 +1,55 @@
+
+if(CODE_GENERATION)
+  macro (generate_benchmark_functor FUNCTOR_NAME FUNCTOR_FILE)
+    ceres_generate_cost_function_implementation_for_functor(
+      NAME ${FUNCTOR_NAME}
+      INPUT_FILE ${FUNCTOR_FILE}
+      OUTPUT_DIRECTORY benchmarks
+      NAMESPACE ceres
+      )
+  endmacro()
+  include(CeresCodeGeneration)
+
+  generate_benchmark_functor(SnavelyReprojectionError snavely_reprojection_error.h)
+  generate_benchmark_functor(Linear1CostFunction linear_cost_functions.h)
+  generate_benchmark_functor(Linear10CostFunction linear_cost_functions.h)
+
+  list(APPEND CERES_BENCHMARK_FLAGS "-DWITH_CODE_GENERATION")
+endif()
+
+
+# TODO: Add support for other compilers
+if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+  list(APPEND CERES_BENCHMARK_FLAGS "-mllvm" "-inline-threshold=1000000")
+
+  # All other flags + fast-math
+  list(APPEND CERES_BENCHMARK_FAST_MATH_FLAGS ${CERES_BENCHMARK_FLAGS} "-ffast-math")
+endif()
+
+add_executable(rat43_benchmark rat43_benchmark.cc)
+add_dependencies_to_benchmark(rat43_benchmark)
+
+
+add_executable(snavely_reprojection_error_benchmark snavely_reprojection_error_benchmark.cc)
+add_dependencies_to_benchmark(snavely_reprojection_error_benchmark)
+target_compile_options(snavely_reprojection_error_benchmark PRIVATE ${CERES_BENCHMARK_FLAGS})
+
+add_executable(snavely_reprojection_error_benchmark_fast_math snavely_reprojection_error_benchmark.cc)
+add_dependencies_to_benchmark(snavely_reprojection_error_benchmark_fast_math)
+target_compile_options(snavely_reprojection_error_benchmark_fast_math PRIVATE ${CERES_BENCHMARK_FAST_MATH_FLAGS})
+
+add_executable(linear_benchmark linear_benchmark.cc)
+add_dependencies_to_benchmark(linear_benchmark)
+target_compile_options(linear_benchmark PRIVATE ${CERES_BENCHMARK_FLAGS})
+
+add_executable(linear_benchmark_fast_math linear_benchmark.cc)
+add_dependencies_to_benchmark(linear_benchmark_fast_math)
+target_compile_options(linear_benchmark_fast_math PRIVATE ${CERES_BENCHMARK_FAST_MATH_FLAGS})
+
+if(CODE_GENERATION)
+  target_link_libraries(snavely_reprojection_error_benchmark PUBLIC SnavelyReprojectionError)
+  target_link_libraries(snavely_reprojection_error_benchmark_fast_math PUBLIC SnavelyReprojectionError)
+  target_link_libraries(linear_benchmark PUBLIC Linear1CostFunction Linear10CostFunction)
+  target_link_libraries(linear_benchmark_fast_math PUBLIC Linear1CostFunction Linear10CostFunction)
+endif()
+
diff --git a/internal/ceres/autodiff_benchmarks/linear_benchmark.cc b/internal/ceres/autodiff_benchmarks/linear_benchmark.cc
new file mode 100644
index 0000000..25f6bd6
--- /dev/null
+++ b/internal/ceres/autodiff_benchmarks/linear_benchmark.cc
@@ -0,0 +1,119 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2020 Google Inc. All rights reserved.
+// http://ceres-solver.org/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: darius.rueckert@fau.de (Darius Rueckert)
+
+#include <memory>
+
+#include "benchmark/benchmark.h"
+#include "ceres/ceres.h"
+#include "codegen/test_utils.h"
+#include "linear_cost_functions.h"
+
+namespace ceres {
+
+#ifdef WITH_CODE_GENERATION
+static void BM_Linear1CodeGen(benchmark::State& state) {
+  double parameter_block1[] = {1.};
+  double* parameters[] = {parameter_block1};
+
+  double jacobian1[1];
+  double residuals[1];
+  double* jacobians[] = {jacobian1};
+
+  std::unique_ptr<ceres::CostFunction> cost_function(new Linear1CostFunction());
+
+  while (state.KeepRunning()) {
+    cost_function->Evaluate(parameters, residuals, jacobians);
+  }
+}
+BENCHMARK(BM_Linear1CodeGen);
+#endif
+
+static void BM_Linear1AutoDiff(benchmark::State& state) {
+  using FunctorType =
+      ceres::internal::CostFunctionToFunctor<Linear1CostFunction>;
+
+  double parameter_block1[] = {1.};
+  double* parameters[] = {parameter_block1};
+
+  double jacobian1[1];
+  double residuals[1];
+  double* jacobians[] = {jacobian1};
+
+  std::unique_ptr<ceres::CostFunction> cost_function(
+      new ceres::AutoDiffCostFunction<FunctorType, 1, 1>(new FunctorType()));
+
+  while (state.KeepRunning()) {
+    cost_function->Evaluate(parameters, residuals, jacobians);
+  }
+}
+BENCHMARK(BM_Linear1AutoDiff);
+
+#ifdef WITH_CODE_GENERATION
+static void BM_Linear10CodeGen(benchmark::State& state) {
+  double parameter_block1[] = {1., 2., 3., 4., 5., 6., 7., 8., 9., 10.};
+  double* parameters[] = {parameter_block1};
+
+  double jacobian1[10 * 10];
+  double residuals[10];
+  double* jacobians[] = {jacobian1};
+
+  std::unique_ptr<ceres::CostFunction> cost_function(
+      new Linear10CostFunction());
+
+  while (state.KeepRunning()) {
+    cost_function->Evaluate(parameters, residuals, jacobians);
+  }
+}
+BENCHMARK(BM_Linear10CodeGen);
+#endif
+
+static void BM_Linear10AutoDiff(benchmark::State& state) {
+  using FunctorType =
+      ceres::internal::CostFunctionToFunctor<Linear10CostFunction>;
+
+  double parameter_block1[] = {1., 2., 3., 4., 5., 6., 7., 8., 9., 10.};
+  double* parameters[] = {parameter_block1};
+
+  double jacobian1[10 * 10];
+  double residuals[10];
+  double* jacobians[] = {jacobian1};
+
+  std::unique_ptr<ceres::CostFunction> cost_function(
+      new ceres::AutoDiffCostFunction<FunctorType, 10, 10>(new FunctorType()));
+
+  while (state.KeepRunning()) {
+    cost_function->Evaluate(parameters, residuals, jacobians);
+  }
+}
+BENCHMARK(BM_Linear10AutoDiff);
+
+}  // namespace ceres
+
+BENCHMARK_MAIN();
diff --git a/internal/ceres/autodiff_benchmarks/linear_cost_functions.h b/internal/ceres/autodiff_benchmarks/linear_cost_functions.h
new file mode 100644
index 0000000..2a9c06c
--- /dev/null
+++ b/internal/ceres/autodiff_benchmarks/linear_cost_functions.h
@@ -0,0 +1,75 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2020 Google Inc. All rights reserved.
+// http://ceres-solver.org/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: darius.rueckert@fau.de (Darius Rueckert)
+//
+//
+#ifndef CERES_INTERNAL_AUTODIFF_BENCHMARKS_LINEAR_COST_FUNCTIONS_H_
+#define CERES_INTERNAL_AUTODIFF_BENCHMARKS_LINEAR_COST_FUNCTIONS_H_
+
+#include "ceres/codegen/codegen_cost_function.h"
+#include "ceres/rotation.h"
+namespace ceres {
+
+struct Linear1CostFunction : public ceres::CodegenCostFunction<1, 1> {
+  template <typename T>
+  bool operator()(const T* const x, T* residuals) const {
+    residuals[0] = x[0] + T(10);
+    return true;
+  }
+#ifdef WITH_CODE_GENERATION
+#include "benchmarks/linear1costfunction.h"
+#else
+  virtual bool Evaluate(double const* const* parameters,
+                        double* residuals,
+                        double** jacobians) const {
+    return false;
+  }
+#endif
+};
+
+struct Linear10CostFunction : public ceres::CodegenCostFunction<10, 10> {
+  template <typename T>
+  bool operator()(const T* const x, T* residuals) const {
+    for (int i = 0; i < 10; ++i) {
+      residuals[i] = x[i] + T(i);
+    }
+    return true;
+  }
+#ifdef WITH_CODE_GENERATION
+#include "benchmarks/linear10costfunction.h"
+#else
+  virtual bool Evaluate(double const* const* parameters,
+                        double* residuals,
+                        double** jacobians) const {
+    return false;
+  }
+#endif
+};
+}
+#endif  // CERES_INTERNAL_AUTODIFF_BENCHMARKS_LINEAR_COST_FUNCTIONS_H_
diff --git a/internal/ceres/autodiff_cost_function_benchmark.cc b/internal/ceres/autodiff_benchmarks/rat43_benchmark.cc
similarity index 100%
rename from internal/ceres/autodiff_cost_function_benchmark.cc
rename to internal/ceres/autodiff_benchmarks/rat43_benchmark.cc
diff --git a/internal/ceres/codegen/snavely_reprojection_error.h b/internal/ceres/autodiff_benchmarks/snavely_reprojection_error.h
similarity index 81%
rename from internal/ceres/codegen/snavely_reprojection_error.h
rename to internal/ceres/autodiff_benchmarks/snavely_reprojection_error.h
index d3c9da0..aa86fd7 100644
--- a/internal/ceres/codegen/snavely_reprojection_error.h
+++ b/internal/ceres/autodiff_benchmarks/snavely_reprojection_error.h
@@ -29,20 +29,19 @@
 // Author: darius.rueckert@fau.de (Darius Rueckert)
 //
 //
-#ifndef CERES_INTERNAL_CODEGEN_SNAVELY_REPROJECTION_ERROR_H_
-#define CERES_INTERNAL_CODEGEN_SNAVELY_REPROJECTION_ERROR_H_
+#ifndef CERES_INTERNAL_AUTODIFF_BENCHMARK_SNAVELY_REPROJECTION_ERROR_H_
+#define CERES_INTERNAL_AUTODIFF_BENCHMARK_SNAVELY_REPROJECTION_ERROR_H_
 
 #include "ceres/codegen/codegen_cost_function.h"
 #include "ceres/rotation.h"
 
-namespace test {
+namespace ceres {
 
-struct SnavelyReprojectionErrorGen
-    : public ceres::CodegenCostFunction<2, 9, 3> {
-  SnavelyReprojectionErrorGen(double observed_x, double observed_y)
+struct SnavelyReprojectionError : public ceres::CodegenCostFunction<2, 9, 3> {
+  SnavelyReprojectionError(double observed_x, double observed_y)
       : observed_x(observed_x), observed_y(observed_y) {}
 
-  SnavelyReprojectionErrorGen() = default;
+  SnavelyReprojectionError() = default;
   template <typename T>
   bool operator()(const T* const camera,
                   const T* const point,
@@ -82,11 +81,17 @@
 
     return true;
   }
-
-#include "tests/snavelyreprojectionerrorgen.h"
+#ifdef WITH_CODE_GENERATION
+#include "benchmarks/snavelyreprojectionerror.h"
+#else
+  virtual bool Evaluate(double const* const* parameters,
+                        double* residuals,
+                        double** jacobians) const {
+    return false;
+  }
+#endif
   double observed_x;
   double observed_y;
 };
-
-}  // namespace test
-#endif  // CERES_INTERNAL_CODEGEN_SNAVELY_REPROJECTION_ERROR_H_
+}  // namespace ceres
+#endif  // CERES_INTERNAL_AUTODIFF_BENCHMARK_SNAVELY_REPROJECTION_ERROR_H_
diff --git a/internal/ceres/codegen/codegen_ba_benchmark.cc b/internal/ceres/autodiff_benchmarks/snavely_reprojection_error_benchmark.cc
similarity index 76%
rename from internal/ceres/codegen/codegen_ba_benchmark.cc
rename to internal/ceres/autodiff_benchmarks/snavely_reprojection_error_benchmark.cc
index 8d37d75..4a58db2 100644
--- a/internal/ceres/codegen/codegen_ba_benchmark.cc
+++ b/internal/ceres/autodiff_benchmarks/snavely_reprojection_error_benchmark.cc
@@ -1,5 +1,5 @@
 // Ceres Solver - A fast non-linear least squares minimizer
-// Copyright 2018 Google Inc. All rights reserved.
+// Copyright 2020 Google Inc. All rights reserved.
 // http://ceres-solver.org/
 //
 // Redistribution and use in source and binary forms, with or without
@@ -26,20 +26,44 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 //
-// Authors: sameeragarwal@google.com (Sameer Agarwal)
-//#define EIGEN_DONT_VECTORIZE
+// Author: darius.rueckert@fau.de (Darius Rueckert)
+
 #include <memory>
 
 #include "benchmark/benchmark.h"
 #include "ceres/ceres.h"
+#include "codegen/test_utils.h"
 #include "snavely_reprojection_error.h"
-#include "test_utils.h"
 
 namespace ceres {
 
-static void BM_BAAutoDiff(benchmark::State& state) {
+#ifdef WITH_CODE_GENERATION
+static void BM_SnavelyReprojectionCodeGen(benchmark::State& state) {
+  double parameter_block1[] = {1., 2., 3., 4., 5., 6., 7., 8., 9.};
+  double parameter_block2[] = {1., 2., 3.};
+  double* parameters[] = {parameter_block1, parameter_block2};
+
+  double jacobian1[2 * 9];
+  double jacobian2[2 * 3];
+  double residuals[2];
+  double* jacobians[] = {jacobian1, jacobian2};
+
+  const double x = 0.2;
+  const double y = 0.3;
+
+  std::unique_ptr<ceres::CostFunction> cost_function(
+      new SnavelyReprojectionError(x, y));
+
+  while (state.KeepRunning()) {
+    cost_function->Evaluate(parameters, residuals, jacobians);
+  }
+}
+BENCHMARK(BM_SnavelyReprojectionCodeGen);
+#endif
+
+static void BM_SnavelyReprojectionAutoDiff(benchmark::State& state) {
   using FunctorType =
-      ceres::internal::CostFunctionToFunctor<test::SnavelyReprojectionErrorGen>;
+      ceres::internal::CostFunctionToFunctor<SnavelyReprojectionError>;
 
   double parameter_block1[] = {1., 2., 3., 4., 5., 6., 7., 8., 9.};
   double parameter_block2[] = {1., 2., 3.};
@@ -57,37 +81,11 @@
           new FunctorType(x, y)));
 
   while (state.KeepRunning()) {
-    cost_function->Evaluate(
-        parameters, residuals, state.range(0) ? jacobians : nullptr);
+    cost_function->Evaluate(parameters, residuals, jacobians);
   }
 }
 
-static void BM_BACodeGen(benchmark::State& state) {
-  double parameter_block1[] = {1., 2., 3., 4., 5., 6., 7., 8., 9.};
-  double parameter_block2[] = {1., 2., 3.};
-  double* parameters[] = {parameter_block1, parameter_block2};
-
-  double jacobian1[2 * 9];
-  double jacobian2[2 * 3];
-  double residuals[2];
-  double* jacobians[] = {jacobian1, jacobian2};
-
-  const double x = 0.2;
-  const double y = 0.3;
-
-  std::unique_ptr<ceres::CostFunction> cost_function(
-      new test::SnavelyReprojectionErrorGen(x, y));
-
-  while (state.KeepRunning()) {
-    cost_function->Evaluate(
-        parameters, residuals, state.range(0) ? jacobians : nullptr);
-  }
-}
-
-BENCHMARK(BM_BAAutoDiff)->ArgName("Residual")->Arg(0);
-BENCHMARK(BM_BAAutoDiff)->ArgName("Residual+Jacobian")->Arg(1);
-BENCHMARK(BM_BACodeGen)->ArgName("Residual")->Arg(0);
-BENCHMARK(BM_BACodeGen)->ArgName("Residual+Jacobian")->Arg(1);
+BENCHMARK(BM_SnavelyReprojectionAutoDiff);
 
 }  // namespace ceres
 
diff --git a/internal/ceres/codegen/CMakeLists.txt b/internal/ceres/codegen/CMakeLists.txt
index 4dd75ed..4bea21f 100644
--- a/internal/ceres/codegen/CMakeLists.txt
+++ b/internal/ceres/codegen/CMakeLists.txt
@@ -88,31 +88,3 @@
     UnaryArithmetic BinaryComparison LogicalOperators ScalarFunctions
     LogicalFunctions Branches)
 endif()
-
-if (BUILD_BENCHMARKS)
-  generate_test_functor(SnavelyReprojectionErrorGen snavely_reprojection_error.h)
-
-  # Benchmark with default compiler flags
-  add_executable(codegen_ba_benchmark codegen_ba_benchmark.cc)
-  add_dependencies_to_benchmark(codegen_ba_benchmark)
-  target_link_libraries(codegen_ba_benchmark PUBLIC SnavelyReprojectionErrorGen)
-
-  # Benchmark with all optimizations (clang only)
-  if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-
-    # Always fine
-    list(APPEND CERES_BENCHMARK_FLAGS "-march=native")
-    list(APPEND CERES_BENCHMARK_FLAGS "-fno-math-errno" "-fno-trapping-math" "-fassociative-math" "-fno-signed-zeros" "-ffp-contract=fast")
-
-    # This enables the 0*x=0 optimization
-    list(APPEND CERES_BENCHMARK_FLAGS "-fno-honor-infinities" "-fno-honor-nans")
-
-    # Total overkill but we want to make sure everything inlineable is inlined.
-    list(APPEND CERES_BENCHMARK_FLAGS "-mllvm" "-inline-threshold=1000000")
-
-    add_executable(codegen_ba_benchmark_fast_math codegen_ba_benchmark.cc)
-    add_dependencies_to_benchmark(codegen_ba_benchmark_fast_math)
-    target_link_libraries(codegen_ba_benchmark_fast_math PUBLIC SnavelyReprojectionErrorGen)
-    target_compile_options(codegen_ba_benchmark_fast_math PRIVATE ${CERES_BENCHMARK_FLAGS})
-  endif()
-endif()