Expand jet_operator_benchmark to all Jet ops Expand jet_operator_benchmark to cover all arithmetic, compound assignment, algebraic, rounding, exponential, logarithmic, trigonometric, inverse trigonometric, hyperbolic, power, norm, hypotenuse, error, Bessel, multi-argument, C++20, comparison, and classification functions on Jet<double, N> across dimensions 3, 10, 15, 25, and 32. Also pass -mllvm -inlinehint-threshold=1000000 and -mno-outline to Clang benchmark targets in add_dependencies_to_benchmark so that EIGEN_STRONG_INLINE is honored and MachineOutliner does not outline instructions inside hot inner benchmark loops. Change-Id: I8fe1b8d7e20b68380cf77d0d010dd1d1aaef6fb1
diff --git a/internal/ceres/CMakeLists.txt b/internal/ceres/CMakeLists.txt index 193d174..286f353 100644 --- a/internal/ceres/CMakeLists.txt +++ b/internal/ceres/CMakeLists.txt
@@ -518,6 +518,12 @@ target_link_libraries(${BENCHMARK_TARGET} PRIVATE ceres_static benchmark::benchmark ${CERES_LIBRARY_PRIVATE_DEPENDENCIES}) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # Honor EIGEN_STRONG_INLINE across deep Jet/Eigen expression templates and + # disable MachineOutliner so hot inner loops are not outlined into calls. + target_compile_options(${BENCHMARK_TARGET} PRIVATE + "-mllvm" "-inlinehint-threshold=1000000" "-mno-outline") + endif() endmacro() if (_Ceres_FEATURE_BENCHMARKS)
diff --git a/internal/ceres/jet_operator_benchmark.cc b/internal/ceres/jet_operator_benchmark.cc index 94b0308..6afc3a4 100644 --- a/internal/ceres/jet_operator_benchmark.cc +++ b/internal/ceres/jet_operator_benchmark.cc
@@ -61,6 +61,15 @@ e_[i].a = T(5.5) * ti; e_[i].v.setRandom(); + // Values strictly inside (-1, 1) for inverse trigonometric functions. + unit_a_[i].a = + T(-0.9) + T(1.8) * static_cast<T>(i) / static_cast<T>(SIZE); + unit_a_[i].v.setRandom(); + + unit_b_[i].a = + T(-0.85) + T(1.7) * static_cast<T>(i) / static_cast<T>(SIZE); + unit_b_[i].v.setRandom(); + scalar_a_[i] = T(1.1) * ti; scalar_b_[i] = T(2.2) * ti; scalar_c_[i] = T(3.3) * ti; @@ -76,6 +85,8 @@ const JetType& c() const { return c_[index_]; } const JetType& d() const { return d_[index_]; } const JetType& e() const { return e_[index_]; } + const JetType& unit_a() const { return unit_a_[index_]; } + const JetType& unit_b() const { return unit_b_[index_]; } T scalar_a() const { return scalar_a_[index_]; } T scalar_b() const { return scalar_b_[index_]; } T scalar_c() const { return scalar_c_[index_]; } @@ -89,6 +100,8 @@ std::array<JetType, SIZE> c_{}; std::array<JetType, SIZE> d_{}; std::array<JetType, SIZE> e_{}; + std::array<JetType, SIZE> unit_a_{}; + std::array<JetType, SIZE> unit_b_{}; std::array<T, SIZE> scalar_a_; std::array<T, SIZE> scalar_b_; std::array<T, SIZE> scalar_c_; @@ -111,6 +124,17 @@ benchmark::DoNotOptimize(out); } +#define CERES_REGISTER_JET_BENCHMARK(Name) \ + BENCHMARK_TEMPLATE(Name, 3)->Arg(1000); \ + BENCHMARK_TEMPLATE(Name, 10)->Arg(1000); \ + BENCHMARK_TEMPLATE(Name, 15)->Arg(1000); \ + BENCHMARK_TEMPLATE(Name, 25)->Arg(1000); \ + BENCHMARK_TEMPLATE(Name, 32)->Arg(1000) + +// --------------------------------------------------------------------------- +// Arithmetic & compound assignment operators +// --------------------------------------------------------------------------- + template <std::size_t JET_SIZE> static void Addition(benchmark::State& state) { using JetType = Jet<double, JET_SIZE>; @@ -119,12 +143,7 @@ out += +d.a() + d.b() + d.c() + d.d() + d.e(); }); } -BENCHMARK_TEMPLATE(Addition, 3)->Arg(1000); -BENCHMARK_TEMPLATE(Addition, 10)->Arg(1000); -BENCHMARK_TEMPLATE(Addition, 15)->Arg(1000); -BENCHMARK_TEMPLATE(Addition, 25)->Arg(1000); -BENCHMARK_TEMPLATE(Addition, 32)->Arg(1000); -BENCHMARK_TEMPLATE(Addition, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(Addition); template <std::size_t JET_SIZE> static void AdditionScalar(benchmark::State& state) { @@ -135,12 +154,7 @@ d.scalar_a() + d.scalar_b() + d.c() + d.scalar_d() + d.scalar_e(); }); } -BENCHMARK_TEMPLATE(AdditionScalar, 3)->Arg(1000); -BENCHMARK_TEMPLATE(AdditionScalar, 10)->Arg(1000); -BENCHMARK_TEMPLATE(AdditionScalar, 15)->Arg(1000); -BENCHMARK_TEMPLATE(AdditionScalar, 25)->Arg(1000); -BENCHMARK_TEMPLATE(AdditionScalar, 32)->Arg(1000); -BENCHMARK_TEMPLATE(AdditionScalar, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(AdditionScalar); template <std::size_t JET_SIZE> static void Subtraction(benchmark::State& state) { @@ -150,12 +164,7 @@ out -= -d.a() - d.b() - d.c() - d.d() - d.e(); }); } -BENCHMARK_TEMPLATE(Subtraction, 3)->Arg(1000); -BENCHMARK_TEMPLATE(Subtraction, 10)->Arg(1000); -BENCHMARK_TEMPLATE(Subtraction, 15)->Arg(1000); -BENCHMARK_TEMPLATE(Subtraction, 25)->Arg(1000); -BENCHMARK_TEMPLATE(Subtraction, 32)->Arg(1000); -BENCHMARK_TEMPLATE(Subtraction, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(Subtraction); template <std::size_t JET_SIZE> static void SubtractionScalar(benchmark::State& state) { @@ -166,27 +175,17 @@ -d.scalar_a() - d.scalar_b() - d.c() - d.scalar_d() - d.scalar_e(); }); } -BENCHMARK_TEMPLATE(SubtractionScalar, 3)->Arg(1000); -BENCHMARK_TEMPLATE(SubtractionScalar, 10)->Arg(1000); -BENCHMARK_TEMPLATE(SubtractionScalar, 15)->Arg(1000); -BENCHMARK_TEMPLATE(SubtractionScalar, 25)->Arg(1000); -BENCHMARK_TEMPLATE(SubtractionScalar, 32)->Arg(1000); -BENCHMARK_TEMPLATE(SubtractionScalar, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(SubtractionScalar); template <std::size_t JET_SIZE> static void Multiplication(benchmark::State& state) { using JetType = Jet<double, JET_SIZE>; JetBenchmarkHelper<JET_SIZE>( state, [](const JetInputData<JetType>& d, JetType& out) { - out *= d.a() * d.b() * d.c() * d.d() * d.e(); + out += d.a() * d.b() * d.c() * d.d() * d.e(); }); } -BENCHMARK_TEMPLATE(Multiplication, 3)->Arg(1000); -BENCHMARK_TEMPLATE(Multiplication, 10)->Arg(1000); -BENCHMARK_TEMPLATE(Multiplication, 15)->Arg(1000); -BENCHMARK_TEMPLATE(Multiplication, 25)->Arg(1000); -BENCHMARK_TEMPLATE(Multiplication, 32)->Arg(1000); -BENCHMARK_TEMPLATE(Multiplication, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(Multiplication); template <std::size_t JET_SIZE> static void MultiplicationLeftScalar(benchmark::State& state) { @@ -197,12 +196,7 @@ (d.scalar_b() * (d.scalar_c() * (d.scalar_d() * d.e()))); }); } -BENCHMARK_TEMPLATE(MultiplicationLeftScalar, 3)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationLeftScalar, 10)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationLeftScalar, 15)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationLeftScalar, 25)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationLeftScalar, 32)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationLeftScalar, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(MultiplicationLeftScalar); template <std::size_t JET_SIZE> static void MultiplicationRightScalar(benchmark::State& state) { @@ -213,27 +207,105 @@ d.scalar_e(); }); } -BENCHMARK_TEMPLATE(MultiplicationRightScalar, 3)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationRightScalar, 10)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationRightScalar, 15)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationRightScalar, 25)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationRightScalar, 32)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplicationRightScalar, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(MultiplicationRightScalar); template <std::size_t JET_SIZE> static void Division(benchmark::State& state) { using JetType = Jet<double, JET_SIZE>; JetBenchmarkHelper<JET_SIZE>( state, [](const JetInputData<JetType>& d, JetType& out) { - out /= d.a() / d.b() / d.c() / d.d() / d.e(); + out += d.a() / d.b() / d.c() / d.d() / d.e(); }); } -BENCHMARK_TEMPLATE(Division, 3)->Arg(1000); -BENCHMARK_TEMPLATE(Division, 10)->Arg(1000); -BENCHMARK_TEMPLATE(Division, 15)->Arg(1000); -BENCHMARK_TEMPLATE(Division, 25)->Arg(1000); -BENCHMARK_TEMPLATE(Division, 32)->Arg(1000); -BENCHMARK_TEMPLATE(Division, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(Division); + +template <std::size_t JET_SIZE> +static void CompoundAdditionScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += d.scalar_a(); + out += d.scalar_b(); + out += d.scalar_c(); + out += d.scalar_d(); + out += d.scalar_e(); + }); +} +CERES_REGISTER_JET_BENCHMARK(CompoundAdditionScalar); + +template <std::size_t JET_SIZE> +static void CompoundSubtractionScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out -= d.scalar_a(); + out -= d.scalar_b(); + out -= d.scalar_c(); + out -= d.scalar_d(); + out -= d.scalar_e(); + }); +} +CERES_REGISTER_JET_BENCHMARK(CompoundSubtractionScalar); + +template <std::size_t JET_SIZE> +static void CompoundMultiplication(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + JetType tmp = d.a(); + tmp *= d.b(); + tmp *= d.c(); + tmp *= d.d(); + tmp *= d.e(); + out += tmp; + }); +} +CERES_REGISTER_JET_BENCHMARK(CompoundMultiplication); + +template <std::size_t JET_SIZE> +static void CompoundMultiplicationScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + JetType tmp = d.a(); + tmp *= d.scalar_b(); + tmp *= d.scalar_c(); + tmp *= d.scalar_d(); + tmp *= d.scalar_e(); + out += tmp; + }); +} +CERES_REGISTER_JET_BENCHMARK(CompoundMultiplicationScalar); + +template <std::size_t JET_SIZE> +static void CompoundDivision(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + JetType tmp = d.a(); + tmp /= d.b(); + tmp /= d.c(); + tmp /= d.d(); + tmp /= d.e(); + out += tmp; + }); +} +CERES_REGISTER_JET_BENCHMARK(CompoundDivision); + +template <std::size_t JET_SIZE> +static void CompoundDivisionScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + JetType tmp = d.a(); + tmp /= d.scalar_b(); + tmp /= d.scalar_c(); + tmp /= d.scalar_d(); + tmp /= d.scalar_e(); + out += tmp; + }); +} +CERES_REGISTER_JET_BENCHMARK(CompoundDivisionScalar); template <std::size_t JET_SIZE> static void DivisionLeftScalar(benchmark::State& state) { @@ -244,12 +316,7 @@ (d.scalar_b() / (d.scalar_c() / (d.scalar_d() / d.e()))); }); } -BENCHMARK_TEMPLATE(DivisionLeftScalar, 3)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionLeftScalar, 10)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionLeftScalar, 15)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionLeftScalar, 25)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionLeftScalar, 32)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionLeftScalar, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(DivisionLeftScalar); template <std::size_t JET_SIZE> static void DivisionRightScalar(benchmark::State& state) { @@ -260,12 +327,7 @@ d.scalar_e(); }); } -BENCHMARK_TEMPLATE(DivisionRightScalar, 3)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionRightScalar, 10)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionRightScalar, 15)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionRightScalar, 25)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionRightScalar, 32)->Arg(1000); -BENCHMARK_TEMPLATE(DivisionRightScalar, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(DivisionRightScalar); template <std::size_t JET_SIZE> static void MultiplyAndAdd(benchmark::State& state) { @@ -277,12 +339,614 @@ d.scalar_e() * d.e(); }); } -BENCHMARK_TEMPLATE(MultiplyAndAdd, 3)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplyAndAdd, 10)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplyAndAdd, 15)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplyAndAdd, 25)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplyAndAdd, 32)->Arg(1000); -BENCHMARK_TEMPLATE(MultiplyAndAdd, 200)->Arg(160); +CERES_REGISTER_JET_BENCHMARK(MultiplyAndAdd); + +// --------------------------------------------------------------------------- +// Unary algebraic & rounding functions +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void Abs(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += abs(d.unit_a()) + abs(d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Abs); + +template <std::size_t JET_SIZE> +static void Sqrt(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += sqrt(d.a()) + sqrt(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Sqrt); + +template <std::size_t JET_SIZE> +static void Cbrt(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += cbrt(d.a()) + cbrt(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Cbrt); + +template <std::size_t JET_SIZE> +static void Norm(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += norm(d.a()) + norm(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Norm); + +template <std::size_t JET_SIZE> +static void Floor(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += floor(d.a()) + floor(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Floor); + +template <std::size_t JET_SIZE> +static void Ceil(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += ceil(d.a()) + ceil(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Ceil); + +// --------------------------------------------------------------------------- +// Exponential & logarithmic functions +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void Exp(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += exp(d.a()) + exp(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Exp); + +template <std::size_t JET_SIZE> +static void Exp2(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += exp2(d.a()) + exp2(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Exp2); + +template <std::size_t JET_SIZE> +static void Expm1(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += expm1(d.unit_a()) + expm1(d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Expm1); + +template <std::size_t JET_SIZE> +static void Log(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += log(d.a()) + log(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Log); + +template <std::size_t JET_SIZE> +static void Log2(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += log2(d.a()) + log2(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Log2); + +template <std::size_t JET_SIZE> +static void Log10(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += log10(d.a()) + log10(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Log10); + +template <std::size_t JET_SIZE> +static void Log1p(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += log1p(d.unit_a()) + log1p(d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Log1p); + +// --------------------------------------------------------------------------- +// Trigonometric & inverse trigonometric functions +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void Sin(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += sin(d.a()) + sin(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Sin); + +template <std::size_t JET_SIZE> +static void Cos(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += cos(d.a()) + cos(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Cos); + +template <std::size_t JET_SIZE> +static void Tan(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += tan(d.a()) + tan(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Tan); + +template <std::size_t JET_SIZE> +static void Asin(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += asin(d.unit_a()) + asin(d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Asin); + +template <std::size_t JET_SIZE> +static void Acos(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += acos(d.unit_a()) + acos(d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Acos); + +template <std::size_t JET_SIZE> +static void Atan(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += atan(d.a()) + atan(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Atan); + +template <std::size_t JET_SIZE> +static void Atan2(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += atan2(d.a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Atan2); + +// --------------------------------------------------------------------------- +// Hyperbolic functions +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void Sinh(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += sinh(d.a()) + sinh(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Sinh); + +template <std::size_t JET_SIZE> +static void Cosh(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += cosh(d.a()) + cosh(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Cosh); + +template <std::size_t JET_SIZE> +static void Tanh(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += tanh(d.a()) + tanh(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Tanh); + +// --------------------------------------------------------------------------- +// Power, norm & hypotenuse functions +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void PowScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += pow(d.a(), 2.3) + pow(d.b(), -0.7); + }); +} +CERES_REGISTER_JET_BENCHMARK(PowScalar); + +template <std::size_t JET_SIZE> +static void PowScalarBase(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += pow(d.scalar_a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(PowScalarBase); + +template <std::size_t JET_SIZE> +static void PowJet(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += pow(d.a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(PowJet); + +template <std::size_t JET_SIZE> +static void Hypot2(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += hypot(d.a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Hypot2); + +template <std::size_t JET_SIZE> +static void Hypot3(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += hypot(d.a(), d.b(), d.c()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Hypot3); + +template <std::size_t JET_SIZE> +static void AccurateNorm2(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += AccurateNorm(d.a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(AccurateNorm2); + +template <std::size_t JET_SIZE> +static void AccurateNorm3(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += AccurateNorm(d.a(), d.b(), d.c()); + }); +} +CERES_REGISTER_JET_BENCHMARK(AccurateNorm3); + +template <std::size_t JET_SIZE> +static void AccurateRNorm2(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += AccurateRNorm(d.a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(AccurateRNorm2); + +template <std::size_t JET_SIZE> +static void AccurateRNorm3(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += AccurateRNorm(d.a(), d.b(), d.c()); + }); +} +CERES_REGISTER_JET_BENCHMARK(AccurateRNorm3); + +// --------------------------------------------------------------------------- +// Special functions (error functions & Bessel functions) +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void Erf(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += erf(d.unit_a()) + erf(d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Erf); + +template <std::size_t JET_SIZE> +static void Erfc(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += erfc(d.unit_a()) + erfc(d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Erfc); + +#if defined(CERES_HAS_CPP17_BESSEL_FUNCTIONS) || \ + defined(CERES_HAS_POSIX_BESSEL_FUNCTIONS) + +template <std::size_t JET_SIZE> +static void BesselJ0Bench(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += BesselJ0(d.a()) + BesselJ0(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(BesselJ0Bench); + +template <std::size_t JET_SIZE> +static void BesselJ1Bench(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += BesselJ1(d.a()) + BesselJ1(d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(BesselJ1Bench); + +template <std::size_t JET_SIZE> +static void BesselJnBench(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += BesselJn(2, d.a()) + BesselJn(3, d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(BesselJnBench); + +#ifdef CERES_HAS_CPP17_BESSEL_FUNCTIONS + +template <std::size_t JET_SIZE> +static void CylBesselJBench(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += cyl_bessel_j(0.0, d.a()) + cyl_bessel_j(2.5, d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(CylBesselJBench); + +#endif // defined(CERES_HAS_CPP17_BESSEL_FUNCTIONS) + +#endif // defined(CERES_HAS_CPP17_BESSEL_FUNCTIONS) || + // defined(CERES_HAS_POSIX_BESSEL_FUNCTIONS) + +// --------------------------------------------------------------------------- +// Multi-argument, branching, & interpolation functions +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void Copysign(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += copysign(d.a(), d.unit_a()) + copysign(d.b(), d.unit_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Copysign); + +template <std::size_t JET_SIZE> +static void Fma(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += fma(d.a(), d.b(), d.c()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Fma); + +template <std::size_t JET_SIZE> +static void Fmax(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += fmax(d.a(), d.b()) + fmax(d.b(), d.a()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Fmax); + +template <std::size_t JET_SIZE> +static void FmaxScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += fmax(d.a(), d.scalar_b()) + fmax(d.b(), d.scalar_a()); + }); +} +CERES_REGISTER_JET_BENCHMARK(FmaxScalar); + +template <std::size_t JET_SIZE> +static void Fmin(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += fmin(d.a(), d.b()) + fmin(d.b(), d.a()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Fmin); + +template <std::size_t JET_SIZE> +static void FminScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += fmin(d.a(), d.scalar_b()) + fmin(d.b(), d.scalar_a()); + }); +} +CERES_REGISTER_JET_BENCHMARK(FminScalar); + +template <std::size_t JET_SIZE> +static void Fdim(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += fdim(d.b(), d.a()) + fdim(d.a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Fdim); + +template <std::size_t JET_SIZE> +static void FdimScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += fdim(d.b(), d.scalar_a()) + fdim(d.a(), d.scalar_b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(FdimScalar); + +#ifdef CERES_HAS_CPP20 + +template <std::size_t JET_SIZE> +static void Lerp(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += lerp(d.a(), d.b(), d.unit_a()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Lerp); + +template <std::size_t JET_SIZE> +static void Midpoint(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + out += midpoint(d.a(), d.b()); + }); +} +CERES_REGISTER_JET_BENCHMARK(Midpoint); + +#endif // defined(CERES_HAS_CPP20) + +// --------------------------------------------------------------------------- +// Comparison & classification functions +// --------------------------------------------------------------------------- + +template <std::size_t JET_SIZE> +static void ComparisonJet(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + const int r = static_cast<int>(d.a() < d.b()) + + static_cast<int>(d.a() <= d.b()) + + static_cast<int>(d.b() > d.a()) + + static_cast<int>(d.b() >= d.a()) + + static_cast<int>(d.a() == d.b()) + + static_cast<int>(d.a() != d.b()); + out.a += static_cast<double>(r); + }); +} +CERES_REGISTER_JET_BENCHMARK(ComparisonJet); + +template <std::size_t JET_SIZE> +static void ComparisonScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + const int r = static_cast<int>(d.a() < d.scalar_b()) + + static_cast<int>(d.scalar_a() <= d.b()) + + static_cast<int>(d.b() > d.scalar_a()) + + static_cast<int>(d.scalar_b() >= d.a()) + + static_cast<int>(d.a() == d.scalar_b()) + + static_cast<int>(d.scalar_a() != d.b()); + out.a += static_cast<double>(r); + }); +} +CERES_REGISTER_JET_BENCHMARK(ComparisonScalar); + +template <std::size_t JET_SIZE> +static void QuietComparisonJet(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + const int r = static_cast<int>(isless(d.a(), d.b())) + + static_cast<int>(islessequal(d.a(), d.b())) + + static_cast<int>(isgreater(d.b(), d.a())) + + static_cast<int>(isgreaterequal(d.b(), d.a())) + + static_cast<int>(islessgreater(d.a(), d.b())) + + static_cast<int>(isunordered(d.a(), d.b())); + out.a += static_cast<double>(r); + }); +} +CERES_REGISTER_JET_BENCHMARK(QuietComparisonJet); + +template <std::size_t JET_SIZE> +static void QuietComparisonScalar(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + const int r = static_cast<int>(isless(d.a(), d.scalar_b())) + + static_cast<int>(islessequal(d.scalar_a(), d.b())) + + static_cast<int>(isgreater(d.b(), d.scalar_a())) + + static_cast<int>(isgreaterequal(d.scalar_b(), d.a())) + + static_cast<int>(islessgreater(d.a(), d.scalar_b())) + + static_cast<int>(isunordered(d.scalar_a(), d.b())); + out.a += static_cast<double>(r); + }); +} +CERES_REGISTER_JET_BENCHMARK(QuietComparisonScalar); + +template <std::size_t JET_SIZE> +static void Classification(benchmark::State& state) { + using JetType = Jet<double, JET_SIZE>; + JetBenchmarkHelper<JET_SIZE>( + state, [](const JetInputData<JetType>& d, JetType& out) { + const int r = static_cast<int>(isfinite(d.a())) + + static_cast<int>(isinf(d.a())) + + static_cast<int>(isnan(d.a())) + + static_cast<int>(isnormal(d.a())) + fpclassify(d.a()) + + static_cast<int>(signbit(d.unit_a())); + out.a += static_cast<double>(r); + }); +} +CERES_REGISTER_JET_BENCHMARK(Classification); + +#undef CERES_REGISTER_JET_BENCHMARK } // namespace ceres