internal/ceres/small_blas_generic.h

// Ceres Solver - A fast non-linear least squares minimizer
// Copyright 2023 Google Inc. All rights reserved.
// http://ceres-solver.org/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
// * Neither the name of Google Inc. nor the names of its contributors may be
//   used to endorse or promote products derived from this software without
//   specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
// Author: yangfan34@lenovo.com (Lenovo Research Device+ Lab - Shanghai)
//
// Optimization for simple blas functions used in the Schur Eliminator.
// These are fairly basic implementations which already yield a significant
// speedup in the eliminator performance.

#ifndef CERES_INTERNAL_SMALL_BLAS_GENERIC_H_
#define CERES_INTERNAL_SMALL_BLAS_GENERIC_H_

namespace ceres::internal {

// The following macros are used to share code
#define CERES_GEMM_OPT_NAIVE_HEADER       \
  double cvec4[4] = {0.0, 0.0, 0.0, 0.0}; \
  const double* pa = a;                   \
  const double* pb = b;                   \
  const int span = 4;                     \
  int col_r = col_a & (span - 1);         \
  int col_m = col_a - col_r;

#define CERES_GEMM_OPT_STORE_MAT1X4 \
  if (kOperation > 0) {             \
    c[0] += cvec4[0];               \
    c[1] += cvec4[1];               \
    c[2] += cvec4[2];               \
    c[3] += cvec4[3];               \
  } else if (kOperation < 0) {      \
    c[0] -= cvec4[0];               \
    c[1] -= cvec4[1];               \
    c[2] -= cvec4[2];               \
    c[3] -= cvec4[3];               \
  } else {                          \
    c[0] = cvec4[0];                \
    c[1] = cvec4[1];                \
    c[2] = cvec4[2];                \
    c[3] = cvec4[3];                \
  }                                 \
  c += 4;

// Matrix-Matrix Multiplication
// Figure out 1x4 of Matrix C in one batch
//
// c op a * B;
// where op can be +=, -=, or =, indicated by kOperation.
//
//  Matrix C              Matrix A                   Matrix B
//
//  C0, C1, C2, C3   op   A0, A1, A2, A3, ...    *   B0, B1, B2, B3
//                                                   B4, B5, B6, B7
//                                                   B8, B9, Ba, Bb
//                                                   Bc, Bd, Be, Bf
//                                                   . , . , . , .
//                                                   . , . , . , .
//                                                   . , . , . , .
//
// unroll for loops
// utilize the data resided in cache
// NOTE: col_a means the columns of A
static inline void MMM_mat1x4(const int col_a,
                              const double* a,
                              const double* b,
                              const int col_stride_b,
                              double* c,
                              const int kOperation) {
  CERES_GEMM_OPT_NAIVE_HEADER
  double av = 0.0;
  int bi = 0;

#define CERES_GEMM_OPT_MMM_MAT1X4_MUL \
  av = pa[k];                         \
  pb = b + bi;                        \
  cvec4[0] += av * pb[0];             \
  cvec4[1] += av * pb[1];             \
  cvec4[2] += av * pb[2];             \
  cvec4[3] += av * pb[3];             \
  pb += 4;                            \
  bi += col_stride_b;                 \
  k++;

  for (int k = 0; k < col_m;) {
    CERES_GEMM_OPT_MMM_MAT1X4_MUL
    CERES_GEMM_OPT_MMM_MAT1X4_MUL
    CERES_GEMM_OPT_MMM_MAT1X4_MUL
    CERES_GEMM_OPT_MMM_MAT1X4_MUL
  }

  for (int k = col_m; k < col_a;) {
    CERES_GEMM_OPT_MMM_MAT1X4_MUL
  }

  CERES_GEMM_OPT_STORE_MAT1X4

#undef CERES_GEMM_OPT_MMM_MAT1X4_MUL
}

// Matrix Transpose-Matrix multiplication
// Figure out 1x4 of Matrix C in one batch
//
// c op a' * B;
// where op can be +=, -=, or = indicated by kOperation.
//
//                        Matrix A
//
//                        A0
//                        A1
//                        A2
//                        A3
//                        .
//                        .
//                        .
//
//  Matrix C              Matrix A'                  Matrix B
//
//  C0, C1, C2, C3   op   A0, A1, A2, A3, ...    *   B0, B1, B2, B3
//                                                   B4, B5, B6, B7
//                                                   B8, B9, Ba, Bb
//                                                   Bc, Bd, Be, Bf
//                                                   . , . , . , .
//                                                   . , . , . , .
//                                                   . , . , . , .
//
// unroll for loops
// utilize the data resided in cache
// NOTE: col_a means the columns of A'
static inline void MTM_mat1x4(const int col_a,
                              const double* a,
                              const int col_stride_a,
                              const double* b,
                              const int col_stride_b,
                              double* c,
                              const int kOperation) {
  CERES_GEMM_OPT_NAIVE_HEADER
  double av = 0.0;
  int ai = 0;
  int bi = 0;

#define CERES_GEMM_OPT_MTM_MAT1X4_MUL \
  av = pa[ai];                        \
  pb = b + bi;                        \
  cvec4[0] += av * pb[0];             \
  cvec4[1] += av * pb[1];             \
  cvec4[2] += av * pb[2];             \
  cvec4[3] += av * pb[3];             \
  pb += 4;                            \
  ai += col_stride_a;                 \
  bi += col_stride_b;

  for (int k = 0; k < col_m; k += span) {
    CERES_GEMM_OPT_MTM_MAT1X4_MUL
    CERES_GEMM_OPT_MTM_MAT1X4_MUL
    CERES_GEMM_OPT_MTM_MAT1X4_MUL
    CERES_GEMM_OPT_MTM_MAT1X4_MUL
  }

  for (int k = col_m; k < col_a; k++) {
    CERES_GEMM_OPT_MTM_MAT1X4_MUL
  }

  CERES_GEMM_OPT_STORE_MAT1X4

#undef CERES_GEMM_OPT_MTM_MAT1X4_MUL
}

// Matrix-Vector Multiplication
// Figure out 4x1 of vector c in one batch
//
// c op A * b;
// where op can be +=, -=, or =, indicated by kOperation.
//
//  Vector c              Matrix A                   Vector b
//
//  C0               op   A0, A1, A2, A3, ...    *   B0
//  C1                    A4, A5, A6, A7, ...        B1
//  C2                    A8, A9, Aa, Ab, ...        B2
//  C3                    Ac, Ad, Ae, Af, ...        B3
//                                                   .
//                                                   .
//                                                   .
//
// unroll for loops
// utilize the data resided in cache
// NOTE: col_a means the columns of A
static inline void MVM_mat4x1(const int col_a,
                              const double* a,
                              const int col_stride_a,
                              const double* b,
                              double* c,
                              const int kOperation) {
  CERES_GEMM_OPT_NAIVE_HEADER
  double bv = 0.0;

  // clang-format off
#define CERES_GEMM_OPT_MVM_MAT4X1_MUL       \
  bv = *pb;                                 \
  cvec4[0] += *(pa                   ) * bv; \
  cvec4[1] += *(pa + col_stride_a    ) * bv; \
  cvec4[2] += *(pa + col_stride_a * 2) * bv; \
  cvec4[3] += *(pa + col_stride_a * 3) * bv; \
  pa++;                                     \
  pb++;
  // clang-format on

  for (int k = 0; k < col_m; k += span) {
    CERES_GEMM_OPT_MVM_MAT4X1_MUL
    CERES_GEMM_OPT_MVM_MAT4X1_MUL
    CERES_GEMM_OPT_MVM_MAT4X1_MUL
    CERES_GEMM_OPT_MVM_MAT4X1_MUL
  }

  for (int k = col_m; k < col_a; k++) {
    CERES_GEMM_OPT_MVM_MAT4X1_MUL
  }

  CERES_GEMM_OPT_STORE_MAT1X4

#undef CERES_GEMM_OPT_MVM_MAT4X1_MUL
}

// Matrix Transpose-Vector multiplication
// Figure out 4x1 of vector c in one batch
//
// c op A' * b;
// where op can be +=, -=, or =, indicated by kOperation.
//
//                        Matrix A
//
//                        A0, A4, A8, Ac
//                        A1, A5, A9, Ad
//                        A2, A6, Aa, Ae
//                        A3, A7, Ab, Af
//                        . , . , . , .
//                        . , . , . , .
//                        . , . , . , .
//
//  Vector c              Matrix A'                  Vector b
//
//  C0               op   A0, A1, A2, A3, ...    *   B0
//  C1                    A4, A5, A6, A7, ...        B1
//  C2                    A8, A9, Aa, Ab, ...        B2
//  C3                    Ac, Ad, Ae, Af, ...        B3
//                                                   .
//                                                   .
//                                                   .
//
// unroll for loops
// utilize the data resided in cache
// NOTE: col_a means the columns of A'
static inline void MTV_mat4x1(const int col_a,
                              const double* a,
                              const int col_stride_a,
                              const double* b,
                              double* c,
                              const int kOperation) {
  CERES_GEMM_OPT_NAIVE_HEADER
  double bv = 0.0;

#define CERES_GEMM_OPT_MTV_MAT4X1_MUL \
  bv = *pb;                           \
  cvec4[0] += pa[0] * bv;             \
  cvec4[1] += pa[1] * bv;             \
  cvec4[2] += pa[2] * bv;             \
  cvec4[3] += pa[3] * bv;             \
  pa += col_stride_a;                 \
  pb++;

  for (int k = 0; k < col_m; k += span) {
    CERES_GEMM_OPT_MTV_MAT4X1_MUL
    CERES_GEMM_OPT_MTV_MAT4X1_MUL
    CERES_GEMM_OPT_MTV_MAT4X1_MUL
    CERES_GEMM_OPT_MTV_MAT4X1_MUL
  }

  for (int k = col_m; k < col_a; k++) {
    CERES_GEMM_OPT_MTV_MAT4X1_MUL
  }

  CERES_GEMM_OPT_STORE_MAT1X4

#undef CERES_GEMM_OPT_MTV_MAT4X1_MUL
}

#undef CERES_GEMM_OPT_NAIVE_HEADER
#undef CERES_GEMM_OPT_STORE_MAT1X4

}  // namespace ceres::internal

#endif  // CERES_INTERNAL_SMALL_BLAS_GENERIC_H_