Fix custom Eigen on macos (EIGEN_INCLUDE_DIR_HINTS)

In ceres' CMakeLists.txt a workaround for custom homebrew locations
sets CMAKE_PREFIX_PATH to the homebrew install path. This overrides
explicit requests for custom locations for third-party dependencies,
in particular for Eigen with EIGEN_INCLUDE_DIR_HINTS.

This commit introduces a workaround to make EIGEN_INCLUDE_DIR_HINTS
work on macos by ignoring CMAKE_PREFIX_PATH in that case.

Fixes https://github.com/ceres-solver/ceres-solver/issues/431

Change-Id: I3ec0ec418d45d41b9e5ebdd6aee60552438afec3

CMakeLists.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6308cfb..be045a2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -87,6 +87,10 @@
 # CMake in find_path & find_library.  This should ensure that we can
 # still build Ceres even if Homebrew is installed in a non-standard
 # location (not /usr/local).
+# Note: This is a hack that breaks HINTS passed to find_... modules
+# for thirdparty dependencies, since CMAKE_PREFIX_PATH takes
+# precendence over HINTS. See FindEigen.cmake for notes on the
+# workaround needed for making EIGEN_INCLUDE_DIR_HINTS work.
 if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
   find_program(HOMEBREW_EXECUTABLE brew)
   mark_as_advanced(FORCE HOMEBREW_EXECUTABLE)

cmake/FindEigen.cmake

diff --git a/cmake/FindEigen.cmake b/cmake/FindEigen.cmake
index f6d2664..d218699 100644
--- a/cmake/FindEigen.cmake
+++ b/cmake/FindEigen.cmake
@@ -192,12 +192,25 @@
     Eigen/include/eigen3 # Windows (for C:/Program Files prefix) < 3.3
     Eigen3/include/eigen3 ) # Windows (for C:/Program Files prefix) >= 3.3
 
+  # On macos with Homebrew, the brew prefix is added
+  # CMAKE_PREFIX_PATH, which overrides EIGEN_INCLUDE_DIR_HINTS in the
+  # following find_path, and the brew Eigen version is found (if
+  # installed) instead of the one provided by the hint. So, in that
+  # case we ignore CMAKE_PREFIX_PATH.  See also the comments in ceres'
+  # main CMakeLists.txt around where HOMEBREW_EXECUTABLE is set.
+  if(HOMEBREW_EXECUTABLE AND EIGEN_INCLUDE_DIR_HINTS)
+    set(FIND_PATH_FLAG NO_CMAKE_PATH)
+  else()
+    set(FIND_PATH_FLAG)
+  endif()
+
   # Search supplied hint directories first if supplied.
   find_path(EIGEN_INCLUDE_DIR
     NAMES Eigen/Core
     HINTS ${EIGEN_INCLUDE_DIR_HINTS}
     PATHS ${EIGEN_CHECK_INCLUDE_DIRS}
-    PATH_SUFFIXES ${EIGEN_CHECK_PATH_SUFFIXES})
+    PATH_SUFFIXES ${EIGEN_CHECK_PATH_SUFFIXES}
+    ${FIND_PATH_FLAG})
 
   if (NOT EIGEN_INCLUDE_DIR OR
       NOT EXISTS ${EIGEN_INCLUDE_DIR})