Skip to content
Snippets Groups Projects
Commit 8df57bc4 authored by Gasperin, Dr. Francesco's avatar Gasperin, Dr. Francesco
Browse files

Merge branch 'new_env_DP3v5' into 'master' 

New env dp3v5

See merge request !1
parents e293177d df1a11b9
No related branches found
No related tags found
1 merge request!1New env dp3v5
Show whitespace changes
Inline Side-by-side
DP3-buildfix.patch
+ 29
− 56
View file @ 8df57bc4
commit 6d66554bcd1e6fbd0362d6805ccc0f0ca19e518d
Author: Dr. Thomas Orgis <thomas.orgis@uni-hamburg.de>
Date: Tue Jun 9 17:32:41 2020 +0200
build: fixup python search and support BLAS_LIBS, LAPACK_LIBS
This is a set of changes we carry at Universität Hamburg to get
the LOFAR toolchain built. I hope it can be included upstream.
Important points are:
- Use current (less deprecated) cmake ways to detect python.
- Less redundancy in Python checking in CMakeLists at deeper level
(Always try to build PythonDPPP as Python and Boost is required
anyway.)
- Add support for BLAS_LIBS and LAPACK_LIBS, as this is
customary in HPC installations to choose the (optimized) BLAS
implementation in an environment that offers multiple options.
The last point is repeated in other patches we carry for the LOFAR
stack. Maybe the common source for the FindCasacore code should
be patched instead?
diff --git a/CMake/FindCasacore.cmake b/CMake/FindCasacore.cmake
index e4689f7..9068fdd 100644
index e4689f7..d6838fa 100644
--- a/CMake/FindCasacore.cmake
+++ b/CMake/FindCasacore.cmake
@@ -17,6 +17,8 @@
#
# Variables used by this module:
# CASACORE_ROOT_DIR - Casacore root directory.
+# BLAS_LIBS - override BLAS library
+# LAPACK_LIBS - override LAPACK library
#
# Variables defined by this module:
# CASACORE_FOUND - System has Casacore, which means that the
@@ -232,7 +234,21 @@ else(NOT CASACORE_INCLUDE_DIR)
@@ -232,7 +232,21 @@ else(NOT CASACORE_INCLUDE_DIR)
elseif(${_comp} STREQUAL fits)
casacore_find_package(CFITSIO REQUIRED)
elseif(${_comp} STREQUAL scimath_f)
......@@ -58,16 +26,14 @@ index e4689f7..9068fdd 100644
endforeach(_comp ${_find_components})
endif(NOT CASACORE_INCLUDE_DIR)
diff --git a/CMake/PythonInstall.cmake b/CMake/PythonInstall.cmake
index 173f995..a1357c7 100644
index 173f995..9fb1b57 100644
--- a/CMake/PythonInstall.cmake
+++ b/CMake/PythonInstall.cmake
@@ -24,10 +24,11 @@
@@ -24,10 +24,9 @@
# $Id: PythonInstall.cmake 32905 2015-11-17 15:31:54Z schaap $
# Search for the Python interpreter.
-find_package(PythonInterp)
+# No. We expect the toplevel CMakeLists.txt to have found python already!
+#find_package(PythonInterp)
# Derive the Python site-packages installation directory and build directory.
-if(PYTHON_EXECUTABLE)
......@@ -75,7 +41,7 @@ index 173f995..a1357c7 100644
set(_cmd
"from distutils.sysconfig import get_python_lib"
"from os.path import join"
@@ -36,7 +37,7 @@ if(PYTHON_EXECUTABLE)
@@ -36,7 +35,7 @@ if(PYTHON_EXECUTABLE)
'site-packages'))"
)
execute_process(
......@@ -84,7 +50,7 @@ index 173f995..a1357c7 100644
OUTPUT_VARIABLE _pydir
ERROR_VARIABLE _pyerr
OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -57,7 +58,7 @@ if(PYTHON_EXECUTABLE)
@@ -57,7 +56,7 @@ if(PYTHON_EXECUTABLE)
message(STATUS "Build directory for Python extensions: ${PYTHON_BUILD_DIR}")
message(STATUS "Installation directory for Python extensions: ${PYTHON_INSTALL_DIR}")
endif()
......@@ -93,7 +59,7 @@ index 173f995..a1357c7 100644
#
@@ -66,9 +67,9 @@ endif(PYTHON_EXECUTABLE)
@@ -66,9 +65,9 @@ endif(PYTHON_EXECUTABLE)
macro(python_install)
# Precondition check.
......@@ -105,11 +71,20 @@ index 173f995..a1357c7 100644
# Parse arguments.
# apart from the python files list, there are two additional arguments
@@ -125,7 +124,7 @@ macro(python_install)
"print('-- Byte-compiling: %s${_inst_dir}/${_py}' % destdir)"
"py_compile.compile('%s${DESTDIR}${_inst_dir}/${_py}' % destdir, doraise=True)")
install(CODE
- "execute_process(COMMAND ${PYTHON_EXECUTABLE} -c \"${_py_code}\"
+ "execute_process(COMMAND ${Python3_EXECUTABLE} -c \"${_py_code}\"
RESULT_VARIABLE _result)
if(NOT _result EQUAL 0)
message(FATAL_ERROR \"Byte-compilation FAILED: \$ENV{DESTDIR}${_inst_dir}/${_py}\")
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2943e17..ae2c423 100644
index 8025cdb..ac9bb03 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -44,13 +44,14 @@ endif(IDGAPI_LIBRARIES AND IDGAPI_INCLUDE_DIRS)
@@ -46,13 +46,13 @@ endif(IDGAPI_LIBRARIES AND IDGAPI_INCLUDE_DIRS)
find_package(CFITSIO REQUIRED)
......@@ -118,9 +93,8 @@ index 2943e17..ae2c423 100644
-message(STATUS "Using python version ${PYTHON_VERSION_STRING}")
+find_package(Python3 COMPONENTS Interpreter Development)
+message(STATUS "Using python version ${Python3_VERSION}")
+
+include_directories(${Python3_INCLUDE_DIRS})
+include_directories(${Python3_INCLUDE_DIRS})
#Prevent accidentally finding old BoostConfig.cmake file from casapy
set(Boost_NO_BOOST_CMAKE ON)
-set(BOOSTPY ${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR})
......@@ -128,46 +102,45 @@ index 2943e17..ae2c423 100644
find_package(Boost COMPONENTS date_time filesystem python${BOOSTPY} numpy${BOOSTPY} program_options system unit_test_framework REQUIRED)
include_directories(${Boost_INCLUDE_DIR})
@@ -236,7 +237,7 @@ set(EXTRA_LIBRARIES ${EXTRA_LIBRARIES}
@@ -220,7 +220,7 @@ set(EXTRA_LIBRARIES
${AOFLAGGER_LIB}
${CASACORE_LIBRARIES}
${Boost_LIBRARIES}
${EXTRA_LIBRARIES}
- ${PYTHON_LIBRARIES}
+ ${Python3_LIBRARIES}
${IDGAPI_LIBRARIES}
Threads::Threads)
diff --git a/PythonDPPP/CMakeLists.txt b/PythonDPPP/CMakeLists.txt
index 72fe59a..7e4bcad 100644
index 72fe59a..69f9fb8 100644
--- a/PythonDPPP/CMakeLists.txt
+++ b/PythonDPPP/CMakeLists.txt
@@ -1,5 +1,5 @@
@@ -1,5 +1,4 @@
-# If Boost-Python3 found, build this package. Otherwise give a warning.
-if(${Boost_PYTHON${BOOSTPY}_FOUND})
+# We set Boost-python as required in toplevel build file.
+if(TRUE)
include(PythonInstall)
@@ -25,20 +25,7 @@ if(${Boost_PYTHON${BOOSTPY}_FOUND})
@@ -25,20 +24,6 @@ if(${Boost_PYTHON${BOOSTPY}_FOUND})
# Install Python modules
python_install(__init__.py DESTINATION lofar/pythondppp)
- find_package(PythonInterp REQUIRED)
- find_package(PythonLibs 3 REQUIRED)
- include_directories(${PYTHON_INCLUDE_DIRS})
-
+ find_package(Casacore COMPONENTS python3)
- if(${PYTHON_VERSION_MAJOR} EQUAL 2)
- find_package(Casacore COMPONENTS python)
- else()
- find_package(Casacore COMPONENTS python3)
- endif()
+ find_package(Casacore COMPONENTS python3)
-
- find_package(Boost REQUIRED COMPONENTS python${BOOSTPY})
- include_directories(${Boost_INCLUDE_DIR})
-
# add_subdirectory(test)
-# add_subdirectory(test)
-else()
- message (WARNING, " Boost-Python not found; PythonDPPP will not be built.")
endif()
LOFARBeam-buildfix.patch
+ 15
− 0
View file @ 8df57bc4
......@@ -34,3 +34,18 @@ index 51d8339..299530f 100644
endif(${_comp} STREQUAL casa)
endforeach(_comp ${_find_components})
endif(NOT CASACORE_INCLUDE_DIR)
# Required to find boost python stuff
diff --git a/CMakeLists.txt b/CMakeLists.txt
index bd1de34..c17d181 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -82,6 +82,7 @@ if (BUILD_PYTHON)
set(oldboostpy "3")
endif()
set(boostpy ${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR})
+ set(Boost_NO_BOOST_CMAKE ON)
message(STATUS "Trying to find boost-python libraries with version suffix ${boostpy}")
find_package(Boost 1.67.0 COMPONENTS python${boostpy} numpy${boostpy})
if(NOT Boost_FOUND)
message(STATUS "GTKMM found.")
aoflagger-buildfix.patch deleted 100644 → 0
+ 0
− 150
View file @ e293177d
diff --git a/CMake/FindCasacore.cmake b/CMake/FindCasacore.cmake
index e4689f7..9068fdd 100644
--- a/CMake/FindCasacore.cmake
+++ b/CMake/FindCasacore.cmake
@@ -17,6 +17,8 @@
#
# Variables used by this module:
# CASACORE_ROOT_DIR - Casacore root directory.
+# BLAS_LIBS - override BLAS library
+# LAPACK_LIBS - override LAPACK library
#
# Variables defined by this module:
# CASACORE_FOUND - System has Casacore, which means that the
@@ -232,7 +234,21 @@ else(NOT CASACORE_INCLUDE_DIR)
elseif(${_comp} STREQUAL fits)
casacore_find_package(CFITSIO REQUIRED)
elseif(${_comp} STREQUAL scimath_f)
- casacore_find_package(LAPACK REQUIRED)
+ # If only looking for LAPACK, no library will be added if LAPACK
+ # is part of BLAS, as it is customary nowadays. So look for both
+ # to avoid confusing linker errors on symbols used in headers/templates.
+ if(DEFINED ENV{BLAS_LIBS})
+ set(BLAS_FOUND YES)
+ list(APPEND CASACORE_LIBRARIES $ENV{BLAS_LIBS})
+ else()
+ casacore_find_package(BLAS REQUIRED)
+ endif()
+ if(DEFINED ENV{LAPACK_LIBS})
+ set(LAPACK_FOUND YES)
+ list(APPEND CASACORE_LIBRARIES $ENV{LAPACK_LIBS})
+ else()
+ casacore_find_package(LAPACK REQUIRED)
+ endif()
endif(${_comp} STREQUAL casa)
endforeach(_comp ${_find_components})
endif(NOT CASACORE_INCLUDE_DIR)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fd1d7c7..4dd733c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,27 +19,61 @@ find_package(CFITSIO REQUIRED)
find_path(FFTW3_INCLUDE_DIR NAMES fftw3.h)
find_library(GSL_LIB NAMES gsl)
-find_library(GSL_CBLAS_LIB NAMES gslcblas)
+if(DEFINED ENV{CBLAS_LIBS})
+ message(STATUS "Using CBLAS_LIBS from environment: $ENV{CBLAS_LIBS}")
+ set(GSL_CBLAS_LIB $ENV{CBLAS_LIBS})
+else()
+ find_library(GSL_CBLAS_LIB NAMES gslcblas)
+endif()
find_path(GSL_INCLUDE_DIR NAMES gsl/gsl_version.h)
find_package(Threads REQUIRED)
find_package(LibXml2 REQUIRED)
find_package(PNG REQUIRED)
+find_package(Python COMPONENTS Interpreter Development)
+if(NOT Python_FOUND)
+ message(STATUS "Falling back to deprecated Python search.")
+ find_package(PythonInterp REQUIRED)
+ find_package(PythonLibs REQUIRED)
+ set(Python_VERSION ${PYTHON_VERSION_STRING})
+ set(Python_VERSION_MAJOR ${PYTHON_VERSION_MAJOR})
+ set(Python_VERSION_MINOR ${PYTHON_VERSION_MINOR})
+ set(Python_LIBRARIES ${PYTHON_LIBRARIES})
+ set(Python_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
+endif()
+message(STATUS "Using python version ${Python_VERSION}")
#Prevent accidentally finding old BoostConfig.cmake file from casapy
set(Boost_NO_BOOST_CMAKE ON)
-#set(PY_VERSION 3)
-#find_package(PythonLibs ${PY_VERSION} REQUIRED)
-find_package(PythonInterp REQUIRED)
-#find_package(PythonLibs ${PYTHON_VERSION_STRING} REQUIRED)
-find_package(PythonLibs 2.7 REQUIRED)
-#message(STATUS "Using python version ${PYTHON_VERSION_STRING}")
# At least the intrusive_ref_counter needs Boost 1.55 (because the header was moved)
-find_package(Boost 1.55.0 REQUIRED COMPONENTS date_time filesystem python system)
-#find_package(Boost REQUIRED COMPONENTS date_time filesystem python-py36 system)
+# Bumped to 1.67 to be able to use proper python suffix. One attempt
+# to support at least two boost_python naming schemes here.
+set(boostpython python${Python_VERSION_MAJOR}${Python_VERSION_MINOR})
+find_package(Boost 1.67.0 COMPONENTS date_time filesystem ${boostpython} system)
+if(NOT Boost_FOUND)
+ message(STATUS "Falling back to old Boost search.")
+ set(boostpython python)
+ find_package(Boost 1.55.0 REQUIRED COMPONENTS date_time filesystem ${boostpython} system)
+endif()
+
find_library(FFTW3_LIB fftw3 REQUIRED)
+# Not sure if this is needed everywhere, but I get linker
+# errors with missing symbols if not explicitly linking to fftw3_threads.
+find_library(FFTW3_THREADS_LIB fftw3_threads OPTIONAL)
enable_language(Fortran OPTIONAL)
-find_package(BLAS REQUIRED)
-find_package(LAPACK REQUIRED)
+if(DEFINED ENV{BLAS_LIBS})
+ message(STATUS "Using BLAS_LIBS from environment: $ENV{BLAS_LIBS}")
+ set(BLAS_FOUND YES)
+ set(BLAS_LIBRARIES $ENV{BLAS_LIBS})
+else()
+ find_package (BLAS REQUIRED)
+endif()
+if(DEFINED ENV{LAPACK_LIBS})
+ message(STATUS "Using LAPACK_LIBS from environment: $ENV{LAPACK_LIBS}")
+ set(LAPACK_FOUND YES)
+ set(LAPACK_LIBRARIES $ENV{LAPACK_LIBS})
+else()
+ find_package (LAPACK REQUIRED)
+endif()
if(CMAKE_SYSTEM_NAME MATCHES "Linux")
find_library(RT_LIBRARY rt)
# list(APPEND LOFAR_EXTRA_LIBRARIES ${RT_LIBRARY})
@@ -53,7 +87,7 @@ include_directories(${FFTW3_INCLUDE_DIR})
if(GSL_INCLUDE_DIR)
include_directories(${GSL_INCLUDE_DIR})
endif(GSL_INCLUDE_DIR)
-include_directories(${PYTHON_INCLUDE_DIRS})
+include_directories(${Python_INCLUDE_DIRS})
# The following stuff will set the "rpath" correctly, so that
# LD_LIBRARY_PATH doesn't have to be set.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 4a36e1c..55713b2 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -235,17 +235,17 @@ set(PYTHON_FILES
set(ALL_LIBRARIES
${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}
- ${Boost_SYSTEM_LIBRARY}
- ${Boost_FILESYSTEM_LIBRARY} ${Boost_DATE_TIME_LIBRARY}
-# ${Boost_PYTHON-PY36_LIBRARIES}
- ${Boost_PYTHON_LIBRARIES}
- ${FFTW3_LIB}
+ Boost::system
+ Boost::filesystem
+ Boost::date_time
+ Boost::${boostpython}
+ ${FFTW3_THREADS_LIB} ${FFTW3_LIB}
${CASACORE_LIBRARIES}
${LAPACK_lapack_LIBRARY}
${CFITSIO_LIBRARY}
${CMAKE_THREAD_LIBS_INIT}
${PNG_LIBRARIES} ${LIBXML2_LIBRARIES}
- ${PYTHON_LIBRARIES})
+ ${Python_LIBRARIES})
if(GTKMM_FOUND)
message(STATUS "GTKMM found.")
build_lofar.sh 100644 → 100755
+ 37
− 11
View file @ 8df57bc4
......@@ -410,13 +410,10 @@ if [ ! -e $prefix/.done ]; then
if [ -e src ]; then
echo "Using existing sources."
else
git clone git://git.code.sf.net/p/aoflagger/code presrc
git clone https://gitlab.com/aroffringa/aoflagger.git src
if [ "$aoflagger_version" != "latest" ]; then
( cd presrc && git checkout tags/v$aoflagger_version )
fi &&
echo "Patching build (for BLAS, Boost, Python, usually) ..." &&
(cd presrc && patch -Np1 < "$scriptdir/aoflagger-buildfix.patch" ) &&
mv presrc src
( cd src && git checkout tags/v$aoflagger_version )
fi
fi &&
mkdir build &&
cd build &&
......@@ -438,6 +435,32 @@ else
echo AOFlagger already installed.
fi >> "$prefix/build.log" 2>&1 || exit 1
begin_pkg EveryBeam
if [ ! -e $prefix/.done ]; then
#
# Install the standalone StationResponse libraries.
#
echo Installing EveryBeam...
cd $prefix &&
rm -rf build bin lib presrc &&
if [ -e src ]; then
echo "Using existing sources."
else
git clone https://git.astron.nl/RD/EveryBeam.git src
fi &&
mkdir build &&
cd build &&
use_prefix $lofar_prefix/casacore $prefix &&
run_cmake \
-DCASACORE_ROOT_DIR=$lofar_prefix/casacore \
-DCMAKE_INSTALL_PREFIX=$prefix \
../src &&
make -j $J &&
make install &&
echo Done with EveryBeam. &&
touch $prefix/.done
fi >> "$prefix/build.log" 2>&1 || exit 1
begin_pkg LOFARBeam
if [ ! -e $prefix/.done ]; then
#
......@@ -519,8 +542,11 @@ if [ ! -e $prefix/.done ]; then
echo "Using existing sources."
else
git clone https://github.com/lofar-astron/DP3.git presrc &&
echo "Patching build (for BLAS, Boost, Python, usually) ..." &&
(cd presrc && patch -Np1 < "$scriptdir/DP3-buildfix.patch" ) &&
if [ "$dp3_version" != "latest" ]; then
( cd presrc && git checkout tags/v$dp3_version )
fi
# echo "Patching build (for BLAS, Boost, Python, usually) ..." &&
#(cd presrc && patch -Np1 < "$scriptdir/DP3-buildfix.patch" ) &&
mv presrc src
fi &&
mkdir build &&
......@@ -528,7 +554,7 @@ if [ ! -e $prefix/.done ]; then
use_prefix \
$lofar_prefix/casacore \
$lofar_prefix/idg \
$lofar_prefix/LOFARBeam \
$lofar_prefix/EveryBeam \
$lofar_prefix/aoflagger \
$prefix &&
# ThOr: It's braindead that I still have to provide things like
......@@ -578,8 +604,8 @@ if [ ! -e $prefix/.done ]; then
else
git clone git://git.code.sf.net/p/wsclean/code presrc
fi &&
echo "Patching build (for BLAS, Boost, Python, usually) ..." &&
(cd presrc && patch -Np1 < "$scriptdir/$pkg-buildfix.patch" ) &&
#echo "Patching build (for BLAS, Boost, Python, usually) ..." &&
#(cd presrc && patch -Np1 < "$scriptdir/$pkg-buildfix.patch" ) &&
mv presrc src
fi &&
mkdir build &&
......
build_lofar.sh.config.rrz
+ 14
− 2
View file @ 8df57bc4
# The setup on the Hamburg LOFAR GPU Cluster.
aoflagger_version=2.14.0
# 2021 01 11:
# This configuration worked, using the following versions:
#aoflagger_version=3.0.2
#casacore_version=3.3.0
#dysco_version=1.2.0
#pybdsf_version=1.9.2
#pycasa_version=3.1.1
#wsclean_version=2.10
#dp3_version=5.0.2
aoflagger_version=latest
casacore_version=3.3.0
dysco_version=1.2.0
pybdsf_version=latest # weird C++ template error with 1.8.12
pycasa_version=3.1.1
wsclean_version=latest
dp3_version=latest
# This gives us CUDA, gcc, Open MPI and dependencies
# from pkgsrc in separate prefixes.
. /sw/profile/init.sh
module switch env env/2019Q4-cuda-gcc-openmpi
module use /sw/TEST
module switch env env/2020Q3-cuda-gcc-openmpi
module load pythonversion/3
# You could experiment with idg using MKL, but it is recommened
# to try a consistent environment where everyone agrees on using
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment