""" * * This file is part of rasdaman community. * * Rasdaman community is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Rasdaman community is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with rasdaman community. If not, see . * * Copyright 2003 - 2016 Peter Baumann / rasdaman GmbH. * * For more information please see * or contact Peter Baumann via . * """ from abc import abstractmethod from util.commands import make_directory, nproc from util.log import log from util.timer import Timer from util.string_util import binary_to_string from services import executor, tomcat_service from tpinstaller.tpinstaller import is_system_cmake_supported import os class Builder: def __init__(self): """ Base class for our builders """ pass @abstractmethod def build(self): """ Builds the component by compiling the code and moving it to the correct position (equivalent in semantics to make && make install) """ pass class BuilderDecorator(Builder): def __init__(self, builder): """ Decorator pattern for builders :param Builder builder: the Builder to """ Builder.__init__(self) self.builder = builder def build(self): self.builder.build() class BuilderBasis(Builder): def __init__(self): """ NO-OP class that is used as a basis when constructing the builder hierarchy """ Builder.__init__(self) def build(self): pass class RasdamanCmakeBuilder(BuilderDecorator): def __init__(self, builder, configurer, source_path, install_path, update, istest): """ Builds rasdaman by compiling the source code using cmake :param Builder builder: the builder to decorate :param Configurer configurer: a configurer to be used in the build process :param str source_path: the path to the sources :param str install_path: the path to the whole rasdaman directory :param bool update: False if this is a first time installation :param bool istest: The systemtest will be executed as well in the install """ BuilderDecorator.__init__(self, builder) self.configurer = configurer self.source_path = source_path self.build_dir = self.source_path + "build/" self.install_path = install_path self.update = update self.istest = istest self.CUSTOM_CMAKE = self.install_path + \ "/third_party/cmake-3.16.2-Linux-x86_64/bin/cmake" def configure(self): """ Configures rasdaman and builds it with the given configuration exposed by the configurer composition :param Configurer configurer: the configurer decorator """ log.info("Configuring rasdaman with cmake...") timer = Timer() if os.path.exists(self.build_dir + "server/rasserver"): log.info("Running make clean as a previous build was found...") executor.execute(["make", "clean"], self.build_dir, throw_on_error=False, warn_on_error=True) make_directory(self.build_dir) configure_params = list(self.configurer.configure()) cmake_cmd = "cmake" if not is_system_cmake_supported(): cmake_cmd = self.CUSTOM_CMAKE allconfigs = " ".join([cmake_cmd] + ["../"] + configure_params) executor.execute(allconfigs, self.build_dir, shell=True) log.info("Rasdaman was configured with params: {0} in {1} s.".format( " ".join(configure_params), timer.elapsed())) def get_most_time_consuming_targets(self, build_output, limit): """ Filter the most consuming targets to build, sort in descending order and limit the results list to the given limit. """ # Line format: # ...... 5s to compile rasnetclientcomm.cc to rasnetclientcomm.cc.o BUILD_TIME_LINE_PREFIX = '...... ' lines = binary_to_string(build_output).split('\n') ret = [l for l in lines if l.startswith(BUILD_TIME_LINE_PREFIX)] ret.sort(reverse=True, key=lambda l: int(l.split(' ')[1][:-1])) return ret[:limit] def __get_env_with_updated_path(self): # pip3 installs sphinx in /usr/local/bin and it is not on the PATH on CentOS 7 import os os.environ['PATH'] = os.environ['PATH'] + ":/usr/local/bin" return os.environ.copy() def make(self): """ Runs the make commands """ timer = Timer() threads = nproc() log.info("Compiling rasdaman with make -j{} in {} (this will take some " "time)...".format(threads, self.build_dir)) out, _, _ = executor.execute(["make", "-j" + threads], self.build_dir, shell=True, env=self.__get_env_with_updated_path()) if self.istest: MAX_LINES_TO_PRINT = 10 targets = self.get_most_time_consuming_targets(out, MAX_LINES_TO_PRINT) if len(targets) > 0: log.info("Top {0} targets taking longest to execute:\n{1}".format( len(targets), "\n".join(targets))) log.info("Rasdaman was compiled successfully in {0} s.".format(timer.elapsed())) log.info("Installing rasdaman...") timer.start() executor.execute(["make", "install"], self.build_dir) log.info("Rasdaman installed successfully in {0} s.".format(timer.elapsed())) if self.istest: log.info("Running unit tests with make unit-check -j{0}...".format(threads)) timer.start() executor.execute(["make", "unit-check", "-j" + threads], self.build_dir) log.info("Unit tests executed successfully in {0} s.".format(timer.elapsed())) def build(self): BuilderDecorator.build(self) self.configure() self.make() class WarFileInstallBuilder(BuilderDecorator): def __init__(self, builder, webapps_path, war_file): """ Installer for a webapp war file :param Builder builder: the builder to decorate :param str webapps_path: the webapps path to deploy to :param str war_file: the path to the war file to be deployed """ BuilderDecorator.__init__(self, builder) self.webapps_path = webapps_path self.war_file = war_file def build(self): BuilderDecorator.build(self) tomcat_service.install_war_file(self.war_file, self.webapps_path) class WarFileUninstallBuilder(BuilderDecorator): def __init__(self, builder, webapps_path, war_file_name): """ Uninstall a webapp war file name, e.g. def.war :param Builder builder: the builder to decorate :param str webapps_path: the webapps path to deploy to :param str war_file_name: the path to the war file to be removed """ BuilderDecorator.__init__(self, builder) self.webapps_path = webapps_path self.war_file_name = war_file_name def build(self): BuilderDecorator.build(self) tomcat_service.uninstall_war_file(self.war_file_name, self.webapps_path) def test(): sample_build_output = """ Scanning dependencies of target directql [ 67%] Building CXX object applications/directql/CMakeFiles/directql.dir/directql.cc.o ...... 28s to compile directql.cc to directql.cc.o [ 67%] Building CXX object applications/directql/CMakeFiles/directql.dir/directql_error.cc.o [ 67%] Building CXX object applications/directql/CMakeFiles/directql.dir/__/__/server/rasserver_config.cc.o [ 67%] Linking CXX executable directql [ 67%] Built target directql Scanning dependencies of target rasql [ 67%] Building CXX object applications/rasql/CMakeFiles/rasql.dir/rasql.cc.o ...... 10s to compile rasql.cc to rasql.cc.o [ 67%] Building CXX object applications/rasql/CMakeFiles/rasql.dir/rasql_error.cc.o [ 67%] Linking CXX executable rasql [ 67%] Built target rasql Scanning dependencies of target rasdapy_generate_errtxts_message [ 67%] Built target rasdapy_generate_errtxts_message Scanning dependencies of target rasj_generate_errtxts_message [ 67%] Built target rasj_generate_errtxts_message Scanning dependencies of target rasj [ 67%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/HealthServiceGrpc.java [ 67%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/ClientRassrvrServiceOuterClass.java [ 67%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/RasmgrClientServiceOuterClass.java [ 67%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/HealthServiceOuterClass.java [ 67%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/Error.java [ 67%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/CommonService.java [ 67%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/ClientRassrvrServiceGrpc.java [ 68%] Generating ../../java/src/main/java/org/rasdaman/rasnet/service/RasmgrClientServiceGrpc.java ...... 41s to build custom target in java: /usr/bin/mvn -q -B package -Prasnet ...... 7s to build custom target in java: /usr/bin/mvn -q install:install-file -Dfile=/opt/rasdaman/source/build/java/target/rasj-jar-with-dependencies.jar -DgroupId=org.rasdaman -DartifactId=rasj -Dversion=9.7.0 -Dpackaging=jar [ 68%] Built target rasj Scanning dependencies of target petascope ...... 176s to build custom target in petascope: /usr/bin/mvn -q -B package -f /opt/rasdaman/source/build/applications/petascope//pom.xml [ 68%] Built target petascope Scanning dependencies of target raswct [ 68%] Generating build/raswct.css [ 68%] Generating build/raswct_lib.js """ exp_result = """...... 176s to build custom target in petascope: /usr/bin/mvn -q -B package -f /opt/rasdaman/source/build/applications/petascope//pom.xml ...... 41s to build custom target in java: /usr/bin/mvn -q -B package -Prasnet ...... 28s to compile directql.cc to directql.cc.o ...... 10s to compile rasql.cc to rasql.cc.o ...... 7s to build custom target in java: /usr/bin/mvn -q install:install-file -Dfile=/opt/rasdaman/source/build/java/target/rasj-jar-with-dependencies.jar -DgroupId=org.rasdaman -DartifactId=rasj -Dversion=9.7.0 -Dpackaging=jar""" builder = RasdamanCmakeBuilder(BuilderBasis(), None, "", "", False, True) l = builder.get_most_time_consuming_targets(sample_build_output, 10) result = "\n".join(l) return result == exp_result if __name__== "__main__": test()