"""
*
* This file is part of rasdaman community.
*
* Rasdaman community is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Rasdaman community is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with rasdaman community. If not, see .
*
* Copyright 2003 - 2016 Peter Baumann / rasdaman GmbH.
*
* For more information please see
* or contact Peter Baumann via .
*
"""
import fileinput
import os
import re
from abc import ABCMeta, abstractmethod
from config_manager import ConfigManager
from util.commands import make_backup, remove, make_directory, sudo_chown, \
sudo_chmod, sudo_mkdir, sudo_copy
from util.log import log
from util.file_util import find_line_containing_token_in_file, write_to_file, \
get_owner, system_user_exists, read_file, get_script_dir, fix_trailing_slash
from util.string_util import strip_whitespace
from util.prompt import ChangeSeverity, user_confirmed_change
from services import executor, tomcat_service, postgres_service, linux_distro
class TPInstaller:
"""
The classes implementing the TPInstaller (third party installer) are in
charge of installing specific external rasdaman dependencies
"""
__metaclass__ = ABCMeta
@abstractmethod
def install(self):
"""
Installs the third party dependency
"""
pass
class TPInstallerBasis(TPInstaller):
def __init__(self):
"""
The basis for all tp installers
"""
pass
def install(self):
pass
class TPInstallerDecorator(TPInstaller):
def __init__(self, installer):
"""
The tp installer decorator allows you to compose several installers into one.
:param TPInstaller installer: the installer that this decorator wraps
"""
self.installer = installer
def install(self):
"""
This method should be overridden to add new installing behavior.
However a call to super will always be required.
"""
self.installer.install()
class MainPathInstaller(TPInstallerDecorator):
def __init__(self, installer, install_path):
"""
Creates the directory where rasdaman is installed (RMANHOME) and makes
sure that it is accessible by the rasdaman user.
:param install_path: the main path where rasdaman package will sit
"""
super(MainPathInstaller, self).__init__(installer)
self.install_path = install_path
def __ensure_directory_tree_access(self):
"""
Makes sure we can access /a/b/c by adding the x flag to each of /a, /a/b, /a/b/c
"""
parent_dir = self.install_path
# Go a maximum of 128 directories up.
# This is to make sure we don't hit some weird symlink structure and loop forever
for i in range(0, 128):
parent_dir = os.path.dirname(parent_dir)
sudo_chmod(parent_dir, "+x", is_dir=False)
if parent_dir == "/":
break
def install(self):
super(MainPathInstaller, self).install()
log.info("Creating installation directory {}...".format(self.install_path))
if not os.path.exists(self.install_path):
sudo_mkdir(self.install_path)
sudo_chown(self.install_path, ConfigManager.default_user)
self.__ensure_directory_tree_access()
third_party_folder = self.install_path + "third_party/"
make_directory(third_party_folder)
log.info("Installation directory created successfully.")
else:
log.info("Installation directory already created.")
if get_owner(self.install_path) != ConfigManager.default_user:
sudo_chown(self.install_path, ConfigManager.default_user)
class DebPackageInstaller(TPInstallerDecorator):
def __init__(self, installer, package_list):
"""
Install a list of debian packages.
:param list[str] package_list: the list of packages to install
"""
super(DebPackageInstaller, self).__init__(installer)
self.package_list = package_list
def __install_ppa_repos(self):
"""Needed for Ubuntu 14.04"""
if 'ruby2.0' in self.package_list:
ppa_repo = 'ppa:brightbox/ruby-ng'
log.info("Installing {0} repo for package ruby2.0... ".format(ppa_repo))
executor.executeSudo(["apt-add-repository", "-y", ppa_repo])
executor.executeSudo(["apt-get", "update"], throw_on_error=False)
log.info("PPA repository installed successfully")
def __install_pkgs(self):
"""
:return True if packages were installed successfully, False otherwise
"""
_, _, rc = executor.executeSudo(
["apt-get", "install", "--yes", "-q", "-m", "--no-install-recommends"] +
self.package_list, throw_on_error=False)
return rc == 0
def install(self):
super(DebPackageInstaller, self).install()
log.info("Installing system packages: " + " ".join(self.package_list))
if user_confirmed_change("Continue?", ChangeSeverity.NORMAL):
self.__install_ppa_repos()
if not self.__install_pkgs():
log.warn("System package installation failed, attempting to fix " \
"apt with potentially broken dependencies.")
executor.executeSudo(["apt-get", "install", "--yes", "-f"])
if not self.__install_pkgs():
msg = "System package installation failed, see " + \
"/tmp/rasdaman.install.log for further details."
log.error(msg)
raise Exception(msg)
log.info("System packages installed successfully.")
else:
log.info("System package installation skipped.")
class RpmPackageInstaller(TPInstallerDecorator):
def __init__(self, installer, package_list, repo_tool='yum'):
"""
Installer for a list of rpm packages
:param list[str] package_list: the list of packages to install
"""
super(RpmPackageInstaller, self).__init__(installer)
self.package_list = package_list
self.repo_tool = repo_tool
def install(self):
super(RpmPackageInstaller, self).install()
log.info("Installing system packages: " + " ".join(self.package_list))
if user_confirmed_change("Continue?", ChangeSeverity.NORMAL):
# --setopt=obsoletes=0 disables removing obsolete packages
executor.executeSudo([self.repo_tool, "install", "-y"] + self.package_list)
log.info("System packages installed successfully.")
else:
log.info("System package installation skipped.")
class PipInstaller(TPInstallerDecorator):
def __init__(self, installer, package_list, pip_tool='pip3'):
"""
Installer for pip packages
:param list[str] package_list: the list of packages to install
"""
super(PipInstaller, self).__init__(installer)
self.package_list = package_list
self.pip_tool = pip_tool
def install(self):
super(PipInstaller, self).install()
if not self.package_list:
return
log.info("Installing python packages with {0}: {1}... "
.format(self.pip_tool, " ".join(self.package_list)))
if user_confirmed_change("Continue?", ChangeSeverity.NORMAL):
_,_,rc =executor.executeSudo([self.pip_tool, "install"] + self.package_list,
throw_on_error=False)
if rc == 0:
log.info("Python packages installed successfully.")
else:
log.warn("Python package installation failed, see " + \
"/tmp/rasdaman.install.log for further details.")
else:
log.info("Python package installation skipped.")
class TomcatInstaller(TPInstallerDecorator):
def __init__(self, installer, tomcat_conf_path=linux_distro.get_tomcat_conf_path()):
"""
Fix configuration of Tomcat if necessary.
"""
super(TomcatInstaller, self).__init__(installer)
self.tomcat_conf_path = tomcat_conf_path
self.MIN_MEMORY_LIMIT_MB = 2000
self.MIN_SETTINGS = '-Xmx{}m -XX:MaxPermSize=256m'.format(str(self.MIN_MEMORY_LIMIT_MB))
def __get_mb_value(self, size, suffix):
"""
Return the size in MB based on the suffix.
"""
if suffix == 'g' or suffix == 'G':
return size * 1000
elif suffix == 'm' or suffix == 'M':
return size
elif suffix == 'k' or suffix == 'K':
return size / 1000
else:
return size / 1000000 # no suffix = bytes
def update_tomcat_conf_contents(self, lines):
"""
Increases the tomcat default memory limit if necessary.
:param str lines: the contents of tomcat.conf as a list of string lines
:rtype: bool
"""
p = re.compile('[^#].*-Xmx(\d+)([gGmMkK\'\"\s]).*')
for i in range(len(lines) - 1, -1, -1):
line = lines[i]
m = p.match(line)
if m is not None:
# get the current -Xmx limit
curr_limit = 0
try:
curr_limit = int(m.group(1))
if len(m.groups()) > 1:
curr_limit = self.__get_mb_value(curr_limit, m.group(2))
except ValueError:
log.warn("Failed extracting -Xmx setting from '{}'.".format(self.tomcat_conf_path))
log.warn("Line: '{}'".format(line))
pass
if curr_limit < self.MIN_MEMORY_LIMIT_MB:
# comment current line
lines[i] = '# ' + line
updated_line = re.sub('-Xmx(\d+)[gGmMkK]?', self.MIN_SETTINGS, line)
# insert updated line below the commented one
lines.insert(i + 1, updated_line)
return lines
# No -Xmx setting was found, append new one at the end
java_opts_line = 'JAVA_OPTS="{}"'.format(self.MIN_SETTINGS)
lines.append(java_opts_line)
return lines
def increase_tomcat_mem_limit(self):
"""
Increases the tomcat default memory limit
"""
if os.path.exists(self.tomcat_conf_path):
with open(self.tomcat_conf_path) as file:
lines = file.readlines()
lines_str = ''.join(lines)
updated_lines_str = ''.join(self.update_tomcat_conf_contents(lines))
if lines_str != updated_lines_str:
log.info("Increasing tomcat memory limit to 2GB...")
if user_confirmed_change("Continue?", ChangeSeverity.TRIVIAL):
backup_file = make_backup(self.tomcat_conf_path)
if backup_file is not None:
log.info("Existing configuration saved at '{}'".format(backup_file))
write_to_file(self.tomcat_conf_path, updated_lines_str)
tomcat_service.restart()
log.info("Tomcat memory increased succesfully.")
else:
log.info("Tomcat configuration file has not been modified.")
log.warn("Make sure to manually update the configuration file allowing " +
"at least 2GB for the JVM (option -Xmx2000m), otherwise petascope " +
"may fail to work correctly.")
else:
log.warn("Tomcat's configuration file '{}' could not be found; " +
"make sure to manually update the configuration file allowing " +
"at least 2GB for the JVM (option -Xmx2000m), otherwise petascope " +
"may fail to work correctly.", self.tomcat_conf_path)
def install(self):
super(TomcatInstaller, self).install()
self.increase_tomcat_mem_limit()
class PathInstaller(TPInstallerDecorator):
def __init__(self, installer, install_path):
"""
Sets up the PATH env variable to include the rasdaman binaries
:param str install_path: the rasdaman install path
"""
super(PathInstaller, self).__init__(installer)
self.install_path = install_path
def install(self):
super(PathInstaller, self).install()
os.environ["RMANHOME"] = self.install_path
os.environ["PATH"] += os.pathsep + self.install_path + "/bin"
profile_path = "/etc/profile.d/rasdaman.sh"
if not os.path.exists(profile_path):
log.info("Adding rasdaman to the PATH in '{}'...".format(profile_path))
write_to_file(profile_path, "export RMANHOME=\"" + self.install_path + "\"\n" +
"export PATH=$PATH:$RMANHOME/bin\n")
sudo_chmod(profile_path, "-x")
log.info("The PATH env variable was set up successfully.")
class EpelRepoInstaller(TPInstallerDecorator):
def __init__(self, installer, install_powertools=False, repo_tool='yum'):
"""
Installs the epel repository on CentOS
"""
super(EpelRepoInstaller, self).__init__(installer)
self.install_powertools = install_powertools # CentOS 8
self.repo_tool = repo_tool
def install(self):
super(EpelRepoInstaller, self).install()
repolist, _, _ = executor.executeSudo(["yum", "repolist"])
if "epel" not in repolist:
log.info("Installing EPEL repository as provider for the GDAL, NetCDF, HDF packages...")
if user_confirmed_change("Continue?", ChangeSeverity.NORMAL):
executor.executeSudo([self.repo_tool, "install", "-y", "wget"])
executor.executeSudo([self.repo_tool, "install", "-y", "epel-release"], "/tmp")
log.info("EPEL repository installed.")
else:
log.info("EPEL repository installation skipped.")
log.warn("rasdaman may fail to build as several dependencies like GDAL, " +
"NetCDF, etc. are not in the standard repository and are " +
"typically found in the EPEL repository.")
if "PowerTools" not in repolist and self.install_powertools:
log.info("Enabling PowerTools as source for rasdaman dependencies...")
if user_confirmed_change("Continue?", ChangeSeverity.NORMAL):
executor.executeSudo([self.repo_tool, "config-manager", "--set-enabled", "PowerTools"])
log.info("PowerTools enabled.")
else:
log.info("PowerTools will not be enabled.")
log.warn("rasdaman may fail to build as several dependencies " +
"cannot be installed if PowerTools is disabled.")
class PostgresqlCentOsInstaller(TPInstallerDecorator):
def __init__(self, installer, petascope_user):
"""
Installer for CentOS like OSs where postgres does not initialize the database normally
:param str petascope_user: the petascope username
"""
super(PostgresqlCentOsInstaller, self).__init__(installer)
self.petascope_user = petascope_user
def fix_postgres_permissions(self, path):
if os.path.exists(path):
check_line = "host all " + self.petascope_user + " localhost md5"
if not find_line_containing_token_in_file(path, check_line):
log.info("Updating access permissions to postgres in '{}'...".format(path))
log.info("The config will be modified to allow localhost TCP/IP access for " +
"the petascope user with md5-hashed password authentication on all databases...")
if user_confirmed_change("Continue?", ChangeSeverity.TRIVIAL):
backup_file = make_backup(path)
if backup_file is not None:
log.info("Existing configuration saved at '{}'".format(backup_file))
fixed = False
for line in fileinput.input(path, inplace=True):
if not fixed and strip_whitespace(line) == "":
print(check_line)
print("host all " + self.petascope_user + " 127.0.0.1/32 md5")
print("host all " + self.petascope_user + " ::1/128 md5")
fixed = True
if self.petascope_user not in line:
print(line.replace("\n", ""))
sudo_chown(path, "postgres")
sudo_chmod(path, "755")
postgres_service.restart()
log.info("Access permissions to postgres updated successfully.")
else:
log.info("Updating access permissions in postgres skipped.")
log.warn("Petascope may fail to connect to postgres.")
else:
log.warn("Postgres access permission config file not found: '{}'.".format(path))
log.warn("Petascope may fail to start.")
def install(self):
super(PostgresqlCentOsInstaller, self).install()
postgres_service.start()
if postgres_service.get_data_dir() is None:
log.info("Initializing postgresql database...")
postgres_service.initdb()
log.info("Postgresql initialized successfully.")
self.fix_postgres_permissions(postgres_service.get_hba_conf_path())
class UserCreationInstaller(TPInstallerDecorator):
def __init__(self, installer, user, home_dir):
"""
Checks if the given user exists and if not creates it in the system
:param str user: the username to check for
:param str home_dir: the home directory of the user
:param
"""
super(UserCreationInstaller, self).__init__(installer)
self.user = user
self.home_dir = home_dir
def install(self):
super(UserCreationInstaller, self).install()
if not system_user_exists(self.user):
log.info("Creating system user " + self.user + "...")
executor.executeSudo(["useradd", "-r", "-d", self.home_dir, "-s", "/bin/bash", self.user])
log.info("System user created successfully.".format(self.user))
class PackagerInstall(TPInstallerDecorator):
def __init__(self, installer):
"""
Installs the fpm utility needed to package things
"""
super(PackagerInstall, self).__init__(installer)
def install(self):
super(PackagerInstall, self).install()
gem_cmd = "gem" if not os.path.exists("/usr/bin/gem2.0") else "gem2.0"
pkg = "fpm" if linux_distro.get_distro_codename() == "jammy" else "fpm:<1.12.0"
executor.executeSudo([gem_cmd, "install", pkg])
class LdconfigInstall(TPInstallerDecorator):
def __init__(self, installer):
"""
Executes ldconfig
"""
super(LdconfigInstall, self).__init__(installer)
def install(self):
super(LdconfigInstall, self).install()
executor.executeSudo(["ldconfig"], throw_on_error=False)
class UpdateJavaAlternativesInstaller(TPInstallerDecorator):
def __init__(self, installer):
"""
Due to https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=678195
we need to set java 7 as default manually on Debian 7
"""
super(UpdateJavaAlternativesInstaller, self).__init__(installer)
def install(self):
super(UpdateJavaAlternativesInstaller, self).install()
executor.executeSudo(["update-alternatives", "--install", "/usr/bin/java", "java",
"/usr/lib/jvm/java-1.7.0-openjdk-amd64/bin/java", "2000"],
throw_on_error=False)
def is_system_cmake_supported():
out, _, exitCode = executor.execute(["cmake", "--version"], throw_on_error=False)
if exitCode:
return False
try:
from packaging import version
return version.parse(re.split(r'\s+', out)[2]) >= version.parse("3.11.0")
except:
return False
class CustomCmakeInstaller(TPInstallerDecorator):
def __init__(self, installer):
"""
Download Cmake version 3.16.2 for Ubuntu14/CentOS7/Debian7
:param Installer installer: the installer to decorate
"""
super(CustomCmakeInstaller, self).__init__(installer)
self.CMAKE_VERSION = "3.16.2"
self.CMAKE_URL = "https://github.com/Kitware/CMake/releases/download/v" + \
self.CMAKE_VERSION + "/cmake-" + \
self.CMAKE_VERSION + "-Linux-x86_64.tar.gz"
self.CUSTOM_CMAKE = "/opt/rasdaman/third_party/cmake-" + self.CMAKE_VERSION + "-Linux-x86_64/bin/cmake"
def install(self):
super(CustomCmakeInstaller, self).install()
if not is_system_cmake_supported() and not os.path.exists(self.CUSTOM_CMAKE):
target_dir = "/opt/rasdaman/third_party"
target_file = "cmake.tar.gz"
log.info("Downloading a supported version of cmake into {}...".format(target_dir))
sudo_mkdir(target_dir)
sudo_chown(target_dir, ConfigManager.default_user)
_, err, rc = executor.execute(["wget", "-q", "-O", target_file, self.CMAKE_URL], target_dir)
if rc != 0:
raise RuntimeError("Failed downloading cmake: " + err)
_, err, rc = executor.execute(["tar", "-xzf", target_file], target_dir)
if rc != 0:
raise RuntimeError("Failed extracting downloaded cmake: " + err)
remove(target_file, working_dir=target_dir)
log.info("Cmake installed in '{}'...".format(target_dir))
class LibgdalJavaInstaller(TPInstallerDecorator):
def __init__(self, installer, subdir):
"""
Install libgdal-java from a resources directory.
:param Installer installer: the installer to decorate
"""
super(LibgdalJavaInstaller, self).__init__(installer)
self.subdir = subdir
def install(self):
super(LibgdalJavaInstaller, self).install()
log.info("Installing libgdal-java...")
fulldir = fix_trailing_slash(get_script_dir(__file__) + self.subdir)
dst = "/usr/share/java/"
sudo_copy(fulldir + "gdal.jar", dst, is_dir=False)
dst = "/usr/lib/jni/"
sudo_mkdir(dst)
sudo_copy(fulldir + "libgdalalljni.so", dst, is_dir=False)
log.info("Installed libgdal-java.")
def test():
install_path = "/tmp/installer_test/"
etc_path = install_path + "etc/"
# mkdir -p returns -1 for some reason.. so set throw_on_error=False
make_directory(etc_path, throw_on_error=False)
sudo_chmod(etc_path, "777")
tomcat_conf_path = etc_path + "tomcat"
tomcat_conf_content = """
# The home directory of the Java development kit (JDK). You need at least
# JDK version 8. If JAVA_HOME is not set, some common directories for
# OpenJDK and the Oracle JDK are tried.
#JAVA_HOME=/usr/lib/jvm/java-8-openjdk
# You may pass JVM startup parameters to Java here. If unset, the default
# options will be: -Djava.awt.headless=true -XX:+UseG1GC
# JAVA_OPTS="-Djava.awt.headless=true -XX:+UseG1GC -Xmx6000m -XX:MaxPermSize=256m"
# JAVA_OPTS="-Djava.awt.headless=true -XX:+UseG1GC -Xmx2000m -XX:MaxPermSize=256m -XX:MaxPermSize=256m"
# JAVA_OPTS="-Djava.awt.headless=true -XX:+UseG1GC -Xmx2000m -XX:MaxPermSize=256m -XX:MaxPermSize=256m -XX:MaxPermSize=256m"
# JAVA_OPTS="-Djava.awt.headless=true -XX:+UseG1GC -Xmx2000m -XX:MaxPermSize=256m -XX:MaxPermSize=256m -XX:MaxPermSize=256m -XX:MaxPermSize=256m"
JAVA_OPTS="-Djava.awt.headless=true -XX:+UseG1GC -Xmx2000m -XX:MaxPermSize=256m -XX:MaxPermSize=256m -XX:MaxPermSize=256m -XX:MaxPermSize=256m -XX:MaxPermSize=256m"
# To enable remote debugging uncomment the following line.
# You will then be able to use a Java debugger on port 8000.
#JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,address=8000,server=y,suspend=n"
# Java compiler to use for translating JavaServer Pages (JSPs). You can use all
# compilers that are accepted by Ant's build.compiler property.
#JSP_COMPILER=javac
# Enable the Java security manager? (true/false, default: false)
#SECURITY_MANAGER=true
# Whether to compress logfiles older than today's
#LOGFILE_COMPRESS=1
"""
write_to_file(tomcat_conf_path, tomcat_conf_content)
tomcat_installer = TomcatInstaller(TPInstallerBasis(), tomcat_conf_path)
tomcat_installer.increase_tomcat_mem_limit()
updated_props_contents = read_file(tomcat_conf_path)
expected_props_contents = tomcat_conf_content
return updated_props_contents == expected_props_contents