#!/usr/bin/env bash
#
# Build and test local package.
#

PROG="$(basename "$0")"

RMANHOME="/opt/rasdaman"
readonly SRC_DIR="$RMANHOME/source"
readonly BUILD_DIR="$SRC_DIR/build"
readonly SYSTEMTEST_DIR="$SRC_DIR/systemtest"
readonly tomcat_url="http://localhost:8080"
readonly demo_coverages="AverageTemperature AverageChlorophyll mean_summer_airtemp"

log()   { echo -e "$PROG: $*"; }
logn()  { echo -e -n "$PROG: $*"; }
error() { echo -e "$*"; exit 1; }

check_root() {
  [ "$EUID" -eq 0 ] || error "This script has to be executed with root user."
}
# sets PYTHONBIN to python3
get_python_binary() {
  PYTHONBIN=python3
  "$PYTHONBIN" --version > /dev/null 2>&1 || error "Please install python."
}
check_deps() {
  wget --help > /dev/null 2>&1 || error "Please install wget."
  curl --help > /dev/null 2>&1 || error "Please install curl."
  get_python_binary
}

get_os() {
  log "Checking OS..."
  tomcat="tomcat9"
  . /etc/os-release
  os="$VERSION_CODENAME"
  repo_type="deb"
  export DEBIAN_FRONTEND=noninteractive
  log "Done, detected OS '$os'."
}

# download url $1 to directory $2
download() {
  local url="$1"
  local dest="$2"
  local retry_after="$3"
  local u=rasguest
  local p=rasguest
  log "Downloading '$url' to '$dest'..."
  local WGET="wget --quiet --no-check-certificate --tries 1 --http-user=$u --http-password=$p"
  if ! $WGET "$url" -O "$dest"; then
    if [ -n "$retry_after" ]; then
      log "... failed, waiting for $retry_after seconds and retrying."
      sleep "$retry_after"
      if $WGET "$url" -O "$dest"; then
        log "... 2nd try succeeded, done."
        return
      fi
    fi
    if ! curl -u $u:$p -sS "$url" 2>&1; then
      error "Failed, terminating."
    fi
  fi
  log "Done."
}

clear_apt_locks() {
  # Ubuntu 16.04/18.04 runs into this error
  # E: Could not get lock /var/lib/apt/lists/lock - open (11: Resource temporarily unavailable)
  # E: Unable to lock directory /var/lib/apt/lists/
  # -> So remove this lock file to fix it
  sudo killall -9 apt apt-get > /dev/null 2>&1
  sudo rm -f /var/lib/apt/lists/lock
  sudo rm -f /var/cache/apt/archives/lock
  for f in /var/lib/dpkg/lock*; do
    sudo rm -f "$f"
  done
  sudo dpkg --configure -a
}
# set global variable pkg_path to the last built package
set_last_built_pkg_path() {
  local pkg_output_dir="/tmp/rasdaman-built-packages/"
  if [ -d "$pkg_output_dir" ]; then
    pkg_path="$(find "$pkg_output_dir" -name "*.$repo_type" | tail -n 1)"
    [ -n "$pkg_path" ]
  else
    return 1
  fi
}
#
# Extract the $current_release from version_releases.txt with the format:
#
# $pkg_type $os_version $pkg_version $current_release
# ...
# $pkg_type $os_version $pkg_version $current_release
#
# e.g.
#
# nightly centos7 10.0.0 4
#
get_version_release() {
  local pkg_version
  pkg_version="$1"
  local version_releases
  version_releases="$iteration_url/version_releases.txt"
  local current_release
  current_release="$(wget --no-check-certificate -qO- "$version_releases" | grep "${pkg_type} ${os} ${pkg_version}" | awk '{ print $4; }')"
  [ -n "$current_release" ] || current_release=1
  echo "$current_release"
}
run_installer() {
  local pkg_name_ext="$1"
  local pkg_version="$2"
  local iteration="$3"
  local PROFILE="$4"
  log "Building $pkg_name_ext $pkg_type package from $repo (branch $branch), version $pkg_version, release $iteration..."
  if "$PYTHONBIN" main.py "$PROFILE"; then
    if [ "$pkg_name_ext" = "rasdaman" ]; then
      if set_last_built_pkg_path; then
        log "$pkg_name_ext $pkg_type package built: $pkg_path"
      else
        log "skipped building a package."
        exit 0
      fi
    else
      log "$pkg_name_ext $pkg_type package build."
    fi
  else
    error "Failed building package $pkg_name_ext."
  fi
}
# builds the package according to specified options, and exports global
# absolute path to the package: pkg_path
build_package() {
  log "------------------------------------------------------------------------"
  log "                    Building new package                                "
  log "------------------------------------------------------------------------"
  echo
  [ "$repo_type" == "deb" ] && clear_apt_locks

  local PROFILE_TEMPLATE="profiles/package/generate.toml"
  [ "$pkg_type" == stable ] && PROFILE_TEMPLATE="profiles/package/generate_stable.toml"
  local PROFILE="profiles/package/generate.toml.instantiated"

  # get the package version, e.g. 10.0.0
  local pkg_version
  pkg_version="$(grep -E 'version .*"[0-9]+\.[0-9]+\.[0-9]+"' "${PROFILE_TEMPLATE}" | sed -e 's/.*= *//g' -e 's/"//g')"

  typeset -i iteration
  iteration="$(get_version_release "$pkg_version")"

  sed "s/{iteration}/${iteration}/g" "${PROFILE_TEMPLATE}" > "${PROFILE}"
  sed -i "s/version = \"master\"/version = \"${branch}\"/g" "${PROFILE}"
  sed -i "s|repository = .*|repository = \"${repo}\"|g" "${PROFILE}"
  sed -i "s|user = \"rasdaman\"|user = \"${user}\"|g" "${PROFILE}"
  if [ "$pkg_type" = nightly ]; then
    sed -i "s|nightly = .*|nightly = true|g" "${PROFILE}"
  else
    sed -i "s|nightly = .*|nightly = false|g" "${PROFILE}"
  fi

  run_installer rasdaman "$pkg_version" "$iteration" "$PROFILE"

  if [ -n "$simd_extensions" ]; then
    for ext in $simd_extensions; do
      echo
      echo
      log "------------------------------------------------------------------------"
      sed -i 's/simd_extensions = .*//g' "${PROFILE}"  # clear any previous extensions set
      sed -i 's/generate_docs = true/generate_docs = true\nsimd_extensions = "'"$ext"'"/' "${PROFILE}"
      pkg_name_ext="rasdaman-$(echo "$ext" | tr '[:upper:]' '[:lower:]')"
      sed -i 's/name = ".*"/name = "'"$pkg_name_ext"'"/' "${PROFILE}"
      run_installer "$pkg_name_ext" "$pkg_version" "$iteration" "$PROFILE"
    done
  fi
}

uninstall_rasdaman() {
  log "------------------------------------------------------------------------"
  log "                    Removing previous rasdaman installation             "
  log "------------------------------------------------------------------------"
  echo
  service rasdaman stop > /dev/null 2>&1
  service $tomcat stop > /dev/null 2>&1
  if [ "$repo_type" == "deb" ]; then
    apt-get purge -qq -y "$pkg_name" "$tomcat"
  else
    yum erase -q -y "$pkg_name" "$tomcat"
  fi
  dropdb petascopedb > /dev/null 2>&1
  rm -rf /opt/rasdaman/etc /etc/init.d/rasdaman /etc/profile.d/rasdaman.sh
}

install_deb() {
  local pkg_version="$1" # nightly/testing/stable
  log "Installing $pkg_version deb repo..."

  log "Importing GPG public key..."
  wget --no-check-certificate -q -t 2 -O - "$pkgs_url/rasdaman.gpg" | apt-key add - \
    || error "Failed importing GPG key."

  local repo_file="/etc/apt/sources.list.d/rasdaman.list"
  echo "deb [arch=amd64] $pkgs_url/deb $os $pkg_version" > "$repo_file"
  clear_apt_locks
  apt-get -qq update
  # automate the configuration update dialog
  log "Installing $pkg_name from DEB package..."
  apt-get -o Dpkg::Options::="--force-confdef" install -q -y $pkg_name \
    || error "Failed installing $pkg_name."
  log "Done."
}
install_yum() {
  local pkg_version="$1" # nightly/testing/stable
  log "Installing $pkg_version yum repo..."
  local repo_file="/etc/yum.repos.d/rasdaman.repo"
  download "$pkgs_url/rpm/$pkg_version/CentOS/7/x86_64/rasdaman.repo" "$repo_file"
  yum clean all -q
  yum install -q -y epel-release
  log "Installing $pkg_name from RPM package..."
  # --setopt=obsoletes=0 disables removal of obsolete packages
  yum install -q --setopt=obsoletes=0 -y $pkg_name \
    || error "Failed installing $pkg_name."
  log "Done."
}
install_pkg() {
  local pkg_version="$1" # nightly/testing/stable
  log "------------------------------------------------------------------------"
  log "                    Installing $pkg_name $pkg_version                    "
  log "------------------------------------------------------------------------"
  echo

  case "$repo_type" in
    deb) install_deb "$pkg_version";;
    rpm) install_yum "$pkg_version";;
    *)   error "Unknown package type: $repo_type.";;
  esac
  log "Sourcing /etc/profile.d/rasdaman.sh..."
  source /etc/profile.d/rasdaman.sh \
    || error "source /etc/profile.d/rasdaman.sh failed."

  sleep 60 # wait for petascope to initialize
}

install_local_pkg() {
  log "------------------------------------------------------------------------"
  log "                    Installing built rasdaman $repo_type package        "
  log "------------------------------------------------------------------------"
  echo

  if [ "$repo_type" = "deb" ]; then
    # dpkg may fail because it cannot install dependencies; in this case
    # the installation can be continued with apt-get install -f
    dpkg --force-confdef -i "$pkg_path" || \
      apt-get install -f -y -q -o Dpkg::Options::="--force-confdef"
  elif [ "$repo_type" = "rpm" ]; then
    yum install -q --setopt=obsoletes=0 -y "$pkg_path"
  else
    error "Unknown package type: $repo_type."
  fi
  [ $? -eq 0 ] || error "Failed installing $pkg_path."

  if [ -f /etc/profile.d/rasdaman.sh ]; then
    log "Sourcing /etc/profile.d/rasdaman.sh..."
    source /etc/profile.d/rasdaman.sh \
      || error "source /etc/profile.d/rasdaman.sh failed."
  else
    log "Warning: /etc/profile.d/rasdaman.sh not found"
  fi

  # on CentOS it seems that somehow rasdaman is stopped after installing it as
  # a local package
  service rasdaman start
}

dropcov() {
  wget -q "$tomcat_url/rasdaman/ows?service=WCS&version=2.0.1&request=DeleteCoverage&coverageId=$1" -O /dev/null \
    && echo ok. || echo "Failed deleting $1."
}
install_test_dependencies() {
  log "Installing packages needed by the systemtest..."
  apt-get install netcdf-bin gdal-bin bc python3-pip python3-protobuf -q -y
  pip3 install grpcio
}
test_install() {
  log "------------------------------------------------------------------------"
  log "                    Testing rasdaman installation                       "
  log "------------------------------------------------------------------------"
  echo

  log "Testing rasdaman query..."
  rasql -q 'SELECT C FROM RAS_COLLECTIONNAMES AS C' --out string > /dev/null \
    || error "rasdaman test query failed."

  log "Inserting rasdaman demo images..."
  local images_path=/opt/rasdaman/share/rasdaman/examples/images
  rasdaman_insertdemo.sh localhost 7001 "$images_path" rasadmin rasadmin > /dev/null \
    || error "rasdaman_insertdemo.sh failed."

  log "Testing SECORE request..."
  local secore_test_url="$tomcat_url/rasdaman/def/"
  if find /var/lib -name def.war 2>&1 | grep -q def.war; then
    secore_test_url="$tomcat_url/def/"
  fi
  download "$secore_test_url" "/dev/null" 60

  log "Testing petascope request..."
  local get_cap_request="service=WCS&request=GetCapabilities&version=2.0.1"
  download "$tomcat_url/rasdaman/ows?$get_cap_request" "/dev/null"

  log "Inserting petascope demo coverages..."
  petascope_insertdemo.sh || error "petascope_insertdemo.sh failed."

  log "------------------------------------------------------------------------"
  log "                    Starting systemtest                                 "
  log "------------------------------------------------------------------------"

  log "Clearing demo data before running systemtest..."
  for c in $demo_coverages; do dropcov $c; done

  log "Running systemtest..."
  pushd "$BUILD_DIR" > /dev/null || exit 1
  install_test_dependencies
  sudo -u rasdaman make check 2>&1 | sudo -u rasdaman tee -a "$SYSTEMTEST_DIR/test.output"
  rc=$?
  popd > /dev/null || exit 1

  pushd "$SYSTEMTEST_DIR" > /dev/null || exit 1
  tar cfz "$SRC_DIR/test_output.tar.gz" test.output $(find . -name output -o -name test.log) --ignore-failed-read
  popd > /dev/null || exit 1

  log "------------------------------------------------------------------------"
  log "                    Installation testing done                           "
  log "------------------------------------------------------------------------"

  if [ $rc -ne 0 ]; then
    error "systemtest failed."
  fi
}

upload_package() {
  log "------------------------------------------------------------------------"
  log "                    Uploading generated package                         "
  log "------------------------------------------------------------------------"
  echo
  [ "$upload" = true ] || return
  # Copy package in the download area
  local non_root_user=$(grep 1000 /etc/passwd | cut -d: -f1)

  # download_server=download@download.rasdaman.org
  # download_server_dir=/home/download/package_sources
  target_dir="$download_server_dir/$repo_type/$os/$pkg_type"
  pkgs_dir="/tmp/rasdaman-built-packages/"

  log "Uploading package with user $non_root_user to directory $download_server:$target_dir..."
  log " packages: $(ls "$pkgs_dir"/)"

  local private_key=/home/$non_root_user/.ssh/id_rsa
  chmod 600 "$private_key"
  scp -i "$private_key" -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r \
      -P "$download_server_port" "$pkgs_dir"/* \
      "$download_server":"$target_dir" || error "Failed to remote copy!"

  uploaded_pkgs_dir="/tmp/rasdaman-uploaded-packages/"
  mkdir -p "$uploaded_pkgs_dir"/uploaded
  mv -v "$pkgs_dir"/* "$uploaded_pkgs_dir"
}

# ------------------------------------------------------------------------

usage() {
  local rc=0
  if [ -n "$1" ]; then
    log "$*\n" # print error message
    rc=1
  fi

  cat <<MARKER
Description: This script builds a local package from the specified branch,
             installs it, and tests rasdaman's features.
             On success a 0 is returned, otherwise non-zero.

Usage: $PROG [-t|--pkg-type (stable|testing|nightly)]
             [-r|--repo <r>] [-b|--branch <b>] [--user <u>]
             [--pkgs-url <u> ] [--iteration-url <i>]
             [--upgrade-from (stable|testing|nightly)]
             [--no-uninstall ] [--skip-install] [--skip-testing]
             [-u|--upload --download-server <d> --download-server-dir <d>]
             [-h|--help]

Options:

  -h, --help
    display this help and exit

  -t, --pkg-type (none|stable|testing|nightly)
    specify the type of package to build;
    default: nightly

  -r, --repo <r>
    specify git repository to build package from;
    default: git://rasdaman.org/rasdaman.git

  --user <u>
    specify the system user to use for building and installing rasdaman;
    default: rasdaman

  -b, --branch <b>
    specify branch from the rasdaman repository to build;
    default: master

  --upgrade-from (stable|testing|nightly)
    first install a package from the specified repository,
    then upgrade to the newly generated package;
    default: do not do upgrade testing

  --pkgs-url <u>
    specify the URL containing rasdaman packages for download;
    default: https://download.rasdaman.org/packages

  --iteration-url <i>
    specify the URL containing package iteration number;
    default: https://download.rasdaman.org/packages

  --skip-testing
    do not perform any tests on the generated package
    default: basic functionality is checked, and systemtest is executed

  --skip-install
    skip installing the built package
    default: the local package will be installed

  --no-uninstall
    do not uninstall rasdaman and tomcat before installing a new rasdaman package;
    default: rasdaman and tomcat will be uninstalled

  -u, --upload
    upload built package;
    default: the generated package is not uploaded

  --download-server <d>
    specify the download server to which the generated package will be uploaded;
    mandatory if --upload is specified; example: download@download.rasdaman.org

  --download-server-ssh-port <p>
    SSH port for the download server; by default it is 22.

  --download-server-dir <d>
    specify directory with package sources to which the generated package will
    be uploaded on the remote server;
    mandatory if --upload is specified; example: /mnt/rasservice/Download/package_source

  --simd-extensions "<ext> <ext> ..."
    specify the SIMD extensions for which to build the binaries in addition to
    the default target; supported are AVX AVX2 AVX512

Examples:

  Generate nightly local package (but do not upload it), from the master branch
  of rasdaman community repository, install it, and test rasdaman:

    ./$PROG

  Install the latest package generated in /tmp/rasdaman-built-packages and
  run tests on it

    ./$PROG --pkg-type none

MARKER

  exit $rc
}

### parse command-line arguments

pkg_type="nightly"
upload=false
branch="master"
repo="git://rasdaman.org/rasdaman.git"
user="rasdaman"
upgrade_from=
pkgs_url="https://download.rasdaman.org/packages"
iteration_url="https://download.rasdaman.org/packages"
skip_install=false
skip_testing=false
uninstall=true
download_server=
download_server_port=22
download_server_dir=
simd_extensions=
pkg_name="rasdaman"  # rasdaman-<e> if --simd-extensions <e> is specified

option=""
for i in "$@"; do

  if [ -n "$option" ]; then
    # $option could be -t, then $i is the value for that option
    case $option in
      -t|--pkg-type)         pkg_type="$i";;
      -b|--branch)           branch="$i";;
      -r|--repo)             repo="$i";;
      --user)                user="$i";;
      --upgrade-from)        upgrade_from="$i";;
      --pkgs-url)            pkgs_url="$i";;
      --iteration-url)       iteration_url="$i";;
      --download-server)     download_server="$i";;
      --download-server-ssh-port) download_server_port="$i";;
      --download-server-dir) download_server_dir="$i";;
      --simd-extensions)     simd_extensions="$i";;
      *)                     usage "unknown option: $option";;
    esac
    option="" # option processed, clear
  elif [ -n "$i" ]; then
    # handle flag options in the case below (e.g. -h),
    # leave options with value for processing in the case above (e.g. -f)
    option=""
    case $i in
      -h|--help)             usage;;
      -u|--upload)           upload=true;;
      --skip-testing)        skip_testing=true;;
      --skip-install)        skip_install=true;;
      --no-uninstall)        uninstall=false;;
      *)                     option="$i";;
    esac
  fi

done

### validate parsed options

case "$upgrade_from" in
  ""|stable|testing|nightly) ;;
  *) usage "invalid value for option --upgrade-from: '$upgrade_from'";;
esac
case "$pkg_type" in
  none|stable|testing|nightly) ;;
  *) usage "invalid value for option --pkg-type: '$pkg_type'";;
esac
if [ "$upload" = true ]; then
  [ -n "$download_server" ] || \
    usage "--download-server is mandatory if --upload is specified."
  [ -n "$download_server_dir" ] || \
    usage "--download-server-dir is mandatory if --upload is specified."
fi
if [ -n "$simd_extensions" ]; then
  for e in $simd_extensions; do
    case "$e" in
      AVX|AVX2|AVX512) ;;
      *) usage "invalid value in option --simd-extensions: '$simd_extensions'";;
    esac
  done
fi

### preparation

# ensure script runs with root
check_root
# check if wget is available, and python binary
check_deps
# sets `os` and `repo_type` (rpm/deb) global variables
get_os


### ------------------------------------------------------------------------
### execution
### ------------------------------------------------------------------------

# 1. build package if requested, and set $pkg_path global variable
if [ "$pkg_type" != "none" ]; then
  build_package
else
  set_last_built_pkg_path
fi

# 2. clear any previous installation
if [ "$uninstall" = true ]; then
  uninstall_rasdaman
fi

# 3. install stable|testing|nightly package from $pkgs_url
if [ -n "$upgrade_from" ]; then
  install_pkg "$upgrade_from"
  service rasdaman stop
fi

# 4. install package built at $pkg_path in step 1.
if [ "$skip_install" = false ]; then
  install_local_pkg
fi

# 5. run testing
if [ "$skip_testing" = false ]; then
  test_install
fi

# 6. upload package to download server
if [ "$upload" = true ]; then
  upload_package
fi

log "Done."