#!/usr/bin/env bash
#/ Usage: ghe-restore [-cfhv] [--version] [--skip-mysql] [-s <snapshot-id>] [<host>]
#/
#/ Restores a GitHub instance from local backup snapshots.
#/
#/ Note that the GitHub Enterprise host must be reachable and your SSH key must
#/ be setup as described in the following help article:
#/
#/ <https://docs.github.com/enterprise-server/admin/configuration/configuring-your-enterprise/accessing-the-administrative-shell-ssh >
#/
#/ OPTIONS:
#/   -c | --config       Restore appliance settings and license in addition to
#/                       datastores. Settings are not restored by default to
#/                       prevent overwriting different configuration on the
#/                       restore host.
#/   -f | --force        Don't prompt for confirmation before restoring.
#/   -h | --help         Show this message.
#/   -v | --verbose      Enable verbose output.
#/        --skip-mysql   Skip MySQL restore steps. Only applicable to external databases.
#/        --version      Display version information and exit.
#/
#/   -s <snapshot-id>    Restore from the snapshot with the given id. Available
#/                       snapshots may be listed under the data directory.
#/
#/   <host>              The <host> is the hostname or IP of the GitHub Enterprise
#/                       instance. The <host> may be omitted when the
#/                       GHE_RESTORE_HOST config variable is set in backup.config.
#/                       When a <host> argument is provided, it always overrides
#/                       the configured restore host.
#/

set -e

# Parse arguments
: "${RESTORE_SETTINGS:=false}"
export RESTORE_SETTINGS

: "${FORCE:=false}"
export FORCE

: "${SKIP_MYSQL:=false}"
export SKIP_MYSQL

while true; do
  case "$1" in
    --skip-mysql)
      SKIP_MYSQL=true
      shift
      ;;
    -f|--force)
      FORCE=true
      shift
      ;;
    -s)
      snapshot_id="$(basename "$2")"
      shift 2
      ;;
    -c|--config)
      RESTORE_SETTINGS=true
      shift
      ;;
    -h|--help)
      export GHE_SHOW_HELP=true
      shift
      ;;
    --version)
      export GHE_SHOW_VERSION=true
      shift
      ;;
    -v|--verbose)
      export GHE_VERBOSE=true
      shift
      ;;
     -i|--incremental)
      export GHE_INCREMENTAL=true
      shift
      ;;
    -*)
      echo "Error: invalid argument: '$1'" 1>&2
      exit 1
      ;;
    *)
      if [ -n "$1" ]; then
        GHE_RESTORE_HOST_OPT="$1"
        shift
      else
        break
      fi
      ;;
  esac
done



start_cron () {
  log_info "Starting cron ..."
  if $CLUSTER; then
    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo timeout 120s service cron start"; then
      log_warn "Failed to start cron on one or more nodes"
    fi
  else
    if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo timeout 120s service cron start"; then
      log_warn "Failed to start cron"
    fi
  fi
}

cleanup () {
  log_info " Exiting, cleaning up ..."
  if [ -n "$1" ]; then
    update_restore_status "$1"
  fi

  if $ACTIONS_STOPPED && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
    log_info "Restarting Actions after restore ..."
    # In GHES 3.3+, ghe-actions-start no longer has a -f (force) flag. In GHES 3.2 and below, we must provide the
    # force flag to make sure it can start in maintenance mode. Use it conditionally based on whether it exists
    # in the --help output
    if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start --help' | grep -q force; then
      ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start -f' 1>&3
    else
      ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start' 1>&3
    fi
  fi

  if ! $CRON_RUNNING; then
    start_cron
  fi

  # Cleanup SSH multiplexing
  log_info "Cleaning up SSH multiplexing ..."
  if ! ghe-ssh --clean; then
    log_info "Failed to clean up SSH multiplexing"
  fi

  # Remove in-progress file
  log_info "Removing in-progress file ..." 1>&3
  if ! rm -f "${GHE_DATA_DIR}/in-progress-restore"; then
    log_error "Failed to remove in-progress file" 1>&3
  fi

  # Remove progress files
  if [ ! -z $PROGRESS_DIR ] && [ -d $PROGRESS_DIR ]; then
    rm -rf "${PROGRESS_DIR:?}/"*
  fi
}

# This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally.
# because it doesn't run locally does not redirect output to fd 3 or use log_info/log_warn/log_error.
# shellcheck disable=SC2034
cleanup_cluster_nodes() {
  uuid="$1"
  if [ -z "$uuid" ]; then
    log_error "Node UUID required."
    exit 2
  fi

  echo "Cleaning up spokes"
  ghe-spokes server evacuate "git-server-$uuid" 'Removing replica'
  ghe-spokes server destroy "git-server-$uuid"

  echo "Cleaning up storage"
  ghe-storage destroy-host "storage-server-$uuid" --force

  echo "Cleaning up dpages"
  ghe-dpages offline "pages-server-$uuid"
  ghe-dpages remove "pages-server-$uuid"

  echo "Cleaning up redis"
  ghe-redis-cli del "resque:queue:maint_git-server-$uuid"
  ghe-redis-cli srem resque:queues "maint_git-server-$uuid"
}

# Bring in the backup configuration
# shellcheck source=share/github-backup-utils/ghe-backup-config
. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"





# Check to make sure moreutils parallel is installed and working properly
ghe_parallel_check

# Check to make sure another restore process is not running
ghe_restore_check

# Grab the host arg
GHE_HOSTNAME="${GHE_RESTORE_HOST_OPT:-$GHE_RESTORE_HOST}"

# Hostname without any port suffix
hostname=$(echo "$GHE_HOSTNAME" | cut -f 1 -d :)

# Show usage with no <host>
[ -z "$GHE_HOSTNAME" ] && print_usage

# Flag to indicate if this script has stopped Actions.
ACTIONS_STOPPED=false

# ghe-restore-snapshot-path validates it exists, determines what current is,
# and if there's any problem, exit for us
GHE_RESTORE_SNAPSHOT_PATH="$(ghe-restore-snapshot-path "$snapshot_id")"
GHE_RESTORE_SNAPSHOT=$(basename "$GHE_RESTORE_SNAPSHOT_PATH")
export GHE_RESTORE_SNAPSHOT

# Check snapshot is not incomplete
if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/incomplete" ]; then
  log_error "Error: $GHE_RESTORE_SNAPSHOT is incomplete. Please restore from a complete snapshot."
  exit 1
fi

# Check to make sure backup is not running
ghe_backup_check

# Detect if the backup we are restoring has a leaked ssh key
echo "Checking for leaked keys in the backup snapshot that is being restored ..."
ghe-detect-leaked-ssh-keys -s "$GHE_RESTORE_SNAPSHOT_PATH" || true

# Figure out whether to use the tarball or rsync restore strategy based on the
# strategy file written in the snapshot directory.
GHE_BACKUP_STRATEGY=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/strategy")

# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
ghe_remote_version_required "$GHE_HOSTNAME"

# Figure out if this instance has been configured or is entirely new.
instance_configured=false
if is_instance_configured; then
  instance_configured=true
else
  RESTORE_SETTINGS=true
fi

# Figure out if we're restoring into cluster
CLUSTER=false
log_info "Checking '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' for instance configuration..."
if ghe-ssh "$GHE_HOSTNAME" -- \
  "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
  log_info "Cluster configuration detected"
  CLUSTER=true
else
  log_info "Standalone configuration detected"
fi
export CLUSTER

# Restoring a cluster backup to a standalone appliance is not supported
if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
  log_error "Error: Snapshot from a GitHub Enterprise cluster cannot be restored to a standalone appliance. Aborting." >&2
  exit 1
fi

# Restoring a backup snapshot that is more than two minor versions lower than the appliance is not supported
export SNAPSHOT_VERSION=${SNAPSHOT_VERSION:=$(awk -F'.' '{gsub(/[a-zA-Z]/, "", $1); print $1"."$2}' "$GHE_RESTORE_SNAPSHOT_PATH/version")}
if (( ${SNAPSHOT_VERSION%.*} < ${GHE_VERSION_MAJOR} || ( ${SNAPSHOT_VERSION%.*} == ${GHE_VERSION_MAJOR} && ${SNAPSHOT_VERSION#*.} + 2 < ${GHE_VERSION_MINOR} ))); then
  log_error "Error: GitHub appliance being restored is more than two versions ahead of the backup snapshot version $SNAPSHOT_VERSION. Please use a supported snapshot to restore." >&2
  exit 1
fi

# Ensure target appliance and restore snapshot are a compatible combination with respect to BYODB
if ! ghe-restore-external-database-compatibility-check; then
  exit 1
fi

# Figure out if this appliance is in a replication pair
if ghe-ssh "$GHE_HOSTNAME" -- \
  "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then
  log_error "Error: Restoring to an appliance with replication enabled is not supported. Please teardown replication before restoring." >&2
  exit 1
fi

# Prompt to verify the restore host given is correct. Restoring overwrites
# important data on the destination appliance that cannot be recovered. This is
# mostly to prevent accidents where the backup host is given to restore instead
# of a separate restore host since they're used in such close proximity.
if $instance_configured && ! $FORCE; then
  echo
  echo "WARNING: All data on GitHub Enterprise appliance $hostname ($GHE_REMOTE_VERSION)"
  echo "         will be overwritten with data from snapshot ${GHE_RESTORE_SNAPSHOT}."
  echo

  if is_external_database_snapshot && $RESTORE_SETTINGS; then
    echo "WARNING: This operation will also restore the external MySQL connection configuration,"
    echo "         which may be dangerous if the GHES appliance the snapshot was taken from is still online."
    echo
  fi

  prompt_for_confirmation "Please verify that this is the correct restore host before continuing."
fi

# Prompt to verify that restoring BYODB snapshot to unconfigured instance
# will result in BYODB connection information being restored as well.
if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then
  echo
  echo "WARNING: This operation will also restore the external MySQL connection configuration,"
  echo "         which may be dangerous if the GHES appliance the snapshot was taken from is still online."
  echo

  prompt_for_confirmation "Please confirm this before continuing."
fi
# Calculate the actual amounts of steps in the restore process
# taking into account the options passed to the script and the appliance configuration
# calculate restore steps
OPTIONAL_STEPS=0

# Restoring UUID
if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
fi
# Restoring Actions + MSSQL
if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
fi
# Restoring minio
if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
fi
# Restoring Elasticsearch
if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then
  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
fi
# Restoring audit log
if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then
  if [[ "$GHE_RESTORE_SKIP_AUDIT_LOG" != "yes" ]]; then
    OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
  fi
fi
# Replica cleanup
if ! $CLUSTER && $instance_configured; then
  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
fi
# Restoring settings + restore-chat-integration + restore-packages
if $RESTORE_SETTINGS; then
  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3))
fi

# Minimum number of steps is 7
export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))

init-progress
echo "$PROGRESS_TOTAL" > $PROGRESS_DIR/total
export PROGRESS_TYPE="Restore"
echo "$PROGRESS_TYPE" > $PROGRESS_DIR/type
export PROGRESS=0 # Used to track progress of restore
echo "$PROGRESS" > $PROGRESS_DIR/progress

# Log restore start message locally and in /var/log/syslog on remote instance
bm_start "$(basename $0)"
START_TIME=$(date +%s)
log_info "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT"

if [ "$GHE_INCREMENTAL" ]; then
  if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then
  log_error "Can only perform incremental restores on enterprise version 3.10 or higher"
  exit 1
fi
if [ "$GHE_VERSION_MINOR" -lt 10 ]; then
  log_error "Can only perform incremental restores on enterprise version 3.10 or higher"
  exit 1
fi
  log_info "Incremental restore from snapshot $GHE_RESTORE_SNAPSHOT"
  # If we see 'inc_previous' prepended to the snapshot name, then
  # we set $INC_FULL_BACKUP and $INC_SNAPSHOT_DATA to $INC_PREVIOUS_FULL_BACKUP and
  # $INC_PREVIOUS_SNAPSHOT_DATA respectively. Otherwise, leave them at default setting
  # so that incremental restore is from current cycle
  if [[ "$GHE_RESTORE_SNAPSHOT" =~ ^inc_previous ]]; then
    INC_FULL_BACKUP=$INC_PREVIOUS_FULL_BACKUP
    INC_SNAPSHOT_DATA=$INC_PREVIOUS_SNAPSHOT_DATA
    log_info "Incremental restore from previous cycle snapshot. Using $INC_FULL_BACKUP"
    log_info "Incremental restore from previous cycle snapshot. Using $INC_SNAPSHOT_DATA"
  fi
  log_info "Validating snapshot $GHE_RESTORE_SNAPSHOT"
  validate_inc_snapshot_data "$GHE_RESTORE_SNAPSHOT"
fi
ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..."
# Create an in-progress-restore file to prevent simultaneous backup or restore runs
echo "${START_TIME} $$" > "${GHE_DATA_DIR}/in-progress-restore"

# Keep other processes on the VM or cluster in the loop about the restore status.
#
# Other processes will look for these states:
# "restoring" - restore is currently in progress
# "failed"    - restore has failed
# "complete"  - restore has completed successfully
update_restore_status () {
  if $CLUSTER; then
    echo "ghe-cluster-each -- \"echo '$1' | sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null\"" |
    ghe-ssh "$GHE_HOSTNAME" /bin/bash
  else
    echo "$1" |
    ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null"
  fi
}

CRON_RUNNING=true
# Update remote restore state file and setup failure trap
trap "cleanup failed" EXIT
update_restore_status "restoring"

# Make sure the GitHub appliance is in maintenance mode.
if $instance_configured; then
  if ! ghe-maintenance-mode-status "$GHE_HOSTNAME"; then
    log_error "Error: $GHE_HOSTNAME must be put in maintenance mode before restoring. Aborting." 1>&2
    exit 1
  fi
fi

# Get GHES release version in major.minor format
RELEASE_VERSION=$(ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --get core.package-version' | cut -d '.' -f 1,2)

# If the backup being restored is from an appliance with Actions disabled, restoring it onto an appliance with Actions enabled will cause
# mismatches in the secrets needed for Actions which ultimately results in Actions not working properly. Note: xargs is to remove whitespace
ACTIONS_ENABLED_IN_BACKUP=$(git config -f "$GHE_RESTORE_SNAPSHOT_PATH/settings.json" --bool app.actions.enabled | xargs)
if [[ $ACTIONS_ENABLED_IN_BACKUP != true ]] && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
    log_error "Restoring a backup with Actions disabled onto an appliance with Actions enabled is not supported." >&2
    exit 1
fi

# Make sure the GitHub appliance has Actions enabled if the snapshot contains Actions data.
# If above is true, also check if ac is present in appliance then snapshot should also contains ac databases
if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then
  if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
    ac_db_ghe=$(echo 'ghe-mssql-console -y -n -q "SELECT name FROM sys.databases" | grep -i "ArtifactCache" | wc -l | tr -d " "' | ghe-ssh "$GHE_HOSTNAME" /bin/bash)
    ac_db_snapshot=$(find "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql/" -maxdepth 1 -name 'ArtifactCache*.bak' | wc -l | tr -d " ")
    if [[ $ac_db_ghe -gt 0  &&  $ac_db_snapshot -eq 0 ]]; then
      log_error "$GHE_HOSTNAME has Actions Cache service enabled but no Actions Cache data is present in snapshot to restore. Aborting \n Please disable Actions cache service in $GHE_HOSTNAME and retry\nTo disable Actions Cache service run as admin: ghe-actions-cache-disable" 1>&2
      exit 1
    fi
    if [[ $ac_db_ghe -eq 0  &&  $ac_db_snapshot -gt 0 && ! $RESTORE_SETTINGS ]]; then
      log_error "$GHE_HOSTNAME has Actions Cache service disabled but the snapshot is attempting to restore data for the service. Aborting. \n Please enable Actions cache service in $GHE_HOSTNAME and retry \n To enable Actions Cache service run as admin: ghe-actions-cache-enable" 1>&2
      exit 1
    fi
  else
    log_error "$GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting. \n Setup details for enabling Actions can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled" 1>&2
    exit 1
  fi
fi

# Create benchmark file
bm_init > /dev/null

ghe-backup-store-version  ||
log_warn "Warning: storing backup-utils version remotely failed."

# Stop cron and timerd, as scheduled jobs may disrupt the restore process.
log_info "Stopping cron and github-timerd ..."
if $CLUSTER; then
  bm_start "$(basename $0) - Stopping cron and github-timerd on cluster"
  if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then
    log_warn "Failed to stop cron on one or more nodes" 1>&3
  fi
  bm_end "$(basename $0) - Stopping cron and github-timerd on cluster"
  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
    if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
      if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
        log_warn "Failed to stop github-timerd on one or more nodes" 1>&3
      fi
    fi
  else
    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service github-timerd stop"; then
      log_warn "Failed to stop github-timerd on one or more nodes" 1>&3
    fi
  fi
 
else
  bm_start "$(basename $0) - Stopping cron and github-timerd"
  echo "$(basename $0) - Stopping cron and github-timerd"
  if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then
    log_warn "Failed to stop cron" 1>&3
  fi
  bm_end "$(basename $0) - Stopping cron and github-timerd"
  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
    if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
      if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
        log_warn "Failed to stop github-timerd" 1>&3
      fi
    fi
  else
    if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service github-timerd stop"; then
      log_warn "Failed to stop github-timerd" 1>&3
    fi
  fi
  
fi
CRON_RUNNING=false

ghe-restore-secrets "$GHE_HOSTNAME"

# Restore settings and license if restoring to an unconfigured appliance or when
# specified manually.
if $RESTORE_SETTINGS; then
  ghe-restore-settings "$GHE_HOSTNAME"
fi

# Make sure mysql and elasticsearch are prep'd and running before restoring.
# These services will not have been started on appliances that have not been
# configured yet.
if ! $CLUSTER; then
  echo "sudo ghe-service-ensure-mysql && sudo ghe-service-ensure-elasticsearch" |
  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
fi

# Restore UUID if present and not restoring to cluster.
if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
  log_info "Restoring UUID ..."

  bm_start "$(basename $0) - Restore UUID"
  ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" <"$GHE_RESTORE_SNAPSHOT_PATH/uuid"
  ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true
  ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
  bm_end "$(basename $0) - Restore UUID"
fi


if is_external_database_snapshot; then
   appliance_strategy="external"
   backup_snapshot_strategy="external"
else
  if is_binary_backup_feature_on; then
    appliance_strategy="binary"
  else
    appliance_strategy="logical"
  fi

  if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then
    backup_snapshot_strategy="binary"
  else
    backup_snapshot_strategy="logical"
  fi
fi

if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
  log_info "Skipping MySQL restore."
else
  log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
  increment-progress-total-count 2
  ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
fi



if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
  log_info "Stopping Actions before restoring databases ..."
  # We mark Actions as stopped even if the `ghe-actions-stop`
  # fails to ensure that we cleanly start actions when performing cleanup.
  ACTIONS_STOPPED=true
  ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-stop' 1>&3

  log_info "Restoring MSSQL databases ..."
  ghe-restore-mssql "$GHE_HOSTNAME" 1>&3

  log_info "Restoring Actions data ..."
  ghe-restore-actions "$GHE_HOSTNAME" 1>&3
  echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning."
  echo "See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners."
fi

if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
  log_info "Restoring MinIO data ..."
  ghe-restore-minio "$GHE_HOSTNAME" 1>&3
fi

# log input into a variable for the parallel command, as the functions don't work with eval
cmd_title=$(log_info "Restoring Redis database ...")
commands=("
echo \"$cmd_title\"
ghe-restore-redis \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"")

cmd_title=$(log_info "Restoring Git Repositories ...")
commands+=("
echo \"$cmd_title\"
ghe-restore-repositories \"$GHE_HOSTNAME\"")

cmd_title=$(log_info "Restoring Gists ...")
commands+=("
echo \"$cmd_title\"
ghe-restore-repositories-gist \"$GHE_HOSTNAME\"")

if [ "$GHE_BACKUP_PAGES" != "no" ]; then
  cmd_title=$(log_info "Restoring Pages ...")
  commands+=("
  echo \"$cmd_title\"
  ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3")
fi

cmd_title=$(log_info "Restoring SSH authorized keys ...")
commands+=("
echo \"$cmd_title\"
ghe-restore-ssh-keys \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"")

cmd_title=$(log_info "Restoring storage data ...")
commands+=("
echo \"$cmd_title\"
ghe-restore-storage \"$GHE_HOSTNAME\" 1>&3")

cmd_title=$(log_info "Restoring custom Git hooks ...")
commands+=("
echo \"$cmd_title\"
ghe-restore-git-hooks \"$GHE_HOSTNAME\" 1>&3")

if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then
  cmd_title=$(log_info "Restoring Elasticsearch indices ...")
  commands+=("
  echo \"$cmd_title\"
  ghe-restore-es-rsync \"$GHE_HOSTNAME\" 1>&3")
fi

# Restore the audit log migration sentinel file, if it exists in the snapshot
if test -f "$GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete"; then
  log_info "Restoring Elasticsearch audit log migration sentinel file ..." 1>&3
  if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then
    log_info "Failed to restore Elasticsearch audit log migration sentinel file." 1>&3
  fi
fi

# Restore exported audit logs to 2.12.9 and newer single nodes and
# all releases of cluster
if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then
  if [[ "$GHE_RESTORE_SKIP_AUDIT_LOGS" = "yes" ]]; then
    log_info "Skipping restore of audit logs."
  else
    cmd_title=$(log_info "Restoring Audit logs ...")
    commands+=("
    echo \"$cmd_title\"
    ghe-restore-es-audit-log \"$GHE_HOSTNAME\" 1>&3")
  fi

fi

if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
  log_info "Restoring data in parallel ..."
  "$GHE_PARALLEL_COMMAND" "${GHE_PARALLEL_COMMAND_OPTIONS[@]}" -- "${commands[@]}"
else
  log_info "Restoring data serially ..." 1>&3
  for c in "${commands[@]}"; do
    . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/bm.sh"
    eval "$c"
  done
fi

# Restart an already running memcached to reset the cache after restore
log_info  "Restarting memcached ..." 1>&3
bm_start "$(basename $0) - Restarting memcached"
echo "sudo restart -q memcached 2>/dev/null || true" |
ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
bm_end "$(basename $0) - Restarting memcached"

# Prevent GitHub Connect jobs running before we've had a chance to reset
# the configuration by setting the last run date to now.
if ! $RESTORE_SETTINGS; then
  log_info "Setting last run date for GitHub Connect jobs ..." 1>&3
  echo "now=$(date +%s.0000000); ghe-redis-cli mset timer:UpdateConnectInstallationInfo \$now timer:UploadEnterpriseServerUserAccountsJob \$now timer:UploadConnectMetricsJob \$now timer:GitHubConnectPushNewContributionsJob \$now" |
    ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
fi

# When restoring to a host that has already been configured, kick off a
# config run to perform data migrations.
if $CLUSTER; then
  log_info "Configuring cluster ..."
  bm_start "$(basename $0) - configure cluster"
  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
    ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3
  elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
    ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
  fi
  ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
  bm_end "$(basename $0) - configure cluster"
elif $instance_configured; then
  log_info "Configuring appliance ..."
  bm_start "$(basename $0) - configure appliance"
  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
    ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3
  elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
    ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
  fi
  ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3
  bm_end "$(basename $0) - configure appliance"
fi

# Clear GitHub Connect settings stored in the restored database.
# This needs to happen after `ghe-config-apply` to ensure all migrations have run.
if ! $RESTORE_SETTINGS; then
  log_info "Clearing GitHub Connect settings ..." 1>&3
  echo "if [ -f /usr/local/share/enterprise/ghe-reset-gh-connect ]; then /usr/local/share/enterprise/ghe-reset-gh-connect -y; fi" |
  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
fi

# Start cron. Timerd will start automatically as part of the config run.
start_cron
CRON_RUNNING=true

# Clean up all stale replicas on configured instances.
if ! $CLUSTER && $instance_configured; then
  log_info "Cleaning up replicas..." 1>&3
  bm_start "$(basename $0) - Cleanup replicas"
  restored_uuid=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid")
  other_nodes=$(echo "
    set -o pipefail; \
    ghe-spokes server show --json \
    | jq -r '.[] | select(.host | contains(\"git-server\")).host' \
    | sed 's/^git-server-//g' \
    | ( grep -F -x -v \"$restored_uuid\" || true )" \
  | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash)
  if [ -n "$other_nodes" ]; then
    log_info "Cleaning up stale nodes ..."
    for uuid in $other_nodes; do
      # shellcheck disable=SC2034
      echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3
    done
  fi
  bm_end "$(basename $0) - Cleanup replicas"
fi

# Update the remote status to "complete". This has to happen before importing
# ssh host keys because subsequent commands will fail due to the host key
# changing otherwise.
trap "cleanup" EXIT
update_restore_status "complete"

# Log restore complete message in /var/log/syslog on remote instance
ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_RESTORE_SNAPSHOT}."

if ! $CLUSTER; then
  log_info "Restoring SSH host keys ..."
  ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-ssh-host-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3
else
  # This will make sure that Git over SSH host keys (babeld) are
  # copied to all the cluster nodes so babeld uses the same keys.
  log_info "Restoring Git over SSH host keys ..."
  ghe-ssh "$GHE_HOSTNAME" -- "sudo tar -xpf - -C $GHE_REMOTE_DATA_USER_DIR/common" < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3
  ghe-ssh "$GHE_HOSTNAME" -- "sudo chown babeld:babeld $GHE_REMOTE_DATA_USER_DIR/common/ssh_host_*" 1>&3
  echo "if [ -f /usr/local/share/enterprise/ghe-cluster-config-update ]; then /usr/local/share/enterprise/ghe-cluster-config-update -s; else ghe-cluster-config-update -s; fi" |
  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
fi

END_TIME=$(date +%s)
log_info "Runtime: $((END_TIME - START_TIME)) seconds"
log_info "Completed restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT at $(date +"%H:%M:%S")"

log_info "Restore of $GHE_HOSTNAME finished."

if ! $instance_configured; then
  echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration."
fi

