provisioning/core/nulib/clusters/run.nu

#use utils.nu cluster_get_file
#use utils/templates.nu on_template_path

use std
use ../lib_provisioning/config/accessor.nu [is-debug-enabled, is-debug-check-enabled]

def make_cmd_env_temp [
  defs: record
  cluster_env_path: string
  wk_vars: string
]: nothing -> string {
  let cmd_env_temp = $"($cluster_env_path)/cmd_env_(mktemp --tmpdir-path $cluster_env_path --suffix ".sh" | path basename)"
  # export all 'PROVISIONING_' $env vars to SHELL
  ($"export NU_LOG_LEVEL=($env.NU_LOG_LEVEL)\n" +
      ($env | items {|key, value| if ($key | str starts-with "PROVISIONING_") {echo $'export ($key)="($value)"\n'} } | compact --empty | to text)
  ) | save --force $cmd_env_temp
  $cmd_env_temp
}
def run_cmd [
  cmd_name: string
  title: string
  where: string
  defs: record
  cluster_env_path: string
  wk_vars: string
]: nothing -> nothing {
  _print $"($title) for ($defs.cluster.name) on ($defs.server.hostname) ($defs.pos.server) ..."
  if $defs.check { return }
  let runner = (grep "^#!" $"($cluster_env_path)/($cmd_name)" | str trim)
  let run_ops = if (is-debug-enabled) { if ($runner | str contains "bash" ) { "-x" } else { "" } } else { "" }
  let cmd_env_temp = make_cmd_env_temp $defs $cluster_env_path $wk_vars
  if ($wk_vars | path exists) {
    let run_res = if ($runner | str ends-with "bash" ) {
        (^bash -c $"'source ($cmd_env_temp) ; bash ($run_ops) ($cluster_env_path)/($cmd_name) ($wk_vars) ($defs.pos.server) ($defs.pos.cluster) (^pwd)'" | complete)
    } else if ($runner | str ends-with "nu" ) {
        (^bash -c $"'source ($cmd_env_temp); ($env.NU) ($env.NU_ARGS) ($cluster_env_path)/($cmd_name)'" | complete)
    } else {
        (^bash -c $"'source ($cmd_env_temp); ($cluster_env_path)/($cmd_name) ($wk_vars)'" | complete)
    }
    rm -f $cmd_env_temp
    if $run_res.exit_code != 0 {
      (throw-error $"🛑 Error server ($defs.server.hostname) cluster ($defs.cluster.name)
        ($cluster_env_path)/($cmd_name) with ($wk_vars) ($defs.pos.server) ($defs.pos.cluster) (^pwd)"
        $run_res.stdout
        $where --span (metadata $run_res).span)
      exit 1
    }
    if not (is-debug-enabled) { rm -f $"($cluster_env_path)/prepare" }
  }
}
export def run_cluster_library [
    defs: record
    cluster_path: string
    cluster_env_path: string
    wk_vars: string
]: nothing -> bool {
  if  not ($cluster_path | path exists) { return false }
  let prov_resources_path = ($defs.settings.data.prov_resources_path | default "" | str replace  "~" $env.HOME)
  let cluster_server_name = $defs.server.hostname
  rm -rf ($cluster_env_path | path  join "*.k") ($cluster_env_path | path join "kcl")
  mkdir ($cluster_env_path | path join "kcl")

  let err_out = ($cluster_env_path | path join (mktemp --tmpdir-path $cluster_env_path --suffix ".err") | path basename)
  let kcl_temp = ($cluster_env_path | path join "kcl" | path join (mktemp --tmpdir-path $cluster_env_path --suffix ".k" ) | path basename)

  let wk_format = if $env.PROVISIONING_WK_FORMAT == "json" { "json" } else { "yaml" }
  let wk_data = { defs: $defs.settings.data, pos: $defs.pos, server: $defs.server }
  if $wk_format == "json" {
    $wk_data | to json | save --force $wk_vars
  } else {
    $wk_data | to yaml | save --force $wk_vars
  }
  if $env.PROVISIONING_USE_KCL {
    cd ($defs.settings.infra_path | path join $defs.settings.infra)
    let kcl_cluster_path = if ($cluster_path | path join "kcl"| path join $"($defs.cluster.name).k" | path exists) {
      ($cluster_path | path join "kcl"| path join $"($defs.cluster.name).k")
    } else if (($cluster_path | path dirname) | path join "kcl"| path join $"($defs.cluster.name).k" | path exists) {
      (($cluster_path | path dirname) | path join "kcl"| path join $"($defs.cluster.name).k")
    } else { "" }
    if ($kcl_temp | path exists) { rm -f $kcl_temp }
    let res = (^kcl import -m $wk_format $wk_vars -o $kcl_temp | complete)
    if $res.exit_code != 0 {
      print $"❗KCL import (_ansi red_bold)($wk_vars)(_ansi reset) Errors found "
      print $res.stdout
      rm -f $kcl_temp
      cd $env.PWD
      return false
    }
    # Very important! Remove external block for import and re-format it
    # ^sed -i "s/^{//;s/^}//" $kcl_temp
    open $kcl_temp -r | lines | find -v --regex "^{" | find -v --regex "^}" | save -f $kcl_temp
    ^kcl fmt $kcl_temp
    if $kcl_cluster_path != "" and ($kcl_cluster_path | path exists) { cat $kcl_cluster_path | save --append $kcl_temp }
    # } else { print $"❗ No cluster kcl ($defs.cluster.k) path found " ; return false }
    if $env.PROVISIONING_KEYS_PATH != "" {
      #use sops on_sops
      let keys_path = ($defs.settings.src_path | path join $env.PROVISIONING_KEYS_PATH)
      if not ($keys_path | path exists) {
        if (is-debug-enabled) {
          print $"❗Error KEYS_PATH (_ansi red_bold)($keys_path)(_ansi reset) found "
        } else {
          print $"❗Error (_ansi red_bold)KEYS_PATH(_ansi reset) not found "
        }
        return false
      }
      (on_sops d $keys_path) | save --append $kcl_temp
      if ($defs.settings.src_path | path join "clusters" | path join $defs.server.hostname | path join $"($defs.cluster.name).k" | path exists ) {
        cat ($defs.settings.src_path | path join "clusters" | path join $defs.server.hostname| path join $"($defs.cluster.name).k" ) | save --append $kcl_temp
      } else if ($defs.settings.src_path | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).k" | path exists ) {
        cat ($defs.settings.src_path | path join "clusters" | path join $defs.pos.server | path join $"($defs.cluster.name).k" ) | save --append $kcl_temp
      } else if ($defs.settings.src_path | path join "clusters" | path join $"($defs.cluster.name).k" | path exists ) {
        cat ($defs.settings.src_path | path join "clusters" | path join $"($defs.cluster.name).k" ) | save --append $kcl_temp
      }
      let res = (^kcl $kcl_temp -o $wk_vars | complete)
      if $res.exit_code != 0 {
        print $"❗KCL errors (_ansi red_bold)($kcl_temp)(_ansi reset) found "
        print $res.stdout
        rm -f $wk_vars
        cd $env.PWD
        return false
      }
      rm -f $kcl_temp $err_out
    } else if ($defs.settings.src_path | path join "clusters" | path join $"($defs.cluster.name).yaml" | path exists) {
      cat  ($defs.settings.src_path | path join "clusters" | path join $"($defs.cluster.name).yaml" ) | tee { save -a $wk_vars }  | ignore
    }
    cd $env.PWD
  }
  (^sed -i $"s/NOW/($env.NOW)/g" $wk_vars)
  if $defs.cluster_install_mode == "library" {
    let cluster_data = (open $wk_vars)
    let verbose = if (is-debug-enabled) { true } else { false }
    if $cluster_data.cluster.copy_paths? != null {
      #use utils/files.nu *
      for it in $cluster_data.cluster.copy_paths {
        let it_list = ($it | split row "|" | default [])
        let cp_source = ($it_list | get -o 0 | default "")
        let cp_target = ($it_list | get -o 1 | default "")
        if ($cp_source | path exists) {
          copy_prov_files $cp_source ($defs.settings.infra_path | path join $defs.settings.infra) $"($cluster_env_path)/($cp_target)" false $verbose
        } else if ($"($prov_resources_path)/($cp_source)" | path exists) {
          copy_prov_files $prov_resources_path $cp_source $"($cluster_env_path)/($cp_target)" false $verbose
        } else if ($cp_source | file exists) {
          copy_prov_file $cp_source $"($cluster_env_path)/($cp_target)" $verbose
        } else if ($"($prov_resources_path)/($cp_source)" | path exists) {
          copy_prov_file $"($prov_resources_path)/($cp_source)" $"($cluster_env_path)/($cp_target)" $verbose
        }
      }
    }
  }
  rm -f ($cluster_env_path | path join "kcl") ($cluster_env_path | path join "*.k")
  on_template_path $cluster_env_path $wk_vars true true
  if ($cluster_env_path | path join $"env-($defs.cluster.name)" | path exists) {
    ^sed -i 's,\t,,g;s,^ ,,g;/^$/d' ($cluster_env_path | path join $"env-($defs.cluster.name)")
  }
  if ($cluster_env_path | path join "prepare" | path exists) {
    run_cmd "prepare" "Prepare" "run_cluster_library" $defs $cluster_env_path $wk_vars
    if ($cluster_env_path | path join "resources" | path exists) {
      on_template_path ($cluster_env_path | path join "resources") $wk_vars false true
    }
  }
  if not (is-debug-enabled) {
    rm -f ($cluster_env_path | path join "*.j2") $err_out $kcl_temp
  }
  true
}
export def run_cluster [
    defs: record
    cluster_path: string
    env_path: string
]: nothing -> bool {
  if  not ($cluster_path | path exists) { return false }
  if $defs.check { return }
  let prov_resources_path = ($defs.settings.data.prov_resources_path | default "" | str replace  "~" $env.HOME)
  let created_clusters_dirpath = ($defs.settings.data.created_clusters_dirpath | default "/tmp" |
    str replace "~" $env.HOME | str replace "NOW" $env.NOW | str replace "./" $"($defs.settings.src_path)/")
  let cluster_server_name = $defs.server.hostname

  let cluster_env_path = if  $defs.cluster_install_mode == "server" { $"($env_path)_($defs.cluster_install_mode)" } else { $env_path }

  if not ( $cluster_env_path | path exists) { ^mkdir -p $cluster_env_path }
  if not ( $created_clusters_dirpath | path exists) { ^mkdir -p $created_clusters_dirpath }

  (^cp -pr $"($cluster_path)/*" $cluster_env_path)
  rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl"

  let wk_vars = $"($created_clusters_dirpath)/($defs.server.hostname).yaml"
  # if $defs.cluster.name == "kubernetes" and  ("/tmp/k8s_join.sh" | path exists) { cp -pr "/tmp/k8s_join.sh" $cluster_env_path }
  let require_j2 = (^ls ($cluster_env_path | path join "*.j2") err> (if $nu.os-info.name == "windows" { "NUL" } else { "/dev/null" }))


  let res = if  $defs.cluster_install_mode == "library" or $require_j2 != "" {
    (run_cluster_library $defs $cluster_path $cluster_env_path $wk_vars)
  }
  if not $res {
    if not (is-debug-enabled) { rm -f $wk_vars }
    return $res
  }
  let err_out = ($env_path | path join (mktemp --tmpdir-path $env_path --suffix ".err") | path basename)
  let tar_ops = if (is-debug-enabled) { "v" } else { "" }
  let bash_ops = if (is-debug-enabled) { "bash -x" } else { "" }

  let res_tar = (^tar -C $cluster_env_path $"-c($tar_ops)zf" $"/tmp/($defs.cluster.name).tar.gz" . | complete)
  if $res_tar.exit_code != 0 {
    _print (
      $"🛑 Error (_ansi red_bold)tar cluster(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset)" +
      $"  cluster (_ansi yellow_bold)($defs.cluster.name)(_ansi reset) ($cluster_env_path) -> /tmp/($defs.cluster.name).tar.gz"
    )
    _print $res_tar.stdout
    return false
  }
  if $defs.check {
    if not (is-debug-enabled) {
      rm -f $wk_vars
      rm -f $err_out
      rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl"
    }
    return true
  }
  let is_local  = (^ip addr | grep "inet " | grep "$defs.ip")
  if $is_local != "" and not (is-debug-check-enabled) {
    if $defs.cluster_install_mode ==  "getfile" {
      if (cluster_get_file $defs.settings $defs.cluster $defs.server $defs.ip true true) { return false }
      return true
    }
    rm -rf $"/tmp/($defs.cluster.name)"
    mkdir $"/tmp/($defs.cluster.name)"
    cd $"/tmp/($defs.cluster.name)"
    tar x($tar_ops)zf $"/tmp/($defs.cluster.name).tar.gz"
    let res_run = (^sudo $bash_ops $"./install-($defs.cluster.name).sh" err> $err_out  | complete)
    if $res_run.exit_code != 0 {
      (throw-error $"🛑 Error server ($defs.server.hostname) cluster ($defs.cluster.name)
          ./install-($defs.cluster.name).sh ($defs.server_pos) ($defs.cluster_pos) (^pwd)"
          $"($res_run.stdout)\n(cat $err_out)"
          "run_cluster_library" --span (metadata $res_run).span)
      exit 1
    }
    fi
    rm -fr $"/tmp/($defs.cluster.name).tar.gz" $"/tmp/($defs.cluster.name)"
  } else {
    if $defs.cluster_install_mode == "getfile" {
      if (cluster_get_file  $defs.settings $defs.cluster $defs.server $defs.ip true false) { return false }
      return true
    }
    if not (is-debug-check-enabled) {
      #use ssh.nu *
      let scp_list: list<string> = ([] | append $"/tmp/($defs.cluster.name).tar.gz")
      if not (scp_to $defs.settings  $defs.server $scp_list "/tmp" $defs.ip) {
        _print (
          $"🛑 Error (_ansi red_bold)ssh_cp(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset) [($defs.ip)] " +
          $"  cluster (_ansi yellow_bold)($defs.cluster.name)(_ansi reset) /tmp/($defs.cluster.name).tar.gz"
        )
        return false
      }
      let cmd = (
          $"rm -rf /tmp/($defs.cluster.name) ; mkdir /tmp/($defs.cluster.name) ; cd /tmp/($defs.cluster.name) ;" +
          $" sudo tar x($tar_ops)zf /tmp/($defs.cluster.name).tar.gz;" +
          $" sudo ($bash_ops) ./install-($defs.cluster.name).sh "  # ($env.PROVISIONING_MATCH_CMD) "
      )
      if not (ssh_cmd $defs.settings $defs.server true $cmd $defs.ip) {
        _print (
          $"🛑 Error (_ansi red_bold)ssh_cmd(_ansi reset) server (_ansi green_bold)($defs.server.hostname)(_ansi reset) [($defs.ip)] " +
          $"  cluster (_ansi yellow_bold)($defs.cluster.name)(_ansi reset) install_($defs.cluster.name).sh"
        )
        return false
      }
      # if $defs.cluster.name == "kubernetes" { let _res_k8s = (scp_from $defs.settings $defs.server "/tmp/k8s_join.sh" "/tmp" $defs.ip) }
      if not (is-debug-enabled) {
        let rm_cmd = $"sudo rm -f /tmp/($defs.cluster.name).tar.gz; sudo rm -rf /tmp/($defs.cluster.name)"
        let _res = (ssh_cmd $defs.settings  $defs.server true $rm_cmd $defs.ip)
        rm -f $"/tmp/($defs.cluster.name).tar.gz"
      }
    }
  }
  if ($"($cluster_path)/postrun" | path exists ) {
    cp $"($cluster_path)/postrun" $"($cluster_env_path)/postrun"
    run_cmd "postrun" "PostRune" "run_cluster_library" $defs $cluster_env_path $wk_vars
  }
  if not (is-debug-enabled) {
    rm -f $wk_vars
    rm -f $err_out
    rm -rf $"($cluster_env_path)/*.k" $"($cluster_env_path)/kcl"
  }
  true
}