jesus/provisioning
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: add current provisioning state before migration

Browse source
This commit is contained in:
Jesús Pérez 2025-09-22 23:11:41 +01:00
parent a9703b4748
commit 50745b0f22
660 changed files with 88126 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

91
providers/aws/nulib/aws/cache.nu Normal file
View file

@ -0,0 +1,91 @@
#!/usr/bin/env nu
# Info: AWS
use lib.nu *
export def aws_start_cache_info [
settings: record
server: record
] {
( $"# To start from scratch set 'vpc' 'subnet' 'sg.id' to '?' then new AWS settings will be collected. This will create 'sg.perms'.\n" +
$"# Removing 'provider_path' and 'defs/aws_data.k' would fallback to defaults with no settings for 'sg.name' and 'sg.perms', etc.\n"
)
}
export def aws_create_cache [
settings: record
server: record
error_exit: bool
] {
if $settings == null {
if $env.PROVISIONING_DEBUG { print $"❗ No settings found " }
return
}
let provider_path = (get_provider_data_path $settings $server)
#use lib_provisioning/utils/settings.nu load_provider_env
let data = (load_provider_env $settings $server $provider_path)
if ($data | is-empty) or ($data | get -o main | get -o vpc) == "?" {
aws_scan_settings "create" $provider_path $settings $server false
let new_data = (load_provider_env $settings $server $provider_path)
if ($new_data | is-empty) or ($new_data | get -o main | get -o vpc) == "?" {
print $"❗AWS no valid provider settings for (_ansi red)($server.hostname)(_ansi reset)"
exit 1
}
} else {
if $env.PROVISIONING_DEBUG {
print $"aws main data already exists in ($provider_path | path basename)"
}
}
aws_scan_servers $provider_path $settings $server
if $env.PROVISIONING_DEBUG { print $"Cache for ($server.provider) on ($server.hostname) saved in: ($provider_path | path basename)" }
# load_provider_env $settings $server $provider_path
}
export def aws_read_cache [
settings: record
server: record
error_exit: bool
] {
if $settings == null {
print $"❗ No settings found "
return
}
}
export def aws_clean_cache [
settings: record
server: record
error_exit: bool
] {
if $settings == null {
print $"❗ No settings found "
return
}
let provider_path = (get_provider_data_path $settings $server)
let data = if ($provider_path | path exists) {
open $provider_path
} else {
{ servers: null }
}
if ($data.servers? != null) and ($data.servers | where {|it| ($it.hostname? | default "") == $server.hostname} | length) == 0 {
if $env.PROVISIONING_DEBUG {
print $"❗server ($server.hostname) already deleted from ($provider_path | path basename)"
}
}
let all_servers = ( $data.servers? | default [] | where {|it| $it.hostname != $server.hostname})
if $env.PROVISIONING_DEBUG { print $"Cache for ($server.provider) delete ($server.hostname) in: ($provider_path | path basename)" }
let new_data = if ($all_servers | length) == 0 {
aws_delete_settings "all" $provider_path $settings $server
{}
} else {
( $data | merge { servers: $all_servers})
}
save_provider_env $new_data $settings $provider_path
}
export def aws_ip_from_cache [
settings: record
server: record
error_exit: bool
] {
let prov_settings = ($settings.providers | find $server.provider ) #| get -o settings)
if ($prov_settings | is-empty) == null { return "" }
($prov_settings | flatten | find $server.hostname | select -o ip_addresses | find "public"| get -o address | get -o 0 | default "")
}

5
providers/aws/nulib/aws/env.nu Normal file
View file

@ -0,0 +1,5 @@
export-env {
$env.AWS_API_URL = ($env | get -o AWS_API_URL | default "")
$env.AWS_AUTH = ($env | get -o AWS_AUTH | default "")
$env.AWS_INTERFACE = ($env | get -o AWS_INTERFACE | default "CLI") # API or CLI
}

715
providers/aws/nulib/aws/lib.nu Normal file
View file

@ -0,0 +1,715 @@
#!/usr/bin/env nu
# Info: Script to create/delete AWS resources from file settings in bash with template/vars
# Author: JesusPerez
# Release: 1.0
# Date: 26-03-2024
use ../../../../core/nulib/lib_provisioning/utils/templates.nu run_from_template
export def aws_review_credentials [
] {
print $"❗AWS credentials not found for '$PROVIDER_CLI_CMD' command."
print $" Use default profile or env AWS_PROFILE from $HOME/.aws/credentials path or environment variables for settings"
print $" More info: "
print $" Profile mode: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html"
print $" Evironment mode: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html"
}
export def aws_check_region [
zone: string
] {
if ($zone |is-empty) {
print $"❗AWS region zone ($env.AWS_DEFAULT_REGION) not found for '$PROVIDER_CLI_CMD' command."
print $"Use set default profile or use env AWS_PROFILE with $HOME/.aws/credentials path or environment variables for settings"
}
(^aws ec2 describe-availability-zones --region $zone | complete).exit_code
}
export def aws_get_plan_info [
var: string
server: record
] {
let plan = ($server | get -o $var | default "")
if ($plan | is-mepty) { return }
let res = (^aws ec2 describe-instance-types --instance-types $plan
--query 'InstanceTypes[].{ type: InstanceType, cores: VCpuInfo.DefaultCores, memory: MemoryInfo.SizeInMiB, arch: ProcessorInfo.SupportedArchitectures, gen: CurrentGeneration, infaces: NetworkInfo.MaximumNetworkInterfaces, ena: NetworkInfo.EnaSupport }'
--out=json )
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json | get -o 0 | default "")
}
}
export def aws_find_plan [
var: string
server: record
] {
let reqplan = ($server | get -o $var | default "")
if ($reqplan | is-mepty) {
print $"❗No reqplan found in settings for ($var)"
return 1
}
let res = (^ aws ec2 describe-instance-types --filters $"Name=processor-info.supported-architecture,Values=($reqplan.arch | default '')"
$"Name=memory-info.size-in-mib,Values=($reqplan.memory | default '')"
$"Name=vcpu-info.default-cores,Values=($reqplan.cores | default '')"
$"Name=network-info.maximum-network-interfaces,Values=($reqplan.infaces | default '')"
$"Name=network-info.ena-support,Values=($reqplan.ena | default '')"
--query 'InstanceTypes[].{ type: InstanceType, cores: VCpuInfo.DefaultCores, memory: MemoryInfo.SizeInMiB, arch: ProcessorInfo.SupportedArchitectures, gen: CurrentGeneration, infaces: NetworkInfo.MaximumNetworkInterfaces, ena: NetworkInfo.EnaSupport }'
--output json
)
if ($res.exit_code == 0) {
($res.stdout | from json)
}
}
export def aws_compare_plan_reqplan [
var_plan: string
var_reqplan: string
settings: record
server: record
] {
let plan = ($server | get -o $var_plan)
let check_plan = (aws_get_plan_info $var_plan $server)
let reqplan = ($server | get -o $var_reqplan)
if ($plan | is-empty) or ( $check_plan | is-empty) {
print $"❗No valid $plan found for $var_plan in $AWS_DEFAULT_REGION"
return 1
}
if ($reqplan | is-empty) { return }
let plan_memory = ($check_plan | get -o memory | default "")
let reqplan_memory = ($reqplan| get -o memory | default "")
if $plan_memory != $reqplan_memory {
print $"❗$plan memory does not match plan: $plan_memory expected $reqplan_memory"
return 1
}
let plan_cores = ($check_plan | get -o cores | default "")
let reqplan_cores = ($reqplan | get -o cores | default "")
if $plan_cores != $reqplan_cores {
print $"❗($plan) cores does not match plan: ($plan_cores) expected ($reqplan_cores)"
return 1
}
let plan_archs = ($check_plan | get -o arch | default "")
let reqplan_archs = ($reqplan | get -o arch | default "")
if not ($plan_archs | str contains $reqplan_archs ) {
print $"❗($plan) architectures does not match plan: ($plan_archs) expected ($reqplan_archs)"
return 1
}
let plan_infaces = ($check_plan | get -o infaces | default "")
let reqplan_infaces = ($reqplan | get -o infaces | default "")
if $plan_infaces < $reqplan_infaces {
print $"❗($plan) interfaces number does not match plan: ($plan_infaces) expected ($reqplan_infaces)"
return 1
}
0
}
export def aws_get_os_image [
name: string
arch: string
] {
let res = (^aws ec2 describe-images --owners amazon --filters $"'Name=name,Values=*'($name)'*'" $"'Name=architecture,Values=*'($arch)'*'"
--query 'reverse(sort_by(Images,&CreationDate))[:5].{id:ImageId, name: Name, date:CreationDate}[0]' --output json
)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json)
} else { "" }
}
export def aws_delete_private_vpcs [
aws_priv_cidr_block: string
] {
let res = (^aws ec2 describe-vpcs --query Vpcs --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
for it in ($res.stdout | from json | where {|it| $it.CidrBlock == $aws_priv_cidr_block } | get -o VpcId | default []) {
print $"delete vpc id ($it)"
^aws ec2 delete-vpc --vpc-id "$it"
}
}
}
export def aws_create_private_vpc [
aws_priv_cidr_block: string
task: string
] {
let res = (^aws ec2 describe-vpcs --query Vpcs --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-empty) {
print $"❗Error ($task) vpcs ($aws_priv_cidr_block) "
exit 1
}
let aws_priv_vpc = ($res.stdout | from json | where {|it| $it.CidrBlock == $aws_priv_cidr_block } | get -o 0 | get -o VpcId | default "")
match $task {
"create" => {
if ($aws_priv_vpc | is-not-empty) {
print $"Clean up VPC ($aws_priv_vpc)..."
let res = (^aws ec2 delete-vpc --vpc-id $aws_priv_vpc err> /dev/null | complete )
if $res.exit_code != 0 {
print $"vpc ($aws_priv_vpc) delete error ($res.exit_code) ($res.stdout)"
return $aws_priv_vpc
}
}
let res = (^aws ec2 create-vpc --cidr-block $aws_priv_cidr_block --query Vpc.VpcId --output text | complete)
if $res.exit_code == 0 {
($res.stdout | str replace '"' '')
} else {
print $"❗ Error ($task) priv_vpc for ($aws_priv_cidr_block)"
exit 1
}
},
_ => {
$aws_priv_vpc
}
}
}
export def aws_delete_sgs_by_name [
aws_sg_name: string
] {
let res = (^aws ec2 describe-security-groups --query SecurityGroups --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
let aws_sg_id = ($res.stdout | from json | where {|it| $it.GroupName == $aws_sg_name } | get -o GroupId | default "")
if ($aws_sg_id | is-not-empty) {
print $"Clean up SGs ($aws_sg_name)"
^aws ec2 delete-security-group --group-id $aws_sg_id
}
}
}
export def aws_delete_sgs [
aws_vpc: string
] {
let res = (^aws ec2 describe-security-groups --query SecurityGroups --out json)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
for it in ($res.stdout | from json | where {|it| $it.VpcId == $aws_vpc } | where {|it| $it.GroupName != "default" } | get -o GroupId | default "") {
print $"delete security group id ($it)"
^aws ec2 delete-security-group --group-id $it
}
}
}
export def aws_create_sg_id [
aws_vpc: string
aws_sg_name: string
task: string
] {
let res = (^aws ec2 describe-security-groups --query SecurityGroups --out json | complete)
if $res.exit_code != 0 or ($res.stdout | is-empty) {
print $"❗Error ($task) sg_id for ($aws_sg_name) in ($aws_vpc)"
exit 1
}
let aws_sg_id = ($res.stdout | from json | where {|it| $it.VpcId == $aws_vpc and $it.GroupName == $aws_sg_name } |
get -o 0 | get -o GroupId | default "")
match $task {
"create" => {
if ($aws_sg_id | is-not-empty) {
print $"Clean up sg ($aws_sg_id) ..."
let res = (^aws ec2 delete-security-group --group-id $aws_sg_id | complete)
if $res.exit_code != 0 {
print $"❗Error delete ($aws_sg_id) for ($aws_sg_name) in ($aws_vpc)"
return $aws_sg_id
}
}
let res = (^aws ec2 create-security-group --group-name $aws_sg_name --description $"Group ($aws_sg_name)"
--tag-specifications $"ResourceType=security-group,Tags=[{Key=Name,Value=($aws_sg_name)}]"
--vpc-id ($aws_vpc | str trim) --out json | complete )
if $res.exit_code == 0 {
($res.stdout | from json | get -o GroupId | default "")
} else {
print $"❗Error ($task) sg_id for ($aws_sg_name) in ($aws_vpc)"
exit 1
}
},
_ => {
$aws_sg_id
}
}
}
export def aws_add_sg_perms [
sg_data: record
server: record
check_mode: bool
] {
let perms = ($sg_data | get -o perms | default [])
if ($perms | is-empty) { return }
let res = (^aws ec2 describe-security-groups --group-id $sg_data.id --query SecurityGroups[].IpPermissions --out json | complete)
let curr_sg_perms = if $res.exit_code == 0 {
($res.stdout | from json | get -o 0 | default [])
} else { [] }
mut curr_perms = []
for p in $curr_sg_perms {
mut ranges = ""
for rng in ($p | get -o IpRanges) {
if ($ranges | is-not-empty) { $ranges = $"($ranges),"}
$ranges = $"($ranges){CidrIp=($rng.CidrIp)}"
}
let protocol = ($p | get -o IpProtocol | default "")
let from_port = ($p | get -o FromPort | default "")
let to_port = ($p | get -o ToPort | default "")
for it in $perms {
if ($protocol == ($it | get -o protocol ) and $from_port == ($it | get -o fromPort ) and
$to_port == ($it | get -o toPort ) and
$ranges == ($it | get -o ranges | str replace "[" "" | str replace "]" "" )) {
} else {
$curr_perms = ($curr_perms | append $p)
break
}
}
}
if ($curr_perms == $curr_sg_perms) and ($curr_perms| length) == ($perms | length) { return }
if ($perms == $curr_perms) { return }
if $env.PROVISIONING_DEBUG {
print $"(_ansi green)current sg perms(_ansi reset) ($curr_perms | table -e)"
}
let wk_format = if $env.PROVISIONING_WK_FORMAT == "json" { "json" } else { "yaml" }
let wk_vars = ( "/tmp/" | path join (mktemp --tmpdir-path "/tmp" --suffix $".($wk_format)" | path basename))
let data = { sg_name: $sg_data.name, sg_id: $sg_data.id, perms: $perms, curr_perms: $curr_perms }
if $wk_format == "json" {
$data | to json | save --force $wk_vars
} else {
$data | to yaml | save --force $wk_vars
}
let run_file = ("/tmp" | path join $"onaws_run_sg_(mktemp --tmpdir-path "/tmp" --suffix ".sh" | path basename | str replace 'tmp.' '' )")
let sg_template = ($env.PROVISIONING | path join "providers" | path join $server.provider | path join templates | path join "aws_sg.j2" )
if not ($sg_template | path exists) {
print $"❗($sg_template) not found for Security Groups ($sg_data.name)"
exit 1
}
#use ../../../../core/nulib/lib_provisioning/utils/templates.nu run_from_template
let res = if $check_mode {
run_from_template $sg_template $wk_vars $run_file --check_mode
} else {
run_from_template $sg_template $wk_vars $run_file
}
if $res {
if $env.PROVISIONING_DEBUG {
print $"(_ansi green)OK(_ansi reset) (_ansi green_bold)($sg_data.name)(_ansi reset)"
} else {
rm --force $wk_vars $run_file
}
} else {
print $"(_ansi red)Failed(_ansi reset) (_ansi green_bold)($sg_data.name)(_ansi reset)"
}
}
export def aws_delete_private_subnets [
aws_priv_vpc: string
aws_priv_cidr_block: string
] {
let res = (^aws ec2 describe-subnets --query Subnets --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
for it in ($res.stdout | from json | where { |it| $it.VpcId == $aws_priv_vpc and $it.CidrBlock == $aws_priv_cidr_block } | get -o SubnetId | default []) {
print $"Clean up subnet ($it) in ($aws_priv_vpc)..."
let res = (^aws ec2 delete-subnet --subnet-id $it | complete)
if $res.exit_code != 0 { return false }
}
}
true
}
export def aws_create_private_subnet [
aws_priv_cidr_block: string
aws_priv_vpc: string
aws_avail_zone: string
task: string
] {
match $task {
"create" => {
if not (aws_delete_private_subnets $aws_priv_vpc $aws_priv_cidr_block) {
let res = (^aws ec2 describe-subnets --query Subnets --out json | complete)
return ($res.stdout | from json | where { |it| $it.VpcId == $aws_priv_vpc and $it.CidrBlock == $aws_priv_cidr_block } | get -o 0)
}
let res = (^aws ec2 create-subnet --vpc-id $aws_priv_vpc --cidr-block $aws_priv_cidr_block --availability-zone $aws_avail_zone --query "Subnet" --output json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json)
} else {
print $"❗aws_priv_subnet not found for ($aws_priv_vpc) - ($aws_priv_cidr_block)"
exit 1
}
},
_ => {
let res = (^aws ec2 describe-subnets --query Subnets --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json | where { |it| $it.VpcId == $aws_priv_vpc and $it.CidrBlock == $aws_priv_cidr_block } | get -o 0)
} else {
{}
}
}
}
}
def aws_vpc_subnet [
aws_avail_zone: string
aws_priv_subnet: string
task: string
] {
let res = (^aws ec2 describe-subnets --query Subnets --out json | complete)
let aws_vpc_subnet_data = if $res.exit_code == 0 {
let data = ($res.stdout | from json | where {|it| $it.AvailabilityZone == $aws_avail_zone and $it.SubnetId != $aws_priv_subnet } | get -o 0 )
{"vpc": $"($data | get -o VpcId | default '')", "subnet": $"($data | get -o SubnetId | default '')"}
} else {
{}
}
if $task == "create" and ($aws_vpc_subnet_data | is-empty) {
^aws ec2 create-default-subnet --availability-zone $aws_avail_zone
(aws_vpc_subnet $aws_avail_zone $aws_priv_subnet "scan")
} else {
$aws_vpc_subnet_data
}
}
export def aws_delete_private_interfaces [
aws_priv_subnet: string
] {
let res = (^aws ec2 describe-network-interfaces --query NetworkInterfaces --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
for it in ($res.stdout | from json | where {|it| $it.SubnetId == $aws_priv_subnet } | get -o NetworkInterfaceId | default []) {
^aws ec2 delete-network-interface --network-interface-id $it
}
}
}
export def aws_delete_private_interface [
network_interface_id: string
] {
^aws ec2 delete-network-interface --network-interface-id "$network_interface_id"
}
export def aws_get_interface_defs [
ip_interface: string
aws_priv_subnet: string
] {
let res = (^aws ec2 describe-network-interfaces --query NetworkInterfaces --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json | where {|it| $it.SubnetId == $aws_priv_subnet and $it.PrivateIpAddress == $ip_interface } |
get -o 0 | get -o NetworkInterfaceId | default ""
)
}
}
export def aws_get_create_private_interface [
ip_interface: string
aws_priv_subnet: string
] {
(aws_get_interface_defs $ip_interface $aws_priv_subnet)
}
export def aws_get_instance_defs [
instance: string
] {
let res = (^aws ec2 describe-instances --instance-ids $instance --out "json" | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json | get -o "Reservations" | get -o "Instances" | default "" )
}
}
export def attach_private_interface [
interface: string
instance: string
aws_priv_subnet: string
] {
if (aws_get_instance_defs $instance | is-not-empty) and (aws_get_interface_defs $interface $aws_priv_subnet | is-not-empty) {
(^aws ec2 attach-network-interface --network-interface-id $interface --instance-id $instance --device-index 1)
} else { "" }
}
export def detach_private_interface [
interface: string
instance: string
aws_priv_subnet: string
] {
if (aws_get_instance_defs $instance | is-not-empty) and (aws_get_interface_defs $interface $aws_priv_subnet | is-not-empty) {
(^aws ec2 detach-network-interface --network-interface-id $interface --instance-id $instance)
} else { "" }
}
export def aws_delete_target [
target: string
target_id: string
settings: record
] {
mut num = 0
mut res = ""
mut status = ""
let val_timeout = if $settings.running_timeout? != null { $settings.running_timeout } else { 60 }
let wait = if $settings.running_wait? != null { $settings.running_wait } else { 10 }
let wait_duration = ($"($wait)sec"| into duration)
if $env.PROVISIONING_DEBUG { print -n $"Delete ($target) -> ($target_id) " }
while ($status | is-empty) {
let status = match $target {
"securityGroup" => (^aws ec2 describe-security-groups --group-id $target_id err> /dev/null),
"subnet" => (^aws ec2 describe-subnets --subnet-id $target_id err> /dev/null) ,
"vpcs" => (^aws ec2 describe-vpcs --vpc-id $target_id err> /dev/null) ,
"interface" => (^aws ec2 describe-network-interfaces --network-interface-id $target_id err> /dev/null),
}
if ($status | is-empty) {
print $" "
return
}
let res = match $target {
"securityGroup" => (^aws ec2 delete-security-group --group-id $target_id | complete).exit_code,
"subnet" => (^aws ec2 delete-subnet --subnet-id $target_id | complete).exit_code,
"vpcs" => (^aws ec2 delete-vpc --vpc-id $target_id | complete).exit_code,
"interface" => (^aws ec2 delete-network-interface --network-interface-id $target_id | complete).exit_code,
}
if ($res == 0) {
print $" "
return
}
if ($status | is-not-empty) or ($res != 0 ) {
sleep $wait_duration
$num += $wait
if $val_timeout > 0 and $num > $val_timeout { return 1 }
print -n $"($num) "
}
}
print $""
}
export def aws_delete_settings [
target: string
provider_path: string
settings: record
server: record
] {
if not ($provider_path |path exists) {
print $"❗aws_settings not found ($provider_path) no delete settings "
return
}
let prov_settings = (load_provider_env $settings $server $provider_path)
let env_settings = (get_provider_env $settings $server)
if ($prov_settings | is-empty) or $prov_settings.main? == null or $prov_settings.priv? == null {
if $env.PROVISIONING_DEBUG { print $"❗aws_settings (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) no settings main and priv found" }
return
}
let aws_priv_subnet = ($prov_settings.priv.subnet | default "")
let aws_priv_cidr_block = ($server.priv_cidr_block | default "")
print $"Scanning aws resources to clean from (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) ... ($aws_priv_cidr_block)"
if $target == "all" or $target == "interface" {
for server_info in ($settings.data.servers) {
let server = ($server_info | get -o hostname | default "")
let network_private_ip = ($server_info | get -o network_private_ip | default "")
let res = (^aws ec2 describe-network-interfaces --query NetworkInterfaces --out "json" | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
let interface = ($res.stdout | from json | where {|it|($it.PrivateIpAddress == $network_private_ip)} | get -o 0 | get -o NetworkInterfaceId | default "")
if ($interface | is-not-empty) { aws_delete_target "interface" $interface $settings }
}
}
}
if not $server.prov_settings_clean {
print $"❗aws provider settings clean ['prov_settings_clean'] set to ($server.prov_settings_clean)"
return
}
if $target == "all" or $target == "pub_sg" {
let aws_sg_name = ($prov_settings | get -o main | get -o sg | get -o name | default "")
if ($aws_sg_name | is-not-empty) {
let res = (^aws ec2 describe-security-groups --query SecurityGroups --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
let aws_sg_id = ($res.stdout | from json | where {|it| ($it.GroupName == $aws_sg_name) } | get -o 0 | get -o GroupId | default "")
if ($aws_sg_id | is-not-empty) { aws_delete_target "securityGroup" $aws_sg_id $settings }
}
}
}
if ($aws_priv_cidr_block | is-not-empty) {
if $target == "all" or $target == "priv_sg" {
let aws_priv_sg_name = ($prov_settings | get -o priv | get -o sg | get -o name | default "")
if ($aws_priv_sg_name | is-not-empty) {
let res = (^aws ec2 describe-security-groups --query SecurityGroups --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
let aws_priv_sg_id = ($res.stdout | from json | where {|it| ($it.GroupName == $aws_priv_sg_name)} | get -o 0 | get -o GroupId | default "")
if ($aws_priv_sg_id | is-not-empty) { aws_delete_target "securityGroup" $aws_priv_sg_id $settings }
}
}
}
if $target == "all" or $target == "priv_subnet" {
let res = (^aws ec2 describe-subnets --query Subnets --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json | where { |it| $it.CidrBlock == $aws_priv_cidr_block } | get -o 0 | get -o SubnetId | default [] |
each {|it| aws_delete_target "subnet" $it $settings }
)
}
}
if $target == "all" or $target == "priv_vpc" {
let res = (^aws ec2 describe-vpcs --query Vpcs --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json | where { |it| $it.CidrBlock == $aws_priv_cidr_block } | get -o 0 | get -o VpcId | default [] |
each {|it| aws_delete_target "vpcs" $it $settings }
)
}
}
} else {
if $env.PROVISIONING_DEBUG { print $"❗aws_priv_cidr_block not found in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) " }
}
}
export def default_vpc [
] {
let res = (^aws ec2 describe-vpcs --query Vpcs[].VpcId --filters "Name=is-default,Values=true" --out text | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | str trim)
} else {
if $env.PROVISIONING_DEBUG { print$"❗Error get (_ansi red)default Vpc(_ansi reset) " }
{}
}
}
export def default_subnet [
vpc: string
] {
let res = (^aws ec2 describe-subnets --query Subnets[] --filters "Name=default-for-az,Values=true" "Name=vpc-id,Values=vpc-0ffea05634122f3fa" --out json | complete)
if $res.exit_code == 0 and ($res.stdout | is-not-empty) {
($res.stdout | from json | default [] | get -o 0 | default "")
} else {
if $env.PROVISIONING_DEBUG { print$"❗Error get (_ansi red)default subnet(_ansi reset) VPC (_ansi yellow)($vpc)(_ansi reset)" }
""
}
}
export def aws_scan_settings [
in_task: string
provider_path: string
settings: record
server: record
check_mode: bool
] {
let prov_settings = (load_provider_env $settings $server $provider_path)
let env_settings = (get_provider_env $settings $server)
if (($prov_settings | get -o main ) == ($env_settings | get -o main)
and ($prov_settings | get -o priv ) == ($env_settings | get -o priv)
and ($prov_settings | get -o main | get -o vpc) != "?") { return }
let task = if $prov_settings.main? == null or ($prov_settings | get -o main | get -o vpc) == "?" {
"create"
} else if $in_task == "create" {
if $env.PROVISIONING_DEBUG { print $"❗aws_scan_settings task ($in_task) and ($provider_path) has content "}
"scan"
} else { $in_task }
let data_settings = if $prov_settings.main? == null or ($prov_settings | get -o main | get -o vpc) != "?" {
$prov_settings
} else { $env_settings }
print $"Scanning (_ansi green_bold)AWS(_ansi reset) resources to (_ansi purple_bold)($task)(_ansi reset) settings in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) ..."
let res = (^aws ec2 describe-availability-zones --query AvailabilityZones| complete)
if $res.exit_code != 0 {
(throw-error $"🛑 Unable lo load ($server.provider) availability zones" $"($res.exit_code) ($res.stdout)" $"server info ($server.hostname)" --span (metadata $res).span)
exit 1
}
let $aws_vpc = if ($data_settings | get -o main | get -o vpc | length) > 1 { $settings.main.vpc } else { default_vpc }
let $aws_subnet_data = if ($data_settings | get -o main | get -o subnet | length) > 1 {
let res = (^aws ec2 describe-subnets --query Subnets -SubnetId $settings.main.subnet --out json | complete)
if $res.exit_code != 0 {
(throw-error $"🛑 Unable lo load ($server.provider) subnet info ($settings.main.subnet)" $"($res.exit_code) ($res.stdout)" $"server info ($server.hostname)" --span (metadata $res).span)
exit 1
}
($res.stdout | from json)
} else {
default_subnet $aws_vpc
}
let aws_subnet = ($aws_subnet_data | get -o SubnetId | default "")
if ($aws_subnet | is-empty) {
(throw-error $"🛑 Unable lo load ($server.provider) subnet id" $"($aws_subnet_data)" $"server info ($server.hostname)" --span (metadata $aws_subnet_data).span)
exit 1
}
let aws_avail_zone = ($aws_subnet_data | get -o AvailabilityZone | default "")
if ($aws_avail_zone | is-empty) {
(throw-error $"🛑 Unable lo load ($server.provider) subnet availability zone" $"($aws_subnet_data)" $"server info ($server.hostname)" --span (metadata $aws_avail_zone).span)
exit 1
}
let aws_priv_cidr_block = ($server.priv_cidr_block | default "")
let priv = if ($aws_priv_cidr_block | is-not-empty) {
let aws_priv_vpc = (aws_create_private_vpc $aws_priv_cidr_block $task)
if ($aws_priv_vpc | is-empty) {
print $"❗ aws_priv_vpc not found in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) "
exit 1
}
let aws_priv_subnet_data = (aws_create_private_subnet $aws_priv_cidr_block $aws_priv_vpc $aws_avail_zone $task)
if $env.PROVISIONING_DEBUG { print $aws_priv_subnet_data }
let aws_priv_subnet = ($aws_priv_subnet_data | get -o SubnetId | default "")
if ($aws_priv_subnet | is-empty) {
print $"❗aws_priv_subnet not found in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) "
exit 1
}
let aws_priv_avail_zone = ($aws_priv_subnet_data | get -o AvailabilityZone | default "")
let aws_priv_sg_name = ($data_settings | get -o priv | get -o sg | get -o name | default "sg_priv")
if ($aws_priv_sg_name | is-empty) {
print $"❗aws_priv_sg.name not found in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset)"
exit 1
}
let aws_priv_sg_id = (aws_create_sg_id $aws_priv_vpc $aws_priv_sg_name $task)
let aws_priv_sg_data = {
id: $aws_priv_sg_id,
name: $aws_priv_sg_name,
perms: ($env_settings | get -o priv | get -o sg | get -o perms | default [])
}
if $task == "create" or $task == "scan" { aws_add_sg_perms $aws_priv_sg_data $server $check_mode}
{
vpc: $aws_priv_vpc,
subnet: $aws_priv_subnet,
cidr_block: $aws_priv_cidr_block,
avail_zone: $aws_priv_avail_zone,
sg: $aws_priv_sg_data
}
} else {
if $env.PROVISIONING_DEBUG { print$"❗aws_priv_cidr_block not found in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) " }
}
let aws_sg_name = ($data_settings | get -o sg | get -o name | default "sg_pub")
if ($aws_sg_name | is-empty) {
print $"aws_sg_name not found in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset) "
exit 1
}
let aws_vpc_subnet_data = (aws_vpc_subnet $aws_avail_zone $priv.subnet $task)
if $task == "create" and ($aws_vpc_subnet_data | is-empty) {
print $"❗No availability zone ($aws_avail_zone) "
exit 1
}
print $aws_vpc_subnet_data
let aws_sg_id = (aws_create_sg_id $aws_vpc $aws_sg_name $task)
let aws_sg_data = {
id: $aws_sg_id,
name: $aws_sg_name,
perms: ($env_settings | get -o main | get -o sg | get -o perms | default [])
}
if $task == "create" or $task == "scan" { aws_add_sg_perms $aws_sg_data $server $check_mode }
let main = {
vpc: $aws_vpc,
subnet: $aws_subnet,
cidr_block: ($aws_subnet_data | get -o CidrBlock | default ""),
avail_zone: $aws_avail_zone,
sg: $aws_sg_data,
}
let data_settings = if ($aws_priv_cidr_block | is-not-empty) {
{ main: $main, priv: $priv }
} else {
{ main: $main }
}
save_provider_env (load_provider_env $settings $server $provider_path | merge $data_settings) $settings $provider_path
print $"✅ (_ansi green_bold)AWS(_ansi reset) (_ansi cyan_bold)settings(_ansi reset) completed in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset)"
}
export def aws_scan_servers [
provider_path: string
settings: record
server: record
] {
mut servers = []
for server_info in ($settings.data.servers? | default []) {
let hostname = ($server_info | get -o hostname | default "" )
let network_private_ip = ($server_info | get -o network_private_ip | default "")
let res = (^aws ec2 describe-instances --out json --filters $'"Name=tag:hostname,Values=($hostname)"' --filters "Name=instance-state-name,Values=running"
--query "Reservations[*].Instances[].{
id: InstanceId,
priv: NetworkInterfaces[],
pub: PublicIpAddress,
type: InstanceType,
status: State.Name,
keyname: KeyName,
launchtime: LaunchTime,
block_devices: BlockDeviceMappings
}"
--output json | complete)
if $res.exit_code != 0 {
print $"❗No data found for ($hostname) in ($server.provider) "
continue
}
for instance_data in ($res.stdout | from json ) {
if ($instance_data | get -o status | str contains "erminated") { continue }
let instance_id = ($instance_data | get -o id | default "")
mut volumes = []
for device in ($instance_data | get -o block_devices | default []) {
let vol_id = ($device | get -o Ebs | get -o VolumeId | default "")
if ($vol_id | is-empty) { continue }
let res_vols = (^aws ec2 describe-volumes --volume-id $vol_id --filters $"Name=attachment.instance-id,Values=($instance_id)"
--query "Volumes[]" --output=json | complete)
if $res_vols.exit_code == 0 {
$volumes = ($volumes | append ($res_vols.stdout | from json))
}
}
$servers = ($servers | append {
id: $instance_id
hostname: $hostname
keyname: ($instance_data | get -o keyname | default ""),
private_ips: ($instance_data | get -o priv | default []),
puplic_ips: ($instance_data | get -o pub | default []),
volumes: $volumes,
devices: ($instance_data | get -o block_devices | default []),
launchtime: ($instance_data | get -o launchtime | default ""),
info: $server_info
})
}
}
save_provider_env (load_provider_env $settings $server $provider_path | merge { servers: $servers}) $settings $provider_path
print $"✅ (_ansi green_bold)AWS(_ansi reset) (_ansi blue_bold)servers settings(_ansi reset)
completed in (_ansi yellow_bold)($provider_path | path basename)(_ansi reset)"
}

6
providers/aws/nulib/aws/mod.nu Normal file
View file

@ -0,0 +1,6 @@
use env.nu
export use lib.nu *
export use servers.nu *
export use usage.nu *
export use prices.nu *
export use utils.nu *

250
providers/aws/nulib/aws/prices.nu Normal file
View file

@ -0,0 +1,250 @@
use ../../../../core/nulib/lib_provisioning/utils/format.nu money_conversion
def aws_default_store_type [] {
"Provisioned IOPS"
}
export def aws_get_price [
all_data: record
key: string
price_col: string = "pricePerUnit"
] {
let data = ($all_data | get -o item)
let str_price_col = if ($price_col | is-empty) { "pricePerUnit" } else { $price_col }
let value = ($data | get -o $str_price_col | get -o "USD" | default "")
let val = if ($value | is-empty) {
0
} else {
money_conversion "USD" "EUR" ($value | into float)
}
let unit = $"($val) ($data | get -o unit | default "")"
if ($unit | str contains "Hrs") {
match $key {
"month" => (($val * 24) * 30),
"day" => ($val * 24),
"hour" => ($val),
"minute" => ($val / 60),
"unit" => $unit,
}
} else if ($unit | str contains "Mo") {
match $key {
"month" => $val,
"day" => ($val / 30),
"hour" => (($val / 30) / 24),
"minute" => ((($val / 30) / 24) / 60),
"unit" => $unit,
}
} else {
0
}
}
export def aws_get_provider_path [
settings: record
server: record
] {
let data_path = if ($settings.data.prov_data_dirpath | str starts-with "." ) {
($settings.src_path | path join $settings.data.prov_data_dirpath)
} else { $settings.data.prov_data_dirpath }
if not ($data_path | path exists) { mkdir $data_path }
($data_path | path join $"($server.provider)_prices.($env.PROVISIONING_WK_FORMAT)")
}
export def aws_get_item_for_server [
server: record
settings: record
cloud_data: record
] {
let provider_prices_path = (aws_get_provider_path $settings $server)
if not ($provider_prices_path | path exists) { return {} }
let pricing_data = (open $provider_prices_path | default [])
let memory = $"(($server.reqplan.memory | default 1024) / 1024) GiB"
let cores = $"($server.reqplan.cores | default 1)"
let current_gen = if ($server.reqplan.gen | default "") == "current" { "Yes" } else { "No" }
#let arch = if ($server.reqplan.arch | str contains "x86_64") { "Intel" } else { ""}
for item in $pricing_data {
if ($item | get -o data | is-empty) or ($item | get -o plan | is-empty) { continue }
if ($item.plan != $server.plan and $item.zone != $server.zone) { continue }
for it in $item.data {
if ($it | get -o product | is-empty) { continue }
if ( $it.product.attributes.memory == $memory
and $it.product.attributes.vcpu == $cores
and $it.product.attributes.currentGeneration == $current_gen
and ($it.product.attributes.operatingSystem | str contains "Linux")
) {
return ($it.on_demand | get -o priceDimensions | default {})
}
}
}
{}
}
export def aws_get_item_for_storage [
server: record
settings: record
cloud_data: record
] {
let provider_prices_path = (aws_get_provider_path $settings $server)
if not ($provider_prices_path | path exists) { return [] }
let pricing_data = (open $provider_prices_path | default [])
if ($pricing_data | length) == 0 { return [] }
let default_store_type = aws_default_store_type
mut $data = []
for store in ($server | get -o storages | default []) {
let store_type = ($store | get -o prov_type | default $default_store_type)
for item in $pricing_data {
let item_type = ($item | get -o store | default "")
if ($item_type | is-empty) or $item_type != $store_type and $item.zone != $server.zone { continue }
if ($item | get -o data | is-empty) { continue }
let item_type = ($item | get -o store | default "")
return ($item | get data | get -o 0 | get -o on_demand | get -o priceDimensions | default {})
# $data = ($data | append ($item | get data | get -o 0 | get -o on_demand | get -o priceDimensions | default {}))
}
}
{}
#$data
}
export def aws_load_infra_servers_info [
settings: record
server: record
error_exit: bool
] {
let provider_prices_path = (aws_get_provider_path $settings $server)
if ($provider_prices_path | path exists) {
let pricing_data = (open $provider_prices_path)
for it in $pricing_data {
let zone = ($it | get -o zone | default "")
let plan = ($it | get -o plan | default "")
if $zone == $server.zone and $plan == $server.plan {
return {plan: $plan, zone: $zone }
}
}
}
(aws_load_infra_servers $provider_prices_path $settings $server)
}
export def aws_load_infra_storages_info [
settings: record
server: record
error_exit: bool
] {
let provider_prices_path = (aws_get_provider_path $settings $server)
if ($provider_prices_path | path exists) {
let default_store_type = aws_default_store_type
let pricing_data = (open $provider_prices_path)
for it in $pricing_data {
let zone = ($it | get -o zone | default "")
let store = ($it | get -o store | default "")
if $zone == $server.zone and $store == $default_store_type {
return {zone: $zone, store: $store }
}
}
}
aws_load_infra_storages $provider_prices_path $settings $server
}
export def aws_get_price_data [
filter: record
server: record
] {
let res = (^aws pricing get-products --service-code AmazonEC2 --filters
$"Type=TERM_MATCH,Field=($filter.field),Value=($filter.value)" $"Type=TERM_MATCH,Field=regionCode,Value=($server.zone)"
--query "PriceList[]" --region us-east-1 --out json | complete
)
if $res.exit_code != 0 {
print $"❗ Errors on ($server.hostname) ($server.provider) ($server.plan) in ($server.zone) load cloud price data error: ($res.stdout ) "
return
}
# | str replace '\' ''| str replace '"{' '{' | str replace '}"' '}')
mut $data = []
for it in ($res.stdout | from json) {
let it_data = ($it | from json)
let product = ($it_data | get -o product | default {})
if ($product | is-empty) { continue }
#let attributes = ($product | get -o attributes | default {})
let on_demand_key = ($it_data | get -o terms | get -o OnDemand | columns | first)
let on_demand_data = ($it_data | get -o terms | get -o OnDemand | get -o $on_demand_key | default {})
let price_dimension = if ($on_demand_data | is-not-empty) {
let price_dimension_key = ($on_demand_data | get -o priceDimensions | columns | first | default "")
($on_demand_data | get -o priceDimensions | get -o $price_dimension_key | default {})
} else {
{}
}
$data = ( $data | append {
product: $product,
on_demand: {
priceDimensions: $price_dimension,
sku: ($on_demand_data | get -o sku),
effectiveDate: ($on_demand_data | get -o effectiveDate),
offerTermCode: ($on_demand_data | get -o offerTermCode),
termAttributes: ($on_demand_data | get -o termAttributes)
}
})
}
$data
}
export def aws_load_infra_storages [
provider_prices_path: string
settings: record
server: record
] {
let default_store_type = aws_default_store_type
let curr_data = if ($provider_prices_path | path exists) {
(open $provider_prices_path)
} else {
[]
}
$curr_data | where {|it|
if $it.zone == $server.zone and $it.store? != null and $it.store == $default_store_type {
print $it
return
}
}
let filter = {
field: "volumeType",
value: $default_store_type
}
let data = (aws_get_price_data $filter $server)
let srv_data = { zone: $server.zone, store: $default_store_type, data: $data}
let all_data = if ($provider_prices_path | path exists) {
(open $provider_prices_path | append $srv_data)
} else {
[$srv_data]
}
if $env.PROVISIONING_WK_FORMAT == "json" {
$all_data | to json | save -f $provider_prices_path
} else {
$all_data | to yaml | save -f $provider_prices_path
}
if $env.PROVISIONING_DEBUG { print $"Storage prices for ($server.provider) in: ($provider_prices_path | path basename) with ($server.zone) saved" }
}
export def aws_load_infra_servers [
provider_prices_path: string
settings: record
server: record
] {
let curr_data = if ($provider_prices_path | path exists) {
(open $provider_prices_path)
} else {
[]
}
$curr_data | where {|it|
if $it.zone? != null and $it.zone == $server.zone and $it.plan? != null and $it.plan == $server.plan {
return $curr_data
}
}
let filter = {
field: "instanceType",
value: $server.plan
}
let data = (aws_get_price_data $filter $server)
let srv_data = { zone: $server.zone, plan: $server.plan, data: $data}
let all_data = if ($provider_prices_path | path exists) {
(open $provider_prices_path | append $srv_data)
} else {
[$srv_data]
}
if $env.PROVISIONING_WK_FORMAT == "json" {
$all_data | to json | save -f $provider_prices_path
} else {
$all_data | to yaml | save -f $provider_prices_path
}
if $env.PROVISIONING_DEBUG { print $"Server prices for ($server.provider) in: ($provider_prices_path | path basename) with ($server.plan)/($server.zone) saved" }
{ plan: $server.plan, zone: $server.zone }
}

1100
providers/aws/nulib/aws/servers.nu Normal file
View file

File diff suppressed because it is too large Load diff

41
providers/aws/nulib/aws/usage.nu Normal file
View file

@ -0,0 +1,41 @@
#!/usr/bin/env nu
# myscript.nu
export def usage [provider: string, infra: string] {
let info = if ( $env.CURRENT_FILE? | into string ) != "" { (^grep "^# Info:" $env.CURRENT_FILE ) | str replace "# Info: " "" } else { "" }
# $(declare -F _usage_options >/dev/null && _usage_options)
$"
USAGE provisioning ($provider) -k cloud-path file-settings.yaml provider-options
DESCRIPTION
AWS ($info)
OPTIONS
-s server-hostname
with server-hostname target selection
-p provider-name
use provider name
do not need if 'current directory path basename' is not one of providers available
-new | new [provisioning-name]
create a new provisioning-directory-name by a copy of ($infra)
-k cloud-path-item
use cloud-path-item as base directory for settings
-x
Trace script with 'set -x'
providerslist | providers-list | providers list
Get available providers list
taskslist | tasks-list | tasks list
Get available tasks list
serviceslist | service-list
Get available services list
tools
Run core/on-tools info
-i
About this
-v
Print version
-h, --help
Print this help and exit.
"
# ["hello" $name $title]
}

24
providers/aws/nulib/aws/utils.nu Normal file
View file

@ -0,0 +1,24 @@
export def aws_check_requirements [
settings: record
fix_error: bool
] {
let has_aws = (^bash -c "type -P aws")
if ($has_aws | path exists) == false and $fix_error {
( ^($env.PROVISIONING_NAME) "tools" "install" "aws")
}
let has_aws = (^bash -c "type -P aws")
if ($has_aws | path exists) == false {
(throw-error $"🛑 CLI command aws not found"
"aws_check_requirements" --span (metadata $has_aws).span)
exit 1
}
let aws_version = (^aws --version | cut -f1 -d" " | sed 's,aws-cli/,,g')
let req_version = (open $env.PROVISIONING_REQ_VERSIONS).aws?.version? | default "")
if ($aws_version != $req_version ) and $fix_error {
( ^($env.PROVISIONING_NAME) "tools" "update" "aws")
}
let aws_version = (^aws --version | cut -f1 -d" " | sed 's,aws-cli/,,g')
if $aws_version != $req_version {
print $"warning❗ aws command as CLI for AWS ($aws_version) with Provisioning is not ($req_version)"
}
}