provisioning/core/nulib/lib_provisioning/utils/templates.nu
Jesús Pérez 6c538b62c8
feat: Complete config-driven architecture migration v2.0.0
Transform provisioning system from ENV-based to hierarchical config-driven architecture.
This represents a complete system redesign with breaking changes requiring migration.

## Migration Summary
- 65+ files migrated across entire codebase
- 200+ ENV variables replaced with 476 config accessors
- 29 syntax errors fixed across 17 files
- 92% token efficiency maintained during migration

## Core Features Added

### Hierarchical Configuration System
- 6-layer precedence: defaults → user → project → infra → env → runtime
- Deep merge strategy with intelligent precedence rules
- Multi-environment support (dev/test/prod) with auto-detection
- Configuration templates for all environments

### Enhanced Interpolation Engine
- Dynamic variables: {{paths.base}}, {{env.HOME}}, {{now.date}}
- Git context: {{git.branch}}, {{git.commit}}, {{git.remote}}
- SOPS integration: {{sops.decrypt()}} for secrets management
- Path operations: {{path.join()}} for dynamic construction
- Security: circular dependency detection, injection prevention

### Comprehensive Validation
- Structure, path, type, semantic, and security validation
- Code injection and path traversal detection
- Detailed error reporting with actionable messages
- Configuration health checks and warnings

## Architecture Changes

### Configuration Management (core/nulib/lib_provisioning/config/)
- loader.nu: 1600+ line hierarchical config loader with validation
- accessor.nu: 476 config accessor functions replacing ENV vars

### Provider System (providers/)
- AWS, UpCloud, Local providers fully config-driven
- Unified middleware system with standardized interfaces

### Task Services (core/nulib/taskservs/)
- Kubernetes, storage, networking, registry services migrated
- Template-driven configuration generation

### Cluster Management (core/nulib/clusters/)
- Complete lifecycle management through configuration
- Environment-specific cluster templates

## New Configuration Files
- config.defaults.toml: System defaults (84 lines)
- config.*.toml.example: Environment templates (400+ lines each)
- Enhanced CLI: validate, env, multi-environment support

## Security Enhancements
- Type-safe configuration access through validated functions
- SOPS integration for encrypted secrets management
- Input validation preventing injection attacks
- Environment isolation and access controls

## Breaking Changes
⚠️  ENV variables no longer supported as primary configuration
⚠️  Function signatures require --config parameter
⚠️  CLI arguments and return types modified
⚠️  Provider authentication now config-driven

## Migration Path
1. Backup current environment variables
2. Copy config.user.toml.example → config.user.toml
3. Migrate ENV vars to TOML format
4. Validate: ./core/nulib/provisioning validate config
5. Test functionality with new configuration

## Validation Results
 Structure valid
 Paths valid
 Types valid
 Semantic rules valid
 File references valid

System ready for production use with config-driven architecture.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-09-23 03:36:50 +01:00

171 lines
7.4 KiB
Plaintext

use ../config/accessor.nu *
export def run_from_template [
template_path: string # Template path
vars_path: string # Variable file with settings for template
run_file: string # File to run
out_file?: string # Out file path
--check_mode # Use check mode to review and not create server
--only_make # not run
] {
# Check if nu_plugin_tera is available
if not (get-use-tera-plugin) {
_print $"🛑 (_ansi red)Error(_ansi reset) nu_plugin_tera not available - template rendering not supported"
return false
}
if not ( $template_path | path exists ) {
_print $"🛑 (_ansi red)Error(_ansi reset) template ($template_path) (_ansi red)not found(_ansi reset)"
return false
}
if not ( $vars_path | path exists ) {
_print $"🛑 (_ansi red)Error(_ansi reset) vars file ($vars_path) (_ansi red)not found(_ansi reset)"
return false
}
let out_file_name = ($out_file | default "")
# Debug: Show what file we're trying to open
if (is-debug-enabled) {
_print $"🔍 Template vars file: ($vars_path)"
if ($vars_path | path exists) {
_print "📄 File preview (first 3 lines):"
_print (open $vars_path --raw | lines | take 3 | str join "\n")
} else {
_print $"❌ File does not exist!"
}
}
# Load variables from YAML/JSON file
let vars = if ($vars_path | path exists) {
if (is-debug-enabled) {
_print $"🔍 Parsing YAML configuration: ($vars_path)"
}
# Check for common YAML syntax issues before attempting to parse
let content = (open $vars_path --raw)
let unquoted_vars = ($content | lines | enumerate | where {|line| $line.item =~ '\s+\w+:\s+\$\w+'})
if ($unquoted_vars | length) > 0 {
_print ""
_print $"🛑 (_ansi red_bold)INFRASTRUCTURE CONFIGURATION ERROR(_ansi reset)"
_print $"📄 Failed to parse YAML variables file: (_ansi yellow)($vars_path | path basename)(_ansi reset)"
_print ""
_print $"(_ansi blue_bold)Diagnosis:(_ansi reset)"
_print "• Found unquoted variable references (invalid YAML syntax):"
for $var in $unquoted_vars {
let line_num = ($var.index + 1)
let line_content = ($var.item | str trim)
_print $" Line ($line_num): (_ansi red)($line_content)(_ansi reset)"
}
_print ""
_print $"(_ansi blue_bold)Root Cause:(_ansi reset)"
_print $"KCL-to-YAML conversion is not properly handling string variables."
# Extract variable names from the problematic lines
let sample_vars = ($unquoted_vars | take 3 | each {|line|
($line.item | str trim | split row " " | last)
} | str join ", ")
if ($sample_vars | is-not-empty) {
_print $"Example variables: ($sample_vars) should be quoted or resolved."
} else {
_print "String variables should be quoted or resolved during conversion."
}
_print ""
_print $"(_ansi blue_bold)Fix Required:(_ansi reset)"
_print $"1. Check KCL configuration generation process"
_print $"2. Ensure variables are properly quoted or resolved during YAML generation"
_print $"3. Source KCL files appear correct, issue is in conversion step"
_print ""
_print $"(_ansi blue_bold)Infrastructure file:(_ansi reset) ($vars_path)"
exit 1
}
# If no obvious issues found, attempt to parse YAML
open $vars_path
} else {
_print $"❌ Variables file not found: ($vars_path)"
return false
}
# Use nu_plugin_tera for template rendering
let result = (render_template $template_path $vars)
# let result = if $result.exit_code == 0 {
# {exit_code: 0, stdout: $result.stdout, stderr: ""}
# } else {
# {exit_code: 1, stdout: "", stderr: $"Template rendering failed for ($template_path)"}
# }
#if $result.exit_code != 0 {
if ($result | is-empty) {
let text = $"(_ansi yellow)template(_ansi reset): ($template_path)\n(_ansi yellow)vars(_ansi reset): ($vars_path)\n(_ansi red)Failed(_ansi reset)"
print $result
print $"(_ansi red)ERROR(_ansi red) nu_plugin_tera render:\n($text)"
exit
}
if not $only_make and (is-debug-enabled) or ($check_mode and ($out_file_name | is-empty)) {
if (is-debug-enabled) and not $check_mode {
_print $"Result running: \n (_ansi default_dimmed)nu_plugin_tera render ($template_path) ($vars_path)(_ansi reset)"
# _print $"\n(_ansi yellow_bold)exit code: ($result.exit_code)(_ansi reset)"
}
let cmd = ((get-file-viewer) | default (if (^bash -c "type -P bat" | is-not-empty) { "bat" } else { "cat" }))
if $cmd != "bat" { _print $"(_ansi magenta_bold)----------------------------------------------------------------------------------------------------------------(_ansi reset)"}
(echo $result | run-external $cmd -)
if $cmd != "bat" { _print $"(_ansi magenta_bold)----------------------------------------------------------------------------------------------------------------(_ansi reset)"}
_print $"Saved in (_ansi green_bold)($run_file)(_ansi reset)"
}
$result | str replace --all "\\ " "\\" | save --append $run_file
if $only_make {
if ($out_file_name | is-not-empty) {
(cat $run_file | tee { save -f $out_file_name } | ignore)
}
return true
}
if $check_mode and not $only_make {
if $out_file_name == "" {
_print $"✅ No errors found !\nTo save command to a file, run next time adding: (_ansi blue)--outfile \(-o\)(_ansi reset) file-path-to-save "
} else {
(cat $run_file | tee { save -f $out_file_name } | ignore)
_print $"✅ No errors found !\nSave in (_ansi green_bold)(_ansi i)($out_file_name)(_ansi reset)"
}
return true
}
if $out_file_name != "" and ($out_file_name | path type) == "file" {
(^bash $run_file | save --force $out_file_name)
} else {
let res = if (is-debug-enabled) {
(^bash -x $run_file | complete)
} else {
(^bash $run_file | complete)
}
if $res.exit_code != 0 {
_print $"\n🛑 (_ansi red)Error(_ansi reset) run from template ($template_path | path basename) (_ansi green_bold)($run_file)(_ansi reset) (_ansi red_bold)failed(_ansi reset) "
_print $"\n($res.stdout)"
return false
}
}
true
}
export def on_template_path [
source_path: string
vars_path: string
remove_path: bool
on_error_exit: bool
] {
for it in (^ls ...(glob $"($source_path)/*")| lines) {
let item = ($it | str trim | str replace -r ':$' '')
if ($item | is-empty) or ($item | path basename | str starts-with "tmp.") or ($item | path basename | str starts-with "_") { continue }
if ($item | path type) == "dir" {
if (ls $item | length) == 0 { continue }
(on_template_path $item $vars_path $remove_path $on_error_exit)
continue
}
if not ($item | str ends-with ".j2") or not ($item | path exists) { continue }
if not (run_from_template $item $vars_path ($item | str replace ".j2" "") --only_make) {
echo $"🛑 Error on_template_path (_ansi red_bold)($item)(_ansi reset) and vars (_ansi yellow_bold)($vars_path)(_ansi reset)"
if $on_error_exit { exit 1 }
}
if $remove_path { rm -f $item }
}
}