feat(taskserv): implement real-time version checking with configurable HTTP client

- Add: GitHub API integration for live version checking in taskserv management
- Add: HTTP client configuration option (http.use_curl) in config.defaults.toml
- Add: Helper function fetch_latest_version with curl/http get support
- Fix: Settings path structure for prov_data_dirpath access pattern
- Remove: Legacy simulation code for version checking
- Update: Core configuration name from "provisioning-system" to "provisioning"
- Clean: Remove obsolete example configs and infrastructure files
This commit is contained in:
Jesús Pérez 2025-09-24 01:55:06 +01:00
parent 38a7470da0
commit 3c3ef47f7f
No known key found for this signature in database
GPG key ID: 9F243E355E0BC939
34 changed files with 5942 additions and 13 deletions

63
core/nulib/lib_provisioning/cache/agent.nu vendored Executable file
View file

@ -0,0 +1,63 @@
#!/usr/bin/env nu
# Dynamic Version Cache Agent
# Token-optimized agent for progressive version caching with infra-aware hierarchy
# Usage: nu agent.nu <command> [args]
use cache_manager.nu *
use version_loader.nu *
use grace_checker.nu *
use batch_updater.nu *
# Main agent entry point
def main [
command: string # Command: init, get, update-all, clear, status
...args # Additional arguments
] {
match $command {
"init" => {
print "🚀 Initializing dynamic version cache system..."
init-cache-system
print "✅ Cache system initialized"
}
"get" => {
if ($args | length) == 0 {
print "❌ Usage: agent.nu get <component-name>"
exit 1
}
let component = ($args | get 0)
print $"🔍 Getting version for ($component)..."
let version = (get-cached-version $component)
print $"📦 ($component): ($version)"
}
"update-all" => {
print "🔄 Updating all cached versions..."
batch-update-cache
print "✅ Cache updated"
}
"clear" => {
print "🗑️ Clearing version cache..."
clear-cache-system
print "✅ Cache cleared"
}
"status" => {
print "📊 Version cache status:"
show-cache-status
}
"sync" => {
print "🔄 Syncing cache from sources..."
sync-cache-from-sources
print "✅ Cache synced"
}
_ => {
print $"❌ Unknown command: ($command)"
print "Available commands: init, get, update-all, clear, status, sync"
exit 1
}
}
}

View file

@ -0,0 +1,166 @@
# Batch Updater - Efficient batch operations for version cache
# Token-optimized batch processing to minimize LLM context usage
# Batch update cache from all sources
export def batch-update-cache [] {
print "🔄 Starting batch cache update..."
# Get all available components
let all_components = (get-all-components)
print $"📦 Found ($all_components | length) components to process"
# Process in batches to be memory efficient
let batch_size = 10
let batches = ($all_components | chunks $batch_size)
print $"⚡ Processing ($batches | length) batches of ($batch_size) components each"
for batch in $batches {
print $"🔄 Processing batch: ($batch | str join ', ')"
process-batch $batch
}
print "✅ Batch update completed"
}
# Process a batch of components
def process-batch [components: list<string>] {
# Load versions for all components in this batch
let versions = (batch-load-versions $components)
# Cache each version
for component in ($versions | columns) {
let version = ($versions | get $component)
# Cache in both provisioning and infra
cache-version $component $version "provisioning"
cache-version $component $version "infra"
print $" ✓ ($component): ($version)"
}
}
# Sync cache from sources (rebuild cache)
export def sync-cache-from-sources [] {
print "🔄 Syncing cache from KCL sources..."
# Clear existing cache
clear-cache-system
# Initialize fresh cache
init-cache-system
# Batch update all components
batch-update-cache
print "✅ Cache sync completed"
}
# Update specific components
export def update-components [
components: list<string> # Specific components to update
] {
print $"🔄 Updating specific components: ($components | str join ', ')"
let versions = (batch-load-versions $components)
for component in ($versions | columns) {
let version = ($versions | get $component)
# Invalidate old cache entries
invalidate-cache-entry $component "infra"
invalidate-cache-entry $component "provisioning"
# Cache new versions
cache-version $component $version "provisioning"
cache-version $component $version "infra"
print $" ✓ Updated ($component): ($version)"
}
print "✅ Component update completed"
}
# Update expired components only
export def update-expired-components [] {
print "🔄 Updating expired cache entries..."
let expired_infra = (get-expired-entries "infra")
let expired_prov = (get-expired-entries "provisioning")
let all_expired = ($expired_infra ++ $expired_prov) | uniq
if ($all_expired | is-empty) {
print "✅ No expired entries found"
return
}
print $"📋 Found ($all_expired | length) expired entries: ($all_expired | str join ', ')"
update-components $all_expired
}
# Auto-update components with check_latest = true
export def auto-update-components [] {
print "🔄 Checking for auto-updates (check_latest = true)..."
let components_needing_update = (get-components-needing-update)
if ($components_needing_update | is-empty) {
print "✅ No components need auto-update"
return
}
print $"📋 Components needing update: ($components_needing_update | str join ', ')"
# For now, just update from sources
# TODO: Add GitHub API integration for latest version checking
update-components $components_needing_update
print "⚠️ Note: GitHub API integration not yet implemented"
}
# Optimize cache (remove duplicates, compress)
export def optimize-cache [] {
print "🔧 Optimizing cache..."
let cache_types = ["infra", "provisioning"]
for cache_type in $cache_types {
let cache_path = if $cache_type == "infra" {
get-infra-cache-path
} else {
get-provisioning-cache-path
}
let cache_file = ($cache_path | path join "versions.json")
if ($cache_file | path exists) {
try {
let cache_data = (open $cache_file)
# Remove empty entries
let cleaned_cache = ($cache_data | items { |key, value|
if ($value.current | is-not-empty) {
{ $key: $value }
} else {
{}
}
} | reduce { |item, acc| $acc | merge $item })
# Save optimized cache
$cleaned_cache | save -f $cache_file
let entry_count = ($cleaned_cache | columns | length)
print $" ✓ Optimized ($cache_type) cache: ($entry_count) entries"
} catch {
print $" ❌ Failed to optimize ($cache_type) cache"
}
}
}
print "✅ Cache optimization completed"
}
# Import required functions
use cache_manager.nu [cache-version, clear-cache-system, init-cache-system, get-infra-cache-path, get-provisioning-cache-path]
use version_loader.nu [batch-load-versions, get-all-components]
use grace_checker.nu [get-expired-entries, get-components-needing-update, invalidate-cache-entry]

View file

@ -0,0 +1,196 @@
# Cache Manager - Progressive version cache with infra hierarchy
# Handles cache lookup, storage, and hierarchy management
use version_loader.nu load-version-from-source
use grace_checker.nu is-cache-valid?
# Get version with progressive cache hierarchy
export def get-cached-version [
component: string # Component name (e.g., kubernetes, containerd)
]: nothing -> string {
# Cache hierarchy: infra -> provisioning -> source
# 1. Try infra cache first (project-specific)
let infra_version = (get-infra-cache $component)
if ($infra_version | is-not-empty) {
if (is-cache-valid? $component "infra") {
return $infra_version
}
}
# 2. Try provisioning cache (system-wide)
let prov_version = (get-provisioning-cache $component)
if ($prov_version | is-not-empty) {
if (is-cache-valid? $component "provisioning") {
return $prov_version
}
}
# 3. Load from source and cache
print $"⚠️ Loading ($component) from source \(cache miss or expired\)"
let version = (load-version-from-source $component)
if ($version | is-not-empty) {
# Cache in both levels
cache-version $component $version "provisioning"
cache-version $component $version "infra"
return $version
}
# 4. Return empty if not found
""
}
# Get version from infra cache
def get-infra-cache [component: string]: nothing -> string {
let cache_path = (get-infra-cache-path)
let cache_file = ($cache_path | path join "versions.json")
if not ($cache_file | path exists) {
return ""
}
try {
let cache_data = (open $cache_file)
let version_data = ($cache_data | get -o $component | default {})
($version_data | get -o current | default "")
} catch {
""
}
}
# Get version from provisioning cache
def get-provisioning-cache [component: string]: nothing -> string {
let cache_path = (get-provisioning-cache-path)
let cache_file = ($cache_path | path join "versions.json")
if not ($cache_file | path exists) {
return ""
}
try {
let cache_data = (open $cache_file)
let version_data = ($cache_data | get -o $component | default {})
($version_data | get -o current | default "")
} catch {
""
}
}
# Cache version data
export def cache-version [
component: string # Component name
version: string # Version string
cache_type: string # "infra" or "provisioning"
] {
let cache_path = if $cache_type == "infra" {
get-infra-cache-path
} else {
get-provisioning-cache-path
}
let cache_file = ($cache_path | path join "versions.json")
# Ensure cache directory exists
mkdir ($cache_file | path dirname)
# Load existing cache or create new
let existing_cache = if ($cache_file | path exists) {
try { open $cache_file } catch { {} }
} else {
{}
}
# Update cache entry
let updated_cache = ($existing_cache | upsert $component {
current: $version
cached_at: (date now | format date '%Y-%m-%dT%H:%M:%SZ')
cache_type: $cache_type
grace_period: (get-default-grace-period)
})
# Save cache
$updated_cache | save -f $cache_file
}
# Get cache paths from config
export def get-infra-cache-path []: nothing -> string {
use ../config/accessor.nu config-get
let infra_path = (config-get "paths.infra" "")
let current_infra = (config-get "infra.current" "default")
if ($infra_path | is-empty) {
return (get-provisioning-cache-path)
}
$infra_path | path join $current_infra "cache"
}
export def get-provisioning-cache-path []: nothing -> string {
use ../config/accessor.nu config-get
config-get "cache.path" ".cache/versions"
}
def get-default-grace-period []: nothing -> int {
use ../config/accessor.nu config-get
config-get "cache.grace_period" 86400
}
# Initialize cache system
export def init-cache-system [] {
let infra_cache = (get-infra-cache-path)
let prov_cache = (get-provisioning-cache-path)
mkdir $infra_cache
mkdir $prov_cache
# Create empty cache files if they don't exist
let infra_file = ($infra_cache | path join "versions.json")
let prov_file = ($prov_cache | path join "versions.json")
if not ($infra_file | path exists) {
{} | save $infra_file
}
if not ($prov_file | path exists) {
{} | save $prov_file
}
}
# Clear cache system
export def clear-cache-system [] {
let infra_cache = (get-infra-cache-path)
let prov_cache = (get-provisioning-cache-path)
try { rm -rf $infra_cache } catch { }
try { rm -rf $prov_cache } catch { }
init-cache-system
}
# Show cache status
export def show-cache-status [] {
let infra_cache = (get-infra-cache-path | path join "versions.json")
let prov_cache = (get-provisioning-cache-path | path join "versions.json")
print "📁 Cache Locations:"
print $" Infra: ($infra_cache)"
print $" Provisioning: ($prov_cache)"
print ""
if ($infra_cache | path exists) {
let infra_data = (open $infra_cache)
let infra_count = ($infra_data | columns | length)
print $"🏗️ Infra cache: ($infra_count) components"
} else {
print "🏗️ Infra cache: not found"
}
if ($prov_cache | path exists) {
let prov_data = (open $prov_cache)
let prov_count = ($prov_data | columns | length)
print $"⚙️ Provisioning cache: ($prov_count) components"
} else {
print "⚙️ Provisioning cache: not found"
}
}

View file

@ -0,0 +1,166 @@
# Grace Period Checker - Validates cache freshness
# Prevents excessive API calls by checking grace periods
# Check if cache entry is still valid (within grace period)
export def is-cache-valid? [
component: string # Component name
cache_type: string # "infra" or "provisioning"
]: nothing -> bool {
let cache_path = if $cache_type == "infra" {
get-infra-cache-path
} else {
get-provisioning-cache-path
}
let cache_file = ($cache_path | path join "versions.json")
if not ($cache_file | path exists) {
return false
}
try {
let cache_data = (open $cache_file)
let version_data = ($cache_data | get -o $component | default {})
if ($version_data | is-empty) {
return false
}
let cached_at = ($version_data | get -o cached_at | default "")
let grace_period = ($version_data | get -o grace_period | default (get-default-grace-period))
if ($cached_at | is-empty) {
return false
}
# Parse cached timestamp
let cached_time = ($cached_at | into datetime)
let current_time = (date now)
let age_seconds = (($current_time - $cached_time) / 1sec)
# Check if within grace period
$age_seconds < $grace_period
} catch {
false
}
}
# Get expired cache entries
export def get-expired-entries [
cache_type: string # "infra" or "provisioning"
]: nothing -> list<string> {
let cache_path = if $cache_type == "infra" {
get-infra-cache-path
} else {
get-provisioning-cache-path
}
let cache_file = ($cache_path | path join "versions.json")
if not ($cache_file | path exists) {
return []
}
try {
let cache_data = (open $cache_file)
$cache_data | columns | where { |component|
not (is-cache-valid? $component $cache_type)
}
} catch {
[]
}
}
# Get components that need update check (check_latest = true and expired)
export def get-components-needing-update []: nothing -> list<string> {
let components = []
# Check infra cache
let infra_expired = (get-expired-entries "infra")
let infra_check_latest = (get-check-latest-components "infra")
let infra_needs_update = ($infra_expired | where { |comp| $comp in $infra_check_latest })
# Check provisioning cache
let prov_expired = (get-expired-entries "provisioning")
let prov_check_latest = (get-check-latest-components "provisioning")
let prov_needs_update = ($prov_expired | where { |comp| $comp in $prov_check_latest })
# Combine and deduplicate
($infra_needs_update ++ $prov_needs_update) | uniq
}
# Get components with check_latest = true
def get-check-latest-components [cache_type: string]: nothing -> list<string> {
let cache_path = if $cache_type == "infra" {
get-infra-cache-path
} else {
get-provisioning-cache-path
}
let cache_file = ($cache_path | path join "versions.json")
if not ($cache_file | path exists) {
return []
}
try {
let cache_data = (open $cache_file)
$cache_data | columns | where { |component|
let comp_data = ($cache_data | get $component)
($comp_data | get -o check_latest | default false)
}
} catch {
[]
}
}
# Invalidate cache entry (force refresh on next access)
export def invalidate-cache-entry [
component: string # Component name
cache_type: string # "infra" or "provisioning"
] {
let cache_path = if $cache_type == "infra" {
get-infra-cache-path
} else {
get-provisioning-cache-path
}
let cache_file = ($cache_path | path join "versions.json")
if ($cache_file | path exists) {
try {
let cache_data = (open $cache_file)
let updated_cache = ($cache_data | upsert $component { |entry|
$entry | upsert cached_at "1970-01-01T00:00:00Z" # Force expiry
})
$updated_cache | save -f $cache_file
} catch {
# Ignore errors
}
}
}
# Helper functions (same as in cache_manager.nu)
def get-infra-cache-path []: nothing -> string {
use ../config/accessor.nu config-get
let infra_path = (config-get "paths.infra" "")
let current_infra = (config-get "infra.current" "default")
if ($infra_path | is-empty) {
return (get-provisioning-cache-path)
}
$infra_path | path join $current_infra "cache"
}
def get-provisioning-cache-path []: nothing -> string {
use ../config/accessor.nu config-get
config-get "cache.path" ".cache/versions"
}
def get-default-grace-period []: nothing -> int {
use ../config/accessor.nu config-get
config-get "cache.grace_period" 86400
}

View file

@ -0,0 +1,247 @@
# Version Loader - Load versions from KCL sources
# Token-optimized loader for version data from various sources
# Load version from source (KCL files)
export def load-version-from-source [
component: string # Component name
]: nothing -> string {
# Try different source locations
let taskserv_version = (load-taskserv-version $component)
if ($taskserv_version | is-not-empty) {
return $taskserv_version
}
let core_version = (load-core-version $component)
if ($core_version | is-not-empty) {
return $core_version
}
let provider_version = (load-provider-version $component)
if ($provider_version | is-not-empty) {
return $provider_version
}
""
}
# Load taskserv version from version.k files
def load-taskserv-version [component: string]: nothing -> string {
# Find version.k file for component
let version_files = [
$"taskservs/($component)/kcl/version.k"
$"taskservs/($component)/default/kcl/version.k"
$"taskservs/($component)/kcl/($component).k"
]
for file in $version_files {
if ($file | path exists) {
let version = (extract-version-from-kcl $file $component)
if ($version | is-not-empty) {
return $version
}
}
}
""
}
# Load core tool version
def load-core-version [component: string]: nothing -> string {
let core_file = "core/versions.k"
if ($core_file | path exists) {
let version = (extract-core-version-from-kcl $core_file $component)
if ($version | is-not-empty) {
return $version
}
}
""
}
# Load provider tool version
def load-provider-version [component: string]: nothing -> string {
# Check provider directories
let providers = ["aws", "upcloud", "local"]
for provider in $providers {
let provider_files = [
$"providers/($provider)/kcl/versions.k"
$"providers/($provider)/versions.k"
]
for file in $provider_files {
if ($file | path exists) {
let version = (extract-version-from-kcl $file $component)
if ($version | is-not-empty) {
return $version
}
}
}
}
""
}
# Extract version from KCL file (taskserv format)
def extract-version-from-kcl [file: string, component: string]: nothing -> string {
try {
let kcl_result = (^kcl $file | complete)
if $kcl_result.exit_code != 0 {
return ""
}
if ($kcl_result.stdout | is-empty) {
return ""
}
let result = ($kcl_result.stdout | from yaml)
# Try different version key patterns
let version_keys = [
$"($component)_version"
"_version"
"version"
]
for key in $version_keys {
let version_data = ($result | get -o $key | default {})
if ($version_data | is-not-empty) {
# Try TaskservVersion format first
let current_version = ($version_data | get -o version.current | default "")
if ($current_version | is-not-empty) {
return $current_version
}
# Try simple format
let simple_version = ($version_data | get -o current | default "")
if ($simple_version | is-not-empty) {
return $simple_version
}
# Try direct string
if ($version_data | describe) == "string" {
return $version_data
}
}
}
""
} catch {
""
}
}
# Extract version from core versions.k file
def extract-core-version-from-kcl [file: string, component: string]: nothing -> string {
try {
let kcl_result = (^kcl $file | complete)
if $kcl_result.exit_code != 0 {
return ""
}
if ($kcl_result.stdout | is-empty) {
return ""
}
let result = ($kcl_result.stdout | from yaml)
# Look for component in core_versions array or individual variables
let core_versions = ($result | get -o core_versions | default [])
if ($core_versions | is-not-empty) {
# Array format
let component_data = ($core_versions | where name == $component | first | default {})
let version = ($component_data | get -o version.current | default "")
if ($version | is-not-empty) {
return $version
}
}
# Individual variable format (e.g., nu_version, kcl_version)
let var_patterns = [
$"($component)_version"
$"($component | str replace '-' '_')_version"
]
for pattern in $var_patterns {
let version_data = ($result | get -o $pattern | default {})
if ($version_data | is-not-empty) {
let current = ($version_data | get -o current | default "")
if ($current | is-not-empty) {
return $current
}
}
}
""
} catch {
""
}
}
# Batch load multiple versions (for efficiency)
export def batch-load-versions [
components: list<string> # List of component names
]: nothing -> record {
mut results = {}
for component in $components {
let version = (load-version-from-source $component)
if ($version | is-not-empty) {
$results = ($results | upsert $component $version)
}
}
$results
}
# Get all available components
export def get-all-components []: nothing -> list<string> {
let taskservs = (get-taskserv-components)
let core_tools = (get-core-components)
let providers = (get-provider-components)
($taskservs ++ $core_tools ++ $providers) | uniq
}
# Get taskserv components
def get-taskserv-components []: nothing -> list<string> {
try {
glob "taskservs/*/kcl/version.k" | each { |file|
$file | path dirname | path dirname | path basename
}
} catch {
[]
}
}
# Get core components
def get-core-components []: nothing -> list<string> {
try {
if ("core/versions.k" | path exists) {
let kcl_result = (^kcl "core/versions.k" | complete)
if $kcl_result.exit_code == 0 and ($kcl_result.stdout | is-not-empty) {
let result = ($kcl_result.stdout | from yaml)
$result | columns | where { |col| $col | str ends-with "_version" } | each { |col|
$col | str replace "_version" ""
}
} else {
[]
}
} else {
[]
}
} catch {
[]
}
}
# Get provider components (placeholder)
def get-provider-components []: nothing -> list<string> {
# TODO: Implement provider component discovery
[]
}

View file

@ -308,9 +308,11 @@ export def load [
if (is-debug-enabled) { _print $"DEBUG work path: ($wk_settings_path)" }
let servers_paths = ($settings_data | get -o servers_paths | default [])
# Set full path for provider data
let data_fullpath = if ($settings_data.prov_data_dirpath | str starts-with "." ) {
($src_dir | path join $settings_data.prov_data_dirpath)
} else { $settings_data.prov_data_dirpath }
let data_fullpath = if (($settings_data | get -o data.prov_data_dirpath) != null and ($settings_data.data.prov_data_dirpath | str starts-with "." )) {
($src_dir | path join $settings_data.data.prov_data_dirpath)
} else {
($settings_data | get -o data.prov_data_dirpath | default "providers")
}
mut list_servers = []
mut providers_settings = []
for it in $servers_paths {

View file

@ -122,7 +122,7 @@ export def detect-version [
let url = ($config | get -o url | default "")
if ($url | is-empty) { return "" }
let result = (http get $url --headers [User-Agent "nushell-version-checker"] | complete)
let result = (http get $url --headers [User-Agent "provisionin-version-checker"] | complete)
if $result.exit_code == 0 and ($result.stdout | length) > 0 {
let response = ($result.stdout | from json)
if ($config | get -o field | is-not-empty) {
@ -166,7 +166,7 @@ export def fetch-versions [
]
for endpoint in $endpoints {
let response = (http get $endpoint --headers [User-Agent "nushell-version-checker"] | default [] | to json | from json | default [])
let response = (http get $endpoint --headers [User-Agent "provisionin-version-checker"] | default [] | to json | from json | default [])
if ($response | length) > 0 {
return ($response
| first $limit
@ -188,7 +188,7 @@ export def fetch-versions [
let repo = ($parts | last)
let url = $"https://hub.docker.com/v2/namespaces/($namespace)/repositories/($repo)/tags"
let result = (http get $url --headers [User-Agent "nushell-version-checker"] | complete)
let result = (http get $url --headers [User-Agent "provisionin-version-checker"] | complete)
if $result.exit_code == 0 and ($result.stdout | length) > 0 {
let response = ($result.stdout | from json)
if ($response | get -o results | is-not-empty) {
@ -208,7 +208,7 @@ export def fetch-versions [
let url = ($config | get -o url | default "")
if ($url | is-empty) { return [] }
let result = (http get $url --headers [User-Agent "nushell-version-checker"] | complete)
let result = (http get $url --headers [User-Agent "provisionin-version-checker"] | complete)
if $result.exit_code == 0 and ($result.stdout | length) > 0 {
let response = ($result.stdout | from json)
let field = ($config | get -o field | default "")
@ -282,4 +282,4 @@ export def check-version [
fixed: $is_fixed
status: $status
}
}
}