diff --git a/src/clouds/monitor_rules.rs b/src/clouds/monitor_rules.rs index c18ff90..cc615ea 100644 --- a/src/clouds/monitor_rules.rs +++ b/src/clouds/monitor_rules.rs @@ -172,9 +172,6 @@ where D: Deserializer<'de> { }); Ok(actions) } -// in kubernetes on wuji-cp-0 if pod with ns rook-ceph and name match "osd-0" is not in "Running state" && is in last "save state info" -// context target(server) selector(type pod - ns - match name - state !Running) - #[allow(clippy::missing_docs_in_private_items)] #[allow(non_snake_case)] #[derive(Clone, Debug, Serialize, Deserialize, Default)] @@ -213,7 +210,6 @@ impl MonitorRules { } } pub fn load(root: &str, path: &str, format: &str) -> Self { - println!("Load Montior Rules ..."); let data_rules = load_fs_content(&root, &path, &format); let err_load = |e|{ eprintln!("Error loading MonitorRules {}/{}.{}: {}",&root,&path,&format,e); @@ -223,7 +219,7 @@ impl MonitorRules { if data_rules.is_empty() { res = MonitorRules::default(); } else { - println!("Parse Montior Rules as: {}",&format); + // println!("Parse Montior Rules as: {}",&format); res = match format { "json" => serde_json::from_str(&data_rules).unwrap_or_else(|e| err_load(format!("{}",e))), "yaml" => serde_yaml::from_str(&data_rules).unwrap_or_else(|e| err_load(format!("{}",e))), @@ -234,7 +230,6 @@ impl MonitorRules { res } pub fn dump(&self, root: &str, path: &str, format: &str) -> Result<()> { - println!("Dump Montior Rules ..."); let err_load = |e| { eprintln!("Error dump MonitorRules {}/{}.{}: {}",&root,&path,&format,e); String::from("") diff --git a/src/monitor.rs b/src/monitor.rs index 1ab3f4c..dbd3334 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -8,14 +8,16 @@ use anyhow::{anyhow,Result}; use std::process::{Command}; use app_env::{ - appenv::AppEnv, + appenv::{AppEnv}, + DataStore, + applogs::AppLogs, // config::{Config} }; use crate::clouds::defs::{ Cloud, }; -use crate::defs::ProviderName; +use crate::defs::{ProviderName}; use crate::clouds::monitor_rules::{ MonitorRules, @@ -55,13 +57,25 @@ pub async fn on_action_server(task: &str, srvr: &str, provider: &str, args: &str _ => Err(anyhow!("Provider {} not defined", &provider_name)) } } -pub async fn on_action_notify(_chnl: &str, _msg: &str, _args: &str, _rule: &MonitorRule) -> Result<()> { +pub async fn on_action_notify(chnl: &str, msg: &str, args: &str, rule: &MonitorRule) -> Result<()> { + let notificator_path = envmnt::get_or("NOTIFICATOR_BIN",""); + if notificator_path.is_empty() || ! Path::new(¬ificator_path).exists() { + return Err(anyhow!("Error notificator:")); + } + let message=format!("Monitor {}: {} {}",&rule.id,&msg,&args); + let cmd = format!("{} {} {}",¬ificator_path,&chnl,&message); + let _ = run_command(&cmd).unwrap_or_else(|e|{ + eprintln!("Error {}: {}",&cmd,e); + String::from("") + }); Ok(()) } -fn get_server(target: &str, val: &str) -> String { +fn parse_target(target: &str, val: &str) -> String { let srvr: String; - if val == "$target" || val == "$server" { - srvr = target.to_owned(); + if val.contains("$target") { + srvr = val.replace("$target", &target.to_owned()); + } else if val.contains("$server") { + srvr = val.replace("$server", &target.to_owned()); } else { srvr = val.to_owned(); } @@ -71,12 +85,12 @@ pub async fn on_monitor_action(action: &MonitorAction, target: &str, rule: &Moni match action { MonitorAction::None => return Ok(()), MonitorAction::Start(v,p,a) => - on_action_server("start",&get_server(target, v),&p,&a,&rule).await?, + on_action_server("start",&parse_target(target, v),&p,&a,&rule).await?, MonitorAction::Stop(v,p,a) => - on_action_server("stop",&get_server(target, v),&p,&a,&rule).await?, + on_action_server("stop",&parse_target(target, v),&p,&a,&rule).await?, MonitorAction::Restart(v,p,a) => - on_action_server("restart",&get_server(target, v),&p,&a,&rule).await?, - MonitorAction::Notify(v,p,a) => on_action_notify(&v,&p,&a,&rule).await?, + on_action_server("restart",&parse_target(target, v),&p,&a,&rule).await?, + MonitorAction::Notify(v,p,a) => on_action_notify(&v,&parse_target(target,&p),&a,&rule).await?, }; Ok(()) } @@ -100,7 +114,6 @@ pub async fn on_service_rule(srvc: &str, out_cmd: &str, rule: &MonitorRule) -> R // println!("target: {}",&target); for action in &rule.actions { on_monitor_action(action, &target, &rule).await?; - } }; } @@ -185,8 +198,26 @@ pub fn run_on_selector(rule: &MonitorRule) -> Result { Err(anyhow!("Selector: failed:\n {}",&rule.id)) } } -pub async fn run_monitor_rule(idx: usize, rule: &MonitorRule ) -> Result<()> { - println!("{} [{}] {}: {}",&idx,&rule.id,&rule.name,&rule.description); +pub async fn log_monitor_state(rule: &MonitorRule, app_env: &AppEnv, env_name: &str, env_state: isize, msg: &str, out_cmd: &str) -> Result<()> { + let now = format!("{}", chrono::Utc::now().timestamp()); + let log_entry = AppLogs { + id: format!("{}_{}", &rule.id,&now), + name: format!("rule: {}", &rule.name), + when: String::from("run_monitor_rule"), + source: format!("{}", &env_name), + target: format!("{}", &out_cmd), + state: format!("{}", &env_state), + msg: format!("{}", &msg), + }; + if app_env.config.logs_store == DataStore::File { + log_entry.write_applogs_entry(format!("{}/monitor.log",&app_env.config.logs_path).as_str()).await?; + } + Ok(()) +} +pub async fn run_monitor_rule(idx: usize, rule: &MonitorRule, app_env: &AppEnv) -> Result<()> { + if envmnt::get_isize("DEBUG",0) > 0 { + println!("{} [{}] {}: {}",&idx,&rule.id,&rule.name,&rule.description); + } let out_cmd: String; if rule.command.is_empty() { out_cmd=run_on_selector(&rule).unwrap_or_else(|e| { @@ -206,48 +237,76 @@ pub async fn run_monitor_rule(idx: usize, rule: &MonitorRule ) -> Result<()> { let env_state = envmnt::get_isize(&env_name,0); if ! on_monitor_operator(&rule.operator,&out_cmd, &rule.value) { if env_state > 0 { - println!("{} Ok in state: {}",&env_name,&env_state); + let msg = format!("{} Ok in state: {}",&env_name,&env_state); + println!("{}",&msg); envmnt::set_isize(&env_name, 0); + log_monitor_state(&rule, &app_env, &env_name, env_state, &msg, &out_cmd).await?; + for item in rule.actions.iter() { + match item { + MonitorAction::Notify(v,p,a) => on_action_notify(&v,&parse_target(format!("Ok state from -> {}",&env_state).as_str(),&p),&a,&rule).await?, + _ => continue, + }; + } } return Ok(()); } if env_state > 0 && env_state < rule.wait_checks { - println!("{} is already set: {}",&env_name,&env_state); + let msg = format!("{} is already set: {} increment counter",&env_name,&env_state); envmnt::increment(&env_name); + println!("{}",&msg); + log_monitor_state(&rule, &app_env, &env_name, env_state, &msg, &out_cmd).await?; return Ok(()); } + let mut err = String::from(""); match &rule.context { RuleContext::Server(v) => { - on_server_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| - eprintln!("Error rule {} on {}: {}",&rule.id,&v,e) - ); + on_server_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| { + err = format!("Error rule {} on {}: {}",&rule.id,&v,e); + eprintln!("{}",err) + }); }, RuleContext::None => { return Ok(()); }, RuleContext::Service(v) => { - on_service_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| - eprintln!("Error rule {} on {}: {}",&rule.id,&v,e) - ); + on_service_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| { + err = format!("Error rule {} on {}: {}",&rule.id,&v,e); + eprintln!("{}",err) + }); }, RuleContext::Ip(v) => { - on_ip_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| - eprintln!("Error rule {} on {}: {}",&rule.id,&v,e) - ); + on_ip_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| { + err = format!("Error rule {} on {}: {}",&rule.id,&v,e); + eprintln!("{}",err) + }); }, } + let msg = match err.is_empty() { + true => format!("{}: 1 done",&env_name), + false => format!("{} error: {}",&env_name,&err), + }; envmnt::set_isize(&env_name,1); + println!("{}",&msg); + log_monitor_state(&rule, &app_env, &env_name, env_state, &msg, &out_cmd).await?; Ok(()) } -pub async fn run_monitor(monitor_rules: MonitorRules, _cloud: Cloud, _app_env: AppEnv) -> Result<()> { +pub async fn run_monitor(monitor_rules: MonitorRules, _cloud: Cloud, app_env: AppEnv) -> Result<()> { // println!("Run {}: {}",&monitor_rules.name,&monitor_rules.description); for (idx,rule) in monitor_rules.rules.iter().enumerate() { match rule.schedule { RuleSchedule::Check => { // dbg!(&rule); - run_monitor_rule(idx,rule).await.unwrap_or_else(|e| - eprintln!("Error rule {}: {}",&rule.id,e) - ); + let mut err = String::from(""); + run_monitor_rule(idx,rule, &app_env).await.unwrap_or_else(|e| { + err = format!("Error rule {}: {}",&rule.id,e); + eprintln!("{}",&err) + }); + match err.is_empty() { + true => continue, + false => log_monitor_state(&rule, &app_env, &rule.name, -1, &err, "").await.unwrap_or_else(|e| + eprintln!("Unable to log monitor entry: {}",e) + ), + }; }, RuleSchedule::OnDemand => continue, RuleSchedule::None => continue,