LibreCloud/lib_clds
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: logs to monitor & notifications

Browse Source
This commit is contained in:
Jesús Pérez Lorenzo 2021-09-14 23:33:46 +01:00
parent 06d9941435
commit 159a3f3caa
2 changed files with 88 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/clouds/monitor_rules.rs
View File

@ -172,9 +172,6 @@ where D: Deserializer<'de> {
}); });
Ok(actions) Ok(actions)
} }
// in kubernetes on wuji-cp-0 if pod with ns rook-ceph and name match "osd-0" is not in "Running state" && is in last "save state info"
// context target(server) selector(type pod - ns - match name - state !Running)
#[allow(clippy::missing_docs_in_private_items)] #[allow(clippy::missing_docs_in_private_items)]
#[allow(non_snake_case)] #[allow(non_snake_case)]
#[derive(Clone, Debug, Serialize, Deserialize, Default)] #[derive(Clone, Debug, Serialize, Deserialize, Default)]
@ -213,7 +210,6 @@ impl MonitorRules {
} }
} }
pub fn load(root: &str, path: &str, format: &str) -> Self { pub fn load(root: &str, path: &str, format: &str) -> Self {
println!("Load Montior Rules ...");
let data_rules = load_fs_content(&root, &path, &format); let data_rules = load_fs_content(&root, &path, &format);
let err_load = |e|{ let err_load = |e|{
eprintln!("Error loading MonitorRules {}/{}.{}: {}",&root,&path,&format,e); eprintln!("Error loading MonitorRules {}/{}.{}: {}",&root,&path,&format,e);
@ -223,7 +219,7 @@ impl MonitorRules {
if data_rules.is_empty() { if data_rules.is_empty() {
res = MonitorRules::default(); res = MonitorRules::default();
} else { } else {
println!("Parse Montior Rules as: {}",&format); // println!("Parse Montior Rules as: {}",&format);
res = match format { res = match format {
"json" => serde_json::from_str(&data_rules).unwrap_or_else(|e| err_load(format!("{}",e))), "json" => serde_json::from_str(&data_rules).unwrap_or_else(|e| err_load(format!("{}",e))),
"yaml" => serde_yaml::from_str(&data_rules).unwrap_or_else(|e| err_load(format!("{}",e))), "yaml" => serde_yaml::from_str(&data_rules).unwrap_or_else(|e| err_load(format!("{}",e))),
@ -234,7 +230,6 @@ impl MonitorRules {
res res
} }
pub fn dump(&self, root: &str, path: &str, format: &str) -> Result<()> { pub fn dump(&self, root: &str, path: &str, format: &str) -> Result<()> {
println!("Dump Montior Rules ...");
let err_load = |e| { let err_load = |e| {
eprintln!("Error dump MonitorRules {}/{}.{}: {}",&root,&path,&format,e); eprintln!("Error dump MonitorRules {}/{}.{}: {}",&root,&path,&format,e);
String::from("") String::from("")

115
src/monitor.rs
View File

@ -8,14 +8,16 @@ use anyhow::{anyhow,Result};
use std::process::{Command}; use std::process::{Command};
use app_env::{ use app_env::{
appenv::AppEnv, appenv::{AppEnv},
DataStore,
applogs::AppLogs,
// config::{Config} // config::{Config}
}; };
use crate::clouds::defs::{ use crate::clouds::defs::{
Cloud, Cloud,
}; };
use crate::defs::ProviderName; use crate::defs::{ProviderName};
use crate::clouds::monitor_rules::{ use crate::clouds::monitor_rules::{
MonitorRules, MonitorRules,
@ -55,13 +57,25 @@ pub async fn on_action_server(task: &str, srvr: &str, provider: &str, args: &str
_ => Err(anyhow!("Provider {} not defined", &provider_name)) _ => Err(anyhow!("Provider {} not defined", &provider_name))
} }
} }
pub async fn on_action_notify(_chnl: &str, _msg: &str, _args: &str, _rule: &MonitorRule) -> Result<()> { pub async fn on_action_notify(chnl: &str, msg: &str, args: &str, rule: &MonitorRule) -> Result<()> {
let notificator_path = envmnt::get_or("NOTIFICATOR_BIN","");
if notificator_path.is_empty() || ! Path::new(&notificator_path).exists() {
return Err(anyhow!("Error notificator:"));
}
let message=format!("Monitor {}: {} {}",&rule.id,&msg,&args);
let cmd = format!("{} {} {}",&notificator_path,&chnl,&message);
let _ = run_command(&cmd).unwrap_or_else(|e|{
eprintln!("Error {}: {}",&cmd,e);
String::from("")
});
Ok(()) Ok(())
} }
fn get_server(target: &str, val: &str) -> String { fn parse_target(target: &str, val: &str) -> String {
let srvr: String; let srvr: String;
if val == "$target" || val == "$server" { if val.contains("$target") {
srvr = target.to_owned(); srvr = val.replace("$target", &target.to_owned());
} else if val.contains("$server") {
srvr = val.replace("$server", &target.to_owned());
} else { } else {
srvr = val.to_owned(); srvr = val.to_owned();
} }
@ -71,12 +85,12 @@ pub async fn on_monitor_action(action: &MonitorAction, target: &str, rule: &Moni
match action { match action {
MonitorAction::None => return Ok(()), MonitorAction::None => return Ok(()),
MonitorAction::Start(v,p,a) => MonitorAction::Start(v,p,a) =>
on_action_server("start",&get_server(target, v),&p,&a,&rule).await?, on_action_server("start",&parse_target(target, v),&p,&a,&rule).await?,
MonitorAction::Stop(v,p,a) => MonitorAction::Stop(v,p,a) =>
on_action_server("stop",&get_server(target, v),&p,&a,&rule).await?, on_action_server("stop",&parse_target(target, v),&p,&a,&rule).await?,
MonitorAction::Restart(v,p,a) => MonitorAction::Restart(v,p,a) =>
on_action_server("restart",&get_server(target, v),&p,&a,&rule).await?, on_action_server("restart",&parse_target(target, v),&p,&a,&rule).await?,
MonitorAction::Notify(v,p,a) => on_action_notify(&v,&p,&a,&rule).await?, MonitorAction::Notify(v,p,a) => on_action_notify(&v,&parse_target(target,&p),&a,&rule).await?,
}; };
Ok(()) Ok(())
} }
@ -100,7 +114,6 @@ pub async fn on_service_rule(srvc: &str, out_cmd: &str, rule: &MonitorRule) -> R
// println!("target: {}",&target); // println!("target: {}",&target);
for action in &rule.actions { for action in &rule.actions {
on_monitor_action(action, &target, &rule).await?; on_monitor_action(action, &target, &rule).await?;
} }
}; };
} }
@ -185,8 +198,26 @@ pub fn run_on_selector(rule: &MonitorRule) -> Result<String> {
Err(anyhow!("Selector: failed:\n {}",&rule.id)) Err(anyhow!("Selector: failed:\n {}",&rule.id))
} }
} }
pub async fn run_monitor_rule(idx: usize, rule: &MonitorRule ) -> Result<()> { pub async fn log_monitor_state(rule: &MonitorRule, app_env: &AppEnv, env_name: &str, env_state: isize, msg: &str, out_cmd: &str) -> Result<()> {
println!("{} [{}] {}: {}",&idx,&rule.id,&rule.name,&rule.description); let now = format!("{}", chrono::Utc::now().timestamp());
let log_entry = AppLogs {
id: format!("{}_{}", &rule.id,&now),
name: format!("rule: {}", &rule.name),
when: String::from("run_monitor_rule"),
source: format!("{}", &env_name),
target: format!("{}", &out_cmd),
state: format!("{}", &env_state),
msg: format!("{}", &msg),
};
if app_env.config.logs_store == DataStore::File {
log_entry.write_applogs_entry(format!("{}/monitor.log",&app_env.config.logs_path).as_str()).await?;
}
Ok(())
}
pub async fn run_monitor_rule(idx: usize, rule: &MonitorRule, app_env: &AppEnv) -> Result<()> {
if envmnt::get_isize("DEBUG",0) > 0 {
println!("{} [{}] {}: {}",&idx,&rule.id,&rule.name,&rule.description);
}
let out_cmd: String; let out_cmd: String;
if rule.command.is_empty() { if rule.command.is_empty() {
out_cmd=run_on_selector(&rule).unwrap_or_else(|e| { out_cmd=run_on_selector(&rule).unwrap_or_else(|e| {
@ -206,48 +237,76 @@ pub async fn run_monitor_rule(idx: usize, rule: &MonitorRule ) -> Result<()> {
let env_state = envmnt::get_isize(&env_name,0); let env_state = envmnt::get_isize(&env_name,0);
if ! on_monitor_operator(&rule.operator,&out_cmd, &rule.value) { if ! on_monitor_operator(&rule.operator,&out_cmd, &rule.value) {
if env_state > 0 { if env_state > 0 {
println!("{} Ok in state: {}",&env_name,&env_state); let msg = format!("{} Ok in state: {}",&env_name,&env_state);
println!("{}",&msg);
envmnt::set_isize(&env_name, 0); envmnt::set_isize(&env_name, 0);
log_monitor_state(&rule, &app_env, &env_name, env_state, &msg, &out_cmd).await?;
for item in rule.actions.iter() {
match item {
MonitorAction::Notify(v,p,a) => on_action_notify(&v,&parse_target(format!("Ok state from -> {}",&env_state).as_str(),&p),&a,&rule).await?,
_ => continue,
};
}
} }
return Ok(()); return Ok(());
} }
if env_state > 0 && env_state < rule.wait_checks { if env_state > 0 && env_state < rule.wait_checks {
println!("{} is already set: {}",&env_name,&env_state); let msg = format!("{} is already set: {} increment counter",&env_name,&env_state);
envmnt::increment(&env_name); envmnt::increment(&env_name);
println!("{}",&msg);
log_monitor_state(&rule, &app_env, &env_name, env_state, &msg, &out_cmd).await?;
return Ok(()); return Ok(());
} }
let mut err = String::from("");
match &rule.context { match &rule.context {
RuleContext::Server(v) => { RuleContext::Server(v) => {
on_server_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| on_server_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| {
eprintln!("Error rule {} on {}: {}",&rule.id,&v,e) err = format!("Error rule {} on {}: {}",&rule.id,&v,e);
); eprintln!("{}",err)
});
}, },
RuleContext::None => { RuleContext::None => {
return Ok(()); return Ok(());
}, },
RuleContext::Service(v) => { RuleContext::Service(v) => {
on_service_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| on_service_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| {
eprintln!("Error rule {} on {}: {}",&rule.id,&v,e) err = format!("Error rule {} on {}: {}",&rule.id,&v,e);
); eprintln!("{}",err)
});
}, },
RuleContext::Ip(v) => { RuleContext::Ip(v) => {
on_ip_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| on_ip_rule(&v, &out_cmd, &rule).await.unwrap_or_else(|e| {
eprintln!("Error rule {} on {}: {}",&rule.id,&v,e) err = format!("Error rule {} on {}: {}",&rule.id,&v,e);
); eprintln!("{}",err)
});
}, },
} }
let msg = match err.is_empty() {
true => format!("{}: 1 done",&env_name),
false => format!("{} error: {}",&env_name,&err),
};
envmnt::set_isize(&env_name,1); envmnt::set_isize(&env_name,1);
println!("{}",&msg);
log_monitor_state(&rule, &app_env, &env_name, env_state, &msg, &out_cmd).await?;
Ok(()) Ok(())
} }
pub async fn run_monitor(monitor_rules: MonitorRules, _cloud: Cloud, _app_env: AppEnv) -> Result<()> { pub async fn run_monitor(monitor_rules: MonitorRules, _cloud: Cloud, app_env: AppEnv) -> Result<()> {
// println!("Run {}: {}",&monitor_rules.name,&monitor_rules.description); // println!("Run {}: {}",&monitor_rules.name,&monitor_rules.description);
for (idx,rule) in monitor_rules.rules.iter().enumerate() { for (idx,rule) in monitor_rules.rules.iter().enumerate() {
match rule.schedule { match rule.schedule {
RuleSchedule::Check => { RuleSchedule::Check => {
// dbg!(&rule); // dbg!(&rule);
run_monitor_rule(idx,rule).await.unwrap_or_else(|e| let mut err = String::from("");
eprintln!("Error rule {}: {}",&rule.id,e) run_monitor_rule(idx,rule, &app_env).await.unwrap_or_else(|e| {
); err = format!("Error rule {}: {}",&rule.id,e);
eprintln!("{}",&err)
});
match err.is_empty() {
true => continue,
false => log_monitor_state(&rule, &app_env, &rule.name, -1, &err, "").await.unwrap_or_else(|e|
eprintln!("Unable to log monitor entry: {}",e)
),
};
}, },
RuleSchedule::OnDemand => continue, RuleSchedule::OnDemand => continue,
RuleSchedule::None => continue, RuleSchedule::None => continue,