recent mize changes
This commit is contained in:
Sebastian Moser
494
src/platform/os/mod.rs
Normal file
494
src/platform/os/mod.rs
Normal file
@@ -0,0 +1,494 @@
|
||||
use ciborium::Value as CborValue;
|
||||
use clap::ArgAction;
|
||||
use clap::ArgMatches;
|
||||
use clap::{crate_version, Arg, Command};
|
||||
use colored::Colorize;
|
||||
use core::fmt;
|
||||
use flate2::bufread::GzDecoder;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256, Sha512};
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::env::VarError;
|
||||
use std::fmt::Write;
|
||||
use std::fs;
|
||||
use std::fs::File;
|
||||
use std::io::copy;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::time::UNIX_EPOCH;
|
||||
use tar::Archive;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::level_filters::LevelFilter;
|
||||
use tracing::Event;
|
||||
use tracing::{debug, error, info, trace, warn};
|
||||
use tracing::{field::Visit, Id, Subscriber};
|
||||
use tracing::{span_enabled, Instrument, Level};
|
||||
use tracing_core::Field;
|
||||
use tracing_subscriber::filter::EnvFilter;
|
||||
use tracing_subscriber::fmt::format;
|
||||
use tracing_subscriber::fmt::FmtContext;
|
||||
use tracing_subscriber::fmt::FormatEvent;
|
||||
use tracing_subscriber::fmt::FormatFields;
|
||||
use tracing_subscriber::fmt::FormattedFields;
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
|
||||
use crate::config::ConfigOpt;
|
||||
use crate::error::{IntoMizeResult, MizeError, MizeResult};
|
||||
use crate::id::MizeId;
|
||||
use crate::instance::store::Store;
|
||||
use crate::instance::EmptyPart;
|
||||
use crate::instance::Mize;
|
||||
use crate::item::{data_from_string, IntoItemData, ItemData};
|
||||
use crate::memstore::MemStore;
|
||||
use crate::{mize_err, MizePart};
|
||||
|
||||
use self::fsstore::FileStore;
|
||||
|
||||
pub mod fsstore;
|
||||
pub mod logging;
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
mod unix_socket;
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
use crate::platform::os::unix_socket::UnixListener;
|
||||
//mod web;
|
||||
|
||||
pub fn os_instance_init(mize: &mut Mize) -> MizeResult<()> {
|
||||
crate::out_macro_stdout()?;
|
||||
// this is the code, that runs to initialize an Instance on a system with an os present.
|
||||
|
||||
////// populate values from $MIZE_CONFIG_FILES
|
||||
if let Ok(config_file_paths) = std::env::var("MIZE_CONFIG_FILES") {
|
||||
for config_file_path in config_file_paths.split(":") {
|
||||
println!("reading config file: {config_file_path}");
|
||||
let content = std::fs::read_to_string(config_file_path)?;
|
||||
let mut data = ItemData::from_toml(content.as_str())?;
|
||||
crate::core::item::set_config_from_data(mize, data)?;
|
||||
}
|
||||
}
|
||||
|
||||
////// load config from MIZE_CONFIG env var
|
||||
match std::env::var("MIZE_CONFIG") {
|
||||
Ok(config_string) => {
|
||||
let data = data_from_string(config_string)?;
|
||||
crate::core::item::set_config_from_data(mize, data)?;
|
||||
}
|
||||
Err(var_err) => match var_err {
|
||||
VarError::NotPresent => {
|
||||
debug!("env var MIZE_CONFIG NOT present")
|
||||
}
|
||||
VarError::NotUnicode(_) => {
|
||||
warn!("env var MIZE_CONFIG is not Unicode, so not reading it")
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// setup loging to stderr if enabled
|
||||
if mize
|
||||
.get_config("log.stderr.enable")
|
||||
.unwrap_or(ItemData::new())
|
||||
.value_bool()
|
||||
.unwrap_or(false)
|
||||
{
|
||||
use tracing::{span_enabled, Instrument, Level};
|
||||
let level_string = mize
|
||||
.get_config("log.level")?
|
||||
.value_string()
|
||||
.unwrap_or("info".to_string());
|
||||
let log_level = match level_string.to_lowercase().as_str() {
|
||||
"info" => Level::INFO,
|
||||
"debug" => Level::DEBUG,
|
||||
"warn" | "warning" => Level::WARN,
|
||||
"trace" => Level::TRACE,
|
||||
"error" => Level::ERROR,
|
||||
_ => Level::INFO, // the default log level
|
||||
};
|
||||
println!("enabling loggging to stderr with log_level: {}", log_level);
|
||||
|
||||
// builder
|
||||
let mut builder = tracing_subscriber::fmt::Subscriber::builder();
|
||||
|
||||
let field_formatter = tracing_subscriber::fmt::format::debug_fn(|writer, field, value| {
|
||||
if field.name() == "message" {
|
||||
write!(writer, "{:?}", value)?;
|
||||
} else {
|
||||
//write!(writer, "{}={:?}", field, value)?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
builder
|
||||
//.with_env_filter(my_env_filter)
|
||||
.without_time()
|
||||
.event_format(MyFormatter)
|
||||
.fmt_fields(field_formatter)
|
||||
.with_max_level(log_level)
|
||||
.init();
|
||||
}
|
||||
|
||||
////// if a config.store_path is set, upgrade to the filestore there
|
||||
let mut store_path = match mize.get_config("store")?.value_string() {
|
||||
Ok(path) => path,
|
||||
Err(e) => {
|
||||
// the default store_path: $HOME/.mize
|
||||
let home_dir = if let Some(dir) = std::env::home_dir() {
|
||||
dir
|
||||
} else {
|
||||
return Err(mize_err!(
|
||||
"could not get home_dir and store_path is not set in the config"
|
||||
));
|
||||
};
|
||||
|
||||
let store_path_buff = home_dir.join(".mize");
|
||||
let store_path = store_path_buff.display().to_string();
|
||||
|
||||
mize.set_blocking(
|
||||
"self/config/store_path",
|
||||
store_path.clone().into_item_data(),
|
||||
)?;
|
||||
|
||||
let store_path = mize.get("self/config/store_path")?.value_string()?;
|
||||
|
||||
println!("store_path: {}", store_path);
|
||||
|
||||
store_path
|
||||
}
|
||||
};
|
||||
|
||||
if FileStore::store_is_opened(store_path.to_owned(), mize)? {
|
||||
// if the store is already opened, connect to the instance, that opened it and join
|
||||
// it's namespace
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
{
|
||||
info!("CONNECTING");
|
||||
use crate::platform::os::unix_socket;
|
||||
unix_socket::connect(mize, store_path.into())?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
{
|
||||
error!("CONNECTING... would connect, but that is not implemented on windows yet");
|
||||
}
|
||||
} else {
|
||||
// else open it ourselves
|
||||
let file_store = FileStore::new(store_path.as_str())?;
|
||||
mize.migrate_to_store(Box::new(file_store))?;
|
||||
|
||||
let path = Path::new(&store_path).to_owned();
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
{
|
||||
mize.add_conn_listener(UnixListener::new(path)?)?;
|
||||
}
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
{
|
||||
warn!("would add a Listener on a local socket, but that is not yet implemented for windows");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn seconds_since_modification(path: &Path) -> MizeResult<u64> {
|
||||
let metadata = fs::metadata(path)?;
|
||||
|
||||
let modified = metadata.modified()?;
|
||||
|
||||
let duration = modified.duration_since(UNIX_EPOCH)?;
|
||||
|
||||
Ok(duration.as_secs())
|
||||
}
|
||||
|
||||
pub fn get_part_hash(
|
||||
mize: &mut Mize,
|
||||
name: &str,
|
||||
mut options: ItemData,
|
||||
) -> MizeResult<(String, String)> {
|
||||
let mut selector_data = mize.get("self/config/selector")?.as_data_full()?;
|
||||
|
||||
// handle the case, when we want a module for not the system we are running on...
|
||||
println!("get_module_hash: name: {}", name);
|
||||
let mut name_parts = name.split(".");
|
||||
if name_parts
|
||||
.next()
|
||||
.ok_or(mize_err!("no 0th element in the modName"))?
|
||||
== "cross"
|
||||
{
|
||||
let system = name_parts
|
||||
.next()
|
||||
.ok_or(mize_err!("no 1th element in the modName"))?;
|
||||
let mod_name: String = name_parts.collect();
|
||||
println!("get_module_hash: mod_name: {}", mod_name);
|
||||
if mod_name == "" {
|
||||
return Err(mize_err!("no 2nd element in the modName"));
|
||||
}
|
||||
|
||||
selector_data.set_path("system", system)?;
|
||||
selector_data.set_path("modName", mod_name)?;
|
||||
} else {
|
||||
selector_data.set_path("modName", name)?;
|
||||
}
|
||||
|
||||
selector_data.merge(options);
|
||||
|
||||
// important to produce the same hash for the sama selector data
|
||||
selector_data.sort_keys();
|
||||
|
||||
let selector_str = selector_data.to_json()?;
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
hasher.update(selector_str.clone().as_bytes());
|
||||
|
||||
let result = hasher.finalize();
|
||||
|
||||
let mut hex: String = result
|
||||
.iter()
|
||||
.map(|b| format!("{:02x}", b).to_string())
|
||||
.collect::<Vec<String>>()
|
||||
.join("");
|
||||
|
||||
hex.truncate(32);
|
||||
|
||||
return Ok((hex, selector_str));
|
||||
}
|
||||
|
||||
fn download_part(
|
||||
mize: &mut Mize,
|
||||
store_path: &str,
|
||||
part_url: &str,
|
||||
part_hash: &str,
|
||||
part_name: &str,
|
||||
part_selector: &str,
|
||||
) -> MizeResult<()> {
|
||||
let tmp_file_path_gz = format!("{}/{}.tar.gz", store_path, part_hash);
|
||||
|
||||
let tmp_gz_file = if !PathBuf::from(tmp_file_path_gz.clone()).exists() {
|
||||
let mut tmp_gz_file = File::create(&tmp_file_path_gz)?;
|
||||
|
||||
debug!("downloading part '{}' with hash: {}", part_name, part_hash);
|
||||
|
||||
let response = http_req::request::get(
|
||||
format!(
|
||||
"http://{}/mize/dist/{}-{}.tar.gz",
|
||||
part_url, part_hash, part_name
|
||||
),
|
||||
&mut tmp_gz_file,
|
||||
)?;
|
||||
|
||||
let code = response.status_code();
|
||||
if code != 200.into() {
|
||||
return Err(mize_err!(
|
||||
"failed to get part '{}' with hash '{}' and selector: {}",
|
||||
part_name,
|
||||
part_hash,
|
||||
part_selector
|
||||
)
|
||||
.msg(format!(
|
||||
"the http request to fetch the part returned a status code of: {}",
|
||||
code
|
||||
)));
|
||||
}
|
||||
|
||||
tmp_gz_file
|
||||
} else {
|
||||
std::fs::File::open(tmp_file_path_gz.clone())?
|
||||
};
|
||||
|
||||
let tar = GzDecoder::new(std::io::BufReader::new(tmp_gz_file));
|
||||
let mut archive = Archive::new(tar);
|
||||
|
||||
//archive.set_mask(umask::Mode::parse("rwxrwxrwx")?.into());
|
||||
|
||||
let target_dir = format!("{}/parts/{}", store_path, part_hash);
|
||||
|
||||
std::fs::create_dir_all(target_dir.as_str())?;
|
||||
|
||||
archive.unpack(target_dir.as_str())?;
|
||||
|
||||
fs::remove_file(&tmp_file_path_gz)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn fetch_part(mize: &mut Mize, part_name: &str) -> MizeResult<String> {
|
||||
let part_name_with_slashes = part_name.replace(".", "/");
|
||||
if let Ok(part_path) = mize
|
||||
.get(format!("self/config/part_dir/{}", part_name_with_slashes))?
|
||||
.value_string()
|
||||
{
|
||||
// in case a dir is configured, which holds the parts output
|
||||
return Ok(part_path);
|
||||
} else {
|
||||
// if null, load it from the url
|
||||
let part_url = mize.get("self/config/part_url")?.value_string()?;
|
||||
|
||||
let store_path = mize.get("self/config/store_path")?.value_string()?;
|
||||
|
||||
let (part_hash, part_selector) = get_part_hash(mize, part_name, ItemData::new())?;
|
||||
|
||||
if !PathBuf::from(format!("{}/parts/{}", store_path, part_hash.as_str())).exists() {
|
||||
// download part if it does not exist
|
||||
download_part(
|
||||
mize,
|
||||
store_path.as_str(),
|
||||
part_url.as_str(),
|
||||
part_hash.as_str(),
|
||||
part_name,
|
||||
part_selector.as_str(),
|
||||
)?;
|
||||
}
|
||||
|
||||
return Ok(format!("{}/parts/{}", store_path, part_hash.as_str()));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_part(mize: &mut Mize, part_name: &str, path: Option<String>) -> MizeResult<()> {
|
||||
let part_path = if path.is_some() {
|
||||
format!("{}/lib/lib{}_part.so", path.unwrap(), part_name)
|
||||
} else {
|
||||
format!(
|
||||
"{}/lib/lib{}_part.so",
|
||||
fetch_part(mize, part_name)?,
|
||||
part_name
|
||||
)
|
||||
};
|
||||
|
||||
let lib = unsafe { libloading::Library::new(part_path)? };
|
||||
|
||||
let func: libloading::Symbol<
|
||||
unsafe extern "C" fn(&mut Box<dyn MizePart + Send + Sync>, Mize) -> (),
|
||||
> = unsafe { lib.get(format!("make_mize_part_{}", part_name).as_bytes())? };
|
||||
|
||||
let mut part: Box<dyn MizePart + Send + Sync> = Box::new(EmptyPart { mize: mize.clone() });
|
||||
|
||||
unsafe { func(&mut part, mize.clone()) };
|
||||
|
||||
part.init(mize)?;
|
||||
|
||||
mize.add_part(part)?;
|
||||
|
||||
unsafe {
|
||||
// dropping the lib, would (i suspect) free all memory, of the library's code, which would
|
||||
// make the part's vtable point into empty memory
|
||||
std::mem::forget(lib);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn config_from_cli_args(matches: &ArgMatches) -> MizeResult<ItemData> {
|
||||
let mut config = ItemData::new();
|
||||
|
||||
if let Some(config_file_path) = matches.get_one::<String>("config-file") {
|
||||
config.merge(config_from_file(config_file_path.to_string())?);
|
||||
trace!("config after --config-file arg: {}", config);
|
||||
}
|
||||
|
||||
if let Some(config_string) = matches.get_one::<String>("config") {
|
||||
config.merge(data_from_string(config_string.to_string())?);
|
||||
trace!("config after --config arg: {}", config);
|
||||
}
|
||||
|
||||
return Ok(config);
|
||||
}
|
||||
|
||||
pub fn config_from_file(file_path: String) -> MizeResult<ItemData> {
|
||||
let toml_string = fs::read_to_string(&file_path)
|
||||
.mres_msg(format!("Could not read file {} as mize config", &file_path))?;
|
||||
|
||||
let item_data = ItemData::from_toml(toml_string.as_str())?;
|
||||
|
||||
return Ok(item_data);
|
||||
}
|
||||
|
||||
struct MyFormatter;
|
||||
|
||||
impl<S, N> FormatEvent<S, N> for MyFormatter
|
||||
where
|
||||
S: tracing::Subscriber + for<'a> LookupSpan<'a>,
|
||||
N: for<'a> FormatFields<'a> + 'static,
|
||||
{
|
||||
fn format_event(
|
||||
&self,
|
||||
ctx: &FmtContext<'_, S, N>,
|
||||
mut writer: format::Writer<'_>,
|
||||
event: &Event<'_>,
|
||||
) -> fmt::Result {
|
||||
// Format values from the event's's metadata:
|
||||
|
||||
let metadata = event.metadata();
|
||||
|
||||
// set format
|
||||
//builder.format(|buf, record| match record.level() {});
|
||||
|
||||
let level_str = match metadata.level() {
|
||||
&Level::TRACE => format!("[ {} ]", "TRACE".yellow()),
|
||||
&Level::DEBUG => format!("[ {} ]", "DEBUG".green()),
|
||||
&Level::INFO => format!("[ {} ]", "INFO".blue()),
|
||||
&Level::WARN => format!("[ {} ]", "WARN".truecolor(245, 164, 66)),
|
||||
&Level::ERROR => format!("[ {} ]", "ERROR".red()),
|
||||
};
|
||||
|
||||
// with target
|
||||
// write!(&mut writer, "{} {}: ", level_str, metadata.target())?;
|
||||
|
||||
write!(&mut writer, "{} ", level_str)?;
|
||||
|
||||
// Format all the spans in the event's span context.
|
||||
if let Some(scope) = ctx.event_scope() {
|
||||
for span in scope.from_root() {
|
||||
write!(writer, "{}", span.name())?;
|
||||
|
||||
// `FormattedFields` is a formatted representation of the span's
|
||||
// fields, which is stored in its extensions by the `fmt` layer's
|
||||
// `new_span` method. The fields will have been formatted
|
||||
// by the same field formatter that's provided to the event
|
||||
// formatter in the `FmtContext`.
|
||||
let ext = span.extensions();
|
||||
let fields = &ext
|
||||
.get::<FormattedFields<N>>()
|
||||
.expect("will never be `None`");
|
||||
|
||||
// Skip formatting the fields if the span had no fields.
|
||||
if !fields.is_empty() {
|
||||
//write!(writer, "{{{}}}", fields)?;
|
||||
}
|
||||
write!(writer, ": ")?;
|
||||
}
|
||||
}
|
||||
|
||||
// Write fields on the event
|
||||
ctx.field_format().format_fields(writer.by_ref(), event)?;
|
||||
|
||||
writeln!(writer);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// thanks: https://stackoverflow.com/a/75546824
|
||||
pub struct StringVisitor<'a> {
|
||||
string: &'a mut String,
|
||||
}
|
||||
impl<'a> StringVisitor<'a> {
|
||||
pub(crate) fn new(string: &'a mut String) -> Self {
|
||||
StringVisitor { string }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Visit for StringVisitor<'a> {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
|
||||
if field.name() == "message" {
|
||||
write!(self.string, "{value:?} ").unwrap();
|
||||
} else {
|
||||
write!(self.string, "{} = {:?}; ", field.name(), value).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user