ppc/mize
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fetching modules should now work

Browse Source
This commit is contained in:
Sebastian Moser
2024-10-05 14:01:30 +02:00
parent de4b9b672f
commit 5607dcaad2
13 changed files with 238 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@@ -898,8 +898,7 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
[[package]]
name = "http_req"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92208b0986f414edaca8ede2c6962c979309346a9e8e19d07d0a7879aae1549e"
source = "git+https://github.com/c2vi/http_req#46b0f93087ef6f68a360b303192e30ecb7b055e2"
dependencies = [
"native-tls",
"unicase",

3
Cargo.toml
View File

@@ -63,7 +63,8 @@ ciborium-io = { version = "0.2.2", optional = true }
tracing-core = { version = "0.1.32", optional = true }
tar = { version = "0.4.42", optional = true }
flate2 = { version = "1.0.33", optional = true }
http_req = { version = "0.12.0", default-features = false, features = [ "native-tls" ], optional = true }
#http_req = { version = "0.12.0", default-features = false, features = [ "rustls" ], optional = true }
http_req = { git = "https://github.com/c2vi/http_req", optional = true, default-features = false, features = [ "native-tls" ] }
# bare dependencies

17
flake.lock generated
View File

@@ -57,18 +57,17 @@
"mize_modules": {
"flake": false,
"locked": {
"dirtyRev": "fa3b462cbe3e63fc9ecada6c4445d366e58a6f86-dirty",
"dirtyShortRev": "fa3b462-dirty",
"lastModified": 1725362504,
"narHash": "sha256-uqk6t8W+2qS6feADN8QaOLEIhz7e3BYn2WRccmtyQws=",
"submodules": true,
"type": "git",
"url": "file:///home/me/work/modules"
"narHash": "sha256-U0IQ+3SiyLb24BTDUvlnE30LdjjJWJ/P6dHPqxArJXk=",
"owner": "c2vi",
"repo": "mize-modules",
"rev": "fa3b462cbe3e63fc9ecada6c4445d366e58a6f86",
"type": "github"
},
"original": {
"submodules": true,
"type": "git",
"url": "file:///home/me/work/modules"
"owner": "c2vi",
"repo": "mize-modules",
"type": "github"
}
},
"nixpkgs": {

32
flake.nix
View File

@@ -18,7 +18,7 @@
};
mize_modules = {
url = "github:c2vi/mize-modules";
url = "github:c2vi/mize-modules?submodules=1";
flake = false;
};
@@ -30,12 +30,13 @@
let
pkgs = nixpkgs.legacyPackages.${system};
wasmToolchain = fenix.packages.${system}.combine [
fenix.packages.${system}.targets.wasm32-unknown-unknown.latest.toolchain
fenix.packages.${system}.latest.toolchain
fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.toolchain
fenix.packages.${system}.stable.toolchain
];
wasmCrane = (crane.mkLib pkgs).overrideToolchain wasmToolchain;
osToolchain = fenix.packages.${system}.latest.toolchain;
osToolchain = fenix.packages.${system}.stable.toolchain;
osCrane = (crane.mkLib pkgs).overrideToolchain osToolchain;
#winToolchain = fenix.packages.${system}.combine [
@@ -152,6 +153,10 @@ in {
############################## DEV SHELLS ##############################
devShells = {
modules = mizeLib.moduleShells system;
modulesCross = pkgs.lib.genAttrs systems (system: mizeLib.moduleShells system);
one = pkgs.stdenv.mkDerivation {
name = "hiiiiiiii";
nativeBuildInputs = [
@@ -213,24 +218,7 @@ in {
};
default = pkgs.mkShell {
buildInputs = with pkgs; [
wasm-pack #pkg-config openssl #cargo rustc
cargo-generate
(fenix.packages.${system}.combine [ wasmToolchain osToolchain ])
lldb gdb
];
MIZE_BUILD_CONFIG = pkgs.writeTextFile {
name = "vic-build-config";
text = builtins.toJSON defaultMizeConfig;
};
shellHook = ''
export MIZE_CONFIG_FILE=${self}/test-config.toml
'';
};
default = (mizeLib.buildMizeForSystem system).devShell;
};
}) // {

94
lib.nix
View File

@@ -174,21 +174,24 @@ rec {
buildModule = path: extraArgs: (pkgs.callPackage path {
buildModule = path: extraArgs: let
pkg = (pkgs.callPackage path {
inherit mkSelString craneLib toolchain_version;
inherit mkMizeModule mkMizeRustModule buildModule findModules crossSystem pkgsCross pkgsNative;
inherit mkMizeModule mkMizeRustModule buildModule findModules crossSystem pkgsCross pkgsNative mkMizeRustShell;
mize_version = main-default.version;
} // extraArgs );
mizeBildConfig = pkgs.writeTextFile {
name = "mize-build-config";
text = builtins.toJSON (defaultMizeConfig // {
selector = {
inherit toolchain_version;
mize_version = main-default.version;
system = crossSystem.nameFull;
in pkg // {
};
mizeBildConfig = let
settingsFormat = pkgs.formats.toml { };
in settingsFormat.generate "mize-build-config.toml" {
config = {
namespace = "mize.buildtime.ns";
module_url = "c2vi.dev";
selector = builtins.fromJSON (mkSelString {});
};
});
};
####### build mize
@@ -199,12 +202,12 @@ rec {
nativeBuildInputs = [
pkgsCross.buildPackages.pkg-config
pkgsCross.buildPackages.nasm
pkgsCross.buildPackages.cmake
#pkgsCross.buildPackages.nasm
#pkgsCross.buildPackages.cmake
];
buildInputs = [
pkgsCross.openssl
(if crossSystem.nameFull == "x86_64-unknown-linux-gnu" then pkgs.openssl else pkgsCross.openssl)
];
MIZE_BUILD_CONFIG = mizeBildConfig;
@@ -270,8 +273,60 @@ rec {
#AR_wasm32_unknown_unknown = "${pkgs.llvmPackages_14.llvm}/bin/llvm-ar";
};
################# dev Shels ###################
mkMizeRustShell = attrs: mkMizeModuleShell (attrs // {
#_shell_type = "rust";
nativeBuildInputs = attrs.nativeBuildInputs or [] ++ [
(fenix.packages.${localSystem}.combine [
fenix.packages.${system}.stable.toolchain
fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.toolchain
fenix.packages.${system}.stable.toolchain
])
];
});
mkMizeModuleShell = attrs: pkgs.mkShell (attrs // {
MIZE_BUILD_CONFIG = mizeBildConfig;
});
mainDevShell = pkgs.mkShell {
buildInputs = with pkgs; [
wasm-pack #pkg-config openssl #cargo rustc
cargo-generate
(fenix.packages.${localSystem}.combine [
fenix.packages.${system}.stable.toolchain
fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.toolchain
fenix.packages.${system}.stable.toolchain
])
openssl
];
nativeBuildInputs = with pkgs; [
pkg-config
];
MIZE_BUILD_CONFIG = mizeBildConfig;
shellHook = ''
export MIZE_CONFIG_FILE=${self}/test-config.toml
'';
};
################# output attrset ###################
in rec {
inherit toolchain_version_drv pkgsCross craneLib;
devShell = mainDevShell;
main =
if crossSystem.kernel.name == "windows"
then main-win
@@ -312,9 +367,9 @@ rec {
echo got module: ${module.name}
echo out: $out
hash=$(echo ${module.selector_string} | sha256sum | cut -c -32)
cp --no-preserve=mode,ownership -r ${module}/* $out/mize/dist/$${hash}-${module.name}
${pkgs.gnutar}/bin/tar -czf $out/mize/dist/$${hash}-${module.name}.tar.gz -C ${module} .
echo '${module.selector_string}' > $out/mize/dist/$hash/selector
cp --no-preserve=mode,ownership -r ${module}/* $out/mize/dist/$hash-${module.modName}
${pkgs.gnutar}/bin/tar -czf $out/mize/dist/$hash-${module.modName}.tar.gz -C ${module} .
echo '${module.selector_string}' > $out/mize/dist/$hash-${module.modName}/selector
'';
webfiles = systems: stdenv.mkDerivation {
@@ -338,6 +393,11 @@ rec {
nativeBuildInputs = [
];
};
moduleShells = system: let
mizeFor = buildMizeForSystem system;
in builtins.listToAttrs ( map ( mod: { name = mod.modName; value = (mod.devShell); } ) mizeFor.modulesList );
}

13
src/core/error.rs
View File

@@ -7,13 +7,14 @@ use std::io;
use colored::Colorize;
use tracing::{trace, debug, info, warn, error};
use crate::proto::MizeMessage;
#[macro_export]
macro_rules! mize_err {
($($arg:tt)*) => { MizeError::new().msg(format!( $($arg)*)) };
}
use crate::proto::MizeMessage;
pub type MizeResult<T> = Result<T, MizeError>;
pub trait MizeResultTrait<T> {
@@ -32,6 +33,7 @@ pub struct MizeError {
pub messages: Vec<String>,
pub caused_by_msg: Option<CausedByMessage>,
pub code_location: Option<MizeCodeLocation>,
pub backtrace: String,
}
#[derive(Debug, Clone)]
@@ -85,6 +87,7 @@ impl MizeError {
messages: Vec::new(),
caused_by_msg: None,
code_location: Some(caller_location.into()),
backtrace: format!("{}", std::backtrace::Backtrace::capture()),
}
}
@@ -110,12 +113,17 @@ impl MizeError {
pub fn log(self) -> MizeError {
error!("MizeError envountered!");
if let Some(ref location) = self.code_location {
error!("[ {} ] {}", "LOCATION".yellow(), location);
};
error!("[ {} ] {}", "BACKTRACE".yellow(), self.backtrace);
for msg in &self.messages {
error!("[ {} ] {}", "MSG".yellow(), msg);
}
self
}
@@ -258,6 +266,7 @@ fn get_error_by_code(code: u32, caller_location: &std::panic::Location) -> MizeE
messages: Vec::new(),
caused_by_msg: None,
code_location: Some(caller_location.into()),
backtrace: format!("{}", std::backtrace::Backtrace::capture()),
};
match code {

11
src/core/instance/mod.rs
View File

@@ -129,6 +129,11 @@ impl Instance {
//instance.spawn("msg_thread", closure)?;
// load the config from build time
let config = ItemData::from_toml(BUILD_TIME_CONFIG)?;
instance.set_blocking("0", config);
return Ok(instance);
}
@@ -146,12 +151,8 @@ impl Instance {
fn init(&mut self) -> MizeResult<()> {
// load the config from build time
let config = ItemData::from_toml(BUILD_TIME_CONFIG)?;
self.set_blocking("0", config);
// platform specific init code
crate::platform::any::instance_init(self);
crate::platform::any::instance_init(self)?;
// end of platform specific init code

41
src/core/instance/tests.rs
View File

@@ -5,6 +5,47 @@ use crate::item::IntoItemData;
use super::*;
#[test]
fn test_get_module_hash() -> MizeResult<()> {
let mut instance = Instance::empty()?;
use crate::platform::os::get_module_hash;
let selector_toml = r#"
mize_version = "0.0.0"
system = "x86_64-unknown-linux-gnu"
toolchain_version = "rustc 1.78.0 (9b00956e5 2024-04-29)"
"#;
instance.set_blocking("0/config/selector", ItemData::from_toml(&selector_toml)?)?;
let hash = get_module_hash(&mut instance, "String", ItemData::new())?;
test_println!("hash: {}", hash);
assert_eq!(hash, "e96edf60216688314efc263c1aa8d6ce".to_owned());
Ok(())
}
#[test]
fn test_set_store_path() -> MizeResult<()> {
let instance = Instance::empty()?;
let store_path = "/home/me/.mize".to_owned();
let config = instance.get("0/config")?.as_data_full()?;
test_println!("data at 0/config: {}", config);
instance.set_blocking("0/config/store_path", store_path.clone().into_item_data())?;
let store_path_new = instance.get("0/config/store_path")?.value_string()?;
assert_eq!(store_path_new, store_path);
Ok(())
}
#[test]
fn test_set_sub_path() -> MizeResult<()> {
let instance = Instance::empty()?;

12
src/core/item.rs
View File

@@ -15,6 +15,7 @@ use crate::error::{MizeError, MizeResult, IntoMizeResult};
use crate::instance::{connection, Instance};
use crate::instance::store::Store;
use crate::id::MizeId;
use crate::mize_err;
use crate::proto::MizeMessage;
use ciborium::Value as CborValue;
use crate::instance::connection::value_raw_con_by_id;
@@ -71,6 +72,7 @@ impl Item<'_> {
},
_ => {},
}
return Err(mize_err!("a /self path, but the next element in the path '{:?}' is not valid", id.nth_part(1)));
}
let self_namespace_inner = self.instance.self_namespace.lock()?;
@@ -258,7 +260,7 @@ pub fn item_data_set_path(data: &mut CborValue, path: Vec<String>, data_to_set:
//trace!("[ {} ] path: {:?}", "ARG".yellow(), path);
//trace!("[ {} ] data_to_set: {}", "ARG".yellow(), data_to_set.clone().into_item_data());
test_println!("###########################");
test_println!("item_data_set_path ###########################");
test_println!("data: {}", ItemData::from_cbor(data.clone()));
test_println!("path: {:?}", path);
test_println!("data_to_set: {}", ItemData::from_cbor(data_to_set.clone()));
@@ -275,7 +277,7 @@ pub fn item_data_set_path(data: &mut CborValue, path: Vec<String>, data_to_set:
};
match data {
CborValue::Map(ref mut map) => {
for (key, val) in map {
for (key, val) in &mut * map {
if let CborValue::Text(key_str) = key {
if key_str == &path_el {
//path_iter.next();
@@ -283,6 +285,12 @@ pub fn item_data_set_path(data: &mut CborValue, path: Vec<String>, data_to_set:
}
}
}
// if we get here, that means, the key (which is the variable path_el) does not exist
// in the map
// so add it
map.push((CborValue::Text(path_el.clone()), data_to_set.to_owned()));
return Ok(());
},
CborValue::Null => {
let mut inner_value = CborValue::Null;

18
src/platform/os/cli.rs
View File

@@ -2,6 +2,7 @@ use std::path::PathBuf;
use clap::ArgMatches;
use home::home_dir;
use crossbeam::channel::bounded;
use std::io::Read;
use mize::error::{IntoMizeResult, MizeError, MizeResult, MizeResultTrait};
use mize::instance::Instance;
@@ -9,7 +10,7 @@ use mize::platform::os::config_from_cli_args;
use mize::instance::subscription::Update;
use mize::platform::os::fsstore::FileStore;
use mize::instance::subscription::Subscription;
use mize::item::IntoItemData;
use mize::item::{IntoItemData, ItemData};
@@ -141,7 +142,20 @@ pub fn gui(sub_matches: &ArgMatches) -> MizeResult<()> {
Ok(())
}
pub fn format_cbor(sub_matches: &ArgMatches) -> MizeResult<()> {
let mut input: Vec<u8> = Vec::new();
let stdin = std::io::stdin();
let mut handle = stdin.lock();
//handle.read_to_end(&mut input);
let cbor_data = ciborium::from_reader(handle)?;
let item_data = ItemData::from_cbor(cbor_data);
println!("{}", item_data);
Ok(())
}

3
src/platform/os/fsstore.rs
View File

@@ -111,7 +111,8 @@ impl Store for FileStore {
if !file_path.exists() {
}
let file = OpenOptions::new().read(true).open(file_path)?;
let file = OpenOptions::new().read(true).open(file_path.clone())
.map_err(|e| mize_err!("fsstoree: Unable to open '{}', err: {}", file_path.display(), e))?;
let cbor_value: CborValue = ciborium::from_reader(file)
.mize_result_msg(format!("could not read file '{}' from FileStore", self.path.join("store").join(id.store_part()).display()))?;

6
src/platform/os/main.rs
View File

@@ -66,6 +66,9 @@ fn main() {
// mi gui
Some(("gui", sub_matches)) => cli::gui(sub_matches),
// mi format-cbor
Some(("format-cbor", sub_matches)) => cli::format_cbor(sub_matches),
// some unknown command passed
Some((cmd, sub_matches)) => Err(MizeError::new().msg(format!("The subcommand: {} is not known. use --help to list availavle commands", cmd))),
@@ -167,6 +170,9 @@ fn cli_matches() -> clap::ArgMatches {
.subcommand(
Command::new("gui")
)
.subcommand(
Command::new("format-cbor")
)
.arg_required_else_help(true);
return main.get_matches();

51
src/platform/os/mod.rs
View File

@@ -72,17 +72,29 @@ pub fn os_instance_init(instance: &mut Instance) -> MizeResult<()> {
}
};
////// if a config.store_path is set, upgrade to the filestore there
let mut test = instance.get("0")?.as_data_full()?;
let mut store_path = match instance.get("0/config/store_path")?.value_string() {
Ok(path) => path,
Ok(path) => {
path
},
Err(e) => {
// the default store_path: $HOME/.mize
let home_dir = env!("HOME");
if home_dir == "" {
return Err(mize_err!("env var $HOME empty"));
}
home_dir.to_owned() + "/.mize"
let store_path = home_dir.to_owned() + "/.mize";
instance.set_blocking("0/config/store_path", store_path.clone().into_item_data())?;
let store_path = instance.get("0/config/store_path")?.value_string()?;
println!("store_path: {}", store_path);
store_path
},
};
@@ -136,7 +148,7 @@ pub fn seconds_since_modification(path: &Path) -> MizeResult<u64> {
pub fn get_module_hash(instance: &mut Instance, name: &str, mut options: ItemData) -> MizeResult<String> {
let mut selector_data = instance.get("self/config/selector")?.as_data_full()?;
let mut selector_data = instance.get("0/config/selector")?.as_data_full()?;
selector_data.set_path("name", name)?;
@@ -150,16 +162,23 @@ pub fn get_module_hash(instance: &mut Instance, name: &str, mut options: ItemDat
let result = hasher.finalize();
return Ok(format!("{:02X?}", result));
let mut hex: String = result.iter()
.map(|b| format!("{:02x}", b).to_string())
.collect::<Vec<String>>()
.join("");
hex.truncate(32);
return Ok(hex);
}
fn fetch_module(instance: &mut Instance, store_path: &str, module_url: &str, module_hash: &str) -> MizeResult<()> {
fn fetch_module(instance: &mut Instance, store_path: &str, module_url: &str, module_hash: &str, module_name: &str) -> MizeResult<()> {
let tmp_file_path_gz = format!("{}/{}.tar.gz", store_path, module_hash);
let mut tmp_gz_file = File::create(&tmp_file_path_gz)?;
http_req::request::get(format!("http://{}/mize/dist/{}.tar.gz", module_url, module_hash), &mut tmp_gz_file)?;
http_req::request::get(format!("http://{}/mize/dist/{}-{}.tar.gz", module_url, module_hash, module_name), &mut tmp_gz_file)?;
let tar = GzDecoder::new(tmp_gz_file);
let mut archive = Archive::new(tar);
@@ -177,29 +196,31 @@ fn fetch_module(instance: &mut Instance, store_path: &str, module_url: &str, mod
Ok(())
}
pub fn load_module(instance: &mut Instance, name: &str, path: Option<PathBuf>) -> MizeResult<()> {
pub fn load_module(instance: &mut Instance, module_name: &str, path: Option<PathBuf>) -> MizeResult<()> {
let module_dir_str = instance.get(format!("0/config/module_dir/{}", name))?.value_string()?;
let module_dir_str = instance.get(format!("0/config/module_dir/{}", module_name))?.value_string();
let module_path = if module_dir_str.clone().into_item_data() == ItemData::new() {
let module_path = match module_dir_str {
Err(_) => {
// if null, load it from the url
let module_url = instance.get("0/config/module_url")?.value_string()?;
let store_path = instance.get("self/store_path")?.value_string()?;
let store_path = instance.get("0/config/store_path")?.value_string()?;
let module_hash = get_module_hash(instance, name, ItemData::new())?;
if !PathBuf::from(format!("{}/modules/{}", store_path, module_hash.as_str())).exists() {
// fetch module if it does not exist
fetch_module(instance, store_path.as_str(), module_url.as_str(), module_hash.as_str())?;
fetch_module(instance, store_path.as_str(), module_url.as_str(), module_hash.as_str(), module_name.as_str())?;
}
// the module_path
format!("{}/modules/{}/lib/libmize_module_{}.so", store_path, module_hash, name)
} else {
format!("{}/modules/{}/lib/libmize_module_{}.so", store_path, module_hash, module_name)
},
Ok(module_dir_str) => {
// the module_path with the module_dir from 0/config
format!("{}/lib/libmize_module_{}.so", module_dir_str.as_str(), name)
format!("{}/lib/libmize_module_{}.so", module_dir_str.as_str(), module_name)
}
};
let lib = unsafe { libloading::Library::new(module_path)? };