fetching modules should now work

Cargo.lock

@@ -898,8 +898,7 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
 [[package]]
 name = "http_req"
 version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/c2vi/http_req#46b0f93087ef6f68a360b303192e30ecb7b055e2"
-checksum = "92208b0986f414edaca8ede2c6962c979309346a9e8e19d07d0a7879aae1549e"
 dependencies = [
  "native-tls",
  "unicase",

Cargo.toml

@@ -63,7 +63,8 @@ ciborium-io = { version = "0.2.2", optional = true }
 tracing-core = { version = "0.1.32", optional = true }
 tar = { version = "0.4.42", optional = true }
 flate2 = { version = "1.0.33", optional = true }
-http_req = { version = "0.12.0", default-features = false, features = [ "native-tls" ], optional = true }
+#http_req = { version = "0.12.0", default-features = false, features = [ "rustls" ], optional = true }
+http_req = { git = "https://github.com/c2vi/http_req", optional = true, default-features = false, features = [ "native-tls" ] }
 
 # bare dependencies
 

flake.lock

@@ -57,18 +57,17 @@
     "mize_modules": {
       "flake": false,
       "locked": {
-        "dirtyRev": "fa3b462cbe3e63fc9ecada6c4445d366e58a6f86-dirty",
-        "dirtyShortRev": "fa3b462-dirty",
         "lastModified": 1725362504,
-        "narHash": "sha256-uqk6t8W+2qS6feADN8QaOLEIhz7e3BYn2WRccmtyQws=",
+        "narHash": "sha256-U0IQ+3SiyLb24BTDUvlnE30LdjjJWJ/P6dHPqxArJXk=",
-        "submodules": true,
+        "owner": "c2vi",
-        "type": "git",
+        "repo": "mize-modules",
-        "url": "file:///home/me/work/modules"
+        "rev": "fa3b462cbe3e63fc9ecada6c4445d366e58a6f86",
+        "type": "github"
       },
       "original": {
-        "submodules": true,
+        "owner": "c2vi",
-        "type": "git",
+        "repo": "mize-modules",
-        "url": "file:///home/me/work/modules"
+        "type": "github"
       }
     },
     "nixpkgs": {

flake.nix

@@ -18,7 +18,7 @@
     };
 
     mize_modules = {
-      url = "github:c2vi/mize-modules";
+      url = "github:c2vi/mize-modules?submodules=1";
       flake = false;
     };
 
@@ -30,12 +30,13 @@
 let
   pkgs = nixpkgs.legacyPackages.${system};
   wasmToolchain = fenix.packages.${system}.combine [
-    fenix.packages.${system}.targets.wasm32-unknown-unknown.latest.toolchain
+    fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.toolchain
-    fenix.packages.${system}.latest.toolchain
+    fenix.packages.${system}.stable.toolchain
   ];
   wasmCrane = (crane.mkLib pkgs).overrideToolchain wasmToolchain;
 
-  osToolchain = fenix.packages.${system}.latest.toolchain;
+  osToolchain = fenix.packages.${system}.stable.toolchain;
+
   osCrane = (crane.mkLib pkgs).overrideToolchain osToolchain;
 
   #winToolchain = fenix.packages.${system}.combine [
@@ -152,6 +153,10 @@ in {
 ############################## DEV SHELLS ##############################
     devShells = {
 
+      modules = mizeLib.moduleShells system;
+
+      modulesCross = pkgs.lib.genAttrs systems (system: mizeLib.moduleShells system);
+
       one = pkgs.stdenv.mkDerivation {
         name = "hiiiiiiii";
         nativeBuildInputs = [
@@ -213,24 +218,7 @@ in {
       };
 
 
-      default = pkgs.mkShell {
+      default = (mizeLib.buildMizeForSystem system).devShell;
-        buildInputs = with pkgs; [
-
-          wasm-pack #pkg-config openssl #cargo rustc
-          cargo-generate
-          (fenix.packages.${system}.combine [ wasmToolchain osToolchain ])
-          lldb gdb
-        ];
-
-        MIZE_BUILD_CONFIG = pkgs.writeTextFile {
-          name = "vic-build-config";
-          text = builtins.toJSON defaultMizeConfig;
-        };
-
-        shellHook = ''
-          export MIZE_CONFIG_FILE=${self}/test-config.toml
-        '';
-      };
     };
 
   }) // {

lib.nix

@@ -174,21 +174,24 @@ rec {
 
 
 
-    buildModule = path:  extraArgs: (pkgs.callPackage path {
+    buildModule = path: extraArgs: let
+      pkg = (pkgs.callPackage path {
         inherit mkSelString craneLib toolchain_version;
-      inherit mkMizeModule mkMizeRustModule buildModule findModules crossSystem pkgsCross pkgsNative;
+        inherit mkMizeModule mkMizeRustModule buildModule findModules crossSystem pkgsCross pkgsNative mkMizeRustShell;
         mize_version = main-default.version;
       } // extraArgs );
-
+    in pkg // {
-    mizeBildConfig = pkgs.writeTextFile {
+    };
-      name = "mize-build-config";
+
-      text = builtins.toJSON (defaultMizeConfig // {
+
-        selector = {
+    mizeBildConfig = let
-          inherit toolchain_version;
+      settingsFormat = pkgs.formats.toml { };
-          mize_version = main-default.version;
+    in settingsFormat.generate "mize-build-config.toml" {
-          system = crossSystem.nameFull;
+      config = {
+        namespace = "mize.buildtime.ns";
+        module_url = "c2vi.dev";
+        selector = builtins.fromJSON (mkSelString {});
       };
-      });
     };
 
     ####### build mize
@@ -199,12 +202,12 @@ rec {
 
       nativeBuildInputs = [ 
         pkgsCross.buildPackages.pkg-config
-        pkgsCross.buildPackages.nasm
+        #pkgsCross.buildPackages.nasm
-        pkgsCross.buildPackages.cmake
+        #pkgsCross.buildPackages.cmake
       ];
 
       buildInputs = [
-          pkgsCross.openssl
+          (if crossSystem.nameFull == "x86_64-unknown-linux-gnu" then pkgs.openssl else pkgsCross.openssl)
       ];
 
       MIZE_BUILD_CONFIG = mizeBildConfig;
@@ -270,8 +273,60 @@ rec {
       #AR_wasm32_unknown_unknown = "${pkgs.llvmPackages_14.llvm}/bin/llvm-ar";
     };
 
+
+
+  ################# dev Shels ################### 
+
+    mkMizeRustShell = attrs: mkMizeModuleShell (attrs // {
+      #_shell_type = "rust";
+      nativeBuildInputs = attrs.nativeBuildInputs or [] ++ [
+        (fenix.packages.${localSystem}.combine [ 
+          fenix.packages.${system}.stable.toolchain
+          fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.toolchain
+          fenix.packages.${system}.stable.toolchain
+        ])
+      ];
+
+    });
+
+    mkMizeModuleShell = attrs: pkgs.mkShell (attrs // {
+      MIZE_BUILD_CONFIG = mizeBildConfig;
+    });
+
+
+    mainDevShell = pkgs.mkShell {
+        buildInputs = with pkgs; [
+
+          wasm-pack #pkg-config openssl #cargo rustc
+          cargo-generate
+          (fenix.packages.${localSystem}.combine [ 
+            fenix.packages.${system}.stable.toolchain
+            fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.toolchain
+            fenix.packages.${system}.stable.toolchain
+          ])
+          openssl
+        ];
+
+        nativeBuildInputs = with pkgs; [
+         pkg-config
+        ];
+
+        MIZE_BUILD_CONFIG = mizeBildConfig;
+
+        shellHook = ''
+          export MIZE_CONFIG_FILE=${self}/test-config.toml
+        '';
+    };
+
+
+
+  ################# output attrset ################### 
+
     in rec {
       inherit toolchain_version_drv pkgsCross craneLib;
+
+      devShell = mainDevShell;
+
       main = 
         if crossSystem.kernel.name == "windows"
           then main-win
@@ -312,9 +367,9 @@ rec {
     echo got module: ${module.name}
     echo out: $out
     hash=$(echo ${module.selector_string} | sha256sum | cut -c -32)
-    cp --no-preserve=mode,ownership -r ${module}/* $out/mize/dist/$${hash}-${module.name}
+    cp --no-preserve=mode,ownership -r ${module}/* $out/mize/dist/$hash-${module.modName}
-    ${pkgs.gnutar}/bin/tar -czf $out/mize/dist/$${hash}-${module.name}.tar.gz -C ${module} .
+    ${pkgs.gnutar}/bin/tar -czf $out/mize/dist/$hash-${module.modName}.tar.gz -C ${module} .
-    echo '${module.selector_string}' > $out/mize/dist/$hash/selector
+    echo '${module.selector_string}' > $out/mize/dist/$hash-${module.modName}/selector
   '';
 
   webfiles = systems: stdenv.mkDerivation {
@@ -338,6 +393,11 @@ rec {
 	  nativeBuildInputs = [
 	  ];
   };
+
+  moduleShells = system: let
+    mizeFor = buildMizeForSystem system;
+  in builtins.listToAttrs ( map ( mod: { name = mod.modName; value = (mod.devShell); } ) mizeFor.modulesList );
+
 }
 
 

src/core/error.rs

@@ -7,13 +7,14 @@ use std::io;
 use colored::Colorize;
 use tracing::{trace, debug, info, warn, error};
 
+use crate::proto::MizeMessage;
+
+
 #[macro_export]
 macro_rules! mize_err {
     ($($arg:tt)*) => { MizeError::new().msg(format!( $($arg)*)) };
 }
 
-use crate::proto::MizeMessage;
-
 pub type MizeResult<T> = Result<T, MizeError>;
 
 pub trait MizeResultTrait<T> {
@@ -32,6 +33,7 @@ pub struct MizeError {
     pub messages: Vec<String>,
     pub caused_by_msg: Option<CausedByMessage>,
     pub code_location: Option<MizeCodeLocation>,
+    pub backtrace: String,
 }
 
 #[derive(Debug, Clone)]
@@ -85,6 +87,7 @@ impl MizeError {
             messages: Vec::new(),
             caused_by_msg: None,
             code_location: Some(caller_location.into()),
+            backtrace: format!("{}", std::backtrace::Backtrace::capture()),
         }
     }
 
@@ -110,12 +113,17 @@ impl MizeError {
 
     pub fn log(self) -> MizeError {
         error!("MizeError envountered!");
+
         if let Some(ref location) = self.code_location {
             error!("[ {} ] {}", "LOCATION".yellow(), location);
         };
+
+        error!("[ {} ] {}", "BACKTRACE".yellow(), self.backtrace);
+
         for msg in &self.messages {
             error!("[ {} ] {}", "MSG".yellow(), msg);
         }
+
         self
     }
 
@@ -258,6 +266,7 @@ fn get_error_by_code(code: u32, caller_location: &std::panic::Location) -> MizeE
         messages: Vec::new(),
         caused_by_msg: None,
         code_location: Some(caller_location.into()),
+        backtrace: format!("{}", std::backtrace::Backtrace::capture()),
     };
 
     match code {

src/core/instance/mod.rs

@@ -129,6 +129,11 @@ impl Instance {
         //instance.spawn("msg_thread", closure)?;
 
 
+        // load the config from build time
+        let config = ItemData::from_toml(BUILD_TIME_CONFIG)?;
+        instance.set_blocking("0", config);
+
+
         return Ok(instance);
     }
 
@@ -146,12 +151,8 @@ impl Instance {
 
     fn init(&mut self) -> MizeResult<()> {
 
-        // load the config from build time
-        let config = ItemData::from_toml(BUILD_TIME_CONFIG)?;
-        self.set_blocking("0", config);
-
         // platform specific init code
-        crate::platform::any::instance_init(self);
+        crate::platform::any::instance_init(self)?;
 
         // end of platform specific init code
 

src/core/instance/tests.rs

@@ -5,6 +5,47 @@ use crate::item::IntoItemData;
 
 use super::*;
 
+#[test]
+fn test_get_module_hash() -> MizeResult<()> {
+    let mut instance = Instance::empty()?;
+
+    use crate::platform::os::get_module_hash;
+    let selector_toml = r#"
+        mize_version = "0.0.0"
+        system = "x86_64-unknown-linux-gnu"
+        toolchain_version = "rustc 1.78.0 (9b00956e5 2024-04-29)"
+    "#;
+
+    instance.set_blocking("0/config/selector", ItemData::from_toml(&selector_toml)?)?;
+
+    let hash = get_module_hash(&mut instance, "String", ItemData::new())?;
+
+    test_println!("hash: {}", hash);
+
+    assert_eq!(hash, "e96edf60216688314efc263c1aa8d6ce".to_owned());
+
+    Ok(())
+}
+
+#[test]
+fn test_set_store_path() -> MizeResult<()> {
+    let instance = Instance::empty()?;
+
+    let store_path = "/home/me/.mize".to_owned();
+
+    let config = instance.get("0/config")?.as_data_full()?;
+
+    test_println!("data at 0/config: {}", config);
+
+    instance.set_blocking("0/config/store_path", store_path.clone().into_item_data())?;
+
+    let store_path_new = instance.get("0/config/store_path")?.value_string()?;
+
+    assert_eq!(store_path_new, store_path);
+
+    Ok(())
+}
+
 #[test]
 fn test_set_sub_path() -> MizeResult<()> {
     let instance = Instance::empty()?;

src/core/item.rs

@@ -15,6 +15,7 @@ use crate::error::{MizeError, MizeResult, IntoMizeResult};
 use crate::instance::{connection, Instance};
 use crate::instance::store::Store;
 use crate::id::MizeId;
+use crate::mize_err;
 use crate::proto::MizeMessage;
 use ciborium::Value as CborValue;
 use crate::instance::connection::value_raw_con_by_id;
@@ -71,6 +72,7 @@ impl Item<'_> {
                 },
                 _ => {},
             }
+            return Err(mize_err!("a /self path, but the next element in the path '{:?}' is not valid", id.nth_part(1)));
         }
 
         let self_namespace_inner = self.instance.self_namespace.lock()?;
@@ -258,7 +260,7 @@ pub fn item_data_set_path(data: &mut CborValue, path: Vec<String>, data_to_set:
     //trace!("[ {} ] path: {:?}", "ARG".yellow(), path);
     //trace!("[ {} ] data_to_set: {}", "ARG".yellow(), data_to_set.clone().into_item_data());
 
-    test_println!("###########################");
+    test_println!("item_data_set_path ###########################");
     test_println!("data: {}", ItemData::from_cbor(data.clone()));
     test_println!("path: {:?}", path);
     test_println!("data_to_set: {}", ItemData::from_cbor(data_to_set.clone()));
@@ -275,7 +277,7 @@ pub fn item_data_set_path(data: &mut CborValue, path: Vec<String>, data_to_set:
     };
     match data {
         CborValue::Map(ref mut map) => {
-            for (key, val) in map {
+            for (key, val) in &mut * map {
                 if let CborValue::Text(key_str) = key {
                     if key_str == &path_el {
                         //path_iter.next();
@@ -283,6 +285,12 @@ pub fn item_data_set_path(data: &mut CborValue, path: Vec<String>, data_to_set:
                     }
                 }
             }
+
+            // if we get here, that means, the key (which is the variable path_el) does not exist
+            // in the map
+            // so add it
+            map.push((CborValue::Text(path_el.clone()), data_to_set.to_owned()));
+            return Ok(());
         },
         CborValue::Null => {
             let mut inner_value = CborValue::Null;

src/platform/os/cli.rs

@@ -2,6 +2,7 @@ use std::path::PathBuf;
 use clap::ArgMatches;
 use home::home_dir;
 use crossbeam::channel::bounded;
+use std::io::Read;
 
 use mize::error::{IntoMizeResult, MizeError, MizeResult, MizeResultTrait};
 use mize::instance::Instance;
@@ -9,7 +10,7 @@ use mize::platform::os::config_from_cli_args;
 use mize::instance::subscription::Update;
 use mize::platform::os::fsstore::FileStore;
 use mize::instance::subscription::Subscription;
-use mize::item::IntoItemData;
+use mize::item::{IntoItemData, ItemData};
 
 
 
@@ -141,7 +142,20 @@ pub fn gui(sub_matches: &ArgMatches) -> MizeResult<()> {
     Ok(())
 }
 
-
+pub fn format_cbor(sub_matches: &ArgMatches) -> MizeResult<()> {
+
+    let mut input: Vec<u8> =  Vec::new();
+    let stdin = std::io::stdin();
+    let mut handle = stdin.lock();
+    //handle.read_to_end(&mut input);
+
+    let cbor_data = ciborium::from_reader(handle)?;
+    let item_data = ItemData::from_cbor(cbor_data);
+
+    println!("{}", item_data);
+
+    Ok(())
+}
 
 
 

src/platform/os/fsstore.rs

@@ -111,7 +111,8 @@ impl Store for FileStore {
         if !file_path.exists() {
             
         }
-        let file = OpenOptions::new().read(true).open(file_path)?;
+        let file = OpenOptions::new().read(true).open(file_path.clone())
+            .map_err(|e| mize_err!("fsstoree: Unable to open '{}', err: {}", file_path.display(), e))?;
 
         let cbor_value: CborValue = ciborium::from_reader(file)
             .mize_result_msg(format!("could not read file '{}' from FileStore", self.path.join("store").join(id.store_part()).display()))?;

src/platform/os/main.rs

@@ -66,6 +66,9 @@ fn main() {
         // mi gui
         Some(("gui", sub_matches)) => cli::gui(sub_matches),
 
+        // mi format-cbor
+        Some(("format-cbor", sub_matches)) => cli::format_cbor(sub_matches),
+
         // some unknown command passed
         Some((cmd, sub_matches)) => Err(MizeError::new().msg(format!("The subcommand: {} is not known. use --help to list availavle commands", cmd))),
 
@@ -167,6 +170,9 @@ fn cli_matches() -> clap::ArgMatches {
         .subcommand(
                 Command::new("gui")
             )
+        .subcommand(
+                Command::new("format-cbor")
+            )
         .arg_required_else_help(true);
 
     return main.get_matches();

src/platform/os/mod.rs

@@ -72,17 +72,29 @@ pub fn os_instance_init(instance: &mut Instance) -> MizeResult<()> {
         }
     };
 
+
     ////// if a config.store_path is set, upgrade to the filestore there
     let mut test = instance.get("0")?.as_data_full()?;
     let mut store_path = match instance.get("0/config/store_path")?.value_string() {
-        Ok(path) => path,
+        Ok(path) => {
+            path
+        },
         Err(e) => {
             // the default store_path: $HOME/.mize
             let home_dir = env!("HOME");
             if home_dir == "" {
                 return Err(mize_err!("env var $HOME empty"));
             }
-            home_dir.to_owned() + "/.mize"
+
+            let store_path = home_dir.to_owned() + "/.mize";
+
+            instance.set_blocking("0/config/store_path", store_path.clone().into_item_data())?;
+
+            let store_path = instance.get("0/config/store_path")?.value_string()?;
+
+            println!("store_path: {}", store_path);
+
+            store_path
         },
     };
 
@@ -136,7 +148,7 @@ pub fn seconds_since_modification(path: &Path) -> MizeResult<u64> {
 
 pub fn get_module_hash(instance: &mut Instance, name: &str, mut options: ItemData) -> MizeResult<String> {
 
-    let mut selector_data = instance.get("self/config/selector")?.as_data_full()?;
+    let mut selector_data = instance.get("0/config/selector")?.as_data_full()?;
 
     selector_data.set_path("name", name)?;
 
@@ -150,16 +162,23 @@ pub fn get_module_hash(instance: &mut Instance, name: &str, mut options: ItemDat
 
     let result = hasher.finalize();
 
-    return Ok(format!("{:02X?}", result));
+    let mut hex: String = result.iter()
+        .map(|b| format!("{:02x}", b).to_string())
+        .collect::<Vec<String>>()
+        .join("");
+
+    hex.truncate(32);
+
+    return Ok(hex);
 
 }
 
-fn fetch_module(instance: &mut Instance, store_path: &str, module_url: &str, module_hash: &str) -> MizeResult<()> {
+fn fetch_module(instance: &mut Instance, store_path: &str, module_url: &str, module_hash: &str, module_name: &str) -> MizeResult<()> {
 
     let tmp_file_path_gz = format!("{}/{}.tar.gz", store_path, module_hash);
     let mut tmp_gz_file = File::create(&tmp_file_path_gz)?;
 
-    http_req::request::get(format!("http://{}/mize/dist/{}.tar.gz", module_url, module_hash), &mut tmp_gz_file)?;
+    http_req::request::get(format!("http://{}/mize/dist/{}-{}.tar.gz", module_url, module_hash, module_name), &mut tmp_gz_file)?;
 
     let tar = GzDecoder::new(tmp_gz_file);
     let mut archive = Archive::new(tar);
@@ -177,29 +196,31 @@ fn fetch_module(instance: &mut Instance, store_path: &str, module_url: &str, mod
     Ok(())
 }
 
-pub fn load_module(instance: &mut Instance, name: &str, path: Option<PathBuf>) -> MizeResult<()> {
+pub fn load_module(instance: &mut Instance, module_name: &str, path: Option<PathBuf>) -> MizeResult<()> {
 
-    let module_dir_str = instance.get(format!("0/config/module_dir/{}", name))?.value_string()?;
+    let module_dir_str = instance.get(format!("0/config/module_dir/{}", module_name))?.value_string();
 
-    let module_path = if module_dir_str.clone().into_item_data() == ItemData::new() {
+    let module_path = match module_dir_str {
+        Err(_) => {
             // if null, load it from the url
             let module_url = instance.get("0/config/module_url")?.value_string()?;
 
-        let store_path = instance.get("self/store_path")?.value_string()?;
+            let store_path = instance.get("0/config/store_path")?.value_string()?;
 
             let module_hash = get_module_hash(instance, name, ItemData::new())?;
 
             if !PathBuf::from(format!("{}/modules/{}", store_path, module_hash.as_str())).exists() {
                 // fetch module if it does not exist
-            fetch_module(instance, store_path.as_str(), module_url.as_str(), module_hash.as_str())?;
+                fetch_module(instance, store_path.as_str(), module_url.as_str(), module_hash.as_str(), module_name.as_str())?;
             }
 
             // the module_path
-        format!("{}/modules/{}/lib/libmize_module_{}.so", store_path, module_hash, name)
+            format!("{}/modules/{}/lib/libmize_module_{}.so", store_path, module_hash, module_name)
-
+        },
-    } else {
+        Ok(module_dir_str) => {
             // the module_path with the module_dir from 0/config
-        format!("{}/lib/libmize_module_{}.so", module_dir_str.as_str(), name)
+            format!("{}/lib/libmize_module_{}.so", module_dir_str.as_str(), module_name)
+        }
     };
 
     let lib = unsafe { libloading::Library::new(module_path)? };