new module loading mechanism for os-target

Cargo.lock

@@ -17,6 +17,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
 [[package]]
 name = "ahash"
 version = "0.8.11"
@@ -137,7 +143,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -227,7 +233,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -240,7 +246,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "miniz_oxide",
+ "miniz_oxide 0.7.3",
  "object",
  "rustc-demangle",
 ]
@@ -324,9 +330,12 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
 
 [[package]]
 name = "cc"
-version = "1.0.98"
+version = "1.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
+checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cfg-if"
@@ -432,6 +441,16 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.6"
@@ -651,12 +670,12 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.30"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
+checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
 dependencies = [
  "crc32fast",
- "miniz_oxide",
+ "miniz_oxide 0.8.0",
 ]
 
 [[package]]
@@ -665,6 +684,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -861,6 +895,16 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
 
+[[package]]
+name = "http_req"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92208b0986f414edaca8ede2c6962c979309346a9e8e19d07d0a7879aae1549e"
+dependencies = [
+ "native-tls",
+ "unicase",
+]
+
 [[package]]
 name = "httparse"
 version = "1.8.0"
@@ -1077,6 +1121,15 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "miniz_oxide"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+dependencies = [
+ "adler2",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.11"
@@ -1103,14 +1156,18 @@ dependencies = [
  "crossbeam",
  "daemonize",
  "dyn-clone",
+ "flate2",
  "home",
+ "http_req",
  "interner",
  "libloading",
  "nix",
  "serde",
  "serde-wasm-bindgen",
  "serde_json",
+ "sha2",
  "sysinfo",
+ "tar",
  "tokio",
  "toml",
  "tracing",
@@ -1141,6 +1198,23 @@ dependencies = [
  "twoway",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "nix"
 version = "0.27.1"
@@ -1220,6 +1294,50 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "openssl"
+version = "0.10.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
+dependencies = [
+ "bitflags 2.5.0",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.77",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -1249,7 +1367,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -1264,6 +1382,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -1508,12 +1632,44 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
 
+[[package]]
+name = "schannel"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "scoped-tls"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
 
+[[package]]
+name = "security-framework"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0"
+dependencies = [
+ "bitflags 2.5.0",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "semver"
 version = "1.0.23"
@@ -1548,7 +1704,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -1610,6 +1766,17 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
 
+[[package]]
+name = "sha2"
+version = "0.10.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
 [[package]]
 name = "sharded-slab"
 version = "0.1.7"
@@ -1619,6 +1786,12 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -1687,9 +1860,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.66"
+version = "2.0.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
+checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1717,6 +1890,17 @@ dependencies = [
  "windows",
 ]
 
+[[package]]
+name = "tar"
+version = "0.4.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.10.1"
@@ -1746,7 +1930,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -1963,7 +2147,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -2140,6 +2324,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -2212,7 +2402,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "wasm-bindgen-shared",
 ]
 
@@ -2299,7 +2489,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -2343,7 +2533,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -2613,6 +2803,17 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "xattr"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f"
+dependencies = [
+ "libc",
+ "linux-raw-sys",
+ "rustix",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.7.35"
@@ -2630,7 +2831,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]

Cargo.toml

@@ -22,7 +22,7 @@ required-features = [ "os-target" ]
 [features]
 we_alloc = []
 default = [ "os-target" ]
-os-target = [ "clap", "home", "nix", "sysinfo", "ciborium/default", "tracing-subscriber", "async", "tracing-subscriber/env-filter", "libloading", "ciborium-io", "tracing-core" ]
+os-target = [ "clap", "home", "nix", "sysinfo", "ciborium/default", "tracing-subscriber", "async", "tracing-subscriber/env-filter", "libloading", "ciborium-io", "tracing-core", "tar", "flate2", "http_req" ]
 wasm-target = [ "wasm-bindgen", "console_error_panic_hook", "wasm-bindgen-cli", "web-sys", "web-sys/Worker", "web-sys/Window", "web-sys/WorkerOptions", "web-sys/WorkerType", "serde-wasm-bindgen" ]
 async = ["tokio/net", "tokio", "tokio/rt-multi-thread", "tokio/io-util"]
 
@@ -61,6 +61,9 @@ tokio = { version = "1.21.0", features = ["sync"], optional = true }
 libloading = { version = "0.8.5", optional = true }
 ciborium-io = { version = "0.2.2", optional = true }
 tracing-core = { version = "0.1.32", optional = true }
+tar = { version = "0.4.42", optional = true }
+flate2 = { version = "1.0.33", optional = true }
+http_req = { version = "0.12.0", default-features = false, features = [ "native-tls" ], optional = true }
 
 # bare dependencies
 
@@ -68,6 +71,7 @@ tracing-core = { version = "0.1.32", optional = true }
 axum = { version = "0.6.1", features = ["ws", "headers"], optional = true }
 axum-extra = { version = "0.4.2", features = ["spa"], optional = true }
 axum-macros = { version = "0.3.0", optional = true }
+sha2 = "0.10.8"
 #tokio-core = "0.1.18"
 #tokio-io = "0.1.13"
 

flake.lock

@@ -60,7 +60,7 @@
         "dirtyRev": "fa3b462cbe3e63fc9ecada6c4445d366e58a6f86-dirty",
         "dirtyShortRev": "fa3b462-dirty",
         "lastModified": 1725362504,
-        "narHash": "sha256-whgym05xZUVEtO6/F4id2XN7MBQTs7VUEhF8PEJV41E=",
+        "narHash": "sha256-uqk6t8W+2qS6feADN8QaOLEIhz7e3BYn2WRccmtyQws=",
         "submodules": true,
         "type": "git",
         "url": "file:///home/me/work/modules"

lib.nix

@@ -122,6 +122,8 @@ rec {
       });
     });
 
+
+
     ########## build Rust Module
     mkMizeRustModule = attrs: craneLib.buildPackage (attrs // {
       MIZE_BUILD_CONFIG = mizeBildConfig;
@@ -129,7 +131,6 @@ rec {
         inherit (attrs) modName;
       });
     }
-
     # linux specific stuff
     // (if crossSystem.kernel.name == "linux" then {
       "CARGO_TARGET_${builtins.replaceStrings ["-"] ["_"] (pkgsCross.lib.strings.toUpper crossSystem.nameFull)}_LINKER" = "${pkgsCross.stdenv.cc.targetPrefix}cc";
@@ -171,6 +172,8 @@ rec {
     } else {})
     );
 
+
+
     buildModule = path:  extraArgs: (pkgs.callPackage path {
       inherit mkSelString craneLib toolchain_version;
       inherit mkMizeModule mkMizeRustModule buildModule findModules crossSystem pkgsCross pkgsNative;
@@ -180,8 +183,11 @@ rec {
     mizeBildConfig = pkgs.writeTextFile {
       name = "mize-build-config";
       text = builtins.toJSON (defaultMizeConfig // {
+        selector = {
           inherit toolchain_version;
           mize_version = main-default.version;
+          system = crossSystem.nameFull;
+          };
       });
     };
 
@@ -191,6 +197,16 @@ rec {
       cargoExtraArgs = "--bin mize --features os-target";
       strictDeps = true;
 
+      nativeBuildInputs = [ 
+        pkgsCross.buildPackages.pkg-config
+        pkgsCross.buildPackages.nasm
+        pkgsCross.buildPackages.cmake
+      ];
+
+      buildInputs = [
+          pkgsCross.openssl
+      ];
+
       MIZE_BUILD_CONFIG = mizeBildConfig;
 
       # patch the interpreter to run on most linux-gnu distros
@@ -214,6 +230,8 @@ rec {
     main-win = craneLib.buildPackage {
       src = craneLib.cleanCargoSource ./.;
 
+      #nativeBuildInputs = [ pkgsCross.buildPackages.nasm pkgsCross.buildPackages.cmake ];
+
       strictDeps = true;
       doCheck = false;
 
@@ -294,7 +312,8 @@ rec {
     echo got module: ${module.name}
     echo out: $out
     hash=$(echo ${module.selector_string} | sha256sum | cut -c -32)
-    cp --no-preserve=mode,ownership -r ${module}/* $out/mize/dist/$hash
+    cp --no-preserve=mode,ownership -r ${module}/* $out/mize/dist/$${hash}-${module.name}
+    ${pkgs.gnutar}/bin/tar -czf $out/mize/dist/$${hash}-${module.name}.tar.gz -C ${module} .
     echo '${module.selector_string}' > $out/mize/dist/$hash/selector
   '';
 

src/core/item.rs

@@ -166,6 +166,15 @@ impl ItemData {
         return ItemData (data);
     }
 
+    pub fn to_json(self) -> MizeResult<String> {
+
+        let mut result: Vec<u8> = Vec::new();
+
+        self.0.serialize(&mut serde_json::Serializer::new(&mut result))?;
+
+        Ok(String::from_utf8(result)?)
+    }
+
     pub fn merge(&mut self, other: ItemData) {
         item_data_merge(&mut self.0, &other.0);
     }

src/platform/os/mod.rs

@@ -6,6 +6,11 @@ use tracing::{debug, error, info, trace, warn};
 use clap::ArgMatches;
 use std::path::{Path, PathBuf};
 use std::time::UNIX_EPOCH;
+use sha2::{Sha256, Sha512, Digest};
+use flate2::read::GzDecoder;
+use tar::Archive;
+use std::fs::File;
+use std::io::copy;
 
 use crate::id::MizeId;
 use crate::instance::store::Store;
@@ -129,39 +134,73 @@ pub fn seconds_since_modification(path: &Path) -> MizeResult<u64> {
     Ok(duration.as_secs())
 }
 
-fn get_correct_module_path(module_dir: &Path, name: &str) -> MizeResult<PathBuf> {
+pub fn get_module_hash(instance: &mut Instance, name: &str, mut options: ItemData) -> MizeResult<String> {
-    let mut release_path = module_dir.join("modules").join(name).join("target").join("release").join(format!("libmize_module_{}.so", name));
+
-    let mut debug_path = module_dir.join("modules").join(name).join("target").join("debug").join(format!("libmize_module_{}.so", name));
+    let mut selector_data = instance.get("self/config/selector")?.as_data_full()?;
+
+    selector_data.set_path("name", name)?;
+
+    selector_data.merge(options);
+
+    let selector_str = selector_data.to_json()?;
+
+    let mut hasher = Sha256::new();
+
+    hasher.update(selector_str.as_bytes());
+
+    let result = hasher.finalize();
+
+    return Ok(format!("{:02X?}", result));
 
-    if !release_path.exists() && !debug_path.exists() {
-        release_path = module_dir.join("modules").join(name).join("target").join("release").join(format!("lib{}.so", name));
-        debug_path = module_dir.join("modules").join(name).join("target").join("debug").join(format!("lib{}.so", name));
 }
 
-    let release_modtime = match seconds_since_modification(&release_path) {
+fn fetch_module(instance: &mut Instance, store_path: &str, module_url: &str, module_hash: &str) -> MizeResult<()> {
-        Ok(secs) => secs,
-        Err(e) => { return Ok(debug_path); },
-    };
 
-    let debug_modtime = match seconds_since_modification(&debug_path) {
+    let tmp_file_path_gz = format!("{}/{}.tar.gz", store_path, module_hash);
-        Ok(secs) => secs,
+    let mut tmp_gz_file = File::create(&tmp_file_path_gz)?;
-        Err(e) => { return Ok(release_path); },
-    };
 
-    if seconds_since_modification(&release_path)? > seconds_since_modification(&debug_path)? {
+    http_req::request::get(format!("http://{}/mize/dist/{}.tar.gz", module_url, module_hash), &mut tmp_gz_file)?;
-        return Ok(debug_path);
-    } else {
-        return Ok(release_path);
-    };
 
+    let tar = GzDecoder::new(tmp_gz_file);
+    let mut archive = Archive::new(tar);
+
+    //archive.set_mask(umask::Mode::parse("rwxrwxrwx")?.into());
+
+    let target_dir = format!("{}/modules/{}", store_path, module_hash);
+
+    std::fs::create_dir_all(target_dir.as_str())?;
+
+    archive.unpack(target_dir.as_str())?;
+
+    fs::remove_file(&tmp_file_path_gz)?;
+
+    Ok(())
 }
 
 pub fn load_module(instance: &mut Instance, name: &str, path: Option<PathBuf>) -> MizeResult<()> {
 
-    let module_dir_str = instance.get("0/config/module_dir")?.value_string()?;
+    let module_dir_str = instance.get(format!("0/config/module_dir/{}", name))?.value_string()?;
-    let module_dir = Path::new(&module_dir_str);
 
-    let module_path = get_correct_module_path(module_dir, name)?;
+    let module_path = if module_dir_str.clone().into_item_data() == ItemData::new() {
+        // if null, load it from the url
+        let module_url = instance.get("0/config/module_url")?.value_string()?;
+
+        let store_path = instance.get("self/store_path")?.value_string()?;
+
+        let module_hash = get_module_hash(instance, name, ItemData::new())?;
+
+        if !PathBuf::from(format!("{}/modules/{}", store_path, module_hash.as_str())).exists() {
+            // fetch module if it does not exist
+            fetch_module(instance, store_path.as_str(), module_url.as_str(), module_hash.as_str())?;
+        }
+
+        // the module_path
+        format!("{}/modules/{}/lib/libmize_module_{}.so", store_path, module_hash, name)
+
+    } else {
+        // the module_path with the module_dir from 0/config
+        format!("{}/lib/libmize_module_{}.so", module_dir_str.as_str(), name)
+    };
 
     let lib = unsafe { libloading::Library::new(module_path)? };
 
@@ -170,8 +209,6 @@ pub fn load_module(instance: &mut Instance, name: &str, path: Option<PathBuf>) -
     let mut module: Box<dyn Module + Send + Sync> = Box::new(EmptyModule {});
 
     unsafe { func(&mut module) };
-    println!("hiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii load module");
-
 
     let mut modules_inner = instance.modules.lock()?;