module loading in wasm

This commit is contained in:
Sebastian Moser
2025-07-01 15:21:39 +02:00
parent f87b2270a5
commit 1ca2dcaf73
10 changed files with 253 additions and 67 deletions

61
Cargo.lock generated
View File

@@ -29,7 +29,7 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"once_cell",
"version_check",
"zerocopy",
@@ -244,7 +244,7 @@ checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11"
dependencies = [
"addr2line",
"cc",
"cfg-if",
"cfg-if 1.0.0",
"libc",
"miniz_oxide 0.7.3",
"object",
@@ -337,6 +337,12 @@ dependencies = [
"shlex",
]
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
@@ -437,7 +443,7 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"wasm-bindgen",
]
@@ -472,7 +478,7 @@ version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@@ -631,7 +637,7 @@ version = "0.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"libc",
"redox_syscall",
"windows-sys 0.52.0",
@@ -746,7 +752,7 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"js-sys",
"libc",
"wasi",
@@ -776,7 +782,7 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"crunchy",
]
@@ -1039,7 +1045,7 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"windows-targets 0.52.5",
]
@@ -1086,6 +1092,12 @@ version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
[[package]]
name = "memory_units"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3"
[[package]]
name = "mime"
version = "0.3.17"
@@ -1169,6 +1181,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-bindgen-test",
"web-sys",
"wee_alloc",
]
[[package]]
@@ -1222,7 +1235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
dependencies = [
"bitflags 2.5.0",
"cfg-if",
"cfg-if 1.0.0",
"libc",
]
@@ -1301,7 +1314,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
dependencies = [
"bitflags 2.5.0",
"cfg-if",
"cfg-if 1.0.0",
"foreign-types",
"libc",
"once_cell",
@@ -1534,7 +1547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
dependencies = [
"cc",
"cfg-if",
"cfg-if 1.0.0",
"getrandom",
"libc",
"spin",
@@ -1761,7 +1774,7 @@ version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"digest",
]
@@ -1778,7 +1791,7 @@ version = "0.10.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"digest",
]
@@ -1879,7 +1892,7 @@ version = "0.30.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"core-foundation-sys",
"libc",
"ntapi",
@@ -1905,7 +1918,7 @@ version = "3.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"fastrand",
"rustix",
"windows-sys 0.52.0",
@@ -1937,7 +1950,7 @@ version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"once_cell",
]
@@ -2378,7 +2391,7 @@ version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"once_cell",
"wasm-bindgen-macro",
]
@@ -2457,7 +2470,7 @@ version = "0.4.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"js-sys",
"once_cell",
"wasm-bindgen",
@@ -2621,6 +2634,18 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "wee_alloc"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e"
dependencies = [
"cfg-if 0.1.10",
"libc",
"memory_units",
"winapi",
]
[[package]]
name = "winapi"
version = "0.3.9"

View File

@@ -20,10 +20,9 @@ path = "src/platform/os/main.rs"
required-features = [ "os-target" ]
[features]
we_alloc = []
default = [ "os-target" ]
os-target = [ "clap", "home", "nix", "sysinfo", "ciborium/default", "tracing-subscriber", "async", "tracing-subscriber/env-filter", "libloading", "ciborium-io", "tracing-core", "tar", "flate2", "http_req" ]
wasm-target = [ "wasm-bindgen", "console_error_panic_hook", "wasm-bindgen-cli", "web-sys", "web-sys/Worker", "web-sys/Window", "web-sys/WorkerOptions", "web-sys/WorkerType", "serde-wasm-bindgen", "wasm-bindgen-futures" ]
wasm-target = [ "wasm-bindgen", "console_error_panic_hook", "wasm-bindgen-cli", "web-sys", "web-sys/Worker", "web-sys/Window", "web-sys/Request", "web-sys/RequestInit", "web-sys/RequestMode", "web-sys/Response", "web-sys/WorkerOptions", "web-sys/WorkerType", "serde-wasm-bindgen", "wasm-bindgen-futures", "wee_alloc"]
async = ["tokio/net", "tokio", "tokio/rt-multi-thread", "tokio/io-util"]
@@ -51,6 +50,7 @@ wasm-bindgen-cli = { version = "0.2.92", optional = true }
web-sys = { version = "0.3.70", optional = true }
serde-wasm-bindgen = { version = "0.6.5", optional = true }
wasm-bindgen-futures = { version = "0.4.49", optional = true }
wee_alloc = { version = "0.4.5", optional = true }
# os dependencies
clap = { version = "4.4.11", features = ["cargo"], optional = true }

12
flake.lock generated
View File

@@ -23,11 +23,11 @@
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1734935689,
"narHash": "sha256-yl/iko/0pvRN3PF6Z4FjQeb6AuGiavMENEisQWJ78h0=",
"lastModified": 1748327829,
"narHash": "sha256-TJwbWCsyT9yJkBRTnqKE/YDTiXuEdYHb37WLK4TcAwE=",
"owner": "nix-community",
"repo": "fenix",
"rev": "30616281e9bfe0883acb3369f2b89aad6850706f",
"rev": "26fbcc33a5e94d6d0dc5f467eca8f7a746a54e6c",
"type": "github"
},
"original": {
@@ -99,11 +99,11 @@
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1734874959,
"narHash": "sha256-NlsVD/fI32wsHFua9Xvc7IFHCUpQIOs6D6RS/3AhMT8=",
"lastModified": 1748295258,
"narHash": "sha256-ejRVtsT7fat4mGgDQpgIGKJP+vbr9r4Mo84jRb2bndk=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "fa4a40bbe867ed54f5a7c905b591fd7d60ba35eb",
"rev": "7fa66d67a72bcad236b50907e14f7464c47ecede",
"type": "github"
},
"original": {

View File

@@ -172,29 +172,11 @@ in {
one = pkgs.stdenv.mkDerivation {
name = "hiiiiiiii";
nativeBuildInputs = [
(fenix.packages.${system}.combine [ wasmToolchain osToolchain winToolchain ])
];
buildInputs = [
pkgs.pkgsCross.mingwW64.stdenv.cc.cc
pkgs.pkgsCross.mingwW64.windows.pthreads
(fenix.packages.${system}.combine [ wasmToolchain osToolchain winToolchain ])
];
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
pkgs.pkgsCross.mingwW64.windows.pthreads
];
RUSTFLAGS="-L ${pkgs.pkgsCross.mingwW64.windows.pthreads}/lib";
CARGO_BUILD_TARGET = "x86_64-pc-windows-gnu";
#fixes issues related to openssl
OPENSSL_DIR = "${pkgs.openssl.dev}";
OPENSSL_LIB_DIR = "${pkgs.openssl.out}/lib";
OPENSSL_INCLUDE_DIR = "${pkgs.openssl.dev}/include/";
depsBuildBuild = with pkgs; [
pkgsCross.mingwW64.stdenv.cc
pkgsCross.mingwW64.windows.pthreads
(fenix.packages.${system}.combine [
fenix.packages.${system}.targets.wasm32-unknown-unknown.latest.toolchain
fenix.packages.${system}.latest.toolchain
fenix.packages.${system}.latest.rust-src
])
];
};

View File

@@ -269,6 +269,7 @@ rec {
mkMizeRustShell = attrs: mkMizeModuleShell (attrs // {
#_shell_type = "rust";
nativeBuildInputs = attrs.nativeBuildInputs or [] ++ [
pkgs.pkg-config
(fenix.packages."x86_64-linux".combine [
fenix.packages."x86_64-linux".stable.toolchain
fenix.packages."x86_64-linux".targets.wasm32-unknown-unknown.stable.toolchain
@@ -289,12 +290,16 @@ rec {
})
];
buildInputs = [
pkgsCross.openssl
];
});
mkMizeModuleShell = attrs: pkgs.mkShell (attrs // {
MIZE_BUILD_CONFIG = mizeBuildConfigStr;
IS_MIZE_DEV_SHELL = "true";
});

View File

@@ -1,5 +0,0 @@
[toolchain]
channel = "nightly-2024-07-30"
components = [ "cargo", "rustc-dev", "rust-src" ]
targets = [ "wasm32-unknown-unknown" ]
profile = "full"

View File

@@ -42,6 +42,26 @@ pub mod msg_thread;
pub mod module;
// console_log macro
// that can be copied into other files for debugging purposes
#[cfg(feature = "wasm-target")]
use wasm_bindgen::prelude::*;
#[cfg(feature = "wasm-target")]
#[wasm_bindgen]
extern "C" {
#[wasm_bindgen(js_namespace = console)]
fn log(s: &str);
}
#[cfg(feature = "wasm-target")]
macro_rules! console_log {
// Note that this is using the `log` function imported above during
// `bare_bones`
($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string())})
}
#[cfg(test)]
@@ -150,11 +170,16 @@ impl Instance {
//let msg_closure = move || msg_thread(op_rx, instance_clone);
//instance.spawn("msg_thread", closure)?;
#[cfg(feature = "wasm-target")]
console_log!("before loading build time config");
// load the config from build time
let config = ItemData::from_toml(BUILD_TIME_CONFIG)?;
instance.set_blocking("0", config);
#[cfg(feature = "wasm-target")]
console_log!("after loading build time config");
return Ok(instance);
}
@@ -187,6 +212,8 @@ impl Instance {
match self.get("0/config/load_modules")?.value_string() {
Ok(modules_to_load) => {
for module in modules_to_load.split(" ") {
#[cfg(feature = "wasm-target")]
console_log!("loading module in Instance::init() ... {}", module);
self.load_module(module)?;
}
},

View File

@@ -24,6 +24,26 @@ pub trait Module: {
}
// console_log macro
// that can be copied into other files for debugging purposes
#[cfg(feature = "wasm-target")]
use wasm_bindgen::prelude::*;
#[cfg(feature = "wasm-target")]
#[wasm_bindgen]
extern "C" {
#[wasm_bindgen(js_namespace = console)]
fn log(s: &str);
}
#[cfg(feature = "wasm-target")]
macro_rules! console_log {
// Note that this is using the `log` function imported above during
// `bare_bones`
($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string())})
}
pub struct EmptyModule {
}
@@ -31,6 +51,9 @@ pub struct EmptyModule {
impl Module for EmptyModule {
fn init(&mut self, instance: &Instance) -> MizeResult<()> {
println!("empty module fn init");
#[cfg(feature = "wasm-target")]
console_log!("empty module fn init");
Ok(())
}
fn exit(&mut self, instance: &Instance) -> MizeResult<()> {

View File

@@ -81,12 +81,14 @@ pub fn os_instance_init(instance: &mut Instance) -> MizeResult<()> {
},
Err(e) => {
// the default store_path: $HOME/.mize
let home_dir = env!("HOME");
if home_dir == "" {
return Err(mize_err!("env var $HOME empty"));
}
let home_dir = if let Some(dir) = std::env::home_dir() {
dir
} else {
return Err(mize_err!("could not get home_dir and store_path is not set in the config"));
};
let store_path = home_dir.to_owned() + "/.mize";
let store_path_buff = home_dir.join(".mize");
let store_path = store_path_buff.display().to_string();
instance.set_blocking("self/config/store_path", store_path.clone().into_item_data())?;

View File

@@ -1,6 +1,8 @@
use core::convert::From;
use core::option::Option;
use std::ptr::NonNull;
use std::panic;
use web_sys::js_sys::{self, eval};
use web_sys::js_sys::{self, eval, Promise};
use crate::id::MizeId;
use crate::item::{Item, ItemData};
use crate::platform::wasm::js_sys::Function;
@@ -12,6 +14,7 @@ use crate::error::MizeResult;
use crate::{mize_err, Module};
use crate::MizeError;
use crate::core::item::IntoItemData;
use crate::instance::module::EmptyModule;
@@ -45,9 +48,6 @@ macro_rules! console_log {
// When the `wee_alloc` feature is enabled, use `wee_alloc` as the global
// allocator.
#[cfg(feature = "wee_alloc")]
#[global_allocator]
static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
@@ -61,13 +61,132 @@ pub fn wasm_instance_init(instance: &mut Instance) -> MizeResult<()> {
pub fn load_module(instance: &mut Instance, name: &str, path: Option<String>) -> MizeResult<()> {
console_log!("loading module: {}", name);
let hi = 1;
let hi = 1;
let hi = 1;
wasm_bindgen_futures::spawn_local(load_module_async(instance.clone(), name.to_owned(), path));
Ok(())
}
async fn load_module_async(mut instance: Instance, name: String, path: Option<String>) -> () {
let name = name.as_str();
let module_url = fetch_module(&mut instance, name).unwrap();
let memory = wasm_bindgen::memory();
let mut empty_module: Box<dyn Module + Send + Sync> = Box::new(EmptyModule {});
let mut mize = instance.clone();
let empty_module_ptr = Box::into_raw(Box::new(empty_module));
let empty_module_usize = empty_module_ptr as usize;
let mize_usize = Box::into_raw(Box::new(mize)) as usize;
console_log!("testttttttttttttt in load_module_async...");
let function_str = format!(r#"
const promise = new Promise((resolve, reject) => {{
import("{module_url}/mize_module_{name}.js").then( module => {{
const wasm_bindgen = module.get_wasm_bindgen();
const {{ wasm_get_mize_module_mme }} = wasm_bindgen;
wasm_bindgen("{module_url}/mize_module_{name}_bg.wasm", memory).then( () => {{
console.log("empty_module_usize:", empty_module_usize)
console.log("mize_usize:", mize_usize)
//resolve()
resolve(wasm_get_mize_module_mme(empty_module_usize, mize_usize))
}})
}})
}})
return promise;
"#);
let function = Function::new_with_args("memory, empty_module_usize, mize_usize", &function_str);
let ret_value: JsValue = match function.call3(&JsValue::null(), &memory, &empty_module_usize.into(), &mize_usize.into()) {
Ok(val) => val,
Err(err) => {
console_log!("failed to load the wasm mize module....");
console_log!("{:?}", err);
return;
},
};
console_log!("got ret_value: {:?}", ret_value);
let ret_promise = Promise::from(ret_value);
let ret_result = match wasm_bindgen_futures::JsFuture::from(ret_promise).await {
Ok(val) => val,
Err(err) => {
console_log!("err awaiting ret_promise");
console_log!("{:?}", err);
return;
},
};
let mut filled_module = unsafe {
Box::from_raw(empty_module_ptr)
};
console_log!("before....");
let a = instance.get("0/config");
console_log!("after....");
//filled_module.init(&instance);
/*
console_log!("got ret_result: {:?}", ret_result);
let ret_f64 = ret_result.as_f64().unwrap();
console_log!("got ret_f64: {:?}", ret_f64);
let ret_usize = ret_f64.floor() as usize;
let mut module: Box<dyn Module + Send + Sync> = Box::new(EmptyModule {});
unsafe {
let get_mize_module_fn_ptr = ret_usize as *const fn(&mut Box<dyn Module + Send + Sync>, Instance) -> ();
let get_mize_module_fn: fn(&mut Box<dyn Module + Send + Sync>, Instance) -> () = unsafe { std::mem::transmute(get_mize_module_fn_ptr) };
get_mize_module_fn(&mut module, instance.clone());
}
*/
}
pub fn fetch_module(instance: &mut Instance, name: &str) -> MizeResult<String> {
let module_name_with_slashes = name.replace(".", "/");
if let Ok(module_path) = instance.get(format!("self/config/module_dir/{}", module_name_with_slashes))?.value_string() {
return Ok(module_path);
} else {
let module_url = instance.get("self/config/module_url")?.value_string()?;
return Ok(format!("{}/{}", module_url, module_name_with_slashes));
}
}
async fn download_module(instance: &mut Instance, name: &str) -> MizeResult<js_sys::ArrayBuffer> {
use wasm_bindgen_futures::JsFuture;
use web_sys::{Request, RequestInit, RequestMode, Response};
let opts = RequestInit::new();
opts.set_method("GET");
opts.set_mode(RequestMode::Cors);
let url = format!("{}/mize_module_{}_bg.wasm", fetch_module(instance, name)?, name);
let request = Request::new_with_str_and_init(&url, &opts).unwrap();
let window = web_sys::window().unwrap();
let resp_value = JsFuture::from(window.fetch_with_request(&request)).await.unwrap();
let resp: Response = resp_value.dyn_into().unwrap();
let wasm_bytes: js_sys::ArrayBuffer = JsFuture::from(resp.array_buffer().unwrap()).await.unwrap().dyn_into().unwrap();
Ok(wasm_bytes)
}
#[wasm_bindgen]
pub struct JsInstance {
inner: NonNull<Instance>,
@@ -82,7 +201,6 @@ pub struct JsItemHandle {
#[wasm_bindgen]
pub async fn new_js_instance(config_json_str: String) -> JsInstance {
console_log!("hooooooooooooooo");
panic::set_hook(Box::new(console_error_panic_hook::hook));
@@ -117,6 +235,15 @@ impl JsInstance {
let item = self.inner.as_mut().get(id)?;
Ok(JsItemHandle { instance: self.inner, id: item.id()})
}
}
impl JsInstance {
pub fn inner(&mut self) -> &mut Instance {
unsafe {
self.inner.as_mut()
}
}
}