updater_thread now works and we listen on unix_socket

src/core/error.rs

@@ -5,7 +5,7 @@ use std::fmt::format;
 use std::string::FromUtf8Error;
 use std::io;
 use colored::Colorize;
-use log::{trace, debug, info, warn, error};
+use tracing::{trace, debug, info, warn, error};
 
 #[macro_export]
 macro_rules! mize_err {

src/core/id.rs

@@ -5,6 +5,7 @@ use std::collections::hash_map::RandomState;
 use std::path::Path;
 
 use crate::instance::{store::Store, Instance};
+use crate::error::{MizeResult, MizeError, MizeResultTrait};
 
 #[derive(Debug, Clone, Hash, Eq, PartialEq)]
 pub struct MizeId {
@@ -22,7 +23,7 @@ impl Namespace {
 }
 
 pub trait IntoMizeId {
-    fn to_mize_id(self, instance: &Instance) -> MizeId;
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId>;
 }
 
 
@@ -54,28 +55,28 @@ impl fmt::Display for MizeId {
 }
 
 impl IntoMizeId for &str {
-    fn to_mize_id(self, instance: &Instance) -> MizeId {
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId> {
         instance.id_from_string(self.to_owned())
     }
 }
 
 impl IntoMizeId for String {
-    fn to_mize_id(self, instance: &Instance) -> MizeId {
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId> {
         instance.id_from_string(self)
     }
 }
 impl IntoMizeId for &String {
-    fn to_mize_id(self, instance: &Instance) -> MizeId {
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId> {
         instance.id_from_string(self.to_owned())
     }
 }
 impl IntoMizeId for Vec<String> {
-    fn to_mize_id(self, instance: &Instance) -> MizeId {
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId> {
         instance.id_from_vec_string(self)
     }
 }
 impl IntoMizeId for &[&str] {
-    fn to_mize_id(self, instance: &Instance) -> MizeId {
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId> {
         let mut vec: Vec<String> = Vec::new();
         for i in self {
             vec.push((*i).to_owned())
@@ -84,7 +85,7 @@ impl IntoMizeId for &[&str] {
     }
 }
 impl IntoMizeId for Vec<&str> {
-    fn to_mize_id(self, instance: &Instance) -> MizeId {
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId> {
         let mut vec: Vec<String> = Vec::new();
         for i in self {
             vec.push((*i).to_owned())
@@ -93,8 +94,8 @@ impl IntoMizeId for Vec<&str> {
     }
 }
 impl IntoMizeId for MizeId {
-    fn to_mize_id(self, instance: &Instance) -> MizeId {
-        self
+    fn to_mize_id(self, instance: &Instance) -> MizeResult<MizeId> {
+        Ok(self)
     }
 }
 

src/core/instance/connection.rs

@@ -1,16 +0,0 @@
-use log::{trace, debug, info, warn, error};
-use tokio::sync::mpsc::{Sender, Receiver};
-
-use crate::proto::MizeMessage;
-use crate::error::{MizeError, MizeResult, IntoMizeResult};
-
-pub trait Connection {
-}
-
-
-//#[derive(Debug)]
-//pub struct Connection {
-    //tx: Sender<MizeMessage>,
-    //rx: Receiver<MizeMessage>,
-//}
-

src/core/instance/mod.rs

@@ -1,59 +1,102 @@
 use core::fmt;
 use std::collections::HashMap;
 use std::fs::create_dir;
+use std::{thread, vec};
+use crossbeam::channel;
 use std::sync::{Arc, Mutex};
-use log::{trace, debug, info, warn, error};
+use tracing::{trace, debug, info, warn, error};
 use uuid::Uuid;
 use std::fs::File;
 use colored::Colorize;
 use std::path::Path;
 use interner::shared::{VecStringPool, StringPool};
 
+
 use crate::error::{MizeError, MizeResult, IntoMizeResult, MizeResultTrait};
-use crate::instance;
 use crate::instance::store::Store;
-use crate::instance::connection::Connection;
+use crate::instance::updater::updater_thread;
+use crate::instance::peer::Peer;
 use crate::id::{IntoMizeId, MizeId, Namespace};
 use crate::instance::subscription::Subscription;
 use crate::item::{Item, ItemData};
 use crate::memstore::MemStore;
+use crate::instance::updater::Operation;
 
-pub mod connection;
+use self::peer::PeerListener;
+
+#[cfg(feature = "async")]
+use tokio::runtime::Runtime;
+use core::future::Future;
+
+pub mod peer;
 pub mod store;
 pub mod subscription;
+pub mod updater;
 
 static MSG_CHANNEL_SIZE: usize = 200;
 
 /// The Instance type is the heart of the mize system
+#[derive(Clone)]
 pub struct Instance {
     // question vec of stores???
     // would mean replication logic and such things is handeled by the instance
     // i think it would be better thogh to then implement a ReplicatedStore
     pub store: Box<dyn Store>,
-    peers: Arc<Mutex<Vec<Box<dyn Connection>>>>,
+    peers: Arc<Mutex<Vec<Peer>>>,
     subs: Arc<Mutex<HashMap<MizeId, Subscription>>>,
-    pub id_pool: VecStringPool,
-    pub namespace_pool: StringPool,
+    pub id_pool: Arc<Mutex<VecStringPool>>,
+    pub namespace_pool: Arc<Mutex<StringPool>>,
     pub namespace: Namespace,
+    op_tx: channel::Sender<Operation>,
+    context: Vec<MizeId>,
+    threads: Vec<String>,
+    #[cfg(feature = "async")]
+    runtime: Arc<Mutex<Runtime>>,
+}
+
+pub struct InstanceRef {
+    inner: Arc<Mutex<Instance>>,
+}
+
+pub struct InstanceAsync {
+    inner: Arc<Mutex<Instance>>,
 }
 
 
 impl Instance {
-    pub fn empty() -> Instance {
+    pub fn empty() -> MizeResult<Instance> {
         let store = MemStore::new();
-        let id_pool = VecStringPool::default();
-        let namespace_pool = StringPool::default();
+        let id_pool = Arc::new(Mutex::new(VecStringPool::default()));
+        let namespace_pool_raw = StringPool::default();
         let peers = Arc::new(Mutex::new(Vec::new()));
         let subs = Arc::new(Mutex::new(HashMap::new()));
-        let namespace = Namespace ( namespace_pool.get("mize.default.namespace") );
-        let mut instance = Instance { store: Box::new(store), peers, subs, id_pool, namespace_pool, namespace };
-        return instance;
+        let namespace = Namespace ( namespace_pool_raw.get("mize.default.namespace") );
+        let (op_tx, op_rx) = channel::unbounded();
+
+        let mut instance = Instance { 
+            store: Box::new(store), 
+            peers, subs, id_pool,
+            namespace, op_tx,
+            namespace_pool: Arc::new(Mutex::new(namespace_pool_raw)),
+            context: vec![],
+            threads: vec![],
+
+            #[cfg(feature = "async")]
+            runtime: Arc::new(Mutex::new(Runtime::new().mize_result_msg("Could not create async runtime")?)),
+        };
+
+        let instance_clone = instance.clone();
+        let closure = move || updater_thread(op_rx, instance_clone);
+        instance.spawn("updater_thread", closure)?;
+
+
+        return Ok(instance);
     }
 
     pub fn new() -> MizeResult<Instance> {
         trace!("[ {} ] Instance::new()", "CALL".yellow());
 
-        let mut instance = Instance::empty();
+        let mut instance = Instance::empty()?;
 
         instance.init();
 
@@ -77,7 +120,7 @@ impl Instance {
     pub fn with_config(config: ItemData) -> MizeResult<Instance> {
         trace!("[ {} ] Instance::with_config()", "CALL".yellow());
         trace!("config: {}", config);
-        let mut instance = Instance::empty();
+        let mut instance = Instance::empty()?;
         instance.set("0", config.clone());
         instance.init()?;
 
@@ -94,56 +137,86 @@ impl Instance {
         println!("MIGRATING");
         let old_store = &self.store;
 
-        let inst_data = self.store.get_value_data_full(self.id_from_string("0".to_owned()))?;
-        new_store.set(self.id_from_string("0".to_owned()), inst_data.to_owned())?;
+        let id = self.id_from_string("0".to_owned())?;
+        let inst_data = self.store.get_value_data_full(id.clone())?;
+        new_store.set(id, inst_data.to_owned())?;
 
         for id in old_store.id_iter()? {
-            let data = self.store.get_value_data_full(self.id_from_string(id?))?;
+            let data = self.store.get_value_data_full(self.id_from_string(id?)?)?;
 
             let id_of_new_store = new_store.new_id()?;
-            new_store.set(self.id_from_string(id_of_new_store), data.to_owned())?;
+            new_store.set(self.id_from_string(id_of_new_store)?, data.to_owned())?;
         };
 
         self.store = new_store;
 
         Ok(())
     }
-}
 
-impl Instance {
     pub fn new_item(&self) -> MizeResult<Item> {
-        let id = self.id_from_string(self.store.new_id()?);
+        let id = self.id_from_string(self.store.new_id()?)?;
         return Ok(Item::new(id, &self));
     }
 
     pub fn get<I: IntoMizeId>(&self, id: I) -> MizeResult<Item> {
-        let id = id.to_mize_id(self);
+        let id = id.to_mize_id(self)?;
         return Ok(Item::new(id, &self));
     }
 
     pub fn set<I: IntoMizeId, V: Into<ItemData>>(&mut self, id: I, value: V) -> MizeResult<()> {
-        let id = id.to_mize_id(self);
-        let item_data = value.into();
-        let mut item = self.get(id.clone())?;
-        item.merge(item_data)?;
+        let id = id.to_mize_id(self)?;
+        self.op_tx.send(Operation::Set(id, value.into()));
         Ok(())
     }
 
-    pub fn new_id<T: IntoMizeId>(&self, value: T) -> MizeId {
+    pub fn new_id<T: IntoMizeId>(&self, value: T) -> MizeResult<MizeId> {
         value.to_mize_id(self)
     }
 
-    pub fn id_from_string(&self, string: String) -> MizeId {
+    pub fn id_from_string(&self, string: String) -> MizeResult<MizeId> {
         let vec_string: Vec<String> = string.split("/").map(|v| v.to_owned()).collect();
-        MizeId { path: self.id_pool.get(vec_string), namespace: self.namespace.clone() }
+        let id_pool_inner = self.id_pool.lock()?;
+
+        let id = MizeId { path: id_pool_inner.get(vec_string), namespace: self.namespace.clone() };
+
+        Ok(id)
     }
 
-    pub fn id_from_vec_string(&self, vec_string: Vec<String>) -> MizeId {
-        MizeId { path: self.id_pool.get(vec_string), namespace: self.namespace.clone() }
+    pub fn id_from_vec_string(&self, vec_string: Vec<String>) -> MizeResult<MizeId> {
+        let id_pool_inner = self.id_pool.lock()?;
+        let id = MizeId { path: id_pool_inner.get(vec_string), namespace: self.namespace.clone() };
+        Ok(id)
     }
 
-    pub fn namespace_from_string(&self, ns_str: String) -> Namespace {
-        Namespace ( self.namespace_pool.get(ns_str) )
+    pub fn namespace_from_string(&self, ns_str: String) -> MizeResult<Namespace> {
+        let namespace_pool_inner = self.namespace_pool.lock()?;
+        let namespace = Namespace ( namespace_pool_inner.get(ns_str) );
+        Ok(namespace)
+    }
+
+    pub fn add_listener<T: PeerListener +'static>(&mut self, listener: T) -> MizeResult<()> {
+        let mut instance_clone = self.clone();
+        self.spawn("some_listener", move || listener.listen(instance_clone));
+        Ok(())
+    }
+
+    pub fn add_peer(&mut self, peer: Peer) -> MizeResult<()> {
+        let mut peer_inner = self.peers.lock()?;
+        peer_inner.push(peer);
+        Ok(())
+    }
+
+    pub fn spawn(&mut self, name: &str, func: impl FnOnce() -> MizeResult<()> + Send + 'static) -> MizeResult<()> {
+        self.threads.push(name.to_owned());
+        thread::spawn(move || func());
+        Ok(())
+    }
+
+    #[cfg(feature = "async")]
+    pub fn spawn_async<F: Future<Output = impl Send + Sync + 'static> + Send + Sync + 'static>(&mut self, name: &str, func: F) {
+        self.threads.push(name.to_owned());
+        let runtime_inner = self.runtime.lock().unwrap();
+        runtime_inner.spawn(func);
     }
 }
 

src/core/instance/peer.rs

@@ -0,0 +1,25 @@
+use crossbeam::channel::{Sender, Receiver};
+use tracing::{trace, debug, info, warn, error};
+
+use crate::proto::MizeMessage;
+use crate::error::{MizeError, MizeResult, IntoMizeResult};
+
+use super::Instance;
+
+pub struct Peer {
+    rx: Receiver<MizeMessage>,
+    tx: Sender<MizeMessage>,
+}
+
+pub trait PeerListener : Send + Sync {
+    fn listen(self, instance: Instance) -> MizeResult<()>;
+}
+
+impl Peer {
+    pub fn new(rx: Receiver<MizeMessage>, tx: Sender<MizeMessage>) -> Peer {
+        Peer { rx , tx }
+    }
+}
+
+
+

src/core/instance/store.rs

@@ -1,14 +1,17 @@
-
 use std::iter::Map;
 use std::option::Iter;
+use dyn_clone::DynClone;
 
 use crate::error::MizeResult;
 use crate::item::{Item, ItemData};
 use crate::id::MizeId;
 use crate::memstore::MemStore;
 use crate::instance::Instance;
+use crate::item::get_raw_from_cbor;
 
-pub trait Store {
+dyn_clone::clone_trait_object!(Store);
+
+pub trait Store: DynClone + Send + Sync {
     // should have the ability to be multi threaded, if the underlying implementation supports 
     // multithreaded IO operations
     // for now all refs to a store hold a Mutex but this mutex is part of where the MizeStore trait

src/core/instance/updater.rs

@@ -0,0 +1,35 @@
+use crossbeam::channel::Receiver;
+use tracing::{trace, error};
+
+use crate::{instance::Instance, item::ItemData};
+use crate::id::MizeId;
+use crate:: error::{MizeResult, MizeError, MizeResultTrait};
+
+
+#[derive(Debug)]
+pub enum Operation {
+    Set(MizeId, ItemData),
+}
+
+pub fn updater_thread(operation_rx : Receiver<Operation>, mut instance: Instance) -> MizeResult<()> {
+    for mut operation in operation_rx {
+        trace!("OPERATION");
+        let result = handle_operation(&mut operation, &mut instance);
+
+        if let Err(err) = result {
+            error!("OPERATION FAILED: {:?}", operation);
+        }
+    }
+    Ok(() )
+}
+
+fn handle_operation(operation: &mut Operation, instance: &mut Instance) -> MizeResult<()> {
+    match operation {
+        Operation::Set(id, value) => {
+            let item_data: ItemData = value.to_owned();
+            let mut item = instance.get(id.clone())?;
+            item.merge(item_data)?;
+        }
+    }
+    Ok(())
+}

src/core/item.rs

@@ -1,4 +1,4 @@
-use log::{trace, debug, info, warn, error};
+use tracing::{debug, error, info, instrument, trace, warn};
 use serde::Serialize;
 use core::fmt;
 use std::{i128, string};
@@ -8,6 +8,7 @@ use tokio::sync::Mutex;
 use std::rc::Rc;
 use colored::Colorize;
 use std::io;
+use tracing::{span, Level};
 
 use crate::error::{MizeError, MizeResult, IntoMizeResult};
 use crate::instance::Instance;
@@ -256,6 +257,74 @@ pub fn item_data_merge(merge_into: &mut CborValue, other: &CborValue){
     }
 }
 
+#[instrument]
+pub fn get_raw_from_cbor<'a>(value: &'a CborValue, path: Vec<&String>) -> MizeResult<&'a [u8]> {
+    trace!("[ {} ] value: {:?}", "ARG".yellow(), value);
+    trace!("[ {} ] path: {:?}", "ARG".yellow(), path);
+
+    let mut path_iter = path.clone().into_iter();
+    let path_el = match path_iter.nth(0) {
+        Some(val) => val,
+        None => {
+            // our base case
+            let ret_val = match value {
+                CborValue::Bytes(vec) => {
+                    trace!("[ {} ] ret value: {:?}", "RET".yellow(), &vec[..]);
+                    return Ok(&vec[..]);
+                },
+                CborValue::Text(string) => { 
+                    trace!("[ {} ] ret value: {:?}", "RET".yellow(), string.as_bytes());
+                    return Ok(string.as_bytes());
+                },
+                other => {
+                    return Err(MizeError::new()
+                        .msg("path is empty and the value is neither Bytes nor Text"));
+                },
+            };
+        }, 
+    };
+    
+    match value {
+        CborValue::Bytes(vec) => {
+            trace!("[ {} ] ret value: {:?}", "RET".yellow(), &vec[..]);
+            return Ok(&vec[..]);
+        },
+        CborValue::Text(string) => { 
+            trace!("[ {} ] ret value: {:?}", "RET".yellow(), string.as_bytes());
+            return Ok(string.as_bytes());
+        },
+        CborValue::Map(map) => {
+            let mut inner_val: &CborValue = &CborValue::Null;
+            let mut inner_val_found = false;
+
+            // find the inner_val at path
+            for (key, val) in map.into_iter() {
+                if let CborValue::Text(key_text) = key {
+                    if key_text == path_el {
+                        inner_val = val;
+                        inner_val_found = true;
+                    }
+                }
+            }
+            
+            // if there is no value at that path
+            if inner_val_found == false {
+                return Err(MizeError::new()
+                    // jesus christ, this is a pfusch...
+                    .msg(format!("get_raw_from_cbor: Path '{}' not Found", path.clone().into_iter().map(|v| v.to_owned()).collect::<Vec<String>>().join("/"))))
+            }
+
+            //path_iter.next();
+            let inner_path = path_iter.collect();
+            let ret_value = get_raw_from_cbor(inner_val, inner_path);
+            trace!("[ {} ] ret value: {:?}", "RET".yellow(), ret_value);
+            return ret_value;
+        },
+        _ => Err(MizeError::new()
+            .msg(format!("get_raw_from_cbor: value is not a map, text or bytes ... value: {:?}", value))),
+    }
+}
+
 trait IntoPath {
     fn into_path(self) -> Vec<String>;
 }

src/core/memstore.rs

@@ -3,7 +3,7 @@ use std::collections::HashMap;
 use std::iter::Map;
 use std::sync::{Arc, Mutex};
 use ciborium::Value as CborValue;
-use log::trace;
+use tracing::{instrument, trace};
 use colored::Colorize;
 
 use crate::id::MizeId;
@@ -11,6 +11,7 @@ use crate::instance::store::{Store, IdIter};
 use crate::error::{MizeError, MizeResult, IntoMizeResult};
 use crate::item::{Item, ItemData};
 use crate::instance::Instance;
+use crate::item::get_raw_from_cbor;
 
 #[derive(Clone, Debug)]
 pub struct MemStore {
@@ -49,7 +50,9 @@ impl Store for MemStore {
         inner.next_id += 1;
         return Ok(id_string);
     }
+    #[instrument(name="MemStore::get_value_raw" skip(self))]
     fn get_value_raw(&self, id: MizeId) -> MizeResult<Vec<u8>> {
+    trace!("[ {} ] id: {:?}", "ARG".yellow(), id);
         let inner = self.inner.lock()?;
 
         let cbor_val = inner.map.get(&id_to_u64(id.clone())?)
@@ -131,74 +134,6 @@ fn str_to_u64(id_str: &str) -> MizeResult<u64> {
         .mize_result_msg(format!("Could not parse the store_part of mizeid {} into a u64 for the MemStore", id_str))
 }
 
-pub fn get_raw_from_cbor<'a>(value: &'a CborValue, path: Vec<&String>) -> MizeResult<&'a [u8]> {
-    trace!("[ {} ] get_raw_from_cbor()", "CALL".yellow());
-    trace!("[ {} ] value: {:?}", "ARG".yellow(), value);
-    trace!("[ {} ] path: {:?}", "ARG".yellow(), path);
-
-    let mut path_iter = path.clone().into_iter();
-    let path_el = match path_iter.nth(0) {
-        Some(val) => val,
-        None => {
-            // our base case
-            let ret_val = match value {
-                CborValue::Bytes(vec) => {
-                    trace!("[ {} ] ret value: {:?}", "RET".yellow(), &vec[..]);
-                    return Ok(&vec[..]);
-                },
-                CborValue::Text(string) => { 
-                    trace!("[ {} ] ret value: {:?}", "RET".yellow(), string.as_bytes());
-                    return Ok(string.as_bytes());
-                },
-                other => {
-                    return Err(MizeError::new()
-                        .msg("path is empty and the value is neither Bytes nor Text"));
-                },
-            };
-        }, 
-    };
-    trace!("hoooooooo: {:?}", path_el);
-    
-    match value {
-        CborValue::Bytes(vec) => {
-            trace!("[ {} ] ret value: {:?}", "RET".yellow(), &vec[..]);
-            return Ok(&vec[..]);
-        },
-        CborValue::Text(string) => { 
-            trace!("[ {} ] ret value: {:?}", "RET".yellow(), string.as_bytes());
-            return Ok(string.as_bytes());
-        },
-        CborValue::Map(map) => {
-            let mut inner_val: &CborValue = &CborValue::Null;
-            let mut inner_val_found = false;
-
-            // find the inner_val at path
-            for (key, val) in map.into_iter() {
-                if let CborValue::Text(key_text) = key {
-                    if key_text == path_el {
-                        inner_val = val;
-                        inner_val_found = true;
-                    }
-                }
-            }
-            
-            // if there is no value at that path
-            if inner_val_found == false {
-                return Err(MizeError::new()
-                    // jesus christ, this is a pfusch...
-                    .msg(format!("get_raw_from_cbor: Path '{}' not Found", path.clone().into_iter().map(|v| v.to_owned()).collect::<Vec<String>>().join("/"))))
-            }
-
-            //path_iter.next();
-            let inner_path = path_iter.collect();
-            trace!("hiiiiii: {:?}", inner_path);
-            let ret_value = get_raw_from_cbor(inner_val, inner_path);
-            trace!("[ {} ] ret value: {:?}", "RET".yellow(), ret_value);
-            return ret_value;
-        },
-        _ => Err(MizeError::new().msg("get_raw_from_cbor: value is not a map, text or bytes")),
-    }
-}
 
 
 

src/core/proto/mod.rs

@@ -4,3 +4,7 @@ use ciborium::Value as CborValue;
 pub struct MizeMessage {
     value: CborValue,
 }
+
+
+
+