major server rewrite now all working

errorcodes.yml

@@ -1,3 +1,6 @@
+Not sure yet:
+  code: 11
+  category: "don't know yet"
 
 Can't delete item 0:
   code: 100
@@ -31,3 +34,32 @@ could not read index of item:
 _commit not 8 bytes long:
   code: 105
   category: "don't know yet"
+
+error while parsing int:
+  code: 106
+  category: "parse_error"
+
+error with message sent to the server:
+  code: 107
+  category: "faulty_message"
+
+missing metadata:
+  code: 108
+  category: "faulty_message"
+
+something that should be utf8 valid was not:
+  code: 109
+  category: "faulty_message"
+
+ParseIntError:
+  code: 110
+  category: "faulty_message"
+
+_commit can't be updated:
+  code: 111
+  category: "faulty_update"
+
+no / on the end of id:
+  code: 112
+  category: "faulty_message"
+

src/error.rs

@@ -1,4 +1,6 @@
 
+use std::string::FromUtf8Error;
+
 use json;
 use crate::server::proto;
 
@@ -19,11 +21,22 @@ impl MizeError {
         return data;
     }
 
-    pub fn to_message(self: MizeError) -> proto::Response {
+    pub fn to_message(self: MizeError, origin: proto::Origin) -> proto::Message {
         let data = self.to_json();
         let mut msg = vec![crate::PROTO_VERSION, proto::MSG_ERROR];
         msg.extend(json::stringify(data).into_bytes());
-        return proto::Response::One(msg);
+        return proto::Message::from_bytes(msg, origin);
+    }
+}
+
+impl From<FromUtf8Error> for MizeError {
+    fn from(utf_err: FromUtf8Error) -> MizeError{
+        let err = MizeError {
+            code: 109,
+            kind: "faulty_message".to_string(),
+            message: "std::string::FromUtf8Error while parsing something from the message".to_string(),
+        };
+        return err;
     }
 }
 

src/integrated_types/test_same.rs

@@ -0,0 +1,2 @@
+
+pub fn main(){}

src/lib.rs

@@ -5,6 +5,9 @@ static PROTO_VERSION: u8 = 1;
 
 pub mod server;
 pub mod error;
+pub mod integrated_types {
+    pub mod test_same;
+}
 
 
 

src/server.rs

@@ -4,23 +4,25 @@ pub mod server_utils;
 pub mod itemstore;
 pub mod proto;
 
-use futures_util::{FutureExt, StreamExt};
+//use futures_util::lock::Mutex;
+use futures_util::{FutureExt, StreamExt, SinkExt};
 use warp::Filter;
 use warp::ws::{WebSocket, self};
+use std::collections::HashMap;
 
-use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
-use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio::sync::mpsc::{Sender, channel, Receiver};
+use tokio_stream::wrappers::ReceiverStream;
 use std::alloc::handle_alloc_error;
 use std::fs;
 use std::sync::{Arc};
 use tokio::sync::Mutex;
 use tokio::sync::mpsc;
-use crate::server::proto::{Response};
 use crate::server::proto::handle_mize_message;
 
 use self::itemstore::Itemstore;
 
 //static API_PATH_STRING: &str = "$api";
+static SOCKET_CHANNEL_SIZE: usize = 200;
 
 // max file sizez in the mize/data folder (in bytes)
 static MAX_INDEX_FILE_SIZE: usize = 1_800_000;
@@ -51,18 +53,58 @@ static MAX_DATA_FILE_SIZE: usize = 3_000_000_000;
 //static VERSION_MESSAGE: &str = "\
 //Version: 0
 //";
-//
 
+
+#[derive(Clone, Debug)]
 pub struct Client {
-    tx: UnboundedSender<Result<ws::Message, warp::Error>>
+//    rx: UnboundedReceiverStream<Result<ws::Message, warp::Error>>,
+    tx: Sender<proto::Message>,
+    id: u64,
 }
 
-pub fn run_server(args: Vec<String>) {
+#[derive(Clone, Debug)]
+pub struct Module {
+    tx: Sender<proto::Message>,
+    name: String,
+    client_id: u64,
+    //later
+    //registered_types: Vec<String>,
+}
+
+//pub struct Upstream maybe???? ... I'd say later....
+
+
+// A collection of all the things, that most functions need
+#[derive(Clone)]
+pub struct Mutexes {
+    next_free_client_id: Arc<Mutex<u64>>,
+    clients: Arc<Mutex<Vec<Client>>>,
+    modules: Arc<Mutex<Vec<Module>>>,
+    subs: Arc<Mutex<HashMap<String, Vec<proto::Origin>>>>,
+    //maybe a list of upstream servers??
+    itemstore: Arc<Mutex<Itemstore>>,
+}
+
+impl Mutexes {
+    pub fn clone(mutexes: &Mutexes) -> Mutexes {
+        Mutexes {
+            next_free_client_id: Arc::clone(&mutexes.next_free_client_id),
+            clients: Arc::clone(&mutexes.clients),
+            modules: Arc::clone(&mutexes.modules),
+            subs: Arc::clone(&mutexes.subs),
+            itemstore: Arc::clone(&mutexes.itemstore),
+        }
+    }
+}
+
+#[tokio::main]
+pub async fn run_server(args: Vec<String>) {
     /*
      * WHAT IT DOES
      *
      */
 
+    //
     //### get the mize_folder
     let mut mize_folder = String::new();
     for i in 1..args.len() {
@@ -74,13 +116,27 @@ pub fn run_server(args: Vec<String>) {
         }
     }
 
+    //create the itemstore
+    let itemstore = crate::server::itemstore::Itemstore::new(mize_folder.clone() + "/db").await.expect("error creating itemstore");
+
+    // A collection of all the things, that most functions need
+    let mutexes: Mutexes = Mutexes {
+        next_free_client_id: Arc::new(Mutex::new(0)),
+        clients: Arc::new(Mutex::new(Vec::new())),
+        subs: Arc::new(Mutex::new(HashMap::new())),
+        modules: Arc::new(Mutex::new(Vec::new())),
+        itemstore: Arc::new(Mutex::new(itemstore)),
+    };
+
+    //listen on the local unix socket
+    //local_socket_server(mize_folder);
+
     //run the webserver
-    warp_server(mize_folder);
+    warp_server(mize_folder, mutexes).await;
 }
 
 
-#[tokio::main]
-async fn warp_server(mize_folder: String) {
+async fn warp_server(mize_folder: String, mutexes: Mutexes) {
     /*
      *
      *
@@ -92,35 +148,30 @@ async fn warp_server(mize_folder: String) {
     //## $api/rest/
  
     // get itemstore
-    let itemstore = crate::server::itemstore::Itemstore::new(mize_folder + "/db").await.expect("error creating itemstore");
 
-    let mut clients: Arc<Mutex<Vec<Client>>> = Arc::new(Mutex::new(Vec::new()));
-    let mut itemstore_mutex: Arc<Mutex<Itemstore>> = Arc::new(Mutex::new(itemstore));
+    let mutexes_clone = Mutexes::clone(&mutexes);
 
     let socket_route = warp::path!("$api" / "socket")
         .and(warp::ws())
         .map(move |ws: warp::ws::Ws| {
-            let clients_clone = Arc::clone(&clients);
-            let itemstore_clone = Arc::clone(&itemstore_mutex);
-            ws.on_upgrade(move |socket| handle_socket_connection(
-                    socket,
-                    clients_clone,
-                    itemstore_clone
-                )
-            )
+            let mutexes_clone = Mutexes::clone(&mutexes_clone);
+            ws.on_upgrade(move |socket| handle_websocket_connection(socket, mutexes_clone))
     });
 
     let render_route = warp::path!("$api" / "render").map(move || "render route");
     let file_route = warp::path!("$api" / "file").map(move || "file route");
+
     //temporary
     let render_route = warp::path!("$api" / "render" / "first").map(move || {
         let answer = fs::read_to_string("/home/sebastian/work/mize/first-render/src/main.js").unwrap();
         warp::http::Response::builder().header("content-type", "text/javascript").body(answer)
     });
+
     let main_route = warp::path!("$api" / "client" / "main.js").map(move || {
         let answer = fs::read_to_string("/home/sebastian/work/mize/js-client/src/main.js").unwrap();
         warp::http::Response::builder().header("content-type", "application/javascript").body(answer)
     });
+
     let defuatl_route = warp::path::full().map(|path|{
         //normally should be included in the binary, but for developing just gonna read the file
         //so I don't have to recompile all the time
@@ -144,54 +195,96 @@ async fn warp_server(mize_folder: String) {
     warp::serve(routes).run(([127, 0, 0, 1], 3000)).await;
 }
 
-async fn handle_socket_connection(
+async fn handle_websocket_connection(
     socket: WebSocket,
-    clients_clone: Arc<Mutex<Vec<Client>>>,
-    itemstore_clone: Arc<Mutex<Itemstore>>,
+    mutexes: Mutexes,
 ){
-    let (socket_tx, mut socket_rx) = socket.split();
-    let (client_tx, client_rx) = mpsc::unbounded_channel();
+    let (mut socket_tx, mut socket_rx) = socket.split();
+    let (msg_tx, msg_rx): (Sender<proto::Message>, Receiver<proto::Message>) = mpsc::channel(SOCKET_CHANNEL_SIZE);
 
-    let client_rx = UnboundedReceiverStream::new(client_rx);
-    tokio::spawn(client_rx.forward(socket_tx));
+    let mut msg_rx = ReceiverStream::new(msg_rx);
 
-    let mut cli = clients_clone.lock().await;
-    cli.push(Client{tx: client_tx.clone()});
+//    tokio::spawn(msg_rx.forward(socket_tx));
+
+    //my own forward
+    tokio::spawn(async move {
+        while let Some(msg) = msg_rx.next().await {
+            socket_tx.send(warp::ws::Message::binary(msg.raw)).await;
+        }
+    });
+
+    let mut cli = mutexes.clients.lock().await;
+    let mut client_id = mutexes.next_free_client_id.lock().await;
+    let client = Client{tx: msg_tx.clone(), id: *client_id};
+    cli.push(client.clone());
+    *client_id += 1;
     drop(cli);
+    drop(client_id);
 
     // Reading and broadcasting messages
     while let Some(result) = socket_rx.next().await {
-        println!("got message ==================================");
-        let msg = result.expect("Error when gettin message from WebSocket");
+        let msg = result.expect("Error when getting message from WebSocket");
+        println!("got message: {:?}", msg);
 
-        let itemstore = &*itemstore_clone.lock().await;
-        let responses = handle_mize_message(proto::Message::new(msg.clone().into_bytes()), itemstore).await;
-        for response in responses {
-            match response {
-                Response::One(response) => {
-                    client_tx.send(Ok(ws::Message::binary(response.clone()))).unwrap()
-                },
-                Response::All(response) => {
-                    send_to_all_clients(ws::Message::binary(response), clients_clone.clone()).await;
-                },
-                Response::AllSubbed(id, response) => {
-                    send_to_all_subed_clients(id, ws::Message::binary(response), clients_clone.clone()).await;
-                },
-                Response::None => {let t = 0;},
-            };
+        if let Err(err) = handle_mize_message(
+            proto::Message::from_bytes(msg.clone().into_bytes(),proto::Origin::Client(client.clone())), mutexes.clone(),
+        ).await {
+            msg_tx.send(err.to_message(proto::Origin::Client(client.clone()))).await;
         };
     };
 }
 
-pub async fn send_to_all_clients(message: ws::Message, clients_clone: Arc<Mutex<Vec<Client>>>){
-    let clients = clients_clone.lock().await;
-    for client in &clients[..] {
-        client.tx.send(Ok(ws::Message::binary(message.clone())));
-    }
-}
+//pub async fn handle_unix_socket_connection(){
+//}
 
-pub async fn send_to_all_subed_clients(id: u64, message: ws::Message, clients: Arc<Mutex<Vec<Client>>>){
-}
+//async fn handle_connection(con: Connection){
+//}
+
+//pub async fn send_to_all_clients(message: ws::Message, mutexes_clone: Mutexes){
+//    let clients = mutexes_clone.clients.lock().await;
+//    for client in &clients.clients[..] {
+//        client.tx.send(Ok(ws::Message::binary(message.clone())));
+//    }
+//    drop(clients);
+
+//    let modules = mutexes_clone.modules.lock().await;
+//    for module in &modules.modules[..] {
+//        module.tx.send(Ok(ws::Message::binary(message.clone())));
+//    }
+//}
+
+//pub async fn send_to_all_subbed_clients(id: String, message: ws::Message, mutexes_clone: Mutexes){
+//    let clients = mutexes_clone.clients.lock().await;
+//    for client in &clients.clients[..] {
+//        if client.sub.contains(&id) {
+//            client.tx.send(Ok(ws::Message::binary(message.clone())));
+//        }
+//    }
+
+//    let modules = mutexes_clone.modules.lock().await;
+//    for module in &modules.modules[..] {
+//        if module.sub.contains(&id) {
+//            module.tx.send(Ok(ws::Message::binary(message.clone())));
+//        }
+//    }
+//}
+
+//that does not work like this
+
+//impl From<proto::Message> for Result<warp::ws::Message, warp::Error> {
+//    fn from(msg: proto::Message) -> Result<warp::ws::Message, warp::Error> { 
+//        let msg = warp::ws::Message::binary(msg.raw);
+//    }
+//}
+
+//impl From<Result<warp::ws::Message, warp::Error>> for proto::Message {
+//    fn from(ws_msg: Result<warp::ws::Message, warp::Error>) -> proto::Message {
+//        let msg = ws_msg.expect("Error in From trait implementation: warp::ws::Messate to proto::Message");
+//    }
+//}
+//191 |     cli.push(Client{tx: client_tx, id: client_id});
+//    = note: expected struct `tokio::sync::mpsc::Sender<proto::Message>`
+//               found struct `tokio::sync::mpsc::Sender<`
 
 
 

src/server/itemstore.rs

@@ -5,7 +5,13 @@ use surrealdb::{Datastore, Key, Val};
 use itertools::Itertools;
 use crate::error;
 use crate::error::MizeError;
+use crate::server::proto;
 
+use super::proto::Delta;
+
+// The struct to do the storage and updates to it
+// is responsible, that no illegal states can occur in the storage, by using transactions
+// currently it's just a surrealdb Datastore. should be replaced by a completely custom File-System in the future.
 pub struct Itemstore {
     db: surrealdb::Datastore,
 }
@@ -94,57 +100,61 @@ impl Itemstore {
         return Ok(new_id)
     }
 
-    pub async fn update(&self, id: u64, new_item: Item) -> Result<(), MizeError>{
+    pub async fn update(&self, update: proto::Update) -> Result<(), MizeError>{
+
         let mut tr = self.db.transaction(true, true).await?;
 
-        let mut keys: Vec<Vec<u8>>  = Vec::new();
+        let old_item_index = tr.get(update.id.clone().into_bytes()).await?;
+        let old_item = for key in old_item_index {
+            let mut key_get = format!("{}:", update.id).into_bytes();
+            key_get.extend(key.clone());
+            tr.get(key_get);
+        };
 
-        for field in new_item {
-            let mut key = format!("{}:", id).into_bytes();
-            key.extend(field[0].clone());
-            let val = field[1].clone();
-            keys.push(field[0].clone());
-            tr.set(key, val).await?;
+        let mut keys: Vec<Vec<u8>> = Vec::new();
+
+        //check the _type of the item and maybe run type-code
+        //TODO: typecode
+
+        //read keys
+        let mut keys = match tr.get(format!("{}", update.id).into_bytes()).await? {
+            Some(keys) => decode(keys),
+            None => {
+                return Err(MizeError{
+                    code: 104,
+                    kind: "data_storage::index_not_found".to_string(),
+                    message: "Internal Datastorage Error: the Index of the Item was not found".to_string(),
+                });
+            },
+        };
+
+        //write new values
+        for (mut key, delta) in update.raw {
+            let mut store_key = (update.id.clone() + ":").into_bytes();
+            store_key.append(&mut key);
+            let mut new_val = Vec::new();
+
+            if let Some(old_val) = tr.get(store_key.clone()).await? {
+                new_val = proto::apply_delta(old_val, delta)?;
+                tr.set(store_key.clone(), new_val.clone()).await?;
+            } else {
+                keys.push(key.clone());
+                new_val = proto::apply_delta(Vec::new(), delta)?;
+            }
+
+            //remove field if new_val is empty
+            if (new_val.len() == 0) {
+                if let Some(key_index) = keys.iter().position(|x| *x == key.clone()){
+                    keys.remove(key_index);
+                }
+                tr.del(key).await?;
+            } else {
+                tr.set(store_key.clone(), new_val).await?;
+            }
         }
 
-        let keys_res = tr.get(format!("{}", id).into_bytes()).await?;
-        if let Some(old_keys) = keys_res {
-            keys.extend(decode(old_keys));
-            keys = keys.into_iter().unique().collect::<Vec<Vec<u8>>>();
-            tr.set(format!("{}", id).into_bytes(), encode(keys)).await?;
-        } else {
-            return Err(MizeError{
-                code: 104,
-                kind: "data_storage::index_not_found".to_string(),
-                message: "Internal Datastorage Error: the Index of the Item was not found".to_string(),
-            });
-        }
-
-        //increment commit number
-        let mut commit_key = format!("{}:_commit", id).into_bytes();
-
-        let commit_res = tr.get(&*commit_key).await?;
-        if let Some(commit_vec) = commit_res {
-            let val: [u8;8] = match commit_vec.try_into() {
-                Ok(val) => val,
-                Err(_) => {
-                    return Err(MizeError{
-                        code: 105,
-                        kind: "don't know yet".to_string(),
-                        message: "_commit of this item is no valid u64 (not 8 bytes long)".to_string(),
-                    });
-                },
-            };
-            let mut commit_num = u64::from_be_bytes(val);
-            commit_num += 1;
-            tr.set(commit_key, commit_num.to_be_bytes()).await?;
-        } else {
-            return Err(MizeError{
-                code: 103,
-                kind: "key_missing::_commit".to_string(),
-                message: "This item has no _commit key, which every item must have.".to_string(),
-            });
-        }
+        //write keys again
+        tr.set(update.id.into_bytes(), encode(keys)).await?;
 
         tr.commit().await?;
         return Ok(());
@@ -152,10 +162,21 @@ impl Itemstore {
 
 
 
-    pub async fn delete(&self, id: u64) -> Result<(), error::MizeError>{
+    pub async fn delete(&self, id_str: String) -> Result<(), error::MizeError>{
+        let id_u64: u64 = match id_str.parse() {
+            Ok(id) => id,
+            Err(err) => {
+                return Err(MizeError {
+                    code: 110,
+                    kind: "faulty_message".to_string(),
+                    message: format!("std::num::ParseIntError while parsing id \"{}\" into an Integer", id_str),
+
+                });
+            }
+        };
         let mut tr = self.db.transaction(true, true).await?;
 
-        let keys_res = tr.get(format!("{}", id).into_bytes()).await?;
+        let keys_res = tr.get(format!("{}", id_u64).into_bytes()).await?;
         if let Some(keys) = keys_res {
             for key in decode(keys) {
                 tr.del(key).await?;
@@ -168,7 +189,7 @@ impl Itemstore {
             });
         }
 
-        tr.del(format!("{}", id)).await?;
+        tr.del(id_str.into_bytes()).await?;
 
         tr.commit().await?;
         return Ok(());