started with using rocksdb to store the data.

- debug binary is no 274MB.... lol
- added encoding and decoding function

Cargo.lock

@@ -14,6 +14,25 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "bindgen"
+version = "0.60.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "clang-sys",
+ "lazy_static",
+ "lazycell",
+ "peeking_take_while",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -60,12 +79,52 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
 
+[[package]]
+name = "bzip2-sys"
+version = "0.1.11+1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+dependencies = [
+ "jobserver",
+]
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "clang-sys"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.5"
@@ -204,6 +263,12 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+
 [[package]]
 name = "h2"
 version = "0.3.14"
@@ -357,12 +422,69 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
 
+[[package]]
+name = "jobserver"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
 [[package]]
 name = "libc"
 version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
 
+[[package]]
+name = "libloading"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd"
+dependencies = [
+ "cfg-if",
+ "winapi",
+]
+
+[[package]]
+name = "librocksdb-sys"
+version = "0.8.0+7.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d"
+dependencies = [
+ "bindgen",
+ "bzip2-sys",
+ "cc",
+ "glob",
+ "libc",
+ "libz-sys",
+ "zstd-sys",
+]
+
+[[package]]
+name = "libz-sys"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.8"
@@ -410,6 +532,12 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "mio"
 version = "0.8.4"
@@ -427,6 +555,7 @@ name = "mize"
 version = "0.1.0"
 dependencies = [
  "futures-util",
+ "rocksdb",
  "tokio",
  "tokio-stream",
  "warp",
@@ -450,6 +579,16 @@ dependencies = [
  "twoway",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.13.1"
@@ -495,6 +634,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
 [[package]]
 name = "percent-encoding"
 version = "2.1.0"
@@ -533,6 +678,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -602,6 +753,21 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "regex"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
+dependencies = [
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
+
 [[package]]
 name = "remove_dir_all"
 version = "0.5.3"
@@ -611,6 +777,22 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rocksdb"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc"
+dependencies = [
+ "libc",
+ "librocksdb-sys",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "ryu"
 version = "1.0.11"
@@ -688,6 +870,12 @@ dependencies = [
  "digest 0.10.3",
 ]
 
+[[package]]
+name = "shlex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.0"
@@ -981,6 +1169,12 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -1097,3 +1291,13 @@ name = "windows_x86_64_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.1+zstd.1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
+dependencies = [
+ "cc",
+ "libc",
+]

Cargo.toml

@@ -7,14 +7,15 @@ edition = "2021"
 
 [[bin]]
 name = "server"
-path = "server-main.rs"
+path = "src/server-main.rs"
 
 [[bin]]
 name = "client"
-path = "client-main.rs"
+path = "src/client-main.rs"
 
 [dependencies]
 futures-util = "0.3.24"
+rocksdb = "0.19.0"
 tokio = {version = "1.21.0", features = ["full"]}
 tokio-stream = "0.1.9"
 warp = "0.3.2"

pitty-commit.sh

@@ -4,7 +4,9 @@
 
 date +"%a %b %d" >> pitty-commit.sh
 git add pitty-commit.sh
-git commit -m "I am sorry. #pitty-commit"
+git commit -m "I am sorry for this. #pitty-commit"
+git push
 
+exit
 # a list of all the dates I did nothing on
-Wed Sep 14
+Wed Sep 13

src/lib.rs

@@ -1,4 +1,5 @@
 
+#![ allow( warnings ) ]
 pub mod server;
 pub mod error;
 

src/server-main.rs

@@ -1,3 +1,6 @@
+
+#![ allow( warnings ) ]
+
 extern crate mize;
 
 static HELP_MESSAGE: &str = "\

src/server.rs

@@ -1,6 +1,7 @@
 
 
 pub mod server_utils;
+pub mod itemstore;
 
 use futures_util::{FutureExt, StreamExt};
 use warp::Filter;
@@ -68,12 +69,14 @@ pub fn run_server(args: Vec<String>) {
     }
 
     //### prep the mize_folder
-    if let Err(err) = crate::server::server_utils::init_mize_folder(mize_folder) {
+    //if let Err(err) = crate::server::server_utils::init_mize_folder(mize_folder) {
-        println!("{}", err.message);
+        //println!("{}", err.message);
-    }
+    //}
+    
+    // get itemstore
+    let itemstore = crate::server::itemstore::itemstore::new(mize_folder + "/db");
 
     //run the webserver
-    //hyper_server();
     warp_server();
 }
 

src/server/itemstore.rs

@@ -0,0 +1,75 @@
+
+use rocksdb::{SingleThreaded, DBCommon, DB, Options};
+use std::io;
+
+pub struct itemstore {
+    db: DB,
+}
+
+impl itemstore {
+    pub fn new(path: String) -> itemstore {
+        let db = DB::open_default(&path).expect(&format!("Could not open rocksdb database at {}", &path)[..]);
+
+        match db.get(b"0") {
+            Ok(Some(val)) => (),
+            Ok(None) => {
+                //db.set(b"0", )
+                println!("hi")
+            },
+            Err(e) => println!("Error reading item 0 from db: {}", e),
+        }
+        return itemstore{db:db};
+    }
+}
+
+pub fn decode(bytes: Vec<u8>) -> Vec<String> {
+    let mut count = 0;
+    let mut index: Vec<String> = Vec::new();
+    let mut key: Vec<u8> = Vec::new();
+    loop {
+        if bytes[count] == '$' as u8 {
+            if key.len() != 0 {
+                index.push(String::from_utf8(key.clone()).expect("Error decoding key"));
+                key.clear();
+            }
+        } else if bytes[count] == '!' as u8 {
+            count += 1;
+            key.push(bytes[count]);
+        } else {
+            key.push(bytes[count]);
+        }
+
+
+
+        if count == bytes.len() -1 {
+            index.push(String::from_utf8(key.clone()).expect("Error decoding key"));
+        }
+
+        count += 1;
+        if count >= bytes.len() {break};
+    }
+    return index;
+}
+
+pub fn encode(index: Vec<String>) -> Vec<u8> {
+    let mut bytes: Vec<u8> = Vec::new();
+    for key in index {
+        let mut b = key.into_bytes();
+        let mut count = 0;
+
+        loop {
+            if b[count] == '!' as u8 || b[count] == '$' as u8 {
+                b.insert(count, '!' as u8);
+                count += 1;
+            }
+
+            count += 1;
+            if count >= b.len() {break};
+        }
+        bytes.push('$' as u8);
+        bytes.append(&mut b);
+    }
+
+    return bytes;
+}
+

src/server/server_utils.rs

@@ -13,7 +13,7 @@ pub struct Fields{
 
 pub struct Index{
     commit_number: u64,
-    fields: Vec<Fields>
+    //fields: Vec<Fields>
 }
 
 pub fn import(args: Vec<String>){
@@ -24,84 +24,5 @@ pub fn update_item(id: u64, key: String, update_string: String){
 
 }
 
-pub fn read_index(id: u64, mize_folder: &str) -> io::Result<Index>{
-    let mut index_folder_num: String = String::new();
-    let mut index_num: String = String::new();
-    let id_len = id.to_string().chars().count();
-
-    if id_len <= 6 {
-        index_folder_num = String::from("0");
-        index_num = id.to_string();
-    } else {
-        index_folder_num = String::from(&id.to_string()[..id_len-6]);
-        index_num = String::from(&id.to_string()[id_len-6..]);
-    }
-
-    let index_file_path = String::from(mize_folder) + "/index" + &index_folder_num + "/" + &index_num;
-
-    let mut file = File::open(index_file_path)?;
-
-    let mut commit_number_buff: [u8; 8];
-    file.read_exact(&mut commit_number_buff[..])?;
-
-    let fields: Vec<Fields> = Vec::new();
-
-    Ok(Index {
-        commit_number: u64::from_le_bytes(commit_number_buff),
-
-    })
-
-}
-
-pub fn init_mize_folder(mize_folder: String) -> Result<(), MizeError>{
-    //##check if the folder resembles a mize folder
-
-    //## check if the index0 file exists and create one if it does not
-    let index_path = mize_folder.clone() + "/index0";
-    if !Path::new(&index_path).exists() {
-        File::create(&index_path).unwrap();
-    }
-    let index_md = fs::metadata(&index_path).unwrap();
-    if index_md.is_dir(){
-        return Err(MizeError{message: String::from("index is a folder and not a file")})
-    }
-
-    let folders: [&str; 5] = ["fields0","keys0","data0","config","other"];
-    for folder in folders {
-        //## check if the folder exists and create one if it does not
-        let path = mize_folder.clone() + "/" + folder;
-        if !Path::new(&path).exists() {
-            fs::create_dir(&path).unwrap();
-        }
-        let index_md = fs::metadata(&path).unwrap();
-        if index_md.is_file(){
-            return Err(MizeError{message: String::from(folder) + " is a file and not a folder"});
-        }
-    }
-
-    let index = read_index(0, &mize_folder);
-
-    //if let Ok(i) = index {
-        
-    //}
-    if let Err(i) = index{
-        println!("{:?}", i.kind());
-        match i.kind() {
-            io::ErrorKind::UnexpectedEof => {
-                //create the first item
-                
-                //create index for first item
-                File::create(mize_folder.clone() + "/keys0/0");
-
-                //create keyfile in keyfile0
-                File::create(mize_folder.clone() + "/keys0/000_000");
-            },
-            _ => return Err(MizeError{
-                message: String::from("sth went wrong when reading the index of item 0")
-            })
-        }
-    }
-    Ok(())
-}