started with using rocksdb to store the data.
- debug binary is no 274MB.... lol - added encoding and decoding function
This commit is contained in:
Sebastian Moser
204
Cargo.lock
generated
204
Cargo.lock
generated
@@ -14,6 +14,25 @@ version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.60.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"peeking_take_while",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@@ -60,12 +79,52 @@ version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
|
||||
|
||||
[[package]]
|
||||
name = "bzip2-sys"
|
||||
version = "0.1.11+1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.73"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||
dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.5"
|
||||
@@ -204,6 +263,12 @@ dependencies = [
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.14"
|
||||
@@ -357,12 +422,69 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "lazycell"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "librocksdb-sys"
|
||||
version = "0.8.0+7.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"bzip2-sys",
|
||||
"cc",
|
||||
"glob",
|
||||
"libc",
|
||||
"libz-sys",
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libz-sys"
|
||||
version = "1.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.8"
|
||||
@@ -410,6 +532,12 @@ dependencies = [
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.8.4"
|
||||
@@ -427,6 +555,7 @@ name = "mize"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"rocksdb",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"warp",
|
||||
@@ -450,6 +579,16 @@ dependencies = [
|
||||
"twoway",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.13.1"
|
||||
@@ -495,6 +634,12 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.1.0"
|
||||
@@ -533,6 +678,12 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.16"
|
||||
@@ -602,6 +753,21 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
|
||||
dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
version = "0.5.3"
|
||||
@@ -611,6 +777,22 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rocksdb"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"librocksdb-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.11"
|
||||
@@ -688,6 +870,12 @@ dependencies = [
|
||||
"digest 0.10.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.0"
|
||||
@@ -981,6 +1169,12 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.4"
|
||||
@@ -1097,3 +1291,13 @@ name = "windows_x86_64_msvc"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.0.1+zstd.1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -7,14 +7,15 @@ edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "server"
|
||||
path = "server-main.rs"
|
||||
path = "src/server-main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "client"
|
||||
path = "client-main.rs"
|
||||
path = "src/client-main.rs"
|
||||
|
||||
[dependencies]
|
||||
futures-util = "0.3.24"
|
||||
rocksdb = "0.19.0"
|
||||
tokio = {version = "1.21.0", features = ["full"]}
|
||||
tokio-stream = "0.1.9"
|
||||
warp = "0.3.2"
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
|
||||
date +"%a %b %d" >> pitty-commit.sh
|
||||
git add pitty-commit.sh
|
||||
git commit -m "I am sorry. #pitty-commit"
|
||||
git commit -m "I am sorry for this. #pitty-commit"
|
||||
git push
|
||||
|
||||
exit
|
||||
# a list of all the dates I did nothing on
|
||||
Wed Sep 14
|
||||
Wed Sep 13
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
|
||||
#![ allow( warnings ) ]
|
||||
pub mod server;
|
||||
pub mod error;
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
|
||||
#![ allow( warnings ) ]
|
||||
|
||||
extern crate mize;
|
||||
|
||||
static HELP_MESSAGE: &str = "\
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
|
||||
|
||||
pub mod server_utils;
|
||||
pub mod itemstore;
|
||||
|
||||
use futures_util::{FutureExt, StreamExt};
|
||||
use warp::Filter;
|
||||
@@ -68,12 +69,14 @@ pub fn run_server(args: Vec<String>) {
|
||||
}
|
||||
|
||||
//### prep the mize_folder
|
||||
if let Err(err) = crate::server::server_utils::init_mize_folder(mize_folder) {
|
||||
println!("{}", err.message);
|
||||
}
|
||||
//if let Err(err) = crate::server::server_utils::init_mize_folder(mize_folder) {
|
||||
//println!("{}", err.message);
|
||||
//}
|
||||
|
||||
// get itemstore
|
||||
let itemstore = crate::server::itemstore::itemstore::new(mize_folder + "/db");
|
||||
|
||||
//run the webserver
|
||||
//hyper_server();
|
||||
warp_server();
|
||||
}
|
||||
|
||||
|
||||
75
src/server/itemstore.rs
Normal file
75
src/server/itemstore.rs
Normal file
@@ -0,0 +1,75 @@
|
||||
|
||||
use rocksdb::{SingleThreaded, DBCommon, DB, Options};
|
||||
use std::io;
|
||||
|
||||
pub struct itemstore {
|
||||
db: DB,
|
||||
}
|
||||
|
||||
impl itemstore {
|
||||
pub fn new(path: String) -> itemstore {
|
||||
let db = DB::open_default(&path).expect(&format!("Could not open rocksdb database at {}", &path)[..]);
|
||||
|
||||
match db.get(b"0") {
|
||||
Ok(Some(val)) => (),
|
||||
Ok(None) => {
|
||||
//db.set(b"0", )
|
||||
println!("hi")
|
||||
},
|
||||
Err(e) => println!("Error reading item 0 from db: {}", e),
|
||||
}
|
||||
return itemstore{db:db};
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode(bytes: Vec<u8>) -> Vec<String> {
|
||||
let mut count = 0;
|
||||
let mut index: Vec<String> = Vec::new();
|
||||
let mut key: Vec<u8> = Vec::new();
|
||||
loop {
|
||||
if bytes[count] == '$' as u8 {
|
||||
if key.len() != 0 {
|
||||
index.push(String::from_utf8(key.clone()).expect("Error decoding key"));
|
||||
key.clear();
|
||||
}
|
||||
} else if bytes[count] == '!' as u8 {
|
||||
count += 1;
|
||||
key.push(bytes[count]);
|
||||
} else {
|
||||
key.push(bytes[count]);
|
||||
}
|
||||
|
||||
|
||||
|
||||
if count == bytes.len() -1 {
|
||||
index.push(String::from_utf8(key.clone()).expect("Error decoding key"));
|
||||
}
|
||||
|
||||
count += 1;
|
||||
if count >= bytes.len() {break};
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
pub fn encode(index: Vec<String>) -> Vec<u8> {
|
||||
let mut bytes: Vec<u8> = Vec::new();
|
||||
for key in index {
|
||||
let mut b = key.into_bytes();
|
||||
let mut count = 0;
|
||||
|
||||
loop {
|
||||
if b[count] == '!' as u8 || b[count] == '$' as u8 {
|
||||
b.insert(count, '!' as u8);
|
||||
count += 1;
|
||||
}
|
||||
|
||||
count += 1;
|
||||
if count >= b.len() {break};
|
||||
}
|
||||
bytes.push('$' as u8);
|
||||
bytes.append(&mut b);
|
||||
}
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ pub struct Fields{
|
||||
|
||||
pub struct Index{
|
||||
commit_number: u64,
|
||||
fields: Vec<Fields>
|
||||
//fields: Vec<Fields>
|
||||
}
|
||||
|
||||
pub fn import(args: Vec<String>){
|
||||
@@ -24,84 +24,5 @@ pub fn update_item(id: u64, key: String, update_string: String){
|
||||
|
||||
}
|
||||
|
||||
pub fn read_index(id: u64, mize_folder: &str) -> io::Result<Index>{
|
||||
let mut index_folder_num: String = String::new();
|
||||
let mut index_num: String = String::new();
|
||||
let id_len = id.to_string().chars().count();
|
||||
|
||||
if id_len <= 6 {
|
||||
index_folder_num = String::from("0");
|
||||
index_num = id.to_string();
|
||||
} else {
|
||||
index_folder_num = String::from(&id.to_string()[..id_len-6]);
|
||||
index_num = String::from(&id.to_string()[id_len-6..]);
|
||||
}
|
||||
|
||||
let index_file_path = String::from(mize_folder) + "/index" + &index_folder_num + "/" + &index_num;
|
||||
|
||||
let mut file = File::open(index_file_path)?;
|
||||
|
||||
let mut commit_number_buff: [u8; 8];
|
||||
file.read_exact(&mut commit_number_buff[..])?;
|
||||
|
||||
let fields: Vec<Fields> = Vec::new();
|
||||
|
||||
Ok(Index {
|
||||
commit_number: u64::from_le_bytes(commit_number_buff),
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
pub fn init_mize_folder(mize_folder: String) -> Result<(), MizeError>{
|
||||
//##check if the folder resembles a mize folder
|
||||
|
||||
//## check if the index0 file exists and create one if it does not
|
||||
let index_path = mize_folder.clone() + "/index0";
|
||||
if !Path::new(&index_path).exists() {
|
||||
File::create(&index_path).unwrap();
|
||||
}
|
||||
let index_md = fs::metadata(&index_path).unwrap();
|
||||
if index_md.is_dir(){
|
||||
return Err(MizeError{message: String::from("index is a folder and not a file")})
|
||||
}
|
||||
|
||||
let folders: [&str; 5] = ["fields0","keys0","data0","config","other"];
|
||||
for folder in folders {
|
||||
//## check if the folder exists and create one if it does not
|
||||
let path = mize_folder.clone() + "/" + folder;
|
||||
if !Path::new(&path).exists() {
|
||||
fs::create_dir(&path).unwrap();
|
||||
}
|
||||
let index_md = fs::metadata(&path).unwrap();
|
||||
if index_md.is_file(){
|
||||
return Err(MizeError{message: String::from(folder) + " is a file and not a folder"});
|
||||
}
|
||||
}
|
||||
|
||||
let index = read_index(0, &mize_folder);
|
||||
|
||||
//if let Ok(i) = index {
|
||||
|
||||
//}
|
||||
if let Err(i) = index{
|
||||
println!("{:?}", i.kind());
|
||||
match i.kind() {
|
||||
io::ErrorKind::UnexpectedEof => {
|
||||
//create the first item
|
||||
|
||||
//create index for first item
|
||||
File::create(mize_folder.clone() + "/keys0/0");
|
||||
|
||||
//create keyfile in keyfile0
|
||||
File::create(mize_folder.clone() + "/keys0/000_000");
|
||||
},
|
||||
_ => return Err(MizeError{
|
||||
message: String::from("sth went wrong when reading the index of item 0")
|
||||
})
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user