From 4cd2dde35c723d64fd621d73db37d261a7cf9c97 Mon Sep 17 00:00:00 2001
From: jake <77554505+brxken128@users.noreply.github.com>
Date: Thu, 13 Oct 2022 23:31:47 +0100
Subject: [PATCH] CAS ID Improvements (#413)

* remove `ring` dependency and use `sha2` instead

* use BLAKE3 and include full file checksum

* update schema comments
---
 Cargo.lock                | Bin 179166 -> 179472 bytes
 core/Cargo.toml           |   3 +--
 core/prisma/schema.prisma |   4 +--
 core/src/object/cas.rs    |  54 ++++++++++++++++++++------------------
 4 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index dd874bb855c68cb770f6d1ed6f50e8be868cb8be..5250160d46835c25154b5ac74648dd029cc00160 100644
GIT binary patch
delta 506
zcmY+AJ!@1!6o#3NyTTebf|9tJBE~|Cpfh*o%txdC17R)2UCzv$ff$XryNK!3%5=&t
zuGr=eC<}YB5p7ZlM$)Corpa#*1&MbT8_Sn-c+PX)!=KGJzndTS=TG<hxYehWKkVXd
zQ~@$0AR}o)GGI_9V-X3E>*R@2^xT9HKtgbqqs+J^l3eo`LFF&!+B4oW3dNuM4DZ;8
zkVUP7;YLt}O_-q|2s(0W69r{FILT$^Gf)Yrf=050pcJU6bGCeOv_Jp)$|>9+S)6~)
zD#_|3$P@rUEAIvPL^@TD+Q+GTmT^cTIboelE^r|<MT}ZG9fXy|*B!QW`QO*8Yt5Ba
zoQ6ty7DC8R9xN?9*&GaUbJ!cMKc*f%EhfL%T7Gh~KI-1sxO(=#S$^)DzomdXLk?Bd
z43Qv7(n*m*3K0N})2LL?N|FVqQ_`Sy)HZSFt?@qpXxsVSdzXsuZ)=^A?98p?UxxKp
zgYkfkua%$Qvop)(k1<<sm9)zm&2q5EdadH%kZm{oqggZE32PUP3$=yf-GqHAj*i)4
X`QnIm>gB;P+ufP&UN85yYpv`rOD(C_

delta 377
zcmYL_Jxjwt0EW3p!PXj-7IjdlIOx)FpSc_e`U9LQb(Kr*qJpFj_6HOfMFl0pO-@#4
zp~6uYx4KrGm39z+fP>(oc8TM&JnzGM@m+ZPD%`ykJDcwDVkd`^rSzvhXErXKTIZ%d
zR>@a&5;fFbvXj0uxlXjkM39I|fhZO#jzb!16(bQ-<;yU}Tm_O790if4B7g)k#RIGv
z#lB)ZZ7s|mwVSrx1dq$(73d+~!_Y=oU}8Q)Gd%@Lw$%k&uDS1lQl5LTD)$d|W!itJ
ztx?9AL|i})1r$uMP)imIA(YPrp`7ptYseXwNQWXILdJ|Sfp~y;h?yClJ2STN1d>9w
zXkVYfwws|@9e@+lAAs_tl85sDg5gowd=5aj^!LXoJ`3{@vbRH!<T8gXe}LX~h9l*Q
NfoD$9x~GnVe*sY=c3c1e

diff --git a/core/Cargo.toml b/core/Cargo.toml
index 3ad5da6a2..c0889def6 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -28,11 +28,10 @@ serde = { version = "1.0", features = ["derive"] }
 chrono = { version = "0.4.22", features = ["serde"] }
 serde_json = "1.0"
 futures = "0.3"
-data-encoding = "2.3.2"
-ring = "0.17.0-alpha.11"
 int-enum = "0.4.0"
 rmp = "^0.8.11"
 rmp-serde = "^1.1.1"
+blake3 = "1.3.1"
 
 # Project dependencies
 rspc = { workspace = true, features = ["uuid", "chrono", "tracing"] }
diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma
index e05e3d6a8..a6cd7f97d 100644
--- a/core/prisma/schema.prisma
+++ b/core/prisma/schema.prisma
@@ -98,9 +98,9 @@ model Location {
 
 model Object {
   id                 Int      @id @default(autoincrement())
-  // content addressable storage id - sha256 sampled checksum
+  // content addressable storage id - blake3 sampled checksum
   cas_id             String   @unique
-  // full byte contents digested into sha256 checksum
+  // full byte contents digested into blake3 checksum
   integrity_checksum String?  @unique
   // basic metadata
   name               String?
diff --git a/core/src/object/cas.rs b/core/src/object/cas.rs
index a671ecea3..58f6858de 100644
--- a/core/src/object/cas.rs
+++ b/core/src/object/cas.rs
@@ -1,5 +1,4 @@
-use data_encoding::HEXLOWER;
-use ring::digest::{Context, SHA256};
+use blake3::Hasher;
 use std::path::PathBuf;
 use tokio::{
 	fs::File,
@@ -18,51 +17,54 @@ async fn read_at(file: &mut File, offset: u64, size: u64) -> Result<Vec<u8>, io:
 	Ok(buf)
 }
 
+fn to_hex_string(b: &[u8]) -> String {
+	b.iter().map(|c| format!("{:02x}", c)).collect::<String>()
+}
+
 pub async fn generate_cas_id(path: PathBuf, size: u64) -> Result<String, io::Error> {
 	// open file reference
 	let mut file = File::open(path).await?;
 
-	let mut context = Context::new(&SHA256);
+	let mut hasher = Hasher::new();
 
 	// include the file size in the checksum
-	context.update(&size.to_le_bytes());
+	hasher.update(&size.to_le_bytes());
 
 	// if size is small enough, just read the whole thing
+
 	if SAMPLE_COUNT * SAMPLE_SIZE > size {
 		let buf = read_at(&mut file, 0, size).await?;
-		context.update(&buf);
+		hasher.update(&buf);
 	} else {
 		// loop over samples
 		for i in 0..SAMPLE_COUNT {
 			let buf = read_at(&mut file, (size / SAMPLE_COUNT) * i, SAMPLE_SIZE).await?;
-			context.update(&buf);
+			hasher.update(&buf);
 		}
 		// sample end of file
 		let buf = read_at(&mut file, size - SAMPLE_SIZE, SAMPLE_SIZE).await?;
-		context.update(&buf);
+		hasher.update(&buf);
 	}
 
-	let digest = context.finish();
-	let hex = HEXLOWER.encode(digest.as_ref());
+	let hex = to_hex_string(hasher.finalize().as_bytes());
 
 	Ok(hex)
 }
 
-// pub fn full_checksum(path: &str) -> Result<String> {
-// 	// read file as buffer and convert to digest
-// 	let mut reader = BufReader::new(File::open(path).unwrap());
-// 	let mut context = Context::new(&SHA256);
-// 	let mut buffer = [0; 1024];
-// 	loop {
-// 		let count = reader.read(&mut buffer)?;
-// 		if count == 0 {
-// 			break;
-// 		}
-// 		context.update(&buffer[..count]);
-// 	}
-// 	let digest = context.finish();
-// 	// create a lowercase hash from
-// 	let hex = HEXLOWER.encode(digest.as_ref());
+pub async fn full_checksum(path: &str) -> Result<String, io::Error> {
+	const BLOCK_SIZE: usize = 1048576;
+	//read file as buffer and convert to digest
+	let mut reader = File::open(path).await?;
+	let mut context = Hasher::new();
+	let mut buffer = [0; 1048576];
+	loop {
+		let read_count = reader.read(&mut buffer).await?;
+		context.update(&buffer[..read_count]);
+		if read_count != BLOCK_SIZE {
+			break;
+		}
+	}
+	let hex = to_hex_string(context.finalize().as_bytes());
 
-// 	Ok(hex)
-// }
+	Ok(hex)
+}