From 7edcc4856400107602c28a58b9eb774d577f0375 Mon Sep 17 00:00:00 2001 From: Rutger Broekhoff Date: Mon, 21 Oct 2024 00:12:46 +0200 Subject: Fix SHA256 checksum encoding --- flake.lock | 41 ++++++-------- flake.nix | 6 +- gitolfs3-server/src/handler.rs | 122 +++++++++++++++++++++++------------------ 3 files changed, 89 insertions(+), 80 deletions(-) diff --git a/flake.lock b/flake.lock index 8e0b00e..6f46ec0 100644 --- a/flake.lock +++ b/flake.lock @@ -17,22 +17,17 @@ } }, "crane": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, "locked": { - "lastModified": 1712513517, - "narHash": "sha256-VuLm5tTMqfS82NZAsNfsW7U+pTZ1+GcOU7gYR/Fb1Z4=", - "rev": "9caad1eb0c69a13ee6467035353b71a76c85ea53", - "revCount": 540, + "lastModified": 1728776144, + "narHash": "sha256-fROVjMcKRoGHofDm8dY3uDUtCMwUICh/KjBFQnuBzfg=", + "rev": "f876e3d905b922502f031aeec1a84490122254b7", + "revCount": 637, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/ipetkov/crane/0.16.4/018eb9c4-47d3-7d6b-9a55-1e0f89eabf96/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/ipetkov/crane/0.19.1/0192831e-af28-72ac-a578-95b4e58acd46/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/ipetkov/crane/0.16.3.tar.gz" + "url": "https://flakehub.com/f/ipetkov/crane/0.19.1.tar.gz" } }, "flake-utils": { @@ -40,30 +35,30 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "revCount": 92, + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "revCount": 101, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/numtide/flake-utils/0.1.92%2Brev-b1d9ab70662946ef0850d488da1c9019f3a9752a/018e2ca5-e5a2-7f80-9261-445a8cecd4d7/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/numtide/flake-utils/0.1.101%2Brev-c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a/0191ff0c-3a46-7ad4-b5e7-9f6806881886/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/numtide/flake-utils/0.1.92.tar.gz" + "url": "https://flakehub.com/f/numtide/flake-utils/0.1.101.tar.gz" } }, "nixpkgs": { "locked": { - "lastModified": 1713828541, - "narHash": "sha256-KtvQeE12MSkCOhvVmnmcZCjnx7t31zWin2XVSDOwBDE=", - "rev": "b500489fd3cf653eafc075f9362423ad5cdd8676", - "revCount": 557957, + "lastModified": 1729181673, + "narHash": "sha256-LDiPhQ3l+fBjRATNtnuDZsBS7hqoBtPkKBkhpoBHv3I=", + "rev": "4eb33fe664af7b41a4c446f87d20c9a0a6321fa3", + "revCount": 635979, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2311.557957%2Brev-b500489fd3cf653eafc075f9362423ad5cdd8676/018f0fd6-81fe-7478-8e97-08ae23c447b1/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2405.635979%2Brev-4eb33fe664af7b41a4c446f87d20c9a0a6321fa3/0192a20e-216f-78c5-8821-f030d9a83220/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2311.%2A.tar.gz" + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2405.%2A.tar.gz" } }, "root": { diff --git a/flake.nix b/flake.nix index 586486c..0b471d0 100644 --- a/flake.nix +++ b/flake.nix @@ -1,10 +1,10 @@ { inputs = { - nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2311.*.tar.gz"; - flake-utils.url = "https://flakehub.com/f/numtide/flake-utils/0.1.92.tar.gz"; + nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2405.*.tar.gz"; + flake-utils.url = "https://flakehub.com/f/numtide/flake-utils/0.1.101.tar.gz"; crane = { - url = "https://flakehub.com/f/ipetkov/crane/0.16.3.tar.gz"; + url = "https://flakehub.com/f/ipetkov/crane/0.19.1.tar.gz"; inputs.nixpkgs.follows = "nixpkgs"; }; diff --git a/gitolfs3-server/src/handler.rs b/gitolfs3-server/src/handler.rs index b9f9bcf..64d5492 100644 --- a/gitolfs3-server/src/handler.rs +++ b/gitolfs3-server/src/handler.rs @@ -33,6 +33,46 @@ pub struct AppState { pub dl_limiter: Arc>, } +enum ObjectStatus { + ExistsOk { content_length: Option }, + ExistsInconsistent, + DoesNotExist, +} + +impl AppState { + async fn check_object(&self, repo: &str, obj: &BatchRequestObject) -> Result { + let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1])); + let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid); + + let result = match self + .s3_client + .head_object() + .bucket(&self.s3_bucket) + .key(full_path) + .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) + .send() + .await + { + Ok(result) => result, + Err(SdkError::ServiceError(e)) if e.err().is_not_found() => { + return Ok(ObjectStatus::DoesNotExist); + } + Err(e) => { + println!("Failed to HeadObject (repo {repo}, OID {}): {e}", obj.oid); + return Err(()); + } + }; + + // Scaleway actually doesn't provide SHA256 support, but maybe in the future :) + if !s3_validate_checksum(obj.oid, &result) || !s3_validate_size(obj.size, &result) { + return Ok(ObjectStatus::ExistsInconsistent); + } + Ok(ObjectStatus::ExistsOk { + content_length: result.content_length(), + }) + } +} + async fn handle_download_object( state: &AppState, repo: &str, @@ -42,18 +82,16 @@ async fn handle_download_object( let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1])); let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid); - let result = match state - .s3_client - .head_object() - .bucket(&state.s3_bucket) - .key(&full_path) - .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) - .send() - .await - { - Ok(result) => result, - Err(e) => { - println!("Failed to HeadObject (repo {repo}, OID {}): {e}", obj.oid); + let content_length = match state.check_object(repo, obj).await { + Ok(ObjectStatus::ExistsOk { content_length }) => content_length, + Ok(_) => { + return BatchResponseObject::error( + obj, + http::StatusCode::UNPROCESSABLE_ENTITY, + "Object corrupted".to_string(), + ); + } + Err(_) => { return BatchResponseObject::error( obj, http::StatusCode::INTERNAL_SERVER_ERROR, @@ -62,22 +100,6 @@ async fn handle_download_object( } }; - // Scaleway actually doesn't provide SHA256 support, but maybe in the future :) - if !s3_validate_checksum(obj.oid, &result) { - return BatchResponseObject::error( - obj, - http::StatusCode::UNPROCESSABLE_ENTITY, - "Object corrupted".to_string(), - ); - } - if !s3_validate_size(obj.size, &result) { - return BatchResponseObject::error( - obj, - http::StatusCode::UNPROCESSABLE_ENTITY, - "Incorrect size specified (or object corrupted)".to_string(), - ); - } - let expires_in = std::time::Duration::from_secs(5 * 60); let expires_at = Utc::now() + expires_in; @@ -122,7 +144,7 @@ async fn handle_download_object( }; } - if let Some(content_length) = result.content_length() { + if let Some(content_length) = content_length { if content_length > 0 { match state .dl_limiter @@ -166,13 +188,6 @@ async fn handle_download_object( ); }; - let upload_path = format!( - "{repo}/info/lfs/objects/{}/{}/{}", - HexByte(obj.oid[0]), - HexByte(obj.oid[1]), - obj.oid, - ); - BatchResponseObject { oid: obj.oid, size: obj.size, @@ -188,7 +203,13 @@ async fn handle_download_object( map }, expires_at, - href: format!("{}/{upload_path}", state.base_url), + href: format!( + "{}/{repo}/info/lfs/objects/{}/{}/{}", + state.base_url, + HexByte(obj.oid[0]), + HexByte(obj.oid[1]), + obj.oid + ), }), ..Default::default() }, @@ -289,23 +310,12 @@ async fn handle_upload_object( let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1])); let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid); - match state - .s3_client - .head_object() - .bucket(&state.s3_bucket) - .key(full_path.clone()) - .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) - .send() - .await - { - Ok(result) => { - if s3_validate_size(obj.size, &result) && s3_validate_checksum(obj.oid, &result) { - return None; - } + match state.check_object(repo, obj).await { + Ok(ObjectStatus::ExistsOk { .. }) => { + return None; } - Err(SdkError::ServiceError(e)) if e.err().is_not_found() => {} - Err(e) => { - println!("Failed to HeadObject (repo {repo}, OID {}): {e}", obj.oid); + Ok(_) => {} + Err(_) => { return Some(BatchResponseObject::error( obj, http::StatusCode::INTERNAL_SERVER_ERROR, @@ -329,7 +339,7 @@ async fn handle_upload_object( .put_object() .bucket(&state.s3_bucket) .key(full_path) - .checksum_sha256(obj.oid.to_string()) + .checksum_sha256(s3_encode_checksum(obj.oid)) .content_length(obj.size) .presigned(config) .await @@ -418,6 +428,10 @@ pub async fn handle_batch( GitLfsJson(Json(resp)).into_response() } +fn s3_encode_checksum(oid: Oid) -> String { + BASE64_STANDARD.encode(oid.as_bytes()) +} + fn s3_validate_checksum(oid: Oid, obj: &HeadObjectOutput) -> bool { if let Some(checksum) = obj.checksum_sha256() { if let Ok(checksum) = BASE64_STANDARD.decode(checksum) { -- cgit v1.2.3