aboutsummaryrefslogtreecommitdiffstats
path: root/gitolfs3-server
diff options
context:
space:
mode:
Diffstat (limited to 'gitolfs3-server')
-rw-r--r--gitolfs3-server/Cargo.toml14
-rw-r--r--gitolfs3-server/src/api.rs28
-rw-r--r--gitolfs3-server/src/authz.rs4
-rw-r--r--gitolfs3-server/src/config.rs20
-rw-r--r--gitolfs3-server/src/handler.rs101
-rw-r--r--gitolfs3-server/src/main.rs6
6 files changed, 85 insertions, 88 deletions
diff --git a/gitolfs3-server/Cargo.toml b/gitolfs3-server/Cargo.toml
index efea78b..91e2c57 100644
--- a/gitolfs3-server/Cargo.toml
+++ b/gitolfs3-server/Cargo.toml
@@ -1,20 +1,20 @@
1[package] 1[package]
2name = "gitolfs3-server" 2name = "gitolfs3-server"
3version = "0.1.0" 3version = "0.1.0"
4edition = "2021" 4edition = "2024"
5license = "MIT" 5license = "MIT"
6 6
7[dependencies] 7[dependencies]
8aws-config = { version = "1.1.2" } 8aws-config = "1.8"
9aws-sdk-s3 = "1.12.0" 9aws-sdk-s3 = "1.128"
10axum = "0.7" 10axum = "0.8"
11base64 = "0.21" 11base64 = "0.22"
12chrono = { version = "0.4", features = ["serde"] } 12chrono = { version = "0.4", features = ["serde"] }
13gitolfs3-common = { path = "../gitolfs3-common" } 13gitolfs3-common = { path = "../gitolfs3-common" }
14mime = "0.3" 14mime = "0.3"
15serde = { version = "1", features = ["derive"] } 15serde = { version = "1", features = ["derive"] }
16serde_json = "1" 16serde_json = "1"
17tokio = { version = "1.35", features = ["full"] } 17tokio = { version = "1.51", features = ["full"] }
18tokio-util = "0.7" 18tokio-util = "0.7"
19tower = "0.4" 19tower = "0.5"
20tracing-subscriber = { version = "0.3", features = ["env-filter"] } 20tracing-subscriber = { version = "0.3", features = ["env-filter"] }
diff --git a/gitolfs3-server/src/api.rs b/gitolfs3-server/src/api.rs
index d71d188..b80c83a 100644
--- a/gitolfs3-server/src/api.rs
+++ b/gitolfs3-server/src/api.rs
@@ -1,15 +1,14 @@
1use std::collections::HashMap; 1use std::collections::HashMap;
2 2
3use axum::{ 3use axum::{
4 async_trait, 4 Extension, Json,
5 extract::{rejection, FromRequest, FromRequestParts, Request}, 5 extract::{FromRequest, FromRequestParts, Request, rejection},
6 http, 6 http,
7 response::{IntoResponse, Response}, 7 response::{IntoResponse, Response},
8 Extension, Json,
9}; 8};
10use chrono::{DateTime, Utc}; 9use chrono::{DateTime, Utc};
11use gitolfs3_common::{Oid, Operation}; 10use gitolfs3_common::{Oid, Operation};
12use serde::{de::DeserializeOwned, Deserialize, Serialize}; 11use serde::{Deserialize, Serialize, de::DeserializeOwned};
13 12
14// ----------------------- Generic facilities ---------------------- 13// ----------------------- Generic facilities ----------------------
15 14
@@ -20,7 +19,10 @@ pub struct GitLfsErrorData<'a> {
20 pub message: &'a str, 19 pub message: &'a str,
21} 20}
22 21
23pub const fn make_error_resp(code: http::StatusCode, message: &str) -> GitLfsErrorResponse { 22pub const fn make_error_resp<'a>(
23 code: http::StatusCode,
24 message: &'a str,
25) -> GitLfsErrorResponse<'a> {
24 (code, GitLfsJson(Json(GitLfsErrorData { message }))) 26 (code, GitLfsJson(Json(GitLfsErrorData { message })))
25} 27}
26 28
@@ -76,7 +78,6 @@ fn has_git_lfs_json_content_type(req: &Request) -> bool {
76 is_git_lfs_json_mimetype(content_type) 78 is_git_lfs_json_mimetype(content_type)
77} 79}
78 80
79#[async_trait]
80impl<T, S> FromRequest<S> for GitLfsJson<T> 81impl<T, S> FromRequest<S> for GitLfsJson<T>
81where 82where
82 T: DeserializeOwned, 83 T: DeserializeOwned,
@@ -122,7 +123,6 @@ impl IntoResponse for RepositoryNameRejection {
122 } 123 }
123} 124}
124 125
125#[async_trait]
126impl<S: Send + Sync> FromRequestParts<S> for RepositoryName { 126impl<S: Send + Sync> FromRequestParts<S> for RepositoryName {
127 type Rejection = RepositoryNameRejection; 127 type Rejection = RepositoryNameRejection;
128 128
@@ -168,25 +168,15 @@ fn default_transfers() -> Vec<TransferAdapter> {
168 vec![TransferAdapter::Basic] 168 vec![TransferAdapter::Basic]
169} 169}
170 170
171#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] 171#[derive(Default, Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)]
172pub enum HashAlgo { 172pub enum HashAlgo {
173 #[default]
173 #[serde(rename = "sha256")] 174 #[serde(rename = "sha256")]
174 Sha256, 175 Sha256,
175 #[serde(other)] 176 #[serde(other)]
176 Unknown, 177 Unknown,
177} 178}
178 179
179impl Default for HashAlgo {
180 fn default() -> Self {
181 Self::Sha256
182 }
183}
184
185#[derive(Debug, Serialize, Deserialize, Clone)]
186struct BatchRef {
187 name: String,
188}
189
190#[derive(Debug, Serialize, Clone)] 180#[derive(Debug, Serialize, Clone)]
191pub struct BatchResponse { 181pub struct BatchResponse {
192 pub transfer: TransferAdapter, 182 pub transfer: TransferAdapter,
diff --git a/gitolfs3-server/src/authz.rs b/gitolfs3-server/src/authz.rs
index 8a5f21f..c4cb6df 100644
--- a/gitolfs3-server/src/authz.rs
+++ b/gitolfs3-server/src/authz.rs
@@ -2,10 +2,10 @@ use std::collections::HashSet;
2 2
3use axum::http; 3use axum::http;
4use chrono::{DateTime, Utc}; 4use chrono::{DateTime, Utc};
5use gitolfs3_common::{generate_tag, Claims, Digest, Oid, Operation, SpecificClaims}; 5use gitolfs3_common::{Claims, Digest, Oid, Operation, SpecificClaims, generate_tag};
6 6
7use crate::{ 7use crate::{
8 api::{make_error_resp, GitLfsErrorResponse, REPO_NOT_FOUND}, 8 api::{GitLfsErrorResponse, REPO_NOT_FOUND, make_error_resp},
9 config::AuthorizationConfig, 9 config::AuthorizationConfig,
10}; 10};
11 11
diff --git a/gitolfs3-server/src/config.rs b/gitolfs3-server/src/config.rs
index c6a51a5..5167cca 100644
--- a/gitolfs3-server/src/config.rs
+++ b/gitolfs3-server/src/config.rs
@@ -1,6 +1,6 @@
1use std::collections::HashSet; 1use std::collections::HashSet;
2 2
3use gitolfs3_common::{load_key, Key}; 3use gitolfs3_common::{Key, load_key};
4 4
5pub struct Config { 5pub struct Config {
6 pub listen_addr: (String, u16), 6 pub listen_addr: (String, u16),
@@ -18,19 +18,11 @@ pub struct AuthorizationConfig {
18 18
19impl Config { 19impl Config {
20 pub fn load() -> Result<Self, String> { 20 pub fn load() -> Result<Self, String> {
21 let env = match Env::load() { 21 let env = Env::load().map_err(|e| format!("failed to load configuration: {e}"))?;
22 Ok(env) => env, 22 let s3_client =
23 Err(e) => return Err(format!("failed to load configuration: {e}")), 23 create_s3_client(&env).map_err(|e| format!("failed to create S3 client: {e}"))?;
24 }; 24 let key =
25 25 load_key(&env.key_path).map_err(|e| format!("failed to load Gitolfs3 key: {e}"))?;
26 let s3_client = match create_s3_client(&env) {
27 Ok(s3_client) => s3_client,
28 Err(e) => return Err(format!("failed to create S3 client: {e}")),
29 };
30 let key = match load_key(&env.key_path) {
31 Ok(key) => key,
32 Err(e) => return Err(format!("failed to load Gitolfs3 key: {e}")),
33 };
34 26
35 let trusted_forwarded_hosts: HashSet<String> = env 27 let trusted_forwarded_hosts: HashSet<String> = env
36 .trusted_forwarded_hosts 28 .trusted_forwarded_hosts
diff --git a/gitolfs3-server/src/handler.rs b/gitolfs3-server/src/handler.rs
index 64d5492..1f47c9e 100644
--- a/gitolfs3-server/src/handler.rs
+++ b/gitolfs3-server/src/handler.rs
@@ -2,24 +2,24 @@ use std::{collections::HashMap, sync::Arc};
2 2
3use aws_sdk_s3::{error::SdkError, operation::head_object::HeadObjectOutput}; 3use aws_sdk_s3::{error::SdkError, operation::head_object::HeadObjectOutput};
4use axum::{ 4use axum::{
5 Json,
5 extract::{Path, State}, 6 extract::{Path, State},
6 http, 7 http,
7 response::{IntoResponse, Response}, 8 response::{IntoResponse, Response},
8 Json,
9}; 9};
10use base64::{prelude::BASE64_STANDARD, Engine}; 10use base64::{Engine, prelude::BASE64_STANDARD};
11use chrono::Utc; 11use chrono::Utc;
12use gitolfs3_common::{generate_tag, Claims, HexByte, Oid, Operation, SpecificClaims}; 12use gitolfs3_common::{Claims, HexByte, Oid, Operation, SpecificClaims, generate_tag};
13use serde::{de, Deserialize}; 13use serde::{Deserialize, de};
14use tokio::sync::Mutex; 14use tokio::sync::Mutex;
15 15
16use crate::{ 16use crate::{
17 api::{ 17 api::{
18 is_git_lfs_json_mimetype, make_error_resp, BatchRequest, BatchRequestObject, BatchResponse, 18 BatchRequest, BatchRequestObject, BatchResponse, BatchResponseObject,
19 BatchResponseObject, BatchResponseObjectAction, BatchResponseObjectActions, GitLfsJson, 19 BatchResponseObjectAction, BatchResponseObjectActions, GitLfsJson, HashAlgo, LFS_MIME,
20 HashAlgo, RepositoryName, TransferAdapter, LFS_MIME, REPO_NOT_FOUND, 20 REPO_NOT_FOUND, RepositoryName, TransferAdapter, is_git_lfs_json_mimetype, make_error_resp,
21 }, 21 },
22 authz::{authorize_batch, authorize_get, Trusted}, 22 authz::{Trusted, authorize_batch, authorize_get},
23 config::AuthorizationConfig, 23 config::AuthorizationConfig,
24 dlimit::DownloadLimiter, 24 dlimit::DownloadLimiter,
25}; 25};
@@ -42,7 +42,7 @@ enum ObjectStatus {
42impl AppState { 42impl AppState {
43 async fn check_object(&self, repo: &str, obj: &BatchRequestObject) -> Result<ObjectStatus, ()> { 43 async fn check_object(&self, repo: &str, obj: &BatchRequestObject) -> Result<ObjectStatus, ()> {
44 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1])); 44 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1]));
45 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid); 45 let full_path = format!("{repo}/lfs/objects/{oid0}/{oid1}/{}", obj.oid);
46 46
47 let result = match self 47 let result = match self
48 .s3_client 48 .s3_client
@@ -57,6 +57,14 @@ impl AppState {
57 Err(SdkError::ServiceError(e)) if e.err().is_not_found() => { 57 Err(SdkError::ServiceError(e)) if e.err().is_not_found() => {
58 return Ok(ObjectStatus::DoesNotExist); 58 return Ok(ObjectStatus::DoesNotExist);
59 } 59 }
60 Err(SdkError::ServiceError(e)) => {
61 println!(
62 "Failed to HeadObject (repo {repo}, OID {}): {}",
63 e.err(),
64 obj.oid
65 );
66 return Err(());
67 }
60 Err(e) => { 68 Err(e) => {
61 println!("Failed to HeadObject (repo {repo}, OID {}): {e}", obj.oid); 69 println!("Failed to HeadObject (repo {repo}, OID {}): {e}", obj.oid);
62 return Err(()); 70 return Err(());
@@ -80,7 +88,7 @@ async fn handle_download_object(
80 trusted: bool, 88 trusted: bool,
81) -> BatchResponseObject { 89) -> BatchResponseObject {
82 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1])); 90 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1]));
83 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid); 91 let full_path = format!("{repo}/lfs/objects/{oid0}/{oid1}/{}", obj.oid);
84 92
85 let content_length = match state.check_object(repo, obj).await { 93 let content_length = match state.check_object(repo, obj).await {
86 Ok(ObjectStatus::ExistsOk { content_length }) => content_length, 94 Ok(ObjectStatus::ExistsOk { content_length }) => content_length,
@@ -144,31 +152,31 @@ async fn handle_download_object(
144 }; 152 };
145 } 153 }
146 154
147 if let Some(content_length) = content_length { 155 if let Some(content_length) = content_length
148 if content_length > 0 { 156 && content_length > 0
149 match state 157 {
150 .dl_limiter 158 match state
151 .lock() 159 .dl_limiter
152 .await 160 .lock()
153 .request(content_length as u64) 161 .await
154 .await 162 .request(content_length as u64)
155 { 163 .await
156 Ok(true) => {} 164 {
157 Ok(false) => { 165 Ok(true) => {}
158 return BatchResponseObject::error( 166 Ok(false) => {
159 obj, 167 return BatchResponseObject::error(
160 http::StatusCode::SERVICE_UNAVAILABLE, 168 obj,
161 "Public LFS downloads temporarily unavailable".to_string(), 169 http::StatusCode::SERVICE_UNAVAILABLE,
162 ); 170 "Public LFS downloads temporarily unavailable".to_string(),
163 } 171 );
164 Err(e) => { 172 }
165 println!("Failed to request {content_length} bytes from download limiter: {e}"); 173 Err(e) => {
166 return BatchResponseObject::error( 174 println!("Failed to request {content_length} bytes from download limiter: {e}");
167 obj, 175 return BatchResponseObject::error(
168 http::StatusCode::INTERNAL_SERVER_ERROR, 176 obj,
169 "Internal server error".to_string(), 177 http::StatusCode::INTERNAL_SERVER_ERROR,
170 ); 178 "Internal server error".to_string(),
171 } 179 );
172 } 180 }
173 } 181 }
174 } 182 }
@@ -259,7 +267,7 @@ pub async fn handle_obj_download(
259 return e.into_response(); 267 return e.into_response();
260 } 268 }
261 269
262 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, oid); 270 let full_path = format!("{repo}/lfs/objects/{oid0}/{oid1}/{oid}");
263 let result = match state 271 let result = match state
264 .s3_client 272 .s3_client
265 .get_object() 273 .get_object()
@@ -270,6 +278,14 @@ pub async fn handle_obj_download(
270 .await 278 .await
271 { 279 {
272 Ok(result) => result, 280 Ok(result) => result,
281 Err(SdkError::ServiceError(e)) => {
282 println!("Failed to GetObject (repo {repo}, OID {oid}): {}", e.err());
283 return (
284 http::StatusCode::INTERNAL_SERVER_ERROR,
285 "Failed to query object information",
286 )
287 .into_response();
288 }
273 Err(e) => { 289 Err(e) => {
274 println!("Failed to GetObject (repo {repo}, OID {oid}): {e}"); 290 println!("Failed to GetObject (repo {repo}, OID {oid}): {e}");
275 return ( 291 return (
@@ -308,7 +324,7 @@ async fn handle_upload_object(
308 obj: &BatchRequestObject, 324 obj: &BatchRequestObject,
309) -> Option<BatchResponseObject> { 325) -> Option<BatchResponseObject> {
310 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1])); 326 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1]));
311 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid); 327 let full_path = format!("{repo}/lfs/objects/{oid0}/{oid1}/{}", obj.oid);
312 328
313 match state.check_object(repo, obj).await { 329 match state.check_object(repo, obj).await {
314 Ok(ObjectStatus::ExistsOk { .. }) => { 330 Ok(ObjectStatus::ExistsOk { .. }) => {
@@ -433,12 +449,11 @@ fn s3_encode_checksum(oid: Oid) -> String {
433} 449}
434 450
435fn s3_validate_checksum(oid: Oid, obj: &HeadObjectOutput) -> bool { 451fn s3_validate_checksum(oid: Oid, obj: &HeadObjectOutput) -> bool {
436 if let Some(checksum) = obj.checksum_sha256() { 452 if let Some(checksum) = obj.checksum_sha256()
437 if let Ok(checksum) = BASE64_STANDARD.decode(checksum) { 453 && let Ok(checksum) = BASE64_STANDARD.decode(checksum)
438 if let Ok(checksum32b) = TryInto::<[u8; 32]>::try_into(checksum) { 454 && let Ok(checksum32b) = TryInto::<[u8; 32]>::try_into(checksum)
439 return Oid::from(checksum32b) == oid; 455 {
440 } 456 return Oid::from(checksum32b) == oid;
441 }
442 } 457 }
443 true 458 true
444} 459}
diff --git a/gitolfs3-server/src/main.rs b/gitolfs3-server/src/main.rs
index 46e840a..c88de76 100644
--- a/gitolfs3-server/src/main.rs
+++ b/gitolfs3-server/src/main.rs
@@ -9,12 +9,12 @@ use config::Config;
9use dlimit::DownloadLimiter; 9use dlimit::DownloadLimiter;
10 10
11use axum::{ 11use axum::{
12 Router, ServiceExt,
12 extract::OriginalUri, 13 extract::OriginalUri,
13 http::{self, Uri}, 14 http::{self, Uri},
14 routing::{get, post}, 15 routing::{get, post},
15 Router, ServiceExt,
16}; 16};
17use handler::{handle_batch, handle_obj_download, AppState}; 17use handler::{AppState, handle_batch, handle_obj_download};
18use std::{process::ExitCode, sync::Arc}; 18use std::{process::ExitCode, sync::Arc};
19use tokio::net::TcpListener; 19use tokio::net::TcpListener;
20use tower::Layer; 20use tower::Layer;
@@ -41,7 +41,7 @@ async fn main() -> ExitCode {
41 }); 41 });
42 let app = Router::new() 42 let app = Router::new()
43 .route("/batch", post(handle_batch)) 43 .route("/batch", post(handle_batch))
44 .route("/:oid0/:oid1/:oid", get(handle_obj_download)) 44 .route("/{oid0}/{oid1}/{oid}", get(handle_obj_download))
45 .with_state(shared_state); 45 .with_state(shared_state);
46 46
47 let middleware = axum::middleware::map_request(rewrite_url); 47 let middleware = axum::middleware::map_request(rewrite_url);