aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLibravatar Rutger Broekhoff2024-01-22 22:52:01 +0100
committerLibravatar Rutger Broekhoff2024-01-22 22:52:01 +0100
commitf5ff2803af0e03f57ab3093a9384d91abb9de083 (patch)
treeedb5a5c28c183c421bdf2b3e2c05c36ecb77bacd
parentefe9d55ea6c01c718d3c53dbcee6b48899b48267 (diff)
downloadgitolfs3-f5ff2803af0e03f57ab3093a9384d91abb9de083.tar.gz
gitolfs3-f5ff2803af0e03f57ab3093a9384d91abb9de083.zip
Finish basic implementation of Rust LFS server
-rw-r--r--rs/Cargo.lock1
-rw-r--r--rs/common/src/lib.rs43
-rw-r--r--rs/git-lfs-authenticate/src/main.rs15
-rw-r--r--rs/server/Cargo.toml1
-rw-r--r--rs/server/src/main.rs548
5 files changed, 483 insertions, 125 deletions
diff --git a/rs/Cargo.lock b/rs/Cargo.lock
index 29a2eac..5a83471 100644
--- a/rs/Cargo.lock
+++ b/rs/Cargo.lock
@@ -1884,6 +1884,7 @@ dependencies = [
1884 "serde", 1884 "serde",
1885 "serde_json", 1885 "serde_json",
1886 "tokio", 1886 "tokio",
1887 "tokio-util",
1887 "tower", 1888 "tower",
1888 "tower-service", 1889 "tower-service",
1889] 1890]
diff --git a/rs/common/src/lib.rs b/rs/common/src/lib.rs
index aafe7f1..27205bd 100644
--- a/rs/common/src/lib.rs
+++ b/rs/common/src/lib.rs
@@ -37,8 +37,9 @@ impl FromStr for Operation {
37} 37}
38 38
39#[repr(u8)] 39#[repr(u8)]
40pub enum AuthType { 40enum AuthType {
41 GitLfsAuthenticate = 1, 41 BatchApi = 1,
42 Download = 2,
42} 43}
43 44
44/// None means out of range. 45/// None means out of range.
@@ -156,6 +157,12 @@ impl<const N: usize> SafeByteArray<N> {
156 } 157 }
157} 158}
158 159
160impl<const N: usize> Default for SafeByteArray<N> {
161 fn default() -> Self {
162 Self::new()
163 }
164}
165
159impl<const N: usize> AsRef<[u8]> for SafeByteArray<N> { 166impl<const N: usize> AsRef<[u8]> for SafeByteArray<N> {
160 fn as_ref(&self) -> &[u8] { 167 fn as_ref(&self) -> &[u8] {
161 &self.inner 168 &self.inner
@@ -184,10 +191,18 @@ impl<const N: usize> FromStr for SafeByteArray<N> {
184 } 191 }
185} 192}
186 193
194pub type Oid = Digest<32>;
195
196#[derive(Debug, Copy, Clone)]
197pub enum SpecificClaims {
198 BatchApi(Operation),
199 Download(Oid),
200}
201
202#[derive(Debug, Copy, Clone)]
187pub struct Claims<'a> { 203pub struct Claims<'a> {
188 pub auth_type: AuthType, 204 pub specific_claims: SpecificClaims,
189 pub repo_path: &'a str, 205 pub repo_path: &'a str,
190 pub operation: Operation,
191 pub expires_at: DateTime<Utc>, 206 pub expires_at: DateTime<Utc>,
192} 207}
193 208
@@ -198,10 +213,18 @@ pub fn generate_tag(claims: Claims, key: impl AsRef<[u8]>) -> Option<Digest<32>>
198 } 213 }
199 214
200 let mut hmac = hmac_sha256::HMAC::new(key); 215 let mut hmac = hmac_sha256::HMAC::new(key);
201 hmac.update([claims.auth_type as u8]); 216 match claims.specific_claims {
217 SpecificClaims::BatchApi(operation) => {
218 hmac.update([AuthType::BatchApi as u8]);
219 hmac.update([operation as u8]);
220 }
221 SpecificClaims::Download(oid) => {
222 hmac.update([AuthType::Download as u8]);
223 hmac.update(oid.as_bytes());
224 }
225 }
202 hmac.update([claims.repo_path.len() as u8]); 226 hmac.update([claims.repo_path.len() as u8]);
203 hmac.update(claims.repo_path.as_bytes()); 227 hmac.update(claims.repo_path.as_bytes());
204 hmac.update([claims.operation as u8]);
205 hmac.update(claims.expires_at.timestamp().to_be_bytes()); 228 hmac.update(claims.expires_at.timestamp().to_be_bytes());
206 Some(hmac.finalize().into()) 229 Some(hmac.finalize().into())
207} 230}
@@ -280,9 +303,9 @@ impl<const N: usize> From<[u8; N]> for Digest<N> {
280 } 303 }
281} 304}
282 305
283impl<const N: usize> Into<[u8; N]> for Digest<N> { 306impl<const N: usize> From<Digest<N>> for [u8; N] {
284 fn into(self) -> [u8; N] { 307 fn from(val: Digest<N>) -> Self {
285 self.inner 308 val.inner
286 } 309 }
287} 310}
288 311
@@ -304,7 +327,7 @@ impl<const N: usize> ConstantTimeEq for Digest<N> {
304 327
305impl<const N: usize> PartialEq for Digest<N> { 328impl<const N: usize> PartialEq for Digest<N> {
306 fn eq(&self, other: &Self) -> bool { 329 fn eq(&self, other: &Self) -> bool {
307 self.ct_eq(&other).into() 330 self.ct_eq(other).into()
308 } 331 }
309} 332}
310 333
diff --git a/rs/git-lfs-authenticate/src/main.rs b/rs/git-lfs-authenticate/src/main.rs
index db95923..36d7818 100644
--- a/rs/git-lfs-authenticate/src/main.rs
+++ b/rs/git-lfs-authenticate/src/main.rs
@@ -148,30 +148,30 @@ struct Config {
148 148
149#[derive(Debug, Eq, PartialEq, Copy, Clone)] 149#[derive(Debug, Eq, PartialEq, Copy, Clone)]
150enum LoadConfigError { 150enum LoadConfigError {
151 BaseUrlMissing, 151 BaseUrlNotProvided,
152 BaseUrlSlashSuffixMissing, 152 BaseUrlSlashSuffixMissing,
153 KeyPathMissing, 153 KeyPathNotProvided,
154} 154}
155 155
156impl fmt::Display for LoadConfigError { 156impl fmt::Display for LoadConfigError {
157 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 157 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
158 match self { 158 match self {
159 Self::BaseUrlMissing => write!(f, "base URL not provided"), 159 Self::BaseUrlNotProvided => write!(f, "base URL not provided"),
160 Self::BaseUrlSlashSuffixMissing => write!(f, "base URL does not end with slash"), 160 Self::BaseUrlSlashSuffixMissing => write!(f, "base URL does not end with slash"),
161 Self::KeyPathMissing => write!(f, "key path not provided"), 161 Self::KeyPathNotProvided => write!(f, "key path not provided"),
162 } 162 }
163 } 163 }
164} 164}
165 165
166fn load_config() -> Result<Config, LoadConfigError> { 166fn load_config() -> Result<Config, LoadConfigError> {
167 let Ok(href_base) = std::env::var("GITOLFS3_HREF_BASE") else { 167 let Ok(href_base) = std::env::var("GITOLFS3_HREF_BASE") else {
168 return Err(LoadConfigError::BaseUrlMissing); 168 return Err(LoadConfigError::BaseUrlNotProvided);
169 }; 169 };
170 if !href_base.ends_with('/') { 170 if !href_base.ends_with('/') {
171 return Err(LoadConfigError::BaseUrlSlashSuffixMissing); 171 return Err(LoadConfigError::BaseUrlSlashSuffixMissing);
172 } 172 }
173 let Ok(key_path) = std::env::var("GITOLFS3_KEY_PATH") else { 173 let Ok(key_path) = std::env::var("GITOLFS3_KEY_PATH") else {
174 return Err(LoadConfigError::KeyPathMissing); 174 return Err(LoadConfigError::KeyPathNotProvided);
175 }; 175 };
176 Ok(Config { 176 Ok(Config {
177 href_base, 177 href_base,
@@ -213,10 +213,9 @@ fn main() -> ExitCode {
213 let expires_at = Utc::now() + Duration::from_secs(5 * 60); 213 let expires_at = Utc::now() + Duration::from_secs(5 * 60);
214 let Some(tag) = common::generate_tag( 214 let Some(tag) = common::generate_tag(
215 common::Claims { 215 common::Claims {
216 auth_type: common::AuthType::GitLfsAuthenticate, 216 specific_claims: common::SpecificClaims::BatchApi(operation),
217 repo_path: &repo_name, 217 repo_path: &repo_name,
218 expires_at, 218 expires_at,
219 operation,
220 }, 219 },
221 key, 220 key,
222 ) else { 221 ) else {
diff --git a/rs/server/Cargo.toml b/rs/server/Cargo.toml
index 9a2a9a9..987e154 100644
--- a/rs/server/Cargo.toml
+++ b/rs/server/Cargo.toml
@@ -15,5 +15,6 @@ mime = "0.3"
15serde = { version = "1", features = ["derive"] } 15serde = { version = "1", features = ["derive"] }
16serde_json = "1" 16serde_json = "1"
17tokio = { version = "1.35", features = ["full"] } 17tokio = { version = "1.35", features = ["full"] }
18tokio-util = "0.7"
18tower = "0.4" 19tower = "0.4"
19tower-service = "0.3" 20tower-service = "0.3"
diff --git a/rs/server/src/main.rs b/rs/server/src/main.rs
index 0266f61..99805a4 100644
--- a/rs/server/src/main.rs
+++ b/rs/server/src/main.rs
@@ -1,7 +1,9 @@
1use std::collections::HashMap; 1use std::collections::HashMap;
2use std::collections::HashSet; 2use std::collections::HashSet;
3use std::process::ExitCode;
3use std::sync::Arc; 4use std::sync::Arc;
4 5
6use aws_sdk_s3::error::SdkError;
5use aws_sdk_s3::operation::head_object::HeadObjectOutput; 7use aws_sdk_s3::operation::head_object::HeadObjectOutput;
6use axum::extract::rejection; 8use axum::extract::rejection;
7use axum::extract::FromRequest; 9use axum::extract::FromRequest;
@@ -15,7 +17,6 @@ use axum::Json;
15use axum::ServiceExt; 17use axum::ServiceExt;
16use base64::prelude::*; 18use base64::prelude::*;
17use chrono::DateTime; 19use chrono::DateTime;
18use chrono::Duration;
19use chrono::Utc; 20use chrono::Utc;
20use common::HexByte; 21use common::HexByte;
21use serde::de; 22use serde::de;
@@ -29,12 +30,10 @@ use axum::{
29 extract::{FromRequestParts, OriginalUri, Request}, 30 extract::{FromRequestParts, OriginalUri, Request},
30 http::{request::Parts, StatusCode, Uri}, 31 http::{request::Parts, StatusCode, Uri},
31 response::IntoResponse, 32 response::IntoResponse,
32 routing::{get, post, put}, 33 routing::{get, post},
33 Extension, Router, 34 Extension, Router,
34}; 35};
35 36
36use serde_json::json;
37
38#[derive(Clone)] 37#[derive(Clone)]
39struct RepositoryName(String); 38struct RepositoryName(String);
40 39
@@ -65,7 +64,10 @@ async fn rewrite_url<B>(
65 let uri = req.uri(); 64 let uri = req.uri();
66 let original_uri = OriginalUri(uri.clone()); 65 let original_uri = OriginalUri(uri.clone());
67 66
68 let path_and_query = uri.path_and_query().unwrap(); 67 let Some(path_and_query) = uri.path_and_query() else {
68 // L @ no path & query
69 return Err(StatusCode::BAD_REQUEST);
70 };
69 let Some((repo, path)) = path_and_query.path().split_once("/info/lfs/objects") else { 71 let Some((repo, path)) = path_and_query.path().split_once("/info/lfs/objects") else {
70 return Err(StatusCode::NOT_FOUND); 72 return Err(StatusCode::NOT_FOUND);
71 }; 73 };
@@ -73,7 +75,7 @@ async fn rewrite_url<B>(
73 .trim_start_matches('/') 75 .trim_start_matches('/')
74 .trim_end_matches('/') 76 .trim_end_matches('/')
75 .to_string(); 77 .to_string();
76 if !path.starts_with("/") || !repo.ends_with(".git") { 78 if !path.starts_with('/') || !repo.ends_with(".git") {
77 return Err(StatusCode::NOT_FOUND); 79 return Err(StatusCode::NOT_FOUND);
78 } 80 }
79 81
@@ -82,7 +84,9 @@ async fn rewrite_url<B>(
82 None => path.try_into().ok(), 84 None => path.try_into().ok(),
83 Some(q) => format!("{path}?{q}").try_into().ok(), 85 Some(q) => format!("{path}?{q}").try_into().ok(),
84 }; 86 };
85 let new_uri = Uri::from_parts(parts).unwrap(); 87 let Ok(new_uri) = Uri::from_parts(parts) else {
88 return Err(StatusCode::INTERNAL_SERVER_ERROR);
89 };
86 90
87 *req.uri_mut() = new_uri; 91 *req.uri_mut() = new_uri;
88 req.extensions_mut().insert(original_uri); 92 req.extensions_mut().insert(original_uri);
@@ -95,11 +99,45 @@ struct AppState {
95 s3_client: aws_sdk_s3::Client, 99 s3_client: aws_sdk_s3::Client,
96 s3_bucket: String, 100 s3_bucket: String,
97 authz_conf: AuthorizationConfig, 101 authz_conf: AuthorizationConfig,
102 // Should not end with a slash.
103 base_url: String,
104}
105
106struct Env {
107 s3_access_key_id: String,
108 s3_secret_access_key: String,
109 s3_bucket: String,
110 s3_endpoint: String,
111 base_url: String,
112 key_path: String,
113 listen_host: String,
114 listen_port: String,
115 trusted_forwarded_hosts: String,
116}
117
118fn require_env(name: &str) -> Result<String, String> {
119 std::env::var(name).map_err(|_| format!("environment variable {name} should be defined and valid"))
120}
121
122impl Env {
123 fn load() -> Result<Env, String> {
124 Ok(Env {
125 s3_secret_access_key: require_env("GITOLFS3_S3_ACCESS_KEY_FILE")?,
126 s3_access_key_id: require_env("GITOLFS3_S3_ACCESS_KEY_ID_FILE")?,
127 s3_endpoint: require_env("GITOLFS3_S3_ENDPOINT")?,
128 s3_bucket: require_env("GITOLFS3_S3_BUCKET")?,
129 base_url: require_env("GITOLFS3_BASE_URL")?,
130 key_path: require_env("GITOLFS3_KEY_PATH")?,
131 listen_host: require_env("GITOLFS3_LISTEN_HOST")?,
132 listen_port: require_env("GITOLFS3_LISTEN_PORT")?,
133 trusted_forwarded_hosts: std::env::var("GITOLFS3_TRUSTED_FORWARDED_HOSTS").unwrap_or_default(),
134 })
135 }
98} 136}
99 137
100fn get_s3_client() -> aws_sdk_s3::Client { 138fn get_s3_client(env: &Env) -> Result<aws_sdk_s3::Client, std::io::Error> {
101 let access_key_id = std::env::var("S3_ACCESS_KEY_ID").unwrap(); 139 let access_key_id = std::fs::read_to_string(&env.s3_access_key_id)?;
102 let secret_access_key = std::env::var("S3_SECRET_ACCESS_KEY").unwrap(); 140 let secret_access_key = std::fs::read_to_string(&env.s3_secret_access_key)?;
103 141
104 let credentials = aws_sdk_s3::config::Credentials::new( 142 let credentials = aws_sdk_s3::config::Credentials::new(
105 access_key_id, 143 access_key_id,
@@ -109,51 +147,85 @@ fn get_s3_client() -> aws_sdk_s3::Client {
109 "gitolfs3-env", 147 "gitolfs3-env",
110 ); 148 );
111 let config = aws_config::SdkConfig::builder() 149 let config = aws_config::SdkConfig::builder()
112 .endpoint_url(std::env::var("S3_ENDPOINT").unwrap()) 150 .endpoint_url(&env.s3_endpoint)
113 .credentials_provider(aws_sdk_s3::config::SharedCredentialsProvider::new( 151 .credentials_provider(aws_sdk_s3::config::SharedCredentialsProvider::new(
114 credentials, 152 credentials,
115 )) 153 ))
116 .build(); 154 .build();
117 aws_sdk_s3::Client::new(&config) 155 Ok(aws_sdk_s3::Client::new(&config))
118} 156}
119 157
120#[tokio::main] 158#[tokio::main]
121async fn main() { 159async fn main() -> ExitCode {
122 // run our app with hyper, listening globally on port 3000 160 let env = match Env::load() {
123 let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); 161 Ok(env) => env,
124 162 Err(e) => {
125 let key_path = std::env::var("GITOLFS3_KEY_PATH").unwrap(); 163 println!("Failed to load configuration: {e}");
126 let key = common::load_key(&key_path).unwrap(); 164 return ExitCode::from(2);
127 let trusted_forwarded_hosts = std::env::var("GITOLFS3_TRUSTED_FORWARDED_HOSTS").unwrap(); 165 }
128 let trusted_forwarded_hosts: HashSet<String> = trusted_forwarded_hosts 166 };
167
168 let s3_client = match get_s3_client(&env) {
169 Ok(s3_client) => s3_client,
170 Err(e) => {
171 println!("Failed to create S3 client: {e}");
172 return ExitCode::FAILURE;
173 },
174 };
175 let key = match common::load_key(&env.key_path) {
176 Ok(key) => key,
177 Err(e) => {
178 println!("Failed to load Gitolfs3 key: {e}");
179 return ExitCode::FAILURE;
180 }
181 };
182
183 let trusted_forwarded_hosts: HashSet<String> = env.trusted_forwarded_hosts
129 .split(',') 184 .split(',')
130 .map(|s| s.to_owned()) 185 .map(|s| s.to_owned())
186 .filter(|s| !s.is_empty())
131 .collect(); 187 .collect();
188 let base_url = env.base_url.trim_end_matches('/').to_string();
132 189
133 let authz_conf = AuthorizationConfig { 190 let authz_conf = AuthorizationConfig {
134 key, 191 key,
135 trusted_forwarded_hosts, 192 trusted_forwarded_hosts,
136 }; 193 };
137 194
138 let s3_client = get_s3_client();
139 let s3_bucket = std::env::var("S3_BUCKET").unwrap();
140 let shared_state = Arc::new(AppState { 195 let shared_state = Arc::new(AppState {
141 s3_client, 196 s3_client,
142 s3_bucket, 197 s3_bucket: env.s3_bucket,
143 authz_conf, 198 authz_conf,
199 base_url,
144 }); 200 });
145 let app = Router::new() 201 let app = Router::new()
146 .route("/batch", post(batch)) 202 .route("/batch", post(batch))
147 .route("/:oid0/:oid1/:oid", get(obj_download)) 203 .route("/:oid0/:oid1/:oid", get(obj_download))
148 .route("/:oid0/:oid1/:oid", put(obj_upload))
149 .with_state(shared_state); 204 .with_state(shared_state);
150 205
151 let middleware = axum::middleware::map_request(rewrite_url); 206 let middleware = axum::middleware::map_request(rewrite_url);
152 let app_with_middleware = middleware.layer(app); 207 let app_with_middleware = middleware.layer(app);
153 208
154 axum::serve(listener, app_with_middleware.into_make_service()) 209 let Ok(listen_port): Result<u16, _> = env.listen_port.parse() else {
155 .await 210 println!("Configured LISTEN_PORT should be an unsigned integer no higher than 65535");
156 .unwrap(); 211 return ExitCode::from(2);
212 };
213 let addr: (String, u16) = (env.listen_host, listen_port);
214 let listener = match tokio::net::TcpListener::bind(addr).await {
215 Ok(listener) => listener,
216 Err(e) => {
217 println!("Failed to listen: {e}");
218 return ExitCode::FAILURE;
219 }
220 };
221
222 match axum::serve(listener, app_with_middleware.into_make_service()).await {
223 Ok(_) => ExitCode::SUCCESS,
224 Err(e) => {
225 println!("Error serving: {e}");
226 ExitCode::FAILURE
227 }
228 }
157} 229}
158 230
159#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] 231#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)]
@@ -174,11 +246,9 @@ impl Default for HashAlgo {
174 } 246 }
175} 247}
176 248
177type Oid = common::Digest<32>;
178
179#[derive(Debug, Deserialize, Clone)] 249#[derive(Debug, Deserialize, Clone)]
180struct BatchRequestObject { 250struct BatchRequestObject {
181 oid: Oid, 251 oid: common::Oid,
182 size: i64, 252 size: i64,
183} 253}
184 254
@@ -196,8 +266,6 @@ struct BatchRequest {
196 operation: common::Operation, 266 operation: common::Operation,
197 #[serde(default = "default_transfers")] 267 #[serde(default = "default_transfers")]
198 transfers: Vec<TransferAdapter>, 268 transfers: Vec<TransferAdapter>,
199 #[serde(rename = "ref")]
200 reference: Option<BatchRef>,
201 objects: Vec<BatchRequestObject>, 269 objects: Vec<BatchRequestObject>,
202 #[serde(default)] 270 #[serde(default)]
203 hash_algo: HashAlgo, 271 hash_algo: HashAlgo,
@@ -206,7 +274,7 @@ struct BatchRequest {
206#[derive(Clone)] 274#[derive(Clone)]
207struct GitLfsJson<T>(Json<T>); 275struct GitLfsJson<T>(Json<T>);
208 276
209const LFS_MIME: &'static str = "application/vnd.git-lfs+json"; 277const LFS_MIME: &str = "application/vnd.git-lfs+json";
210 278
211enum GitLfsJsonRejection { 279enum GitLfsJsonRejection {
212 Json(rejection::JsonRejection), 280 Json(rejection::JsonRejection),
@@ -246,7 +314,7 @@ fn has_git_lfs_json_content_type(req: &Request) -> bool {
246 let Ok(content_type) = content_type.to_str() else { 314 let Ok(content_type) = content_type.to_str() else {
247 return false; 315 return false;
248 }; 316 };
249 return is_git_lfs_json_mimetype(content_type); 317 is_git_lfs_json_mimetype(content_type)
250} 318}
251 319
252#[async_trait] 320#[async_trait]
@@ -287,7 +355,7 @@ struct GitLfsErrorData<'a> {
287 355
288type GitLfsErrorResponse<'a> = (StatusCode, GitLfsJson<GitLfsErrorData<'a>>); 356type GitLfsErrorResponse<'a> = (StatusCode, GitLfsJson<GitLfsErrorData<'a>>);
289 357
290const fn make_error_resp<'a>(code: StatusCode, message: &'a str) -> GitLfsErrorResponse { 358const fn make_error_resp(code: StatusCode, message: &str) -> GitLfsErrorResponse {
291 (code, GitLfsJson(Json(GitLfsErrorData { message }))) 359 (code, GitLfsJson(Json(GitLfsErrorData { message })))
292} 360}
293 361
@@ -309,13 +377,36 @@ struct BatchResponseObjectActions {
309 verify: Option<BatchResponseObjectAction>, 377 verify: Option<BatchResponseObjectAction>,
310} 378}
311 379
380#[derive(Debug, Clone, Serialize)]
381struct BatchResponseObjectError {
382 code: u16,
383 message: String,
384}
385
312#[derive(Debug, Serialize, Clone)] 386#[derive(Debug, Serialize, Clone)]
313struct BatchResponseObject { 387struct BatchResponseObject {
314 oid: Oid, 388 oid: common::Oid,
315 size: i64, 389 size: i64,
316 #[serde(skip_serializing_if = "Option::is_none")] 390 #[serde(skip_serializing_if = "Option::is_none")]
317 authenticated: Option<bool>, 391 authenticated: Option<bool>,
318 actions: BatchResponseObjectActions, 392 actions: BatchResponseObjectActions,
393 #[serde(skip_serializing_if = "Option::is_none")]
394 error: Option<BatchResponseObjectError>,
395}
396
397impl BatchResponseObject {
398 fn error(obj: &BatchRequestObject, code: StatusCode, message: String) -> BatchResponseObject {
399 BatchResponseObject {
400 oid: obj.oid,
401 size: obj.size,
402 authenticated: None,
403 actions: Default::default(),
404 error: Some(BatchResponseObjectError {
405 code: code.as_u16(),
406 message,
407 }),
408 }
409 }
319} 410}
320 411
321#[derive(Debug, Serialize, Clone)] 412#[derive(Debug, Serialize, Clone)]
@@ -325,11 +416,11 @@ struct BatchResponse {
325 hash_algo: HashAlgo, 416 hash_algo: HashAlgo,
326} 417}
327 418
328fn validate_checksum(oid: Oid, obj: &HeadObjectOutput) -> bool { 419fn validate_checksum(oid: common::Oid, obj: &HeadObjectOutput) -> bool {
329 if let Some(checksum) = obj.checksum_sha256() { 420 if let Some(checksum) = obj.checksum_sha256() {
330 if let Ok(checksum) = BASE64_STANDARD.decode(checksum) { 421 if let Ok(checksum) = BASE64_STANDARD.decode(checksum) {
331 if let Ok(checksum32b) = TryInto::<[u8; 32]>::try_into(checksum) { 422 if let Ok(checksum32b) = TryInto::<[u8; 32]>::try_into(checksum) {
332 return Oid::from(checksum32b) == oid; 423 return common::Oid::from(checksum32b) == oid;
333 } 424 }
334 } 425 }
335 } 426 }
@@ -343,11 +434,15 @@ fn validate_size(expected: i64, obj: &HeadObjectOutput) -> bool {
343 true 434 true
344} 435}
345 436
346async fn handle_download_object(state: &AppState, repo: &str, obj: &BatchRequestObject, trusted: bool) -> BatchResponseObject { 437async fn handle_upload_object(
438 state: &AppState,
439 repo: &str,
440 obj: &BatchRequestObject,
441) -> Option<BatchResponseObject> {
347 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1])); 442 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1]));
348 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid); 443 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid);
349 444
350 let result = state 445 match state
351 .s3_client 446 .s3_client
352 .head_object() 447 .head_object()
353 .bucket(&state.s3_bucket) 448 .bucket(&state.s3_bucket)
@@ -355,36 +450,189 @@ async fn handle_download_object(state: &AppState, repo: &str, obj: &BatchRequest
355 .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled) 450 .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
356 .send() 451 .send()
357 .await 452 .await
358 .unwrap(); // TODO: don't unwrap() 453 {
454 Ok(result) => {
455 if validate_size(obj.size, &result) && validate_checksum(obj.oid, &result) {
456 return None;
457 }
458 }
459 Err(SdkError::ServiceError(e)) if e.err().is_not_found() => {}
460 _ => {
461 return Some(BatchResponseObject::error(
462 obj,
463 StatusCode::INTERNAL_SERVER_ERROR,
464 "Failed to query object information".to_string(),
465 ));
466 }
467 };
468
469 let expires_in = std::time::Duration::from_secs(5 * 60);
470 let expires_at = Utc::now() + expires_in;
471
472 let Ok(config) = aws_sdk_s3::presigning::PresigningConfig::expires_in(expires_in) else {
473 return Some(BatchResponseObject::error(
474 obj,
475 StatusCode::INTERNAL_SERVER_ERROR,
476 "Failed to generate upload URL".to_string(),
477 ));
478 };
479 let Ok(presigned) = state
480 .s3_client
481 .put_object()
482 .checksum_sha256(obj.oid.to_string())
483 .content_length(obj.size)
484 .presigned(config)
485 .await
486 else {
487 return Some(BatchResponseObject::error(
488 obj,
489 StatusCode::INTERNAL_SERVER_ERROR,
490 "Failed to generate upload URL".to_string(),
491 ));
492 };
493 Some(BatchResponseObject {
494 oid: obj.oid,
495 size: obj.size,
496 authenticated: Some(true),
497 actions: BatchResponseObjectActions {
498 download: Some(BatchResponseObjectAction {
499 header: presigned
500 .headers()
501 .map(|(k, v)| (k.to_owned(), v.to_owned()))
502 .collect(),
503 expires_at,
504 href: presigned.uri().to_string(),
505 }),
506 ..Default::default()
507 },
508 error: None,
509 })
510}
511
512async fn handle_download_object(
513 state: &AppState,
514 repo: &str,
515 obj: &BatchRequestObject,
516 trusted: bool,
517) -> BatchResponseObject {
518 let (oid0, oid1) = (HexByte(obj.oid[0]), HexByte(obj.oid[1]));
519 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, obj.oid);
520
521 let result = match state
522 .s3_client
523 .head_object()
524 .bucket(&state.s3_bucket)
525 .key(full_path)
526 .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
527 .send()
528 .await
529 {
530 Ok(result) => result,
531 Err(_) => {
532 return BatchResponseObject::error(
533 obj,
534 StatusCode::INTERNAL_SERVER_ERROR,
535 "Failed to query object information".to_string(),
536 )
537 }
538 };
539
359 // Scaleway actually doesn't provide SHA256 suport, but maybe in the future :) 540 // Scaleway actually doesn't provide SHA256 suport, but maybe in the future :)
360 if !validate_checksum(obj.oid, &result) { 541 if !validate_checksum(obj.oid, &result) {
361 todo!(); 542 return BatchResponseObject::error(
543 obj,
544 StatusCode::UNPROCESSABLE_ENTITY,
545 "Object corrupted".to_string(),
546 );
362 } 547 }
363 if !validate_size(obj.size, &result) { 548 if !validate_size(obj.size, &result) {
364 todo!(); 549 return BatchResponseObject::error(
550 obj,
551 StatusCode::UNPROCESSABLE_ENTITY,
552 "Incorrect size specified (or object corrupted)".to_string(),
553 );
365 } 554 }
366 555
367 let expires_in = std::time::Duration::from_secs(5 * 60); 556 let expires_in = std::time::Duration::from_secs(5 * 60);
368 let expires_at = Utc::now() + expires_in; 557 let expires_at = Utc::now() + expires_in;
369 558
370 if trusted { 559 if trusted {
371 let config = aws_sdk_s3::presigning::PresigningConfig::expires_in(expires_in).unwrap(); 560 let Ok(config) = aws_sdk_s3::presigning::PresigningConfig::expires_in(expires_in) else {
372 let presigned = state.s3_client.get_object().presigned(config).await.unwrap(); 561 return BatchResponseObject::error(
373 return BatchResponseObject{ 562 obj,
563 StatusCode::INTERNAL_SERVER_ERROR,
564 "Failed to generate upload URL".to_string(),
565 );
566 };
567 let Ok(presigned) = state.s3_client.get_object().presigned(config).await else {
568 return BatchResponseObject::error(
569 obj,
570 StatusCode::INTERNAL_SERVER_ERROR,
571 "Failed to generate upload URL".to_string(),
572 );
573 };
574 return BatchResponseObject {
374 oid: obj.oid, 575 oid: obj.oid,
375 size: obj.size, 576 size: obj.size,
376 authenticated: Some(true), 577 authenticated: Some(true),
377 actions: BatchResponseObjectActions { 578 actions: BatchResponseObjectActions {
378 download: Some(BatchResponseObjectAction{ 579 download: Some(BatchResponseObjectAction {
379 header: presigned.headers().map(|(k, v)| (k.to_owned(), v.to_owned())).collect(), 580 header: presigned
581 .headers()
582 .map(|(k, v)| (k.to_owned(), v.to_owned()))
583 .collect(),
380 expires_at, 584 expires_at,
381 href: presigned.uri().to_string(), 585 href: presigned.uri().to_string(),
382 }), 586 }),
383 ..Default::default() 587 ..Default::default()
384 } 588 },
589 error: None,
385 }; 590 };
386 } 591 }
387 todo!(); 592
593 let Some(tag) = common::generate_tag(
594 common::Claims {
595 specific_claims: common::SpecificClaims::Download(obj.oid),
596 repo_path: repo,
597 expires_at,
598 },
599 &state.authz_conf.key,
600 ) else {
601 return BatchResponseObject::error(
602 obj,
603 StatusCode::INTERNAL_SERVER_ERROR,
604 "Internal server error".to_string(),
605 );
606 };
607
608 let upload_path = format!(
609 "{repo}/info/lfs/objects/{}/{}/{}",
610 HexByte(obj.oid[0]),
611 HexByte(obj.oid[1]),
612 obj.oid,
613 );
614
615 BatchResponseObject {
616 oid: obj.oid,
617 size: obj.size,
618 authenticated: Some(true),
619 actions: BatchResponseObjectActions {
620 download: Some(BatchResponseObjectAction {
621 header: {
622 let mut map = HashMap::new();
623 map.insert(
624 "Authorization".to_string(),
625 format!("Gitolfs3-Hmac-Sha256 {tag} {}", expires_at.timestamp()),
626 );
627 map
628 },
629 expires_at,
630 href: format!("{}/{upload_path}", state.base_url),
631 }),
632 ..Default::default()
633 },
634 error: None,
635 }
388} 636}
389 637
390struct AuthorizationConfig { 638struct AuthorizationConfig {
@@ -410,17 +658,17 @@ fn forwarded_for_trusted_host(
410 )); 658 ));
411 } 659 }
412 } 660 }
413 return Ok(false); 661 Ok(false)
414} 662}
415const REPO_NOT_FOUND: GitLfsErrorResponse = 663const REPO_NOT_FOUND: GitLfsErrorResponse =
416 make_error_resp(StatusCode::NOT_FOUND, "Repository not found"); 664 make_error_resp(StatusCode::NOT_FOUND, "Repository not found");
417 665
418fn authorize( 666fn authorize_batch(
419 conf: &AuthorizationConfig, 667 conf: &AuthorizationConfig,
420 headers: &HeaderMap,
421 repo_path: &str, 668 repo_path: &str,
422 public: bool, 669 public: bool,
423 operation: common::Operation, 670 operation: common::Operation,
671 headers: &HeaderMap,
424) -> Result<Trusted, GitLfsErrorResponse<'static>> { 672) -> Result<Trusted, GitLfsErrorResponse<'static>> {
425 // - No authentication required for downloading exported repos 673 // - No authentication required for downloading exported repos
426 // - When authenticated: 674 // - When authenticated:
@@ -428,46 +676,12 @@ fn authorize(
428 // - When accessing over Tailscale: 676 // - When accessing over Tailscale:
429 // - No authentication required for downloading from any repo 677 // - No authentication required for downloading from any repo
430 678
431 const INVALID_AUTHZ_HEADER: GitLfsErrorResponse = 679 let claims = VerifyClaimsInput {
432 make_error_resp(StatusCode::BAD_REQUEST, "Invalid authorization header"); 680 specific_claims: common::SpecificClaims::BatchApi(operation),
433 681 repo_path,
434 if let Some(authz) = headers.get(header::AUTHORIZATION) { 682 };
435 if let Ok(authz) = authz.to_str() { 683 if verify_claims(conf, &claims, headers)? {
436 if let Some(val) = authz.strip_prefix("Gitolfs3-Hmac-Sha256 ") { 684 return Ok(Trusted(true));
437 let Some((tag, expires_at)) = val.split_once(' ') else {
438 return Err(INVALID_AUTHZ_HEADER);
439 };
440 let Ok(tag): Result<common::Digest<32>, _> = tag.parse() else {
441 return Err(INVALID_AUTHZ_HEADER);
442 };
443 let Ok(expires_at): Result<i64, _> = expires_at.parse() else {
444 return Err(INVALID_AUTHZ_HEADER);
445 };
446 let Some(expires_at) = DateTime::<Utc>::from_timestamp(expires_at, 0) else {
447 return Err(INVALID_AUTHZ_HEADER);
448 };
449 let Some(expected_tag) = common::generate_tag(
450 common::Claims {
451 auth_type: common::AuthType::GitLfsAuthenticate,
452 repo_path,
453 expires_at,
454 operation,
455 },
456 &conf.key,
457 ) else {
458 return Err(INVALID_AUTHZ_HEADER);
459 };
460 if tag == expected_tag {
461 return Ok(Trusted(true));
462 } else {
463 return Err(INVALID_AUTHZ_HEADER);
464 }
465 } else {
466 return Err(INVALID_AUTHZ_HEADER);
467 }
468 } else {
469 return Err(INVALID_AUTHZ_HEADER);
470 }
471 } 685 }
472 686
473 let trusted = forwarded_for_trusted_host(headers, &conf.trusted_forwarded_hosts)?; 687 let trusted = forwarded_for_trusted_host(headers, &conf.trusted_forwarded_hosts)?;
@@ -495,7 +709,7 @@ fn repo_exists(name: &str) -> bool {
495 let Ok(metadata) = std::fs::metadata(name) else { 709 let Ok(metadata) = std::fs::metadata(name) else {
496 return false; 710 return false;
497 }; 711 };
498 return metadata.is_dir(); 712 metadata.is_dir()
499} 713}
500 714
501fn is_repo_public(name: &str) -> Option<bool> { 715fn is_repo_public(name: &str) -> Option<bool> {
@@ -517,12 +731,12 @@ async fn batch(
517 let Some(public) = is_repo_public(&repo) else { 731 let Some(public) = is_repo_public(&repo) else {
518 return REPO_NOT_FOUND.into_response(); 732 return REPO_NOT_FOUND.into_response();
519 }; 733 };
520 let Trusted(trusted) = match authorize( 734 let Trusted(trusted) = match authorize_batch(
521 &state.authz_conf, 735 &state.authz_conf,
522 &headers,
523 &repo, 736 &repo,
524 public, 737 public,
525 payload.operation, 738 payload.operation,
739 &headers,
526 ) { 740 ) {
527 Ok(authn) => authn, 741 Ok(authn) => authn,
528 Err(e) => return e.into_response(), 742 Err(e) => return e.into_response(),
@@ -547,16 +761,24 @@ async fn batch(
547 return make_error_resp(StatusCode::CONFLICT, message).into_response(); 761 return make_error_resp(StatusCode::CONFLICT, message).into_response();
548 } 762 }
549 763
550 let resp: BatchResponse; 764 let mut resp = BatchResponse {
765 transfer: TransferAdapter::Basic,
766 objects: vec![],
767 hash_algo: HashAlgo::Sha256,
768 };
551 for obj in payload.objects { 769 for obj in payload.objects {
552 handle_download_object(&state, &repo, &obj, trusted).await; 770 match payload.operation {
553 // match payload.operation { 771 common::Operation::Download => resp
554 // Operation::Download => resp.objects.push(handle_download_object(repo, obj));, 772 .objects
555 // Operation::Upload => resp.objects.push(handle_upload_object(repo, obj)), 773 .push(handle_download_object(&state, &repo, &obj, trusted).await),
556 // }; 774 common::Operation::Upload => {
775 if let Some(obj_resp) = handle_upload_object(&state, &repo, &obj).await {
776 resp.objects.push(obj_resp);
777 }
778 }
779 };
557 } 780 }
558 781 GitLfsJson(Json(resp)).into_response()
559 format!("hi from {repo}\n").into_response()
560} 782}
561 783
562#[derive(Deserialize, Copy, Clone)] 784#[derive(Deserialize, Copy, Clone)]
@@ -564,7 +786,7 @@ async fn batch(
564struct FileParams { 786struct FileParams {
565 oid0: HexByte, 787 oid0: HexByte,
566 oid1: HexByte, 788 oid1: HexByte,
567 oid: Oid, 789 oid: common::Oid,
568} 790}
569 791
570impl<'de> Deserialize<'de> for FileParams { 792impl<'de> Deserialize<'de> for FileParams {
@@ -591,6 +813,118 @@ impl<'de> Deserialize<'de> for FileParams {
591 } 813 }
592} 814}
593 815
594async fn obj_download(Path(FileParams { oid0, oid1, oid }): Path<FileParams>) {} 816pub struct VerifyClaimsInput<'a> {
817 pub specific_claims: common::SpecificClaims,
818 pub repo_path: &'a str,
819}
820
821// Note: expires_at is ignored.
822fn verify_claims(
823 conf: &AuthorizationConfig,
824 claims: &VerifyClaimsInput,
825 headers: &HeaderMap,
826) -> Result<bool, GitLfsErrorResponse<'static>> {
827 const INVALID_AUTHZ_HEADER: GitLfsErrorResponse =
828 make_error_resp(StatusCode::BAD_REQUEST, "Invalid authorization header");
829
830 if let Some(authz) = headers.get(header::AUTHORIZATION) {
831 if let Ok(authz) = authz.to_str() {
832 if let Some(val) = authz.strip_prefix("Gitolfs3-Hmac-Sha256 ") {
833 let (tag, expires_at) = val.split_once(' ').ok_or(INVALID_AUTHZ_HEADER)?;
834 let tag: common::Digest<32> = tag.parse().map_err(|_| INVALID_AUTHZ_HEADER)?;
835 let expires_at: i64 = expires_at.parse().map_err(|_| INVALID_AUTHZ_HEADER)?;
836 let expires_at =
837 DateTime::<Utc>::from_timestamp(expires_at, 0).ok_or(INVALID_AUTHZ_HEADER)?;
838 let Some(expected_tag) = common::generate_tag(
839 common::Claims {
840 specific_claims: claims.specific_claims,
841 repo_path: claims.repo_path,
842 expires_at,
843 },
844 &conf.key,
845 ) else {
846 return Err(make_error_resp(
847 StatusCode::INTERNAL_SERVER_ERROR,
848 "Internal server error",
849 ));
850 };
851 if tag == expected_tag {
852 return Ok(true);
853 }
854 }
855 }
856 return Err(INVALID_AUTHZ_HEADER);
857 }
858 Ok(false)
859}
860
861fn authorize_get(
862 conf: &AuthorizationConfig,
863 repo_path: &str,
864 oid: common::Oid,
865 headers: &HeaderMap,
866) -> Result<(), GitLfsErrorResponse<'static>> {
867 let claims = VerifyClaimsInput {
868 specific_claims: common::SpecificClaims::Download(oid),
869 repo_path,
870 };
871 if !verify_claims(conf, &claims, headers)? {
872 return Err(make_error_resp(
873 StatusCode::UNAUTHORIZED,
874 "Repository not found",
875 ));
876 }
877 Ok(())
878}
595 879
596async fn obj_upload(Path(FileParams { oid0, oid1, oid }): Path<FileParams>) {} 880async fn obj_download(
881 State(state): State<Arc<AppState>>,
882 headers: HeaderMap,
883 RepositoryName(repo): RepositoryName,
884 Path(FileParams { oid0, oid1, oid }): Path<FileParams>,
885) -> Response {
886 if let Err(e) = authorize_get(&state.authz_conf, &repo, oid, &headers) {
887 return e.into_response();
888 }
889
890 let full_path = format!("{repo}/lfs/objects/{}/{}/{}", oid0, oid1, oid);
891 let result = match state
892 .s3_client
893 .get_object()
894 .bucket(&state.s3_bucket)
895 .key(full_path)
896 .checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
897 .send()
898 .await
899 {
900 Ok(result) => result,
901 Err(_) => {
902 return (
903 StatusCode::INTERNAL_SERVER_ERROR,
904 "Failed to query object information",
905 )
906 .into_response();
907 }
908 };
909
910 let mut headers = header::HeaderMap::new();
911 if let Some(content_type) = result.content_type {
912 let Ok(header_value) = content_type.try_into() else {
913 return (
914 StatusCode::INTERNAL_SERVER_ERROR,
915 "Object has invalid content type",
916 )
917 .into_response();
918 };
919 headers.insert(header::CONTENT_TYPE, header_value);
920 }
921 if let Some(content_length) = result.content_length {
922 headers.insert(header::CONTENT_LENGTH, content_length.into());
923 }
924
925 let async_read = result.body.into_async_read();
926 let stream = tokio_util::io::ReaderStream::new(async_read);
927 let body = axum::body::Body::from_stream(stream);
928
929 (headers, body).into_response()
930}