From 77f852e06d3be7cb558be73d6edc98d30cb52d65 Mon Sep 17 00:00:00 2001 From: Rutger Broekhoff Date: Fri, 29 Dec 2023 20:29:22 +0100 Subject: Write basic read-only public Git LFS server The 'integration' with Gitolite is honestly pretty bad and should not be taken very seriously: it runs the 'gitolite access' command to check if some user (e.g., daemon/nobody) should be able to read from the repository. Based on this, it grants access to objects stored in S3, by generating Presigned GetObject URLs using the S3 API. Of course, this integration with Gitolite (especially when using the daemon user to check if the user should be able to read) is not very 'high-value': 1. If we already make use of the daemon pseudo-user to control access to public repositories, we may as well check for the existence of git-daemon-export-ok files. In case they exist, we simply assume that the repository is meant to be shown on the public internet and that therefore the LFS archive should also be considered 'open to the public'. 2. The way that Gitolite commands are currently run, this program breaks when not running under the git user without extra configuration; Gitolite decides where repositories are based on the HOME environment variable. This program currently does not set this. This could be set by the CGI server (or fcgiwrap) and would unbreak the system. There's no support for any more advanced kind of authn/authz. Uploading is also not supported yet. That's still to come. --- cmd/git-lfs-server/main.go | 329 +++++++++++++++++++++++++++++++++++++++++++++ go.mod | 21 +++ go.sum | 51 +++++++ 3 files changed, 401 insertions(+) create mode 100644 go.sum diff --git a/cmd/git-lfs-server/main.go b/cmd/git-lfs-server/main.go index 06ab7d0..c5b47a3 100644 --- a/cmd/git-lfs-server/main.go +++ b/cmd/git-lfs-server/main.go @@ -1 +1,330 @@ package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/cgi" + "net/url" + "os" + "os/exec" + "path" + "regexp" + "slices" + "strconv" + "strings" + "time" + "unicode" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type operation string +type transferAdapter string +type hashAlgo string + +const ( + operationDownload operation = "download" + operationUpload operation = "upload" + transferAdapterBasic transferAdapter = "basic" + hashAlgoSHA256 hashAlgo = "sha256" +) + +const lfsMIME = "application/vnd.git-lfs+json" + +type batchRef struct { + Name string `json:"name"` +} + +type batchRequestObject struct { + OID string `json:"oid"` + Size uint64 `json:"size"` +} + +type batchRequest struct { + Operation operation `json:"operation"` + Transfers []transferAdapter `json:"transfers,omitempty"` + Ref *batchRef `json:"ref,omitempty"` + Objects []batchRequestObject `json:"objects"` + HashAlgo hashAlgo `json:"hash_algo,omitempty"` +} + +type RFC3339SecondsTime time.Time + +func (t RFC3339SecondsTime) MarshalJSON() ([]byte, error) { + b := make([]byte, 0, len(time.RFC3339)+len(`""`)) + b = append(b, '"') + b = time.Time(t).AppendFormat(b, time.RFC3339) + b = append(b, '"') + return b, nil +} + +type SecondDuration time.Duration + +func (d SecondDuration) MarshalJSON() ([]byte, error) { + var b []byte + b = strconv.AppendInt(b, int64(time.Duration(d).Seconds()), 10) + return b, nil +} + +type batchAction struct { + HRef *url.URL `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresIn *SecondDuration `json:"expires_in,omitempty"` + ExpiresAt *RFC3339SecondsTime `json:"expires_at,omitempty"` +} + +type batchError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type batchResponseObject struct { + OID string `json:"oid"` + Size uint64 `json:"size"` + Authenticated *bool `json:"authenticated"` + Actions map[operation]batchAction `json:"actions,omitempty"` + Error *batchError `json:"error,omitempty"` +} + +type batchResponse struct { + Transfer transferAdapter `json:"transfer,omitempty"` + Objects []batchResponseObject `json:"objects"` + HashAlgo hashAlgo `json:"hash_algo,omitempty"` +} + +var re = regexp.MustCompile(`^([a-zA-Z0-9-_/]+)\.git/info/lfs/objects/batch$`) + +type handler struct { + mc *minio.Client + bucket string + anonUser string +} + +// Requires lowercase hash +func isValidSHA256Hash(hash string) bool { + if len(hash) != 64 { + return false + } + for _, c := range hash { + if !unicode.Is(unicode.ASCII_Hex_Digit, c) { + return false + } + } + return true +} + +type lfsError struct { + Message string `json:"message"` + DocumentationURL string `json:"documentation_url,omitempty"` + RequestID string `json:"request_id,omitempty"` +} + +func makeRespError(w http.ResponseWriter, message string, code int) { + w.Header().Set("Content-Type", lfsMIME) + w.WriteHeader(code) + json.NewEncoder(w).Encode(lfsError{Message: message}) +} + +func makeObjError(obj parsedBatchObject, message string, code int) batchResponseObject { + return batchResponseObject{ + OID: obj.fullHash, + Size: obj.size, + Error: &batchError{ + Message: message, + Code: code, + }, + } +} + +func (h *handler) handleDownloadObject(ctx context.Context, repo string, obj parsedBatchObject) batchResponseObject { + fullPath := path.Join(repo, obj.firstByte, obj.secondByte, obj.fullHash) + expiresIn := time.Hour * 24 + expiresInSeconds := SecondDuration(expiresIn) + + info, err := h.mc.StatObject(ctx, h.bucket, fullPath, minio.StatObjectOptions{Checksum: true}) + if err != nil { + var resp minio.ErrorResponse + if errors.As(err, &resp) && resp.StatusCode == http.StatusNotFound { + return makeObjError(obj, "Object does not exist", http.StatusNotFound) + } + // TODO: consider not making this an object-specific, but rather a + // generic error such that the entire Batch API request fails. + return makeObjError(obj, "Failed to query object information", http.StatusInternalServerError) + } + if info.ChecksumSHA256 != "" && strings.ToLower(info.ChecksumSHA256) != obj.fullHash { + return makeObjError(obj, "Corrupted file", http.StatusUnprocessableEntity) + } + if uint64(info.Size) != obj.size { + return makeObjError(obj, "Incorrect size specified for object", http.StatusUnprocessableEntity) + } + + presigned, err := h.mc.PresignedGetObject(ctx, h.bucket, fullPath, expiresIn, url.Values{}) + if err != nil { + // TODO: consider not making this an object-specific, but rather a + // generic error such that the entire Batch API request fails. + return makeObjError(obj, "Failed to generate action href", http.StatusInternalServerError) + } + + authenticated := true + return batchResponseObject{ + OID: obj.fullHash, + Size: obj.size, + Authenticated: &authenticated, + Actions: map[operation]batchAction{ + operationDownload: { + HRef: presigned, + ExpiresIn: &expiresInSeconds, + }, + }, + } +} + +type parsedBatchObject struct { + firstByte string + secondByte string + fullHash string + size uint64 +} + +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + submatches := re.FindStringSubmatch(r.URL.Path) + if len(submatches) != 1 { + makeRespError(w, "Not found", http.StatusNotFound) + return + } + repo := strings.TrimPrefix("/", path.Clean(submatches[0])) + + if !slices.Contains(r.Header.Values("Accept"), lfsMIME) { + makeRespError(w, "Expected "+lfsMIME+" in list of acceptable response media types", http.StatusNotAcceptable) + return + } + if r.Header.Get("Content-Type") != lfsMIME { + makeRespError(w, "Expected request Content-Type to be "+lfsMIME, http.StatusUnsupportedMediaType) + return + } + + var body batchRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + makeRespError(w, "Failed to parse request body as JSON", http.StatusBadRequest) + return + } + + if body.HashAlgo != hashAlgoSHA256 { + makeRespError(w, "Unsupported hash algorithm specified", http.StatusConflict) + return + } + + // TODO: handle authentication + // right now, we're just trying to make everything publically accessible + if body.Operation == operationUpload { + makeRespError(w, "Upload operations are currently not supported", http.StatusForbidden) + return + } + + if len(body.Transfers) != 0 && !slices.Contains(body.Transfers, transferAdapterBasic) { + makeRespError(w, "Unsupported transfer adapter specified (supported: basic)", http.StatusConflict) + return + } + + gitoliteArgs := []string{"access", "-q", repo, h.anonUser, "R"} + if body.Ref != nil && body.Ref.Name != "" { + gitoliteArgs = append(gitoliteArgs, body.Ref.Name) + } + cmd := exec.Command("gitolite", gitoliteArgs...) + err := cmd.Run() + permGranted := err == nil + var exitErr *exec.ExitError + if err != nil && !errors.As(err, &exitErr) { + makeRespError(w, "Failed to query access information", http.StatusInternalServerError) + return + } + if !permGranted { + // TODO: when handling authorization, make sure to return 403 Forbidden + // here when the user *does* have read permissions, but is not allowed + // to write when requesting an upload operation. + makeRespError(w, "Repository not found", http.StatusNotFound) + return + } + + var objects []parsedBatchObject + for _, obj := range body.Objects { + oid := strings.ToLower(obj.OID) + if !isValidSHA256Hash(oid) { + makeRespError(w, "Invalid hash format in object ID", http.StatusBadRequest) + return + } + objects = append(objects, parsedBatchObject{ + firstByte: oid[:2], + secondByte: oid[2:4], + fullHash: oid, + size: obj.Size, + }) + } + + resp := batchResponse{ + Transfer: transferAdapterBasic, + HashAlgo: hashAlgoSHA256, + } + for _, obj := range objects { + resp.Objects = append(resp.Objects, h.handleDownloadObject(r.Context(), repo, obj)) + } + + w.Header().Set("Content-Type", lfsMIME) + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(resp) +} + +func die(msg string, args ...any) { + fmt.Fprint(os.Stderr, "Error: ") + fmt.Fprintf(os.Stderr, msg, args...) + fmt.Fprint(os.Stderr, "\n") + os.Exit(1) +} + +func main() { + endpoint := os.Getenv("S3_ENDPOINT") + accessKeyID := os.Getenv("S3_ACCESS_KEY_ID") + secretAccessKey := os.Getenv("S3_SECRET_ACCESS_KEY") + bucket := os.Getenv("S3_BUCKET") + anonUser := os.Getenv("ANON_USER") + + if endpoint == "" { + die("Expected environment variable S3_ENDPOINT to be set") + } + if accessKeyID == "" { + die("Expected environment variable S3_ACCESS_KEY_ID to be set") + } + if secretAccessKey == "" { + die("Expected environment variable S3_SECRET_ACCESS_KEY to be set") + } + if bucket == "" { + die("Expected environment variable S3_BUCKET to be set") + } + if anonUser == "" { + die("Expected environment variable ANON_USER to be set") + } + + mc, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: true, + }) + if err != nil { + die("Failed to create S3 client") + } + + if err = cgi.Serve(&handler{mc, bucket, anonUser}); err != nil { + die("Failed to serve CGI: %s", err) + } +} + +// Directory stucture: +// - lfs/ +// - locks/ +// - objects/ +// - <1st OID byte> +// - <2nd OID byte> +// - <- this is the object diff --git a/go.mod b/go.mod index e5c27f4..751d2c6 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,24 @@ module git.fautchen.eu/gitolfs3 go 1.21.5 + +require github.com/minio/minio-go/v7 v7.0.66 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..839d4c1 --- /dev/null +++ b/go.sum @@ -0,0 +1,51 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= +github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -- cgit v1.2.3