Add database migrations

models/migrations/migrations.go

@@ -319,6 +319,8 @@ var migrations = []Migration{
 	NewMigration("Create PushMirror table", createPushMirrorTable),
 	// v184 -> v185
 	NewMigration("Rename Task errors to message", renameTaskErrorsToMessage),
+	// v185 -> v186
+	NewMigration("Add new table repo_archiver", addRepoArchiver),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v181.go

@@ -1,3 +1,4 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 

models/migrations/v185.go

@@ -0,0 +1,21 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"xorm.io/xorm"
+)
+
+func addRepoArchiver(x *xorm.Engine) error {
+	// RepoArchiver represents all archivers
+	type RepoArchiver struct {
+		ID          int64  `xorm:"pk autoincr"`
+		RepoID      int64  `xorm:"index unique(s)"`
+		Type        int    `xorm:"unique(s)"`
+		CommitID    string `xorm:"VARCHAR(40) unique(s)"`
+		CreatedUnix int64  `xorm:"INDEX NOT NULL created"`
+	}
+	return x.Sync2(new(RepoArchiver))
+}

models/repo_archiver.go

@@ -5,20 +5,59 @@
 package models
 
 import (
+	"fmt"
+
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/timeutil"
 )
 
+type RepoArchiverStatus int
+
+const (
+	RepoArchiverGenerating = iota // the archiver is generating
+	RepoArchiverReady             // it's ready
+)
+
 // RepoArchiver represents all archivers
 type RepoArchiver struct {
 	ID          int64           `xorm:"pk autoincr"`
 	RepoID      int64           `xorm:"index unique(s)"`
+	Repo        *Repository     `xorm:"-"`
 	Type        git.ArchiveType `xorm:"unique(s)"`
-	CommitID    string          `xorm:"VARCHAR(40) unique(s)"`
-	Name        string
+	Status      RepoArchiverStatus
+	CommitID    string             `xorm:"VARCHAR(40) unique(s)"`
 	CreatedUnix timeutil.TimeStamp `xorm:"INDEX NOT NULL created"`
 }
 
+// LoadRepo loads repository
+func (archiver *RepoArchiver) LoadRepo() (*Repository, error) {
+	if archiver.Repo != nil {
+		return archiver.Repo, nil
+	}
+
+	var repo Repository
+	has, err := x.ID(archiver.RepoID).Get(&repo)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, ErrRepoNotExist{
+			ID: archiver.RepoID,
+		}
+	}
+	return &repo, nil
+}
+
+// RelativePath returns relative path
+func (archiver *RepoArchiver) RelativePath() (string, error) {
+	repo, err := archiver.LoadRepo()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s/%s/%s.%s", repo.RepoPath(), archiver.CommitID[:2], archiver.CommitID, archiver.Type.String()), nil
+}
+
 // GetRepoArchiver get an archiver
 func GetRepoArchiver(ctx DBContext, repoID int64, tp git.ArchiveType, commitID string) (*RepoArchiver, error) {
 	var archiver RepoArchiver

models/unit_tests.go

@@ -74,6 +74,8 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
 
 	setting.RepoAvatar.Storage.Path = filepath.Join(setting.AppDataPath, "repo-avatars")
 
+	setting.RepoArchive.Storage.Path = filepath.Join(setting.AppDataPath, "repo-archives")
+
 	if err = storage.Init(); err != nil {
 		fatalTestError("storage.Init: %v\n", err)
 	}

modules/setting/repository.go

@@ -333,6 +333,7 @@ func newRepository() {
 		Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath)
 	}
 
-	repoAvatarSec := Cfg.Section("repository.archives")
-	RepoArchive.Storage = getStorage("repo-archive", "", repoAvatarSec)
+	repoAvatarSec := Cfg.Section("repo-archive")
+	storageType := repoAvatarSec.Key("STORAGE_TYPE").MustString("")
+	RepoArchive.Storage = getStorage("repo-archive", storageType, repoAvatarSec)
 }

routers/web/repo/repo.go

@@ -367,7 +367,7 @@ func RedirectDownload(ctx *context.Context) {
 // Download an archive of a repository
 func Download(ctx *context.Context) {
 	uri := ctx.Params("*")
-	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.RepoID, ctx.Repo.GitRepo, uri)
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri)
 	if err != nil {
 		ctx.ServerError("archiver_service.NewRequest", err)
 		return
@@ -377,24 +377,31 @@ func Download(ctx *context.Context) {
 		return
 	}
 
-	downloadName := ctx.Repo.Repository.Name + "-" + aReq.GetArchiveName()
-
-	if err := archiver_service.ArchiveRepository(aReq); err != nil {
+	archiver, err := archiver_service.ArchiveRepository(aReq)
+	if err != nil {
 		ctx.ServerError("ArchiveRepository", err)
 		return
 	}
 
+	rPath, err := archiver.RelativePath()
+	if err != nil {
+		ctx.ServerError("archiver.RelativePath", err)
+		return
+	}
+
+	downloadName := ctx.Repo.Repository.Name + "-" + aReq.GetArchiveName()
+
 	if setting.RepoArchive.ServeDirect {
 		//If we have a signed url (S3, object storage), redirect to this directly.
-		u, err := storage.RepoArchives.URL(aReq.GetArchivePath(), downloadName)
+		u, err := storage.RepoArchives.URL(rPath, downloadName)
 		if u != nil && err == nil {
 			ctx.Redirect(u.String())
 			return
 		}
 	}
 
 	//If we have matched and access to release or issue
-	fr, err := storage.RepoArchives.Open(aReq.GetArchivePath())
+	fr, err := storage.RepoArchives.Open(rPath)
 	if err != nil {
 		ctx.ServerError("Open", err)
 		return
@@ -418,13 +425,13 @@ func InitiateDownload(ctx *context.Context) {
 		return
 	}
 
-	err = archiver_service.ArchiveRepository(aReq)
+	archiver, err := archiver_service.ArchiveRepository(aReq)
 	if err != nil {
 		ctx.ServerError("archiver_service.ArchiveRepository", err)
 		return
 	}
 
 	ctx.JSON(http.StatusOK, map[string]interface{}{
-		"complete": true,
+		"complete": archiver.Status == models.RepoArchiverReady,
 	})
 }

services/archiver/archiver.go

@@ -8,12 +8,10 @@ package archiver
 import (
 	"fmt"
 	"io"
-	"path"
 	"regexp"
 	"strings"
 
 	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/setting"
@@ -31,9 +29,8 @@ type ArchiveRequest struct {
 	repo        *git.Repository
 	refName     string
 	ext         string
-	archivePath string
 	archiveType git.ArchiveType
-	commit      *git.Commit
+	commitID    string
 }
 
 // SHA1 hashes will only go up to 40 characters, but SHA256 hashes will go all
@@ -53,105 +50,106 @@ func NewRequest(repoID int64, repo *git.Repository, uri string) (*ArchiveRequest
 	switch {
 	case strings.HasSuffix(uri, ".zip"):
 		r.ext = ".zip"
-		r.archivePath = path.Join(r.repo.Path, "archives/zip")
 		r.archiveType = git.ZIP
 	case strings.HasSuffix(uri, ".tar.gz"):
 		r.ext = ".tar.gz"
-		r.archivePath = path.Join(r.repo.Path, "archives/targz")
 		r.archiveType = git.TARGZ
 	default:
 		return nil, fmt.Errorf("Unknown format: %s", uri)
 	}
 
 	r.refName = strings.TrimSuffix(r.uri, r.ext)
-	var err error
 
+	var err error
 	// Get corresponding commit.
 	if r.repo.IsBranchExist(r.refName) {
-		r.commit, err = r.repo.GetBranchCommit(r.refName)
+		r.commitID, err = r.repo.GetBranchCommitID(r.refName)
 		if err != nil {
 			return nil, err
 		}
 	} else if r.repo.IsTagExist(r.refName) {
-		r.commit, err = r.repo.GetTagCommit(r.refName)
+		r.commitID, err = r.repo.GetTagCommitID(r.refName)
 		if err != nil {
 			return nil, err
 		}
 	} else if shaRegex.MatchString(r.refName) {
-		r.commit, err = r.repo.GetCommit(r.refName)
-		if err != nil {
-			return nil, err
-		}
+		r.commitID = r.refName
 	} else {
 		return nil, fmt.Errorf("Unknow ref %s type", r.refName)
 	}
 
-	r.archivePath = path.Join(r.archivePath, base.ShortSha(r.commit.ID.String())+r.ext)
-	if err != nil {
-		return nil, err
-	}
 	return r, nil
 }
 
-// GetArchivePath returns the path from which we can serve this archive.
-func (aReq *ArchiveRequest) GetArchivePath() string {
-	return aReq.archivePath
-}
-
 // GetArchiveName returns the name of the caller, based on the ref used by the
 // caller to create this request.
 func (aReq *ArchiveRequest) GetArchiveName() string {
 	return aReq.refName + aReq.ext
 }
 
-func doArchive(r *ArchiveRequest) error {
+func doArchive(r *ArchiveRequest) (*models.RepoArchiver, error) {
 	ctx, commiter, err := models.TxDBContext()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer commiter.Close()
 
-	archiver, err := models.GetRepoArchiver(ctx, r.repoID, r.archiveType, r.commit.ID.String())
+	archiver, err := models.GetRepoArchiver(ctx, r.repoID, r.archiveType, r.commitID)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if archiver != nil {
-		return nil
+		return archiver, nil
 	}
 
-	if err := models.AddArchiver(ctx, &models.RepoArchiver{
+	archiver = &models.RepoArchiver{
 		RepoID:   r.repoID,
 		Type:     r.archiveType,
-		CommitID: r.commit.ID.String(),
-		Name:     r.GetArchiveName(),
-	}); err != nil {
-		return err
+		CommitID: r.commitID,
+	}
+	if err := models.AddArchiver(ctx, archiver); err != nil {
+		return nil, err
 	}
 
-	rd, w := io.Pipe()
-	var done chan error
+	rPath, err := archiver.RelativePath()
+	if err != nil {
+		return nil, err
+	}
 
-	go func(done chan error, w io.Writer) {
+	rd, w := io.Pipe()
+	defer func() {
+		w.Close()
+		rd.Close()
+	}()
+	var done = make(chan error)
+
+	go func(done chan error, w *io.PipeWriter) {
+		defer func() {
+			if r := recover(); r != nil {
+				done <- fmt.Errorf("%v", r)
+			}
+		}()
 		err := r.repo.CreateArchive(
 			graceful.GetManager().ShutdownContext(),
 			r.archiveType,
 			w,
 			setting.Repository.PrefixArchiveFiles,
-			r.commit.ID.String(),
+			r.commitID,
 		)
+		w.CloseWithError(err)
 		done <- err
 	}(done, w)
 
-	if _, err := storage.RepoArchives.Save(r.archivePath, rd, -1); err != nil {
-		return fmt.Errorf("Unable to write archive: %v", err)
+	if _, err := storage.RepoArchives.Save(rPath, rd, -1); err != nil {
+		return nil, fmt.Errorf("unable to write archive: %v", err)
 	}
 
 	err = <-done
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return commiter.Commit()
+	return archiver, commiter.Commit()
 }
 
 // ArchiveRepository satisfies the ArchiveRequest being passed in.  Processing
@@ -160,6 +158,6 @@ func doArchive(r *ArchiveRequest) error {
 // anything.  In all cases, the caller should be examining the *ArchiveRequest
 // being returned for completion, as it may be different than the one they passed
 // in.
-func ArchiveRepository(request *ArchiveRequest) error {
+func ArchiveRepository(request *ArchiveRequest) (*models.RepoArchiver, error) {
 	return doArchive(request)
 }