fix: lookup of pipelines for older builds

router/middleware/pipeline/pipeline.go

@@ -9,11 +9,14 @@ import (
 	"net/http"
 
 	"github.com/gin-gonic/gin"
+	"github.com/go-vela/server/compiler"
 	"github.com/go-vela/server/database"
 	"github.com/go-vela/server/router/middleware/org"
 	"github.com/go-vela/server/router/middleware/repo"
 	"github.com/go-vela/server/router/middleware/user"
+	"github.com/go-vela/server/scm"
 	"github.com/go-vela/server/util"
+	"github.com/go-vela/types"
 	"github.com/go-vela/types/library"
 	"github.com/sirupsen/logrus"
 )
@@ -47,6 +50,8 @@ func Establish() gin.HandlerFunc {
 			return
 		}
 
+		entry := fmt.Sprintf("%s/%s", r.GetFullName(), p)
+
 		// update engine logger with API metadata
 		//
 		// https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields
@@ -55,15 +60,34 @@ func Establish() gin.HandlerFunc {
 			"pipeline": p,
 			"repo":     r.GetName(),
 			"user":     u.GetName(),
-		}).Debugf("reading pipeline %s/%s", r.GetFullName(), p)
+		}).Debugf("reading pipeline %s", entry)
 
 		pipeline, err := database.FromContext(c).GetPipelineForRepo(p, r)
-		if err != nil {
-			retErr := fmt.Errorf("unable to read pipeline %s/%s: %w", r.GetFullName(), p, err)
-
-			util.HandleError(c, http.StatusNotFound, retErr)
-
-			return
+		if err != nil { // assume the pipeline doesn't exist in the database yet (before pipeline support was added)
+			// send API call to capture the pipeline configuration file
+			config, err := scm.FromContext(c).ConfigBackoff(u, r, p)
+			if err != nil {
+				retErr := fmt.Errorf("unable to get pipeline configuration for %s: %w", entry, err)
+
+				util.HandleError(c, http.StatusNotFound, retErr)
+
+				return
+			}
+
+			// parse and compile the pipeline configuration file
+			_, pipeline, err = compiler.FromContext(c).
+				Duplicate().
+				WithMetadata(c.MustGet("metadata").(*types.Metadata)).
+				WithRepo(r).
+				WithUser(u).
+				Compile(config)
+			if err != nil {
+				retErr := fmt.Errorf("unable to compile pipeline configuration for %s: %w", entry, err)
+
+				util.HandleError(c, http.StatusInternalServerError, retErr)
+
+				return
+			}
 		}
 
 		ToContext(c, pipeline)

router/middleware/pipeline/pipeline_test.go

@@ -5,17 +5,28 @@
 package pipeline
 
 import (
+	"flag"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/go-vela/server/compiler"
+	"github.com/go-vela/server/compiler/native"
 	"github.com/go-vela/server/database"
 	"github.com/go-vela/server/database/sqlite"
 	"github.com/go-vela/server/router/middleware/org"
 	"github.com/go-vela/server/router/middleware/repo"
+	"github.com/go-vela/server/router/middleware/token"
+	"github.com/go-vela/server/router/middleware/user"
+	"github.com/go-vela/server/scm"
+	"github.com/go-vela/server/scm/github"
+	"github.com/go-vela/types"
 	"github.com/go-vela/types/library"
+	"github.com/urfave/cli/v2"
 )
 
 func TestPipeline_Retrieve(t *testing.T) {
@@ -197,6 +208,8 @@ func TestPipeline_Establish_NoPipelineParameter(t *testing.T) {
 
 func TestPipeline_Establish_NoPipeline(t *testing.T) {
 	// setup types
+	secret := "superSecret"
+
 	r := new(library.Repo)
 	r.SetID(1)
 	r.SetUserID(1)
@@ -206,28 +219,86 @@ func TestPipeline_Establish_NoPipeline(t *testing.T) {
 	r.SetFullName("foo/bar")
 	r.SetVisibility("public")
 
+	u := new(library.User)
+	u.SetID(1)
+	u.SetName("foo")
+	u.SetToken("bar")
+	u.SetHash("baz")
+	u.SetAdmin(true)
+
+	m := &types.Metadata{
+		Database: &types.Database{
+			Driver: "foo",
+			Host:   "foo",
+		},
+		Queue: &types.Queue{
+			Channel: "foo",
+			Driver:  "foo",
+			Host:    "foo",
+		},
+		Source: &types.Source{
+			Driver: "foo",
+			Host:   "foo",
+		},
+		Vela: &types.Vela{
+			Address:    "foo",
+			WebAddress: "foo",
+		},
+	}
+
+	tok, err := token.CreateAccessToken(u, time.Minute*15)
+	if err != nil {
+		t.Errorf("unable to create access token: %v", err)
+	}
+
+	comp, err := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	if err != nil {
+		t.Errorf("unable to create compiler: %v", err)
+	}
+
 	// setup database
 	db, _ := sqlite.NewTest()
 
 	defer func() {
 		db.Sqlite.Exec("delete from repos;")
+		db.Sqlite.Exec("delete from users;")
 		_sql, _ := db.Sqlite.DB()
 		_sql.Close()
 	}()
 
 	_ = db.CreateRepo(r)
+	_ = db.CreateUser(u)
 
 	// setup context
 	gin.SetMode(gin.TestMode)
 
 	resp := httptest.NewRecorder()
 	context, engine := gin.CreateTestContext(resp)
 	context.Request, _ = http.NewRequest(http.MethodGet, "/pipelines/foo/bar/148afb5bdc41ad69bf22588491333f7cf71135163", nil)
+	context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok))
 
-	// setup mock server
+	// setup github mock server
+	engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) {
+		c.Header("Content-Type", "application/json")
+		c.Status(http.StatusOK)
+		c.File("testdata/yml.json")
+	})
+
+	s := httptest.NewServer(engine)
+	defer s.Close()
+
+	// setup client
+	client, _ := github.NewTest(s.URL)
+
+	// setup vela mock server
+	engine.Use(func(c *gin.Context) { c.Set("metadata", m) })
+	engine.Use(func(c *gin.Context) { c.Set("secret", secret) })
+	engine.Use(func(c *gin.Context) { compiler.WithGinContext(c, comp) })
 	engine.Use(func(c *gin.Context) { database.ToContext(c, db) })
+	engine.Use(func(c *gin.Context) { scm.ToContext(c, client) })
 	engine.Use(org.Establish())
 	engine.Use(repo.Establish())
+	engine.Use(user.Establish())
 	engine.Use(Establish())
 	engine.GET("/pipelines/:org/:repo/:pipeline", func(c *gin.Context) {
 		c.Status(http.StatusOK)
@@ -236,7 +307,7 @@ func TestPipeline_Establish_NoPipeline(t *testing.T) {
 	// run test
 	engine.ServeHTTP(context.Writer, context.Request)
 
-	if resp.Code != http.StatusNotFound {
-		t.Errorf("Establish returned %v, want %v", resp.Code, http.StatusNotFound)
+	if resp.Code != http.StatusOK {
+		t.Errorf("Establish returned %v, want %v", resp.Code, http.StatusOK)
 	}
 }

router/middleware/pipeline/testdata/yml.json

@@ -0,0 +1,18 @@
+{
+  "type": "file",
+  "encoding": "base64",
+  "size": 5362,
+  "name": ".vela.yml",
+  "path": ".vela.yml",
+  "content": "LS0tCnZlcnNpb246ICIxIgoKbWV0YWRhdGE6CiAgb3M6IGxpbnV4CgpzdGVwczoKICAtIG5hbWU6IGJ1aWxkCiAgICBpbWFnZTogb3BlbmpkazpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIGVudmlyb25tZW50OgogICAgICBHUkFETEVfVVNFUl9IT01FOiAuZ3JhZGxlCiAgICAgIEdSQURMRV9PUFRTOiAtRG9yZy5ncmFkbGUuZGFlbW9uPWZhbHNlIC1Eb3JnLmdyYWRsZS53b3JrZXJzLm1heD0xIC1Eb3JnLmdyYWRsZS5wYXJhbGxlbD1mYWxzZQogICAgY29tbWFuZHM6CiAgICAgIC0gLi9ncmFkbGV3IGJ1aWxkIGRpc3RUYXIK",
+  "sha": "3d21ec53a331a6f037a91c368710b99387d012c1",
+  "url": "https://api.github.com/repos/octokit/octokit.rb/contents/.vela.yml",
+  "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1",
+  "html_url": "https://github.com/octokit/octokit.rb/blob/master/.vela.yml",
+  "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/.vela.yml",
+  "_links": {
+    "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1",
+    "self": "https://api.github.com/repos/octokit/octokit.rb/contents/.vela.yml",
+    "html": "https://github.com/octokit/octokit.rb/blob/master/.vela.yml"
+  }
+}