diff --git a/i18n/cmd/commands/transifex/pull_transifex.go b/i18n/cmd/commands/transifex/pull_transifex.go index af9c90b19b1..cabfb8d6fee 100644 --- a/i18n/cmd/commands/transifex/pull_transifex.go +++ b/i18n/cmd/commands/transifex/pull_transifex.go @@ -16,13 +16,17 @@ package transifex import ( + "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "os" - "path" + "sync" + "time" + "github.com/arduino/go-paths-helper" "github.com/spf13/cobra" ) @@ -33,99 +37,246 @@ var pullTransifexCommand = &cobra.Command{ } func getLanguages() []string { - req, err := http.NewRequest( - "GET", - fmt.Sprintf( - "https://www.transifex.com/api/2/project/%s/resource/%s/stats/", - project, resource, - ), nil) + url := mainEndpoint + fmt.Sprintf("projects/o:%s:p:%s/languages", organization, project) + req, err := http.NewRequest("GET", url, nil) if err != nil { fmt.Println(err.Error()) os.Exit(1) } - req.SetBasicAuth("api", apiKey) - - resp, err := http.DefaultClient.Do(req) + addHeaders(req) + res, err := http.DefaultClient.Do(req) if err != nil { fmt.Println(err.Error()) os.Exit(1) } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) + defer res.Body.Close() + b, err := ioutil.ReadAll(res.Body) if err != nil { fmt.Println(err.Error()) os.Exit(1) } - var jsonResp map[string]interface{} - if err := json.Unmarshal(b, &jsonResp); err != nil { + var jsonRes struct { + Data []struct { + Attributes struct { + Code string `json:"code"` + } `json:"attributes"` + } `json:"data"` + } + if err := json.Unmarshal(b, &jsonRes); err != nil { fmt.Println(err.Error()) os.Exit(1) } - var langs []string - for key := range jsonResp { - langs = append(langs, key) + var languages []string + for _, object := range jsonRes.Data { + languages = append(languages, object.Attributes.Code) } - - return langs + return languages } -func pullCatalog(cmd *cobra.Command, args []string) { - languages := getLanguages() - fmt.Println("translations found:", languages) +// startTranslationDownload notifies Transifex that we want to start downloading +// the resources file for the specified languageCode. +// Returns an id to monitor the download status. +func startTranslationDownload(languageCode string) string { + url := mainEndpoint + "resource_translations_async_downloads" - folder := args[0] + type jsonReq struct { + Data struct { + Relationships struct { + Language struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + } `json:"data"` + } `json:"language"` + Resource struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + } `json:"data"` + } `json:"resource"` + } `json:"relationships"` + Type string `json:"type"` + } `json:"data"` + } - for _, lang := range languages { + jsonData := jsonReq{} + jsonData.Data.Type = "resource_translations_async_downloads" + jsonData.Data.Relationships.Language.Data.ID = fmt.Sprintf("l:%s", languageCode) + jsonData.Data.Relationships.Language.Data.Type = "languages" + jsonData.Data.Relationships.Resource.Data.ID = fmt.Sprintf("o:%s:p:%s:r:%s", organization, project, resource) + jsonData.Data.Relationships.Resource.Data.Type = "resources" + + jsonBytes, err := json.Marshal(jsonData) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + req, err := http.NewRequest( + "POST", + url, + bytes.NewBuffer(jsonBytes), + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + addHeaders(req) + + res, err := http.DefaultClient.Do(req) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + defer res.Body.Close() + body, err := io.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + var jsonRes struct { + Data struct { + ID string `json:"id"` + } `json:"data"` + } + if err = json.Unmarshal(body, &jsonRes); err != nil { + fmt.Println(err) + os.Exit(1) + } + return jsonRes.Data.ID +} + +// getDownloadURL checks for the download status of the languageCode file specified +// by downloadID. +// It return a URL to download the file when ready. +func getDownloadURL(languageCode, downloadID string) string { + url := mainEndpoint + "resource_translations_async_downloads/" + downloadID + // The download request status must be asked from time to time, if it's + // still pending we try again using exponentional backoff starting from 2.5 seconds. + backoff := 2500 * time.Millisecond + for { req, err := http.NewRequest( "GET", - fmt.Sprintf( - "https://www.transifex.com/api/2/project/%s/resource/%s/translation/%s/?mode=reviewed&file=po", - project, resource, lang, - ), nil) - + url, + nil, + ) if err != nil { - fmt.Println(err.Error()) + fmt.Println(err) os.Exit(1) } - req.SetBasicAuth("api", apiKey) - - resp, err := http.DefaultClient.Do(req) + addHeaders(req) + client := http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // We handle redirection manually + return http.ErrUseLastResponse + }, + } + res, err := client.Do(req) if err != nil { - fmt.Println(err.Error()) + fmt.Println(err) os.Exit(1) } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) + if res.StatusCode == 303 { + // Return the URL to download translation file + return res.Header.Get("location") + } + body, err := io.ReadAll(res.Body) if err != nil { - fmt.Println(err.Error()) + fmt.Println(err) os.Exit(1) } + res.Body.Close() - os.Remove(path.Join(folder, lang+".po")) - file, err := os.OpenFile(path.Join(folder, lang+".po"), os.O_CREATE|os.O_RDWR, 0644) - - if err != nil { - fmt.Println(err.Error()) + var jsonRes struct { + Data struct { + Attributes struct { + Status string `json:"status"` + Errors []struct { + Code string `json:"code"` + Detail string `json:"detail"` + } `json:"errors"` + } `json:"attributes"` + } `json:"data"` + } + if err = json.Unmarshal(body, &jsonRes); err != nil { + fmt.Println(err) os.Exit(1) } - _, err = file.Write(b) - if err != nil { - fmt.Println(err.Error()) + status := jsonRes.Data.Attributes.Status + switch status { + case "succeeded": + return "" + case "pending": + fallthrough + case "processing": + fmt.Printf("Current status for language %s: %s\n", languageCode, status) + time.Sleep(backoff) + backoff = backoff * 2 + // Request the status again + continue + case "failed": + for _, err := range jsonRes.Data.Attributes.Errors { + fmt.Printf("%s: %s\n", err.Code, err.Detail) + } os.Exit(1) } + fmt.Printf("Status request for language %s failed in an unforeseen way\n", languageCode) + os.Exit(1) + } +} + +// download file from url and saves it in folder with the specified fileName +func download(folder, fileName, url string) { + fmt.Printf("Starting download of %s\n", fileName) + filePath := paths.New(folder, fileName) + + res, err := http.DefaultClient.Get(url) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + data, err := io.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + filePath.WriteFile(data) + fmt.Printf("Finished download of %s\n", fileName) +} + +func pullCatalog(cmd *cobra.Command, args []string) { + languages := getLanguages() + fmt.Println("translations found:", languages) + + folder := args[0] + + var wg sync.WaitGroup + for _, lang := range languages { + wg.Add(1) + go func(lang string) { + downloadID := startTranslationDownload(lang) + url := getDownloadURL(lang, downloadID) + download(folder, lang+".po", url) + wg.Done() + }(lang) } + wg.Wait() + fmt.Println("Translation files downloaded") } diff --git a/i18n/cmd/commands/transifex/push_transifex.go b/i18n/cmd/commands/transifex/push_transifex.go index 40a28a2bbb6..ffb7e42a6bc 100644 --- a/i18n/cmd/commands/transifex/push_transifex.go +++ b/i18n/cmd/commands/transifex/push_transifex.go @@ -17,14 +17,15 @@ package transifex import ( "bytes" + "encoding/base64" + "encoding/json" "fmt" "io" - "mime/multipart" "net/http" "os" - "path" - "path/filepath" + "time" + "github.com/arduino/go-paths-helper" "github.com/spf13/cobra" ) @@ -35,56 +36,163 @@ var pushTransifexCommand = &cobra.Command{ Run: pushCatalog, } -func pushFile(folder, lang, url string) { - filename := path.Join(folder, lang+".po") - file, err := os.Open(filename) +// uploadSourceFile starts an async upload of resourceFile. +// Returns an id to monitor the upload status. +func uploadSourceFile(resourceFile *paths.Path) string { + url := mainEndpoint + "resource_strings_async_uploads" + data, err := resourceFile.ReadFile() if err != nil { - fmt.Println(err.Error()) + fmt.Println(err) os.Exit(1) } - defer file.Close() - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile(lang, filepath.Base(filename)) + type jsonReq struct { + Data struct { + Attributes struct { + Content string `json:"content"` + ContentEncoding string `json:"content_encoding"` + } `json:"attributes"` + Relationships struct { + Resource struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + } `json:"data"` + } `json:"resource"` + } `json:"relationships"` + Type string `json:"type"` + } `json:"data"` + } + jsonData := jsonReq{} + jsonData.Data.Type = "resource_strings_async_uploads" + jsonData.Data.Attributes.Content = base64.StdEncoding.EncodeToString(data) + jsonData.Data.Attributes.ContentEncoding = "base64" + jsonData.Data.Relationships.Resource.Data.ID = fmt.Sprintf("o:%s:p:%s:r:%s", organization, project, resource) + jsonData.Data.Relationships.Resource.Data.Type = "resources" + jsonBytes, err := json.Marshal(jsonData) if err != nil { - fmt.Println(err.Error()) + fmt.Println(err) os.Exit(1) } - _, err = io.Copy(part, file) - writer.WriteField("file_type", "po") + req, err := http.NewRequest( + "POST", + url, + bytes.NewBuffer(jsonBytes), + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } - req, err := http.NewRequest("PUT", url, body) - req.Header.Set("Content-Type", writer.FormDataContentType()) + addHeaders(req) + res, err := http.DefaultClient.Do(req) if err != nil { - fmt.Println(err.Error()) + fmt.Println(err) + os.Exit(1) + } + + defer res.Body.Close() + body, err := io.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + var jsonRes struct { + Data struct { + ID string `json:"id"` + } `json:"data"` + } + if err = json.Unmarshal(body, &jsonRes); err != nil { + fmt.Println(err) os.Exit(1) } + fmt.Printf("Started upload of resource file %s\n", resourceFile) + return jsonRes.Data.ID +} - req.SetBasicAuth("api", apiKey) +func checkUploadStatus(uploadID string) { + url := mainEndpoint + "resource_strings_async_uploads/" + uploadID + // The upload request status must be asked from time to time, if it's + // still pending we try again using exponentional backoff starting from 2.5 seconds. + backoff := 2500 * time.Millisecond - resp, err := http.DefaultClient.Do(req) + for { + req, err := http.NewRequest( + "GET", + url, + nil, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } - if err != nil { - fmt.Println(err.Error()) + addHeaders(req) + + res, err := http.DefaultClient.Do(req) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + body, err := io.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + res.Body.Close() + + var jsonRes struct { + Data struct { + Attributes struct { + Status string `json:"status"` + Errors []struct { + Code string `json:"code"` + Detail string `json:"detail"` + } `json:"errors"` + } `json:"attributes"` + } `json:"data"` + } + if err = json.Unmarshal(body, &jsonRes); err != nil { + fmt.Println(err) + os.Exit(1) + } + + status := jsonRes.Data.Attributes.Status + switch status { + case "succeeded": + fmt.Println("Resource file uploaded") + return + case "pending": + fallthrough + case "processing": + fmt.Printf("Current status: %s\n", status) + time.Sleep(backoff) + backoff = backoff * 2 + // Request the status again + continue + case "failed": + for _, err := range jsonRes.Data.Attributes.Errors { + fmt.Printf("%s: %s\n", err.Code, err.Detail) + } + os.Exit(1) + } + fmt.Println("Status request failed in an unforeseen way") os.Exit(1) } - resp.Body.Close() +} + +func pushFile(resourceFile *paths.Path) { + uploadID := uploadSourceFile(resourceFile) + checkUploadStatus(uploadID) } func pushCatalog(cmd *cobra.Command, args []string) { folder := args[0] - pushFile( - folder, - "en", - fmt.Sprintf( - "https://www.transifex.com/api/2/project/%s/resource/%s/content/", - project, - resource, - ), - ) + pushFile(paths.New(folder, "en.po")) } diff --git a/i18n/cmd/commands/transifex/transifex.go b/i18n/cmd/commands/transifex/transifex.go index 63fb3750ba2..a7428f55336 100644 --- a/i18n/cmd/commands/transifex/transifex.go +++ b/i18n/cmd/commands/transifex/transifex.go @@ -17,6 +17,7 @@ package transifex import ( "fmt" + "net/http" "os" "github.com/spf13/cobra" @@ -29,6 +30,9 @@ var Command = &cobra.Command{ PersistentPreRun: preRun, } +const mainEndpoint = "https://rest.api.transifex.com/" + +var organization string var project string var resource string var apiKey string @@ -39,9 +43,10 @@ func init() { } func preRun(cmd *cobra.Command, args []string) { - project = os.Getenv("TRANSIFEX_PROJECT") - resource = os.Getenv("TRANSIFEX_RESOURCE") - apiKey = os.Getenv("TRANSIFEX_RESOURCE") + if organization = os.Getenv("TRANSIFEX_ORGANIZATION"); organization == "" { + fmt.Println("missing TRANSIFEX_ORGANIZATION environment variable") + os.Exit(1) + } if project = os.Getenv("TRANSIFEX_PROJECT"); project == "" { fmt.Println("missing TRANSIFEX_PROJECT environment variable") @@ -57,6 +62,9 @@ func preRun(cmd *cobra.Command, args []string) { fmt.Println("missing TRANSIFEX_API_KEY environment variable") os.Exit(1) } +} - return +func addHeaders(req *http.Request) { + req.Header.Set("Content-Type", "application/vnd.api+json") + req.Header.Set("Authorization", "Bearer "+apiKey) }