Skip to content

Commit 6b73c5d

Browse files
uuartttuuarttt
committed
1.add
1 parent f53803a commit 6b73c5d

File tree

103 files changed

+13428
-8163
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

103 files changed

+13428
-8163
lines changed

‎di/commons/config/config.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,10 @@ import (
2828
"github.com/alecthomas/units"
2929
"google.golang.org/grpc/grpclog"
3030

31-
"runtime"
32-
3331
log "github.com/sirupsen/logrus"
3432
"github.com/spf13/viper"
3533
v1core "k8s.io/api/core/v1"
34+
"runtime"
3635
)
3736

3837
const (
@@ -147,6 +146,12 @@ const (
147146
// learner pods should not be cleaned up
148147
NoCleanup = "nocleanup"
149148

149+
MongoAddressKey = "MONGO_ADDRESS"
150+
MongoDatabaseKey = "MONGO_DATABASE"
151+
MongoUsernameKey = "MONGO_USERNAME"
152+
MongoPasswordKey = "MONGO_PASSWORD"
153+
MongoAuthenticationDatabase = "MONGO_Authentication_Database"
154+
150155
DlaasResourceLimit = "resource.limit"
151156
DlaasResourceLimitQuerySize = "resource.limit.query.size"
152157

@@ -195,6 +200,7 @@ func InitViper() {
195200
// FfDL Change:Most likely be "standard" in Minikube and "ibmc-s3fs-standard" in DIND, (other value is "default" or "")
196201
viper.SetDefault(SharedVolumeStorageClassKey, "")
197202

203+
198204
// enable ENV vars and defaults
199205
viper.AutomaticEnv()
200206
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
@@ -203,6 +209,11 @@ func InitViper() {
203209
setLogLevel()
204210
viper.SetDefault(PortKey, 8080)
205211

212+
viper.SetDefault(MongoAddressKey,os.Getenv(MongoAddressKey))
213+
viper.SetDefault(MongoDatabaseKey,os.Getenv(MongoDatabaseKey))
214+
viper.SetDefault(MongoUsernameKey,os.Getenv(MongoUsernameKey))
215+
viper.SetDefault(MongoPasswordKey,os.Getenv(MongoPasswordKey))
216+
viper.SetDefault(MongoAuthenticationDatabase,os.Getenv(MongoAuthenticationDatabase))
206217
viper.SetDefault(ServicesTagKey, "prod") // or "latest" as the default?
207218

208219
viper.SetDefault(LearnerTagKey, "prod")
@@ -463,6 +474,11 @@ func GetDataStoreConfig() map[string]string {
463474
if val != "" {
464475
m[RegionKey] = val
465476
}
477+
// FfDL Change: This was in the older PR, supposing not needed?
478+
//val = viper.GetString("objectstore." + UsernameKey)
479+
//if val != "" {
480+
// m[UsernameKey] = val
481+
//}
466482

467483
val = viper.GetString("objectstore." + ProjectKey)
468484
if val != "" {
@@ -473,6 +489,7 @@ func GetDataStoreConfig() map[string]string {
473489
m[StorageType] = val
474490
}
475491

492+
476493
return m
477494
}
478495

‎di/commons/constants/constants.go‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,13 @@ const (
6464

6565
const TFOS = "tfos"
6666

67-
// const BDAPSAFE_SS = "bdapsafe-ss"
68-
// const BDAP_SS = "bdap-ss"
69-
// const BDP_SS = "bdp-ss"
67+
const BDAPSAFE_SS = "bdapsafe-ss"
68+
const BDAP_SS = "bdap-ss"
69+
const BDP_SS = "bdp-ss"
7070

71-
// const BDAP = "bdap"
72-
// const BDP = "bdp"
73-
// const SAFE = "safe"
71+
const BDAP = "bdap"
72+
const BDP = "bdp"
73+
const SAFE = "safe"
7474

7575
const ENVIR = "envir"
7676

@@ -79,11 +79,14 @@ const FLUENT_BIT_LOG_PATH string = "FLUENT_BIT_LOG_PATH"
7979

8080
const JOBTYPE string = "jobType"
8181
const SINGLE string = "single"
82-
83-
const MongoAddressKey = "mongo.address"
84-
const MongoDatabaseKey = "mongo.database"
85-
const MongoUsernameKey = "mongo.username"
86-
const MongoPasswordKey = "mongo.password"
82+
const DI_START_NODEPORT string = "DI_START_NODEPORT"
83+
const DI_END_NODEPORT string = "DI_END_NODEPORT"
84+
85+
const MongoAddressKey = "MONGO_ADDRESS"
86+
const MongoDatabaseKey = "MONGO_DATABASE"
87+
const MongoUsernameKey = "MONGO_USERNAME"
88+
const MongoPasswordKey = "MONGO_PASSWORD"
89+
const MongoAuthenticationDatabase = "MONGO_Authentication_Database"
8790

8891
const (
8992
LOG_STORING_KEY = "store_status"

‎di/commons/controlcenter/client/cc_client.go‎

Lines changed: 126 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,21 @@ type UserResponse struct {
3030
Token string `json:"token,omitempty"`
3131
}
3232

33+
type ProxyUserResponse struct {
34+
Name string `json:"name"`
35+
GID int `json:"gid"`
36+
UID int `json:"uid"`
37+
Token string `json:"token,omitempty"`
38+
UserId int `json:"user_id,omitempty"`
39+
Path string `json:"path,omitempty"`
40+
}
41+
42+
type ProxyUserCheckResponse struct {
43+
Code string `json:"code"`
44+
Message int `json:"message"`
45+
Result bool `json:"result"`
46+
}
47+
3348
const CcAuthToken = "MLSS-Token"
3449

3550
// FIXME MLSS Change: get models filter by username and namespace
@@ -137,6 +152,50 @@ func (cc *CcClient) UserNamespaceCheck(token string, userName string, namespace
137152
return nil
138153
}
139154

155+
func (cc *CcClient) GetProxyUserFromCC(token string, proxyUserId string, userId string) (*ProxyUserResponse, error) {
156+
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/proxyUser/%v/user/%v", proxyUserId, userId)
157+
body, err := cc.accessCheckFromCc(token, apiUrl)
158+
if err != nil {
159+
return nil, err
160+
}
161+
var u = new(ProxyUserResponse)
162+
//json.Unmarshal(body, &u)
163+
err = models.GetResultData(body, &u)
164+
if err != nil {
165+
return nil, errors.New("GetResultData failed" + err.Error())
166+
}
167+
log.Debugf("response=%+v, GID=%v, UID=%v", u, u.GID, u.UID)
168+
if (u.GID == 0 || u.UID == 0) {
169+
return nil, errors.New("error GUIDCheck is on and GID or UID is 0")
170+
}
171+
//gid := strconv.Itoa(u.GID)
172+
//uid := strconv.Itoa(u.UID)
173+
174+
return u, nil
175+
}
176+
177+
func (cc *CcClient) GetGUIDFromProxyUserId(token string, proxyUserId string) (*string, *string, error) {
178+
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/proxyUser/%v", proxyUserId)
179+
body, err := cc.accessCheckFromCc(token, apiUrl)
180+
if err != nil {
181+
return nil, nil, err
182+
}
183+
var u = new(UserResponse)
184+
//json.Unmarshal(body, &u)
185+
err = models.GetResultData(body, &u)
186+
if err != nil {
187+
return nil, nil, errors.New("GetResultData failed")
188+
}
189+
log.Debugf("response=%+v, GID=%v, UID=%v", u, u.GID, u.UID)
190+
if u.GUIDCheck && (u.GID == 0 || u.UID == 0) {
191+
return nil, nil, errors.New("error GUIDCheck is on and GID or UID is 0")
192+
}
193+
gid := strconv.Itoa(u.GID)
194+
uid := strconv.Itoa(u.UID)
195+
196+
return &gid, &uid, nil
197+
}
198+
140199
func (cc *CcClient) GetGUIDFromUserId(token string, userId string) (*string, *string, error) {
141200
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/users/name/%v", userId)
142201
body, err := cc.accessCheckFromCc(token, apiUrl)
@@ -159,6 +218,7 @@ func (cc *CcClient) GetGUIDFromUserId(token string, userId string) (*string, *st
159218
return &gid, &uid, nil
160219
}
161220

221+
162222
func (cc *CcClient) GetUserByName(token string, userId string) (*UserResponse, error) {
163223
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/users/name/%v", userId)
164224
body, err := cc.accessCheckFromCc(token, apiUrl)
@@ -176,6 +236,24 @@ func (cc *CcClient) GetUserByName(token string, userId string) (*UserResponse, e
176236
return u, nil
177237
}
178238

239+
func (cc *CcClient) ProxyUserCheck(token string, userId string, proxyUser string) (bool, error) {
240+
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/proxyUserCheck/%v/%v", proxyUser, userId)
241+
body, err := cc.accessCheckFromCc(token, apiUrl)
242+
if err != nil {
243+
return false, err
244+
}
245+
var auth bool
246+
//json.Unmarshal(body, &u)
247+
err = models.GetResultData(body, &auth)
248+
249+
if err != nil {
250+
log.Error("Unmarshal Error", err)
251+
return false, errors.New("Check Proxy User failed")
252+
}
253+
254+
return auth, nil
255+
}
256+
179257
func (cc *CcClient) AdminUserCheck(token string, adminUserId string, userId string) error {
180258
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/auth/access/users/%v/users/%v", adminUserId, userId)
181259
_, err := cc.accessCheckFromCc(token, apiUrl)
@@ -303,8 +381,8 @@ func (cc *CcClient) CheckNamespaceUser(token string, namespace string, user stri
303381
return nil
304382
}
305383

306-
func (cc *CcClient) GetCurrentUserNamespaceWithRole(token string, roleId string, clusterName string) ([]byte, error) {
307-
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/groups/users/roles/%v/namespaces/clusterName/%v", roleId, clusterName)
384+
func (cc *CcClient) GetCurrentUserNamespaceWithRole(token string, roleId string) ([]byte, error) {
385+
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/groups/users/roles/%v/namespaces", roleId)
308386
log.Debugf("get apiUrl: %v", apiUrl)
309387
req, err := http.NewRequest("GET", apiUrl, strings.NewReader(""))
310388
if err != nil {
@@ -330,3 +408,49 @@ func (cc *CcClient) GetCurrentUserNamespaceWithRole(token string, roleId string,
330408
defer resp.Body.Close()
331409
return body, nil
332410
}
411+
412+
413+
type GetGroupFromUserRequest struct {
414+
415+
}
416+
417+
type GetGroupFromUserResponse struct {
418+
ID int64 `gorm:"column:id; PRIMARY_KEY" json:"id"`
419+
Name string `json:"name"`
420+
EnableFlag int8 `json:"enable_flag"`
421+
UserId int64 `json:"user_id"`
422+
RoleId int64 `json:"role_id"`
423+
GroupId int64 `json:"group_id"`
424+
Remarks string `json:"remarks"`
425+
}
426+
427+
428+
429+
func (cc *CcClient) GetGroupFromUser(username string) (*GetGroupFromUserResponse,error){
430+
apiUrl := cc.ccUrl + fmt.Sprintf("/cc/v1/group/username/%v", username)
431+
432+
req,err := http.NewRequest("GET",apiUrl,nil)
433+
if err != nil {
434+
return nil,err
435+
}
436+
437+
res,err := cc.httpClient.Do(req)
438+
if err != nil {
439+
return nil, err
440+
}
441+
442+
resByte, err := ioutil.ReadAll(res.Body)
443+
if err != nil {
444+
return nil, err
445+
}
446+
447+
group := GetGroupFromUserResponse{}
448+
err = json.Unmarshal(resByte,&group)
449+
if err != nil {
450+
return nil, err
451+
}
452+
453+
return &group,nil
454+
455+
456+
}

‎di/commons/logger/logger.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ const (
6060
LogkeyRestAPIService = "rest-api"
6161
LogkeyStorageService = "storage-service"
6262
LogkeyTrainerService = "trainer-service"
63+
LogkeyStorerService = "storer-service"
6364
LogkeyVolumeManagerService = "volume-manager-service"
6465

6566
LogkeyJobMonitor = "jobmonitor"
@@ -225,23 +226,23 @@ func Config() {
225226

226227
loggingType := viper.GetString(config.LoggingType)
227228
switch loggingType {
228-
case config.LoggingTypeJson:
229-
log.SetFormatter(&log.JSONFormatter{})
230-
case config.LoggingTypeText:
231-
log.SetFormatter(&log.TextFormatter{})
232-
default: // any other env will use local settings (assuming outside SL)
233-
env := viper.GetString(config.EnvKey)
234-
235-
switch env {
236-
case config.DevelopmentEnv:
237-
log.SetFormatter(&log.JSONFormatter{})
238-
case config.StagingEnv:
239-
log.SetFormatter(&log.JSONFormatter{})
240-
case config.ProductionEnv:
229+
case config.LoggingTypeJson:
241230
log.SetFormatter(&log.JSONFormatter{})
231+
case config.LoggingTypeText:
232+
log.SetFormatter(&log.TextFormatter{})
242233
default: // any other env will use local settings (assuming outside SL)
234+
//env := viper.GetString(config.EnvKey)
243235
log.SetFormatter(&log.JSONFormatter{})
244-
}
236+
//switch env {
237+
// case config.DevelopmentEnv:
238+
// log.SetFormatter(&log.JSONFormatter{})
239+
// case config.StagingEnv:
240+
// log.SetFormatter(&log.JSONFormatter{})
241+
// case config.ProductionEnv:
242+
// log.SetFormatter(&log.JSONFormatter{})
243+
// default: // any other env will use local settings (assuming outside SL)
244+
// log.SetFormatter(&log.JSONFormatter{})
245+
// }
245246
}
246247

247248
viper.SetDefault(LogCategoryGetTrainingLogStream, LogCategoryGetTrainingLogStreamDefaultValue)

‎di/commons/models/experiment.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ type Experiment struct {
2121
BaseModel
2222
ExpDesc *string `json:"exp_desc"`
2323
ExpName *string `json:"exp_name"`
24+
GroupName string `json:"group_name"`
25+
MlflowExpID *string `json:"mlflow_exp_id" gorm:"column:mlflow_exp_id"`
2426
DssProjectID int64 `json:"dss_project_id"`
2527
DssProjectVersionID int64 `json:"dss_project_version"`
2628
DssWorkspaceID int64 `json:"dss_workspace_id"`

‎di/commons/service/client/lcm.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func NewLcm(lcm service.LifecycleManagerClient) (LcmClient, error) {
7676
if conn == nil{
7777
log.Error("conn is nil")
7878
}
79-
log.Error("conn is not nil")
79+
log.Debugf("conn is not nil")
8080
return &lcmClient{
8181
conn: conn,
8282
client: service.NewLifecycleManagerClient(conn),

0 commit comments

Comments
 (0)