Skip to content

Commit b6c301b

Browse files
carddev81carddev81
committed
fix: merge latest changes to branch
1 parent d1c0600 commit b6c301b

2 files changed

Lines changed: 186 additions & 34 deletions

File tree

‎provider-middleware/brightspace.go‎

Lines changed: 125 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ func (srv *BrightspaceService) getBrightspaceBulkData(pluginName, zipFileName st
151151
}
152152

153153
func (srv *BrightspaceService) GetUsers(db *gorm.DB) ([]models.ImportUser, error) {
154-
csvFile, err := srv.getBrightspaceBulkData("Users", "Users.zip")
154+
csvFile, err := srv.getBrightspaceBulkData("Users", "users/Users.zip")
155155
if err != nil {
156156
log.Errorf("error attempting to get bulk data for Brightspace courses, error is: %v", err)
157157
return nil, err
@@ -176,19 +176,36 @@ func (srv *BrightspaceService) GetUsers(db *gorm.DB) ([]models.ImportUser, error
176176
}
177177

178178
func (srv *BrightspaceService) ImportCourses(db *gorm.DB) error {
179-
csvFile, err := srv.getBrightspaceBulkData("Organizational Units", "OrganizationalUnits.zip")
179+
contentObjCsvFile, err := srv.getBrightspaceBulkData("Content Objects", "courses/ContentObjects.zip")
180+
if err != nil {
181+
log.Errorf("error attempting to get bulk data for Brightspace content objects for importing activities, error is: %v", err)
182+
return err
183+
}
184+
organizationalCsvFile, err := srv.getBrightspaceBulkData("Organizational Units", "courses/OrganizationalUnits.zip")
180185
if err != nil {
181186
log.Errorf("error attempting to get bulk data for Brightspace courses, error is: %v", err)
182187
return err
183188
}
184189
bsCourses := []BrightspaceCourse{}
185-
readCSV(&bsCourses, csvFile)
186-
cleanUpFiles(csvFile)
187-
fields := log.Fields{"provider": srv.ProviderPlatformID, "Function": "ImportCourses", "csvFile": csvFile}
190+
readCSV(&bsCourses, organizationalCsvFile)
191+
contentObjects := []BrightspaceContentObject{}
192+
readCSV(&contentObjects, contentObjCsvFile)
193+
totalProgressMilestonesMap := make(map[string]int)
194+
for _, obj := range contentObjects {
195+
if value, ok := totalProgressMilestonesMap[obj.OrgUnitId]; ok {
196+
totalProgressMilestonesMap[obj.OrgUnitId] = value + 1
197+
} else {
198+
totalProgressMilestonesMap[obj.OrgUnitId] = 1
199+
}
200+
}
201+
cleanUpFiles(organizationalCsvFile, contentObjCsvFile)
202+
fields := log.Fields{"provider": srv.ProviderPlatformID, "Function": "ImportCourses", "organizationalCsvFile": organizationalCsvFile, "contentObjCsvFile": contentObjCsvFile}
188203
log.WithFields(fields).Info("importing courses from provider using csv file")
189204
for _, bsCourse := range bsCourses {
190-
if strings.ToUpper(bsCourse.IsActive) == "TRUE" && strings.ToUpper(bsCourse.IsDeleted) == "FALSE" && bsCourse.Type == "Course Offering" {
205+
if strings.ToUpper(bsCourse.IsActive) == "TRUE" && strings.ToUpper(bsCourse.IsDeleted) == "FALSE" && bsCourse.Type == "Course Offering" && totalProgressMilestonesMap[bsCourse.OrgUnitId] > 0 {
206+
bsCourse.TotalContentCount = totalProgressMilestonesMap[bsCourse.OrgUnitId]
191207
if db.Where("provider_platform_id = ? AND external_id = ?", srv.ProviderPlatformID, bsCourse.OrgUnitId).First(&models.Course{}).Error == nil {
208+
srv.updateTotalProgress(db, bsCourse.TotalContentCount, bsCourse.OrgUnitId)
192209
continue
193210
}
194211
log.Infof("importing course named %v with external id %v", bsCourse.Name, bsCourse.OrgUnitId)
@@ -202,6 +219,14 @@ func (srv *BrightspaceService) ImportCourses(db *gorm.DB) error {
202219
return nil
203220
}
204221

222+
func (srv *BrightspaceService) updateTotalProgress(db *gorm.DB, totalProgressMilestones int, externalId string) {
223+
if db.Where("provider_platform_id = ? AND external_id = ? AND total_progress_milestones = ?", srv.ProviderPlatformID, externalId, totalProgressMilestones).First(&models.Course{}).Error != nil {
224+
if err := db.Table("courses as c").Where("provider_platform_id = ? AND external_id = ?", srv.ProviderPlatformID, externalId).Update("total_progress_milestones", totalProgressMilestones).Error; err != nil {
225+
log.Errorln("error updating course total_progress_milestones in db, error is: ", err) //just logging the error if it occurs, logic will continue
226+
}
227+
}
228+
}
229+
205230
func (srv *BrightspaceService) ImportMilestones(course map[string]interface{}, users []map[string]interface{}, db *gorm.DB, lastRun time.Time) error {
206231
usersMap := make(map[string]uint)
207232
for _, user := range users {
@@ -212,7 +237,7 @@ func (srv *BrightspaceService) ImportMilestones(course map[string]interface{}, u
212237
usersMap: usersMap,
213238
}
214239
if !srv.IsDownloaded {
215-
cleanUpCsvFiles()
240+
cleanUpCsvFiles("milestones")
216241
}
217242
err := importBSEnrollmentMilestones(srv, paramObj, db)
218243
if err != nil {
@@ -236,7 +261,7 @@ func importBSEnrollmentMilestones(srv *BrightspaceService, po milestonePO, db *g
236261
case true:
237262
csvFile = srv.CsvFileMap["enrollments"]
238263
case false:
239-
csvFile, err := srv.getBrightspaceBulkData("User Enrollments", "UserEnrollments.zip")
264+
csvFile, err := srv.getBrightspaceBulkData("User Enrollments", "milestones/UserEnrollments.zip")
240265
if err != nil {
241266
log.Errorf("error attempting to get bulk data for Brightspace enrollments, error is: %v", err)
242267
return err
@@ -251,15 +276,15 @@ func importBSEnrollmentMilestones(srv *BrightspaceService, po milestonePO, db *g
251276
usersMap := po.usersMap
252277
courseId := uint(course["course_id"].(float64))
253278
externalCourseId := course["external_course_id"].(string)
254-
filteredBSEnrollments := findSlice(bsEnrollments, func(bsCourse BrightspaceEnrollment) bool {
255-
return bsCourse.OrgUnitId == externalCourseId
279+
filteredBSEnrollments := findSlice(bsEnrollments, func(bsEnrollment BrightspaceEnrollment) bool {
280+
return bsEnrollment.OrgUnitId == externalCourseId && usersMap[bsEnrollment.UserId] != 0
256281
})
257282
var id string
258283
var userId uint
259-
for _, bsCourse := range filteredBSEnrollments {
260-
id = bsCourse.OrgUnitId
261-
userId = usersMap[bsCourse.UserId]
262-
if db.Where("user_id = ? AND course_id = ? AND external_id = ?", usersMap[bsCourse.UserId], courseId, id).First(&models.Milestone{}).Error == nil {
284+
for _, bsEnrollment := range filteredBSEnrollments {
285+
id = bsEnrollment.getCompositeKeyId()
286+
userId = usersMap[bsEnrollment.UserId]
287+
if db.Where("user_id = ? AND course_id = ? AND external_id = ?", userId, courseId, id).First(&models.Milestone{}).Error == nil {
263288
continue
264289
}
265290
milestone := models.Milestone{
@@ -282,7 +307,7 @@ func importBSAssignmentSubmissionMilestones(srv *BrightspaceService, po mileston
282307
case true:
283308
csvFile = srv.CsvFileMap["assignments"]
284309
case false:
285-
csvFile, err := srv.getBrightspaceBulkData("Assignment Submissions", "AssignmentSubmissions.zip")
310+
csvFile, err := srv.getBrightspaceBulkData("Assignment Submissions", "milestones/AssignmentSubmissions.zip")
286311
if err != nil {
287312
log.Errorf("error attempting to get bulk data for Brightspace assignment submissions, error is: %v", err)
288313
return err
@@ -299,7 +324,7 @@ func importBSAssignmentSubmissionMilestones(srv *BrightspaceService, po mileston
299324
courseId := uint(course["course_id"].(float64))
300325
externalCourseId := course["external_course_id"].(string)
301326
filteredBSAssignments := findSlice(bsAssignments, func(bsAssignment BrightspaceAssignmentSubmission) bool {
302-
return bsAssignment.OrgUnitId == externalCourseId
327+
return bsAssignment.OrgUnitId == externalCourseId && usersMap[bsAssignment.UserId] != 0
303328
})
304329
var id string
305330
var userId uint
@@ -347,7 +372,7 @@ func importBSQuizSubmissionMilestones(srv *BrightspaceService, po milestonePO, d
347372
case true:
348373
csvFile = srv.CsvFileMap["quizzes"]
349374
case false:
350-
csvFile, err := srv.getBrightspaceBulkData("Quiz Attempts", "QuizAttempts.zip")
375+
csvFile, err := srv.getBrightspaceBulkData("Quiz Attempts", "milestones/QuizAttempts.zip")
351376
if err != nil {
352377
log.Errorf("error attempting to get bulk data for Brightspace quiz attempts, error is: %v", err)
353378
return err
@@ -364,7 +389,7 @@ func importBSQuizSubmissionMilestones(srv *BrightspaceService, po milestonePO, d
364389
courseId := uint(course["course_id"].(float64))
365390
externalCourseId := course["external_course_id"].(string)
366391
filteredBSQuizes := findSlice(bsQuizes, func(bsQuiz BrightspaceQuizSubmission) bool {
367-
return bsQuiz.OrgUnitId == externalCourseId && bsQuiz.IsDeleted == "False" //todo
392+
return bsQuiz.OrgUnitId == externalCourseId && strings.ToUpper(bsQuiz.IsDeleted) == "FALSE" && usersMap[bsQuiz.UserId] != 0
368393
})
369394
var id string
370395
var userId uint
@@ -435,11 +460,91 @@ func addQuizMilestoneIfNotExists(db *gorm.DB, userId, courseId uint, bsQuiz Brig
435460
}
436461
}
437462

438-
func (srv *BrightspaceService) ImportActivityForCourse(coursePair map[string]interface{}, db *gorm.DB) error {
439-
fmt.Println("ImportActivityForCourse...")
463+
func (srv *BrightspaceService) ImportActivityForCourse(course map[string]interface{}, db *gorm.DB) error {
464+
if !srv.IsDownloaded {
465+
cleanUpCsvFiles("activities")
466+
}
467+
var contentObjCsvFile string
468+
var userProgressCsvFile string
469+
switch srv.IsDownloaded {
470+
case true:
471+
contentObjCsvFile = srv.CsvFileMap["contentobjs"]
472+
userProgressCsvFile = srv.CsvFileMap["contentprogress"]
473+
case false:
474+
contentObjCsvFile, err := srv.getBrightspaceBulkData("Content Objects", "activities/ContentObjects.zip")
475+
if err != nil {
476+
log.Errorf("error attempting to get bulk data for Brightspace content objects for importing activities, error is: %v", err)
477+
return err
478+
}
479+
userProgressCsvFile, err := srv.getBrightspaceBulkData("Content User Progress", "activities/ContentUserProgress.zip")
480+
if err != nil {
481+
log.Errorf("error attempting to get bulk data for Brightspace content user progress for importing activities, error is: %v", err)
482+
return err
483+
}
484+
srv.CsvFileMap["contentobjs"] = contentObjCsvFile
485+
srv.CsvFileMap["contentprogress"] = userProgressCsvFile
486+
srv.IsDownloaded = true
487+
}
488+
courseId := uint(course["course_id"].(float64))
489+
externalCourseId := course["external_course_id"].(string)
490+
bsContentDtoMap := srv.getUsersActivity(db, externalCourseId, contentObjCsvFile, userProgressCsvFile)
491+
fields := log.Fields{"provider": srv.ProviderPlatformID, "Function": "ImportActivityForCourse", "contentObjCsvFile": contentObjCsvFile, "userProgressCsvFile": userProgressCsvFile}
492+
log.WithFields(fields).Info("importing activities from provider using csv file")
493+
for userId, bsUserContentDtos := range bsContentDtoMap {
494+
for _, bsDto := range bsUserContentDtos {
495+
var acts []map[string]interface{}
496+
if db.Raw("select external_id from activities where course_id = ? AND external_id = ?", courseId, bsDto.ExternalId).First(&acts).Error == nil {
497+
continue
498+
}
499+
if err := db.Exec("SELECT insert_daily_activity(?, ?, ?, ?, ?)", userId, courseId, models.ContentInteraction, bsDto.TotalTime, bsDto.ExternalId).Error; err != nil {
500+
log.WithFields(log.Fields{"user_id": userId, "course_id": courseId, "error": err}).Error("Failed to create activity using brightspace data")
501+
continue
502+
}
503+
}
504+
}
440505
return nil
441506
}
442507

508+
func (srv *BrightspaceService) getUsersActivity(db *gorm.DB, externalCourseId, contentObjCsvFile, userProgressCsvFile string) map[uint][]BrightspaceUserContentDto {
509+
bsContentDtoMap := make(map[uint][]BrightspaceUserContentDto)
510+
bsContentObjects := []BrightspaceContentObject{}
511+
readCSV(&bsContentObjects, contentObjCsvFile)
512+
filteredContentObjs := findSlice(bsContentObjects, func(bsContentObj BrightspaceContentObject) bool { //get content objects for the course being iterate
513+
return bsContentObj.OrgUnitId == externalCourseId && bsContentObj.IsDeleted == "0"
514+
})
515+
if len(filteredContentObjs) < 1 {
516+
log.Infof("no content objects found for the course with external id of %s", externalCourseId)
517+
return bsContentDtoMap
518+
}
519+
bsContentUserProgress := []BrightspaceContentUserProgress{}
520+
readCSV(&bsContentUserProgress, userProgressCsvFile)
521+
for _, contentUser := range bsContentUserProgress { //separate by user
522+
var userId uint //filter out the users
523+
if err := db.Model(&models.ProviderUserMapping{}).Select("user_id").First(&userId, "external_user_id = ? and provider_platform_id = ?", contentUser.UserId, srv.ProviderPlatformID).Error; err != nil {
524+
log.Errorln("error finding brightspace user by external id in ImportActivityForCourse")
525+
continue
526+
}
527+
contentObjsByUser := findSlice(filteredContentObjs, func(bsContentObj BrightspaceContentObject) bool { //get content objects for the course being iterated
528+
return bsContentObj.ContentObjectId == contentUser.ContentObjectId
529+
})
530+
if _, ok := bsContentDtoMap[userId]; !ok && len(contentObjsByUser) > 0 {
531+
bsContentDtoMap[userId] = []BrightspaceUserContentDto{}
532+
}
533+
for _, contentObj := range contentObjsByUser {
534+
dto := BrightspaceUserContentDto{
535+
OrgUnitId: contentObj.OrgUnitId,
536+
ContentObjectId: contentObj.ContentObjectId,
537+
TotalTime: contentUser.TotalTime,
538+
Title: contentObj.Title,
539+
ContentObjectType: contentObj.ContentObjectType,
540+
ExternalId: contentUser.getCompositeKeyId(),
541+
}
542+
bsContentDtoMap[userId] = append(bsContentDtoMap[userId], dto)
543+
}
544+
}
545+
return bsContentDtoMap
546+
}
547+
443548
func (srv *BrightspaceService) GetJobParams() *map[string]interface{} {
444549
return srv.JobParams
445550
}

‎provider-middleware/brightspace_data.go‎

Lines changed: 61 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,15 @@ type BrightspaceUser struct {
3939
}
4040

4141
type BrightspaceCourse struct {
42-
OrgUnitId string `csv:"OrgUnitId"`
43-
Organization string `csv:"Organization"`
44-
Type string `csv:"Type"`
45-
Name string `csv:"Name"`
46-
Code string `csv:"Code"`
47-
IsActive string `csv:"IsActive"`
48-
IsDeleted string `csv:"IsDeleted"`
49-
OrgUnitTypeId string `csv:"OrgUnitTypeId"`
42+
OrgUnitId string `csv:"OrgUnitId"`
43+
Organization string `csv:"Organization"`
44+
Type string `csv:"Type"`
45+
Name string `csv:"Name"`
46+
Code string `csv:"Code"`
47+
IsActive string `csv:"IsActive"`
48+
IsDeleted string `csv:"IsDeleted"`
49+
OrgUnitTypeId string `csv:"OrgUnitTypeId"`
50+
TotalContentCount int
5051
}
5152

5253
type BrightspaceEnrollment struct {
@@ -56,6 +57,10 @@ type BrightspaceEnrollment struct {
5657
EnrollmentType string `csv:"EnrollmentType"`
5758
}
5859

60+
func (enroll *BrightspaceEnrollment) getCompositeKeyId() string {
61+
return enroll.UserId + "-" + enroll.OrgUnitId
62+
}
63+
5964
type BrightspaceAssignmentSubmission struct {
6065
DropboxId string `csv:"DropboxId"`
6166
OrgUnitId string `csv:"OrgUnitId"`
@@ -88,6 +93,41 @@ func (assign *BrightspaceQuizSubmission) getGradedCompositeKeyId() string {
8893
return "-quiz-" + assign.AttemptId
8994
}
9095

96+
type BrightspaceContentObject struct {
97+
ContentObjectId string `csv:"ContentObjectId"`
98+
OrgUnitId string `csv:"OrgUnitId"`
99+
Title string `csv:"Title"`
100+
ContentObjectType string `csv:"ContentObjectType"`
101+
CompletionType string `csv:"CompletionType"`
102+
StartDate string `csv:"StartDate"`
103+
EndDate string `csv:"EndDate"`
104+
DueDate string `csv:"DueDate"`
105+
LastModified string `csv:"LastModified"`
106+
IsDeleted string `csv:"IsDeleted"`
107+
}
108+
109+
type BrightspaceContentUserProgress struct {
110+
ContentObjectId string `csv:"ContentObjectId"`
111+
UserId string `csv:"UserId"`
112+
CompletedDate string `csv:"CompletedDate"`
113+
LastVisited string `csv:"LastVisited"`
114+
TotalTime string `csv:"TotalTime"`
115+
Version string `csv:"Version"`
116+
}
117+
118+
func (bsProgress *BrightspaceContentUserProgress) getCompositeKeyId() string {
119+
return bsProgress.Version + "-" + bsProgress.ContentObjectId + "-" + bsProgress.UserId + "-" + bsProgress.LastVisited
120+
}
121+
122+
type BrightspaceUserContentDto struct {
123+
OrgUnitId string
124+
ContentObjectId string
125+
TotalTime string
126+
Title string
127+
ContentObjectType string
128+
ExternalId string
129+
}
130+
91131
func (srv *BrightspaceService) IntoImportUser(bsUser BrightspaceUser) *models.ImportUser {
92132
user := models.ImportUser{
93133
Username: bsUser.UserName,
@@ -118,7 +158,7 @@ func (srv *BrightspaceService) IntoCourse(bsCourse BrightspaceCourse) *models.Co
118158
imgPath, err = UploadBrightspaceImage(imgBytes, id)
119159
if err != nil {
120160
log.Errorf("error during upload of Brightspace image to UnlockEd, id used was: %v error is %v", id, err)
121-
imgPath = "/brightspace.png"
161+
imgPath = "/brightspace.jpg"
122162
}
123163
}
124164
}
@@ -130,8 +170,8 @@ func (srv *BrightspaceService) IntoCourse(bsCourse BrightspaceCourse) *models.Co
130170
ThumbnailURL: imgPath,
131171
Type: "fixed_enrollment", //open to discussion
132172
Description: "Brightspace Managed Course: " + bsCourse.Name, //WIP
133-
TotalProgressMilestones: uint(0), //WIP
134-
ExternalURL: srv.BaseURL, //WIP
173+
TotalProgressMilestones: uint(bsCourse.TotalContentCount),
174+
ExternalURL: srv.BaseURL + "/" + "d2l/le/sequenceLauncher/" + bsCourse.OrgUnitId + "/View",//WIP
135175
}
136176
return &course
137177
}
@@ -239,6 +279,12 @@ func (srv *BrightspaceService) downloadAndUnzipFile(targetFileName string, endpo
239279
if resp.StatusCode == http.StatusOK {
240280
log.Infof("succesful request to url %v for downloading file %v", endpointUrl, targetFileName)
241281
zipFilePath := filepath.Join(CsvDownloadPath, targetFileName)
282+
targetDownloadDirectory := filepath.Dir(zipFilePath)
283+
err := os.MkdirAll(targetDownloadDirectory, os.ModePerm)
284+
if err != nil {
285+
log.Errorf("unable to create directory %s for downloading and zipping bulk data from Brightspace, error is %v", targetDownloadDirectory, err)
286+
return destPath, err
287+
}
242288
file, err := os.Create(zipFilePath)
243289
if err != nil {
244290
log.Errorf("error creating file %v used to write csv data into, error is: %v", zipFilePath, err)
@@ -263,7 +309,7 @@ func (srv *BrightspaceService) downloadAndUnzipFile(targetFileName string, endpo
263309
cleanUpFiles(zipFilePath) //delete zipfile here
264310
}()
265311
for _, zippedFile := range zipFile.File {
266-
destPath = filepath.Join(CsvDownloadPath, zippedFile.Name)
312+
destPath = filepath.Join(targetDownloadDirectory, zippedFile.Name)
267313
if zippedFile.FileInfo().IsDir() { //handles all zipped files
268314
if err := os.MkdirAll(destPath, os.ModePerm); err != nil {
269315
log.Errorf("error making directory to destination path %v, error is: %v", destPath, err)
@@ -296,8 +342,9 @@ func (srv *BrightspaceService) downloadAndUnzipFile(targetFileName string, endpo
296342
return destPath, err
297343
}
298344

299-
func cleanUpCsvFiles() {
300-
csvs, err := filepath.Glob(CsvDownloadPath + "/*.csv")
345+
func cleanUpCsvFiles(subDirectory string) {
346+
dirPath := filepath.Join(CsvDownloadPath, subDirectory)
347+
csvs, err := filepath.Glob(dirPath + "/*.csv")
301348
if err != nil {
302349
log.Errorln("error retrieving list of csv file paths, error is ", err)
303350
return

0 commit comments

Comments
 (0)