diff --git a/CHANGELOG.md b/CHANGELOG.md index 2695880..6309d74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,24 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [1.1.0](https://github.com/rdkcentral/sysint-broadband/compare/1.0.0...1.1.0) + +- RDKB-63303, RDKB-63414 : Move RdkLogger Scripts to Cron [`#15`](https://github.com/rdkcentral/sysint-broadband/pull/15) +- RDKB-63514: Optimize and reduce RDKB memory usage - Reducing journalctl logs storage - Update log [`#14`](https://github.com/rdkcentral/sysint-broadband/pull/14) +- RDKB-63373, XB10-1930 : Observing integer expression expected in uplo… [`#10`](https://github.com/rdkcentral/sysint-broadband/pull/10) +- RDKB-63514: Optimize and reduce RDKB memory usage - Reducing journalctl logs storage [`#11`](https://github.com/rdkcentral/sysint-broadband/pull/11) +- RDKB-62753: Crontab output is not expected for RFC_Reboot.sh [`#12`](https://github.com/rdkcentral/sysint-broadband/pull/12) +- Updating update_journal_log.sh [`433739b`](https://github.com/rdkcentral/sysint-broadband/commit/433739b128f406e2754e29d0a99502c81f881762) +- RDKB-63373, XB10-1930 : Observing integer expression expected in uploadRDKBLogs.sh [`41695de`](https://github.com/rdkcentral/sysint-broadband/commit/41695deeffe480b5d6f67012dac7383c095ac05b) +- Updating update_journal_logs.sh [`d76c346`](https://github.com/rdkcentral/sysint-broadband/commit/d76c346da430cbafed9bf9b01fa99fc47dcd6b7e) + #### 1.0.0 +> 29 January 2026 + - Create CODEOWNERS [`#7`](https://github.com/rdkcentral/sysint-broadband/pull/7) - Create CONTRIBUTING.md [`#6`](https://github.com/rdkcentral/sysint-broadband/pull/6) - Create fossid_integration_stateless_diffscan_target_repo.yml [`#5`](https://github.com/rdkcentral/sysint-broadband/pull/5) - Import of source (stable2) [`2fc8051`](https://github.com/rdkcentral/sysint-broadband/commit/2fc8051f1468a67dd0ce3f650105e5678f7ed30b) - Create cla.yml [`ae32072`](https://github.com/rdkcentral/sysint-broadband/commit/ae320721e019c03b5edd620f9f9aa8e5a4c69465) +- Add changelog for release 1.0.0 [`f0fa4a8`](https://github.com/rdkcentral/sysint-broadband/commit/f0fa4a8c00a83b9ade058763419092fd3eb0b27b) diff --git a/RfcRebootCronschedule.sh b/RfcRebootCronschedule.sh index ada0f7f..ccd086d 100644 --- a/RfcRebootCronschedule.sh +++ b/RfcRebootCronschedule.sh @@ -37,17 +37,20 @@ calcRebootExecTime() time_offset=`dmcli eRT getv Device.Time.TimeOffset | grep "value:" | cut -d ":" -f 3 | tr -d ' '` if [ "x$BOX_TYPE" = "xHUB4" ] || [ "x$BOX_TYPE" = "xSR213" ]; then - #Maintence start and end time in UTC + #Maintenance start and end time in UTC main_start_time=$start_time main_end_time=$end_time else - #Maintence start and end time in local - main_start_time=$((start_time-time_offset)) - main_end_time=$((end_time-time_offset)) + #Maintenance start and end time in local. Ensure that start and end times are non-negative. + main_start_time=$(((start_time - time_offset + 86400) % 86400)) + main_end_time=$(((end_time - time_offset + 86400) % 86400)) fi #calculate random time in sec - rand_time_in_sec=`awk -v min=$main_start_time -v max=$main_end_time -v seed="$(date +%N)" 'BEGIN{srand(seed);print int(min+rand()*(max-min+1))}'` + rand_time_in_sec=`awk -v min=$main_start_time -v max=$main_end_time -v seed="$(date +%N)" 'BEGIN{ + if (max < min) max += 86400; #handle maintenance window crossing midnight + srand(seed);print int(min+rand()*(max-min+1)); + }'` # To avoid cron to be set beyond 24 hr clock limit if [ $rand_time_in_sec -ge 86400 ] diff --git a/fileUploadRandom.sh b/fileUploadRandom.sh index c030951..6af521b 100755 --- a/fileUploadRandom.sh +++ b/fileUploadRandom.sh @@ -34,7 +34,18 @@ DCM_PATH="/lib/rdk" SELFHEAL_PATH="/usr/ccsp/tad" CERT_CHECKER_PATH="/lib/rdk" -calcRandTimeandUpload() +CRON_MODE=0 +FILEUPLOAD_TMP_DIR="/tmp/.fileupload_state" +DELAY_REMAINING_FILE="$FILEUPLOAD_TMP_DIR/.delay_remaining_secs" +HEARTBEAT_TICK_FILE="$FILEUPLOAD_TMP_DIR/.heartbeat_tick" +CRON_SETUP_FLAG="$FILEUPLOAD_TMP_DIR/.cron_setup_complete" +NEXT_UPLOAD_TIME_FILE="$FILEUPLOAD_TMP_DIR/.next_upload_timestamp" + +if [ ! -d "$FILEUPLOAD_TMP_DIR" ]; then + mkdir -p "$FILEUPLOAD_TMP_DIR" +fi + +generate_random_delay() { rand_hr=0 rand_min=0 @@ -53,7 +64,59 @@ calcRandTimeandUpload() min_to_sleep=$(($rand_hr*60 + $rand_min)) sec_to_sleep=$(($min_to_sleep*60 + $rand_sec)) - sleep $sec_to_sleep; +} + +calcRandTimeandUpload() +{ + delay_completed=0 + if [ "$CRON_MODE" = "1" ]; then + if [ ! -f "$DELAY_REMAINING_FILE" ]; then + generate_random_delay + echo "$sec_to_sleep" > "$DELAY_REMAINING_FILE" + echo_t "fileupload_random: Initial random delay stored: $sec_to_sleep seconds" >> "$NEXT_UPLOAD_TIME_FILE" + else + echo_t "fileupload_random: Random delay already generated, reusing existing value" >> "$NEXT_UPLOAD_TIME_FILE" + fi + + if [ -f "$HEARTBEAT_TICK_FILE" ]; then + current_tick=$(cat "$HEARTBEAT_TICK_FILE") + else + current_tick=0 + fi + + if [ "$current_tick" = "0" ]; then + remaining=$(cat "$DELAY_REMAINING_FILE" 2>/dev/null) + [ -z "$remaining" ] && remaining=0 + + if [ "$remaining" -le 300 ]; then + echo_t "fileupload_random: Sleeping $remaining seconds now" >> "$NEXT_UPLOAD_TIME_FILE" + rm -f "$DELAY_REMAINING_FILE" "$HEARTBEAT_TICK_FILE" + delay_completed=1 + [ "$remaining" -gt 0 ] && sleep "$remaining" + else + echo_t "fileupload_random: Remaining delay before upload: $remaining seconds" >> "$NEXT_UPLOAD_TIME_FILE" + new_remaining=$((remaining - 300)) + if [ "$new_remaining" -lt 0 ]; then + new_remaining=0 + fi + echo $new_remaining > "$DELAY_REMAINING_FILE" + fi + else + echo_t "fileupload_random: Skipping countdown this minute (tick=$current_tick/4)" >> "$NEXT_UPLOAD_TIME_FILE" + fi + + if [ "$delay_completed" != "1" ]; then + new_tick=$(( (current_tick + 1) % 5 )) + echo "$new_tick" > "$HEARTBEAT_TICK_FILE" + return 0 + fi + fi + + if [ "$CRON_MODE" != "1" ]; then + generate_random_delay + echo_t "fileupload_random: Sleeping for $sec_to_sleep seconds" >> "$NEXT_UPLOAD_TIME_FILE" + sleep $sec_to_sleep; + fi if [ -f "$MAINTENANCEWINDOW" ] then @@ -229,34 +292,79 @@ getTFTPServer() BUILD_TYPE=`getBuildType` SERVER=`getTFTPServer $BUILD_TYPE` -loop=1 -upload_logfile=1 -while [ $loop -eq 1 ] -do - sleep 60 - if [ "$UTC_ENABLE" == "true" ] - then - cur_hr=`LTime H | tr -dc '0-9'` - cur_min=`LTime M | tr -dc '0-9'` - else - cur_hr=`date +"%H"` - cur_min=`date +"%M"` - fi +check_maintenance_window_upload() +{ + upload_logfile=1 + if [ "$UTC_ENABLE" == "true" ]; then + cur_hr=`LTime H | tr -dc '0-9'` + cur_min=`LTime M | tr -dc '0-9'` + else + cur_hr=`date +"%H"` + cur_min=`date +"%M"` + fi - if [ "$cur_hr" -ge "02" ] && [ "$cur_hr" -le "05" ] - then - if [ "$cur_hr" = "05" ] && [ "$cur_min" != "00" ] - then - upload_logfile=1 - else - if [ "$upload_logfile" = "1" ] - then - calcRandTimeandUpload - fi - fi - else - upload_logfile=1 - fi -done + if [ "$cur_hr" -ge "02" ] && [ "$cur_hr" -le "05" ]; then + if [ "$cur_hr" = "05" ] && [ "$cur_min" != "00" ]; then + upload_logfile=1 + else + if [ "$upload_logfile" = "1" ]; then + calcRandTimeandUpload + fi + fi + else + upload_logfile=1 + fi +} +install_cron_entry() { + CRON_LINE="* * * * * /rdklogger/fileUploadRandom.sh" + + if crontab -l 2>/dev/null | grep -q "fileUploadRandom.sh"; then + echo_t "fileUploadRandom.sh - Cron entry already present" + return 0 + fi + + (crontab -l 2>/dev/null; echo "$CRON_LINE") | crontab - + rc=$? + + if [ $rc -eq 0 ]; then + echo_t "fileUploadRandom.sh - Cron installed cleanly: $CRON_LINE" + else + echo_t "fileUploadRandom.sh - Cron install failed (rc=$rc)" + fi +} + +service_mode() { + while [ 1 ]; + do + sleep 60 + CRON_MODE=0 + check_maintenance_window_upload + done +} + +rdklogger_cron_enable=`syscfg get RdkbLogCronEnable` + +if [ "$rdklogger_cron_enable" = "true" ]; then + CRON_MODE=1 + + if [ ! -d "$FILEUPLOAD_TMP_DIR" ]; then + mkdir -p "$FILEUPLOAD_TMP_DIR" + fi + + if [ ! -f "$CRON_SETUP_FLAG" ]; then + install_cron_entry + touch "$CRON_SETUP_FLAG" + fi + + if [ -f "$DELAY_REMAINING_FILE" ]; then + calcRandTimeandUpload + else + check_maintenance_window_upload + fi + exit 0 +else + CRON_MODE=0 + service_mode +fi diff --git a/rdkbLogMonitor.sh b/rdkbLogMonitor.sh index 8f10b41..9b3842a 100644 --- a/rdkbLogMonitor.sh +++ b/rdkbLogMonitor.sh @@ -76,6 +76,23 @@ DeviceUP=0 # Check if upload on reboot flag is ON. If "yes", then we will upload the # log files first before starting monitoring of logs. +CRON_MODE=0 +RDKB_LOGMON_TMP_DIR="/tmp/.rdkb_logmonitor" +BOOTUP_UPLOAD_STATE="/tmp/bootup_upload.state" + +BOOT_PROCESSING_DONE="$RDKB_LOGMON_TMP_DIR/.boot_processing_done" +CRON_INITIALIZED_FLAG="$RDKB_LOGMON_TMP_DIR/.cron_initialized" +DELAY_COUNTDOWN_FILE="$RDKB_LOGMON_TMP_DIR/.delay_countdown" +PROCESS_HEARTBEAT="$RDKB_LOGMON_TMP_DIR/.process_heartbeat" +UPLOAD_SCHEDULE_FILE="$RDKB_LOGMON_TMP_DIR/.upload_schedule" +ARCHIVE_READY_FLAG="$RDKB_LOGMON_TMP_DIR/.archive_ready" +REBOOT_WORKFLOW_DONE="$RDKB_LOGMON_TMP_DIR/.reboot_workflow_done" +BOOT_ARCHIVE_DONE="$RDKB_LOGMON_TMP_DIR/.boot_archive_done" +INITIAL_BOOT_MARKER="$RDKB_LOGMON_TMP_DIR/.initial_boot_done" + +if [ ! -d "$RDKB_LOGMON_TMP_DIR" ]; then + mkdir -p "$RDKB_LOGMON_TMP_DIR" +fi #--------------------------------- # Function declarations @@ -118,13 +135,64 @@ getBuildType() } -random_sleep() +generate_random_sleep() { randomizedNumber=`awk -v min=0 -v max=30 -v seed="$(date +%N)" 'BEGIN{srand(seed);print int(min+rand()*(max-min+1))}'` RANDOM_SLEEP=`expr $randomizedNumber \\* 60` echo_t "Random sleep for $RANDOM_SLEEP" - sleep $RANDOM_SLEEP +} + +random_sleep() +{ + delay_completed=0 + if [ "$CRON_MODE" = "1" ]; then + if [ ! -f "$DELAY_COUNTDOWN_FILE" ]; then + generate_random_sleep + echo "$RANDOM_SLEEP" > "$DELAY_COUNTDOWN_FILE" + echo_t "rdkbLogMonitor: Initial random delay stored: $RANDOM_SLEEP seconds" >> "$UPLOAD_SCHEDULE_FILE" + else + echo_t "rdkbLogMonitor: Random already generated, reusing existing value" >> "$UPLOAD_SCHEDULE_FILE" + fi + + if [ -f "$PROCESS_HEARTBEAT" ]; then + current_tick=$(cat "$PROCESS_HEARTBEAT") + else + current_tick=0 + fi + + if [ "$current_tick" = "0" ]; then + remaining=$(cat "$DELAY_COUNTDOWN_FILE" 2>/dev/null) + [ -z "$remaining" ] && remaining=0 + + if [ "$remaining" -le 300 ]; then + echo_t "rdkbLogMonitor: Sleeping $remaining seconds now" >> "$UPLOAD_SCHEDULE_FILE" + rm -f "$PROCESS_HEARTBEAT" + delay_completed=1 + [ "$remaining" -gt 0 ] && sleep "$remaining" + else + echo_t "rdkbLogMonitor: Remaining sleep before completion: $remaining seconds" >> "$UPLOAD_SCHEDULE_FILE" + new_remaining=$((remaining - 300)) + if [ "$new_remaining" -lt 0 ]; then + new_remaining=0 + fi + echo $new_remaining > "$DELAY_COUNTDOWN_FILE" + echo_t "rdkbLogMonitor: Updated remaining sleep to $new_remaining seconds, exiting" >> "$UPLOAD_SCHEDULE_FILE" + fi + else + echo_t "rdkbLogMonitor: Skipping this minute (tick=$current_tick/4)" >> "$UPLOAD_SCHEDULE_FILE" + fi + + if [ "$delay_completed" != "1" ]; then + new_tick=$(( (current_tick + 1) % 5 )) + echo "$new_tick" > "$PROCESS_HEARTBEAT" + return 0 + fi + else + generate_random_sleep + echo_t "rdkbLogMonitor: Sleeping for $RANDOM_SLEEP seconds" >> "$UPLOAD_SCHEDULE_FILE" + sleep $RANDOM_SLEEP + fi } ## Process the responce and update it in a file DCMSettings.conf @@ -413,206 +481,324 @@ bootup_tarlogs() fi } -bootup_upload() +wait_for_stack() { - - echo_t "RDK_LOGGER: bootup_upload" - - if [ -e "$UPLOAD_ON_REBOOT" ] - then - - if [ "$LOGBACKUP_ENABLE" == "true" ]; then - #Sync log files immediately after reboot - echo_t "RDK_LOGGER: Sync logs to nvram2 after reboot" - syncLogs_nvram2 - cd $TMP_UPLOAD - else - BACKUPENABLE=`syscfg get logbackup_enable` - if [ "$BACKUPENABLE" = "true" ]; then - # First time call syncLogs after boot, - # remove existing log files (in $LOG_FILES_NAMES) in $LOG_BACK_UP_REBOOT - curDir=`pwd` - cd $LOG_BACK_UP_REBOOT - for fileName in $LOG_FILES_NAMES - do - rm 2>/dev/null $fileName #avoid error message - done - cd $curDir - syncLogs - fi - fi - - macOnly=`getMacAddressOnly` - fileToUpload=`ls | grep tgz` - # This check is to handle migration scenario from /nvram to /nvram2 - if [ "$fileToUpload" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ] - then - echo_t "Checking if any file available in $LOG_BACK_UP_REBOOT" - if [ -d $LOG_BACK_UP_REBOOT ]; then - fileToUpload=`ls $LOG_BACK_UP_REBOOT | grep tgz` - fi - fi - - echo_t "File to be uploaded is $fileToUpload ...." - - HAS_WAN_IP="" - - while [ "$loop" = "1" ] - do - echo_t "Waiting for stack to come up completely to upload logs..." - t2CountNotify "SYS_INFO_WaitingFor_Stack_Init" - sleep 30 - WEBSERVER_STARTED=`sysevent get webserver` - if [ "$WEBSERVER_STARTED" == "started" ] - then - echo_t "Webserver $WEBSERVER_STARTED..., uploading logs after 2 mins" - break - fi + while [ "$loop" = "1" ] + do + echo_t "Waiting for stack to come up completely to upload logs..." + t2CountNotify "SYS_INFO_WaitingFor_Stack_Init" + sleep 30 + WEBSERVER_STARTED=`sysevent get webserver` + if [ "$WEBSERVER_STARTED" == "started" ] + then + echo_t "Webserver $WEBSERVER_STARTED..., uploading logs after 2 mins" + break + fi bootup_time_sec=$(cut -d. -f1 /proc/uptime) if [ "$bootup_time_sec" -ge "600" ] ; then echo_t "Boot time is more than 10 min, Breaking Loop" break fi - done - sleep 120 - - if [ "$fileToUpload" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ] - then - echo_t "Checking if any file available in $TMP_LOG_UPLOAD_PATH" - if [ -d $TMP_LOG_UPLOAD_PATH ]; then - fileToUpload=`ls $TMP_LOG_UPLOAD_PATH | grep tgz` - fi - fi - - if [ "$fileToUpload" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ] - then - echo_t "Checking if any file available in $TMP_UPLOAD" - if [ -d $TMP_UPLOAD ]; then - fileToUpload=`ls $TMP_UPLOAD | grep tgz` - fi - fi - - echo_t "File to be uploaded is $fileToUpload ...." - #RDKB-7196: Randomize log upload within 30 minutes - # We will not remove 2 minute sleep above as removing that may again result in synchronization issues with xconf - boot_up_log_synced="true" - if [ "$fileToUpload" != "" ] - then - - random_sleep - $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $TMP_LOG_UPLOAD_PATH "true" - else - echo_t "No log file found in logbackupreboot folder" - fi - - #Venu - logBackupEnable=`syscfg get log_backup_enable` - if [ "$logBackupEnable" = "true" ];then - if [ -d $PRESERVE_LOG_PATH ] ; then - cd $PRESERVE_LOG_PATH - fileToUpload=`ls | grep tgz` - if [ "$fileToUpload" != "" ] - then - sleep 60 - echo_t "Uploading backup logs found in $PRESERVE_LOG_PATH" - $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $PRESERVE_LOG_PATH "true" - else - echo_t "No backup logs found in $PRESERVE_LOG_PATH" - fi - fi #end if [ -d $PRESERVE_LOG_PATH - fi #end if [ "$logBackupEnable" = "true" ] - - UPLOADED_AFTER_REBOOT="true" - sleep 2 - rm $UPLOAD_ON_REBOOT - cd $curDir + done +} + +file_bootup_upload_on_reboot() +{ + if [ "$LOGBACKUP_ENABLE" == "true" ]; then + echo_t "RDK_LOGGER: Sync logs to nvram2 after reboot" + syncLogs_nvram2 + cd $TMP_UPLOAD + else + BACKUPENABLE=`syscfg get logbackup_enable` + if [ "$BACKUPENABLE" = "true" ]; then + curDir=`pwd` + cd $LOG_BACK_UP_REBOOT + for fileName in $LOG_FILES_NAMES + do + rm 2>/dev/null $fileName + done + cd $curDir + syncLogs + fi + fi + + macOnly=`getMacAddressOnly` + fileToUpload=`ls | grep tgz` + + if [ "$fileToUpload" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ]; then + echo_t "Checking if any file available in $LOG_BACK_UP_REBOOT" + if [ -d $LOG_BACK_UP_REBOOT ]; then + fileToUpload=`ls $LOG_BACK_UP_REBOOT | grep tgz` + fi + fi + + echo_t "File to be uploaded is $fileToUpload ...." + HAS_WAN_IP="" + + if [ "$CRON_MODE" = "1" ]; then + WEBSERVER_STARTED=`sysevent get webserver` + bootup_time_sec=$(cut -d. -f1 /proc/uptime) + if [ "$bootup_time_sec" -lt "600" ] && [ "$WEBSERVER_STARTED" != "started" ]; then + echo_t "CRON: Boot <10min ($bootup_time_sec s) + webserver NOT started. Skipping." + return 0 + fi + else + wait_for_stack fi + + sleep 120 - echo_t "Check if any tar file available in /logbackup/ " - curDir=`pwd` + if [ "$fileToUpload" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ]; then + echo_t "Checking if any file available in $TMP_LOG_UPLOAD_PATH" + if [ -d $TMP_LOG_UPLOAD_PATH ]; then + fileToUpload=`ls $TMP_LOG_UPLOAD_PATH | grep tgz` + fi + fi - if [ "$LOGBACKUP_ENABLE" == "true" ]; then - cd $TMP_UPLOAD - else - cd $LOG_BACK_UP_PATH + if [ "$fileToUpload" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ]; then + echo_t "Checking if any file available in $TMP_UPLOAD" + if [ -d $TMP_UPLOAD ]; then + fileToUpload=`ls $TMP_UPLOAD | grep tgz` fi + fi - UploadFile=`ls | grep "tgz"` + echo_t "File to be uploaded is $fileToUpload ...." + boot_up_log_synced="true" +} - if [ "$UploadFile" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ] - then - echo_t "Checking if any file available in $TMP_LOG_UPLOAD_PATH" - if [ -d $TMP_LOG_UPLOAD_PATH ]; then - UploadFile=`ls $TMP_LOG_UPLOAD_PATH | grep tgz` - fi +bootup_upload_on_reboot() +{ + if [ -e "$UPLOAD_ON_REBOOT" ]; then + if [ "$CRON_MODE" = "1" ]; then + if [ -f "$BOOTUP_UPLOAD_STATE" ]; then + source "$BOOTUP_UPLOAD_STATE" 2>/dev/null || true + fi + + if [ -f "$DELAY_COUNTDOWN_FILE" ]; then + random_sleep + if [ $? -eq 0 ] && [ -f "$PROCESS_HEARTBEAT" ]; then + echo_t "Random sleep incomplete bootup_upload_on_reboot in next cron" >> "$UPLOAD_SCHEDULE_FILE" + return 0 + fi + echo_t "Random sleep completed" >> "$UPLOAD_SCHEDULE_FILE" + rm -f "$DELAY_COUNTDOWN_FILE" + if [ "$fileToUpload" != "" ]; then + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $TMP_LOG_UPLOAD_PATH "true" + fi + else + echo_t "RDK_LOGGER: bootup_upload_on_reboot" + file_bootup_upload_on_reboot + if [ "$fileToUpload" != "" ]; then + random_sleep + if [ $? -eq 0 ]&& [ -f "$PROCESS_HEARTBEAT" ]; then + echo_t "Random sleep incomplete file_bootup_upload_on_reboot - next cron" >> "$UPLOAD_SCHEDULE_FILE" + return 0 + fi + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $TMP_LOG_UPLOAD_PATH "true" + fi + fi + else + echo_t "RDK_LOGGER: bootup_upload_on_reboot" + file_bootup_upload_on_reboot + + cat > "$BOOTUP_UPLOAD_STATE" << EOF +fileToUpload="$fileToUpload" +boot_up_log_synced="$boot_up_log_synced" +EOF + if [ "$fileToUpload" != "" ]; then + random_sleep + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $TMP_LOG_UPLOAD_PATH "true" + fi + fi + + echo_t "No log file found in logbackupreboot folder" + + # Venu preserve logs + logBackupEnable=`syscfg get log_backup_enable` + if [ "$logBackupEnable" = "true" ];then + if [ -d $PRESERVE_LOG_PATH ] ; then + cd $PRESERVE_LOG_PATH + fileToUpload=`ls | grep tgz` + if [ "$fileToUpload" != "" ]; then + sleep 60 + echo_t "Uploading backup logs found in $PRESERVE_LOG_PATH" + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $PRESERVE_LOG_PATH "true" + else + echo_t "No backup logs found in $PRESERVE_LOG_PATH" + fi + fi + fi + + UPLOADED_AFTER_REBOOT="true" + sleep 2 + rm $UPLOAD_ON_REBOOT + fi + if [ "$CRON_MODE" = "1" ]; then + touch "$REBOOT_WORKFLOW_DONE" + rm -f "$BOOTUP_UPLOAD_STATE" fi - if [ "$UploadFile" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ] - then - echo_t "Checking if any file available in $LOG_SYNC_BACK_UP_REBOOT_PATH" +} + +tar_file_bootup_upload() +{ + curDir=`pwd` + + if [ "$LOGBACKUP_ENABLE" == "true" ]; then + cd $TMP_UPLOAD + else + cd $LOG_BACK_UP_PATH + fi + + UploadFile=`ls | grep "tgz"` + + if [ "$UploadFile" = "" ] && [ "$LOGBACKUP_ENABLE" == "true" ]; then + echo_t "Checking if any file available in $TMP_LOG_UPLOAD_PATH" + if [ -d $TMP_LOG_UPLOAD_PATH ]; then + UploadFile=`ls $TMP_LOG_UPLOAD_PATH | grep tgz` + fi + fi + if [ "$UploadFile" = "" ] && [ "$LOGBACKUP_ENABLE" == "true" ]; then + echo_t "Checking if any file available in $LOG_SYNC_BACK_UP_REBOOT_PATH" if [ -d $LOG_SYNC_BACK_UP_REBOOT_PATH ]; then UploadFile=`ls $LOG_SYNC_BACK_UP_REBOOT_PATH | grep tgz` fi - fi + fi + + files_exist_in_preserve="false" + if [ "$UploadFile" = "" ] && [ "$LOGBACKUP_ENABLE" == "true" ]; then + echo_t "Checking if any file available in $PRESERVE_LOG_PATH" + if [ -d $PRESERVE_LOG_PATH ]; then + UploadFile=`ls $PRESERVE_LOG_PATH | grep tgz` + if [ "$UploadFile" != "" ]; then + files_exist_in_preserve="true" + fi + fi + fi + + echo_t "File to be uploaded is $UploadFile ...." - files_exist_in_preserve="false" - if [ "$UploadFile" = "" ] && [ "$LOGBACKUP_ENABLE" = "true" ] - then - echo_t "Checking if any file available in $PRESERVE_LOG_PATH" - if [ -d $PRESERVE_LOG_PATH ]; then - UploadFile=`ls $PRESERVE_LOG_PATH | grep tgz` - if [ "$UploadFile" != "" ] - then - files_exist_in_preserve="true" - fi - fi - fi + cd $curDir +} - echo_t "File to be uploaded is $UploadFile ...." +tar_bootup_upload() +{ + curDir=`pwd` + if [ "$CRON_MODE" = "1" ]; then + if [ -f "$BOOTUP_UPLOAD_STATE" ]; then + source "$BOOTUP_UPLOAD_STATE" 2>/dev/null || true + fi - if [ "$UploadFile" != "" ] - then - echo_t "File to be uploaded from logbackup/ is $UploadFile " - if [ "$UPLOADED_AFTER_REBOOT" == "true" ] - then - random_sleep - $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "" $TMP_LOG_UPLOAD_PATH "true" + if [ -f "$DELAY_COUNTDOWN_FILE" ]; then + echo_t "bootup_upload_tar_bootup: Resuming random_sleep" >> "$UPLOAD_SCHEDULE_FILE" + random_sleep + if [ $? -eq 0 ] && [ -f "$PROCESS_HEARTBEAT" ]; then + echo_t "Random sleep incomplete upload logs in next cron" >> "$UPLOAD_SCHEDULE_FILE" + return 0 + fi + rm -f "$DELAY_COUNTDOWN_FILE" + echo_t "Random sleep completed" >> "$UPLOAD_SCHEDULE_FILE" + + if [ "$UPLOADED_AFTER_REBOOT" == "true" ]; then + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "" $TMP_LOG_UPLOAD_PATH "true" + else + if [ "$files_exist_in_preserve" == "true" ]; then + echo_t "Uploading backup logs found in $PRESERVE_LOG_PATH" + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $PRESERVE_LOG_PATH "true" + else + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "" $TMP_LOG_UPLOAD_PATH "true" + fi + UPLOADED_AFTER_REBOOT="true" + fi + if [ "$CRON_MODE" = "1" ]; then + touch "$BOOT_ARCHIVE_DONE" + rm -f "$BOOTUP_UPLOAD_STATE" + fi else - while [ "$loop" = "1" ] - do - echo_t "Waiting for stack to come up completely to upload logs..." - t2CountNotify "SYS_INFO_WaitingFor_Stack_Init" - sleep 30 - WEBSERVER_STARTED=`sysevent get webserver` - if [ "$WEBSERVER_STARTED" == "started" ] - then - echo_t "Webserver $WEBSERVER_STARTED..., uploading logs after 2 mins" - break - fi + echo_t "RDK_LOGGER: bootup_upload_tar_bootup" + echo_t "Check if any tar file available in /logbackup/ " + tar_file_bootup_upload + + cat > "$BOOTUP_UPLOAD_STATE" << EOF +UploadFile="$UploadFile" +files_exist_in_preserve="$files_exist_in_preserve" +UPLOADED_AFTER_REBOOT="$UPLOADED_AFTER_REBOOT" +EOF + + echo_t "File to be uploaded from logbackup/ is $UploadFile " + if [ "$UploadFile" != "" ]; then + if [ "$UPLOADED_AFTER_REBOOT" == "true" ]; then + random_sleep + if [ $? -eq 0 ] && [ -f "$PROCESS_HEARTBEAT" ]; then + echo_t "Random sleep incomplete upload logs in next cron" >> "$UPLOAD_SCHEDULE_FILE" + return 0 + fi + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "" $TMP_LOG_UPLOAD_PATH "true" + else + if [ "$CRON_MODE" = "1" ]; then + WEBSERVER_STARTED=`sysevent get webserver` + bootup_time_sec=$(cut -d. -f1 /proc/uptime) + if [ "$bootup_time_sec" -lt "600" ] && [ "$WEBSERVER_STARTED" != "started" ]; then + echo_t "RDK_LOGGER: Boot <10min ($bootup_time_sec s) + webserver NOT started. Skipping." + return 0 + fi + fi + + sleep 120 - bootup_time_sec=$(cut -d. -f1 /proc/uptime) - if [ "$bootup_time_sec" -ge "600" ] ; then - echo_t "Boot time is more than 10 min, Breaking Loop" - break - fi - done - sleep 120 + if [ "$files_exist_in_preserve" == "true" ]; then + random_sleep + if [ $? -eq 0 ] && [ -f "$PROCESS_HEARTBEAT" ]; then + echo_t "Random sleep incomplete upload preserve log in next cron" >> "$UPLOAD_SCHEDULE_FILE" + return 0 + fi + echo_t "Uploading backup logs found in $PRESERVE_LOG_PATH" + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $PRESERVE_LOG_PATH "true" + else + random_sleep + if [ $? -eq 0 ] && [ -f "$PROCESS_HEARTBEAT" ]; then + echo_t "Random sleep incomplete upload non-preserve log in next cron" >> "$UPLOAD_SCHEDULE_FILE" + return 0 + fi + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "" $TMP_LOG_UPLOAD_PATH "true" + fi + UPLOADED_AFTER_REBOOT="true" + fi + fi + if [ "$CRON_MODE" = "1" ]; then + touch "$BOOT_ARCHIVE_DONE" + rm -f "$BOOTUP_UPLOAD_STATE" + fi + fi + else + echo_t "RDK_LOGGER: bootup_upload_tar_bootup" + echo_t "Check if any tar file available in /logbackup/ " + tar_file_bootup_upload - if [ "$files_exist_in_preserve" == "true" ] - then - random_sleep - echo_t "Uploading backup logs found in $PRESERVE_LOG_PATH" - $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $PRESERVE_LOG_PATH "true" - else + echo_t "File to be uploaded from logbackup/ is $UploadFile " + if [ "$UploadFile" != "" ]; then + if [ "$UPLOADED_AFTER_REBOOT" == "true" ]; then random_sleep $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "" $TMP_LOG_UPLOAD_PATH "true" - fi - UPLOADED_AFTER_REBOOT="true" + else + wait_for_stack + sleep 120 + + if [ "$files_exist_in_preserve" == "true" ]; then + random_sleep + echo_t "Uploading backup logs found in $PRESERVE_LOG_PATH" + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "true" "" $PRESERVE_LOG_PATH "true" + else + random_sleep + $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "" $TMP_LOG_UPLOAD_PATH "true" + fi + UPLOADED_AFTER_REBOOT="true" + fi fi fi - cd $curDir -} + cd $curDir +} #ARRISXB6-5184 : Remove hidden files coming up in /rdklogs/logs and /nvram/logs remove_hidden_files() @@ -648,229 +834,97 @@ fi # RDKB-26588 && TRDKB-355 - Mitigation # # To Ensure only one instance is running # ######################################################## -RDKLOG_LOCK_DIR="/tmp/locking_logmonitor" - -if mkdir $RDKLOG_LOCK_DIR -then - echo "Got the first instance running" +UPLOAD_LOGS=`processDCMResponse` +rdklogger_cron_enable=`syscfg get RdkbLogCronEnable` + +if [ "$rdklogger_cron_enable" = "true" ]; then + LOCKFILE="/tmp/rdkb_cron.lock" + if [ -f "$LOCKFILE" ]; then + old_pid=$(cat "$LOCKFILE" 2>/dev/null) + if [ -n "$old_pid" ] && kill -0 "$old_pid" 2>/dev/null; then + echo_t "Already a cron instance is running; No 2nd instance" + exit 0 + fi + rm -f "$LOCKFILE" + fi + echo $$ > "$LOCKFILE" + cleanup() { rm -f "$LOCKFILE"; } + trap cleanup EXIT else - echo "Already a instance is running; No 2nd instance" - exit -fi -######################################################## + RDKLOG_LOCK_DIR="/tmp/locking_logmonitor" -PEER_COMM_ID="/tmp/elxrretyt-logm.swr" + if mkdir $RDKLOG_LOCK_DIR + then + echo "Got the first instance running" + else + echo "Already a instance is running; No 2nd instance" + exit + fi +fi -RebootReason=`syscfg get X_RDKCENTRAL-COM_LastRebootReason` +install_cron_entry() { + CRON_LINE="* * * * * /rdklogger/rdkbLogMonitor.sh" + + if crontab -l 2>/dev/null | grep -q "rdkbLogMonitor.sh"; then + echo_t "rdkbLogMonitor.sh - Cron entry already present" + return 0 + fi -if [ "$BOX_TYPE" = "XB3" ]; then - if [ "$RebootReason" = "RESET_ORIGIN_ATOM_WATCHDOG" ] || [ "$RebootReason" = "RESET_ORIGIN_ATOM" ]; then - if [ ! -f $PEER_COMM_ID ]; then - GetConfigFile $PEER_COMM_ID - fi - scp -i $PEER_COMM_ID -r root@$ATOM_INTERFACE_IP:$RAM_OOPS_FILE_LOCATION$RAM_OOPS_FILE $LOG_SYNC_PATH > /dev/null 2>&1 - fi - if [ "$RebootReason" = "HOST-OOPS-REBOOT" ]; then - cp $RAM_OOPS_FILE_LOCATION$RAM_OOPS_FILE0 $LOG_SYNC_PATH$RAM_OOPS_FILE0_HOST - cp $RAM_OOPS_FILE_LOCATION$RAM_OOPS_FILE1 $LOG_SYNC_PATH$RAM_OOPS_FILE1_HOST - fi -fi + (crontab -l 2>/dev/null; echo "$CRON_LINE") | crontab - + rc=$? + + if [ $rc -eq 0 ]; then + echo_t "rdkbLogMonitor.sh - Cron installed cleanly: $CRON_LINE" + else + echo_t "rdkbLogMonitor.sh - Cron install failed (rc=$rc)" + fi +} -if [ "$LOGBACKUP_ENABLE" == "true" ]; then +device_state() +{ + if [ "$DeviceUP" = "0" ]; then + #for rdkb-4260 + t2CountNotify "SYS_INFO_bootup" + if [ -f "$SW_UPGRADE_REBOOT" ]; then + echo_t "RDKB_REBOOT: Device is up after reboot due to software upgrade" + t2CountNotify "SYS_INFO_SW_upgrade_reboot" + #deleting reboot_due_to_sw_upgrade file + echo_t "Deleting file /nvram/reboot_due_to_sw_upgrade" + rm -rf /nvram/reboot_due_to_sw_upgrade + DeviceUP=1 + else + echo_t "RDKB_REBOOT: Device is up after reboot" + DeviceUP=1 + fi + fi +} - #ARRISXB6-3045 - This is speific to Axb6. If nvram2 supported hardware found, all syncing should switch to nvram2/logs. - #While switching from nvram to nvram2, old logs should be backed-up, uploaded and cleared from old sync path. -# model=`cat /etc/device.properties | grep MODEL_NUM | cut -f2 -d=` - if [ "$BOX_TYPE" = "XB6" -a "$MANUFACTURE" = "Arris" ];then - isNvram2Mounted=`grep nvram2 /proc/mounts` - if [ -d "/nvram2" ];then - if [ "$isNvram2Mounted" != "" ];then - if [ -d "/nvram/logs" ];then - file_list=`ls /nvram/logs/` - if [ "$file_list" != "" ]; then - echo_t "nvram/logs contains older logs" - if [ ! -d "$LOG_SYNC_PATH" ];then - echo_t "Creating new sync path - nvram2/logs" - mkdir $LOG_SYNC_PATH - fi - echo_t "nvram2 detected first time. Copying nvram/logs to nvram2/logs for boottime logupload" - cp /nvram/logs/* "$LOG_SYNC_PATH" - fi - echo_t "logs copied to nvram2. Removing old log path - nvram/logs" - rm -rf "/nvram/logs" - fi +regular_upload_state() +{ + if [ ! -e $REGULAR_UPLOAD ] + then + getLogfileSize "$LOG_PATH" + if [ "$totalSize" -ge "$MAXSIZE" ]; then + echo_t "Log size max reached" + get_logbackup_cfg + if [ "$UPLOAD_LOGS" = "" ] || [ ! -f "$DCM_SETTINGS_PARSED" ] + then + echo_t "processDCMResponse to get the logUploadSettings" + UPLOAD_LOGS=`processDCMResponse` + fi + + echo_t "UPLOAD_LOGS val is $UPLOAD_LOGS" + if [ "$UPLOAD_LOGS" = "true" ] || [ "$UPLOAD_LOGS" = "" ] + then + UPLOAD_LOGS="true" + # this file is touched to indicate log upload is enabled + # we check this file in logfiles.sh before creating tar ball. + # tar ball will be created only if this file exists. + echo_t "Log upload is enabled. Touching indicator in regular upload" + touch /tmp/.uploadregularlogs else - echo_t "nvram2 available, but not mounted." - fi - fi - fi - - PCIE_REBOOT_INDICATOR="/nvram/pcie_error_reboot_occurred" - PCIE_REBOOT_LOG="/rdklogs/logs/pcie_reboot.txt" - - if [ -f $PCIE_REBOOT_INDICATOR ] - then - echo "Previous Reboot reason:PCIE ENUM failed" > $PCIE_REBOOT_LOG - cat "$PCIE_REBOOT_INDICATOR" >> $PCIE_REBOOT_LOG - rm $PCIE_REBOOT_INDICATOR - if [ -f /nvram/pcie_error_reboot_needed ];then - rm /nvram/pcie_error_reboot_needed - fi - if [ -f /nvram/pcie_error_reboot_counter ];then - rm /nvram/pcie_error_reboot_counter - fi - fi - - #ARRISXB6-2821: - #DOCSIS_TIME_SYNC_NEEDED=yes for devices where DOCSIS and RDKB are in different processors - #and time sync needed before logbackup. - #Checking TimeSync-status before doing backupnvram2logs_on_reboot to ensure uploaded tgz file - #having correct timestamp. - #Will use default time if time not synchronized even after 2 mini of bootup to unblock - #other rdkbLogMonitor.sh functionality - - #TCCBR-4723 To handle all log upload with epoc time cases. Brought this block of code just - #above to the prevoius condition for making all cases to wait for timesync before log upload. - - if [ "`sysevent get wan-status`" != "started" ] || [ "x`sysevent get ntp_time_sync`" != "x1" ];then - loop=1 - retry=1 - while [ "$loop" = "1" ] - do - echo_t "Waiting for time synchronization between processors before logbackup" - WAN_STATUS=`sysevent get wan-status` - NTPD_STATUS=`sysevent get ntp_time_sync` - if [ "$WAN_STATUS" == "started" ] && [ "x$NTPD_STATUS" == "x1" ] - then - echo_t "wan status is $WAN_STATUS, and time sync status $NTPD_STATUS" - echo_t "Time is synced, breaking the loop" - break - elif [ "$retry" -gt "9" ] - then - echo_t "wan status is $WAN_STATUS, and time sync status $NTPD_STATUS" - echo_t "Time is not synced after 3 min retry. Breaking loop and using default time for logbackup" - break - else - echo_t "Time is not synced. Sleeping.. Retry:$retry" - retry=`expr $retry + 1` - sleep 20 - fi - done - fi - - echo_t "RDK_LOGGER: creating $TMP_UPLOAD" - if [ ! -d "$TMP_UPLOAD" ]; then - mkdir -p $TMP_UPLOAD - fi - file_list=`ls $LOG_SYNC_PATH | grep -v tgz` - #TCCBR-4275 to handle factory reset case. - if ( [ "$file_list" != "" ] && [ ! -f "$UPLOAD_ON_REBOOT" ] ) || ( [ "$RebootReason" == "factory-reset" ] ); then - echo_t "RDK_LOGGER: creating tar from nvram2 on reboot" - - #HUB4 uses NTP for syncing time. It doesnt have DOCSIS time sync, Hence waiting for NTP time sync. - if [ "$BOX_TYPE" == "HUB4" ] || [ "$BOX_TYPE" == "SR300" ] || [ "x$BOX_TYPE" == "xSR213" ] || [ "$BOX_TYPE" == "SE501" ] || [ "$BOX_TYPE" == "WNXL11BWL" ] || [ "$BOX_TYPE" == "SCER11BEL" ] || [ "$BOX_TYPE" == "SCXF11BFL" ]; then - loop=1 - retry=1 - while [ "$loop" = "1" ] - do - echo_t "Waiting for time synchronization before logbackup" - TIME_SYNC_STATUS=`timedatectl status | grep "clock synchronized" | cut -d " " -f 4` - if [ "$TIME_SYNC_STATUS" == "yes" ] - then - echo_t "Time synced. Breaking loop" - break - elif [ "$retry" = "12" ] - then - echo_t "Time not synced even after 2 min retry. Breaking loop and using default time for logbackup" - break - else - echo_t "Time not synced yet. Sleeping.. Retry:$retry" - retry=`expr $retry + 1` - sleep 10 - fi - done - fi - backupnvram2logs_on_reboot - #upload_nvram2_logs - - if [ "$LOGBACKUP_ENABLE" == "true" ]; then - #Sync log files immediately after reboot - echo_t "RDK_LOGGER: Sync logs to nvram2 after reboot" - syncLogs_nvram2 - else - BACKUPENABLE=`syscfg get logbackup_enable` - if [ "$BACKUPENABLE" = "true" ]; then - # First time call syncLogs after boot, - # remove existing log files (in $LOG_FILES_NAMES) in $LOG_BACK_UP_REBOOT - curDir=`pwd` - cd $LOG_BACK_UP_REBOOT - for fileName in $LOG_FILES_NAMES - do - rm 2>/dev/null $fileName #avoid error message - done - cd $curDir - syncLogs - fi - fi - elif [ "$file_list" == "" ] && [ ! -f "$UPLOAD_ON_REBOOT" ]; then - if [ "$LOGBACKUP_ENABLE" == "true" ]; then - #Sync log files immediately after reboot - echo_t "RDK_LOGGER: Sync logs to nvram2 after reboot" - syncLogs_nvram2 - fi - fi -fi - -# rdkb-24823 create tar file, then do upload in the background, -bootup_tarlogs -bootup_upload & - -UPLOAD_LOGS=`processDCMResponse` -while [ "$loop" = "1" ] -do - if [ "$DeviceUP" = "0" ]; then - #for rdkb-4260 - t2CountNotify "SYS_INFO_bootup" - if [ -f "$SW_UPGRADE_REBOOT" ]; then - echo_t "RDKB_REBOOT: Device is up after reboot due to software upgrade" - t2CountNotify "SYS_INFO_SW_upgrade_reboot" - #deleting reboot_due_to_sw_upgrade file - echo_t "Deleting file /nvram/reboot_due_to_sw_upgrade" - rm -rf /nvram/reboot_due_to_sw_upgrade - DeviceUP=1 - else - echo_t "RDKB_REBOOT: Device is up after reboot" - DeviceUP=1 - fi - fi - - sleep 60 - if [ ! -e $REGULAR_UPLOAD ] - then - getLogfileSize "$LOG_PATH" - - if [ "$totalSize" -ge "$MAXSIZE" ]; then - echo_t "Log size max reached" - get_logbackup_cfg - - if [ "$UPLOAD_LOGS" = "" ] || [ ! -f "$DCM_SETTINGS_PARSED" ] - then - echo_t "processDCMResponse to get the logUploadSettings" - UPLOAD_LOGS=`processDCMResponse` - fi - - echo_t "UPLOAD_LOGS val is $UPLOAD_LOGS" - if [ "$UPLOAD_LOGS" = "true" ] || [ "$UPLOAD_LOGS" = "" ] - then - UPLOAD_LOGS="true" - # this file is touched to indicate log upload is enabled - # we check this file in logfiles.sh before creating tar ball. - # tar ball will be created only if this file exists. - echo_t "Log upload is enabled. Touching indicator in regular upload" - touch /tmp/.uploadregularlogs - else - echo_t "Log upload is disabled. Removing indicator in regular upload" - rm -rf /tmp/.uploadregularlogs + echo_t "Log upload is disabled. Removing indicator in regular upload" + rm -rf /tmp/.uploadregularlogs fi cd $TMP_UPLOAD @@ -888,7 +942,6 @@ do if [ "$LOGBACKUP_ENABLE" == "true" ]; then createSysDescr syncLogs_nvram2 - # Check if there is any tar ball to be preserved # else tar ball will be removed in backupnvram2logs logBackupEnable=`syscfg get log_backup_enable` @@ -901,7 +954,6 @@ do preserveThisLog $fileName $TMP_UPLOAD fi fi - backupnvram2logs "$TMP_UPLOAD" else syncLogs @@ -914,7 +966,6 @@ do echo_t "Logupload http_ret value = $http_ret" if [ "$http_ret" = "200" ] || [ "$http_ret" = "302" ] ;then logBackupEnable=`syscfg get log_backup_enable` - if [ "$logBackupEnable" = "true" ] ; then if [ -d $PRESERVE_LOG_PATH ] ; then cd $PRESERVE_LOG_PATH @@ -931,7 +982,6 @@ do else echo_t "Preserve Logupload not success because of http val $http_ret" fi - else echo_t "Regular log upload is disabled" fi @@ -983,29 +1033,270 @@ do echo_t "Log upload is disabled. Removing indicator in maintenance window" rm /tmp/.uploadregularlogs fi - syncLogs_nvram2 if [ "$ATOM_SYNC" == "" ]; then syncLogs fi backupnvram2logs "$TMP_UPLOAD" - if [ "$UPLOAD_LOGS" = "true" ] then $RDK_LOGGER_PATH/uploadRDKBLogs.sh $SERVER "HTTP" $URL "false" "true" else echo_t "Regular log upload is disabled" fi - fi fi - -done +} ######################################################## -# RDKB-26588 && TRDKB-355 - Mitigation # -# To Ensure only one instance is running # -######################################################## -rm -rf $RDKLOG_LOCK_DIR -######################################################## +bootup_nvram_upload() +{ +PEER_COMM_ID="/tmp/elxrretyt-logm.swr" + +RebootReason=`syscfg get X_RDKCENTRAL-COM_LastRebootReason` +if [ "$BOX_TYPE" = "XB3" ]; then + if [ "$RebootReason" = "RESET_ORIGIN_ATOM_WATCHDOG" ] || [ "$RebootReason" = "RESET_ORIGIN_ATOM" ]; then + if [ ! -f $PEER_COMM_ID ]; then + GetConfigFile $PEER_COMM_ID + fi + scp -i $PEER_COMM_ID -r root@$ATOM_INTERFACE_IP:$RAM_OOPS_FILE_LOCATION$RAM_OOPS_FILE $LOG_SYNC_PATH > /dev/null 2>&1 + fi + if [ "$RebootReason" = "HOST-OOPS-REBOOT" ]; then + cp $RAM_OOPS_FILE_LOCATION$RAM_OOPS_FILE0 $LOG_SYNC_PATH$RAM_OOPS_FILE0_HOST + cp $RAM_OOPS_FILE_LOCATION$RAM_OOPS_FILE1 $LOG_SYNC_PATH$RAM_OOPS_FILE1_HOST + fi +fi + +if [ "$LOGBACKUP_ENABLE" == "true" ]; then + + #ARRISXB6-3045 - This is speific to Axb6. If nvram2 supported hardware found, all syncing should switch to nvram2/logs. + #While switching from nvram to nvram2, old logs should be backed-up, uploaded and cleared from old sync path. +# model=`cat /etc/device.properties | grep MODEL_NUM | cut -f2 -d=` + if [ "$BOX_TYPE" = "XB6" -a "$MANUFACTURE" = "Arris" ];then + isNvram2Mounted=`grep nvram2 /proc/mounts` + if [ -d "/nvram2" ];then + if [ "$isNvram2Mounted" != "" ];then + if [ -d "/nvram/logs" ];then + file_list=`ls /nvram/logs/` + if [ "$file_list" != "" ]; then + echo_t "nvram/logs contains older logs" + if [ ! -d "$LOG_SYNC_PATH" ];then + echo_t "Creating new sync path - nvram2/logs" + mkdir $LOG_SYNC_PATH + fi + echo_t "nvram2 detected first time. Copying nvram/logs to nvram2/logs for boottime logupload" + cp /nvram/logs/* "$LOG_SYNC_PATH" + fi + echo_t "logs copied to nvram2. Removing old log path - nvram/logs" + rm -rf "/nvram/logs" + fi + else + echo_t "nvram2 available, but not mounted." + fi + fi + fi + + PCIE_REBOOT_INDICATOR="/nvram/pcie_error_reboot_occurred" + PCIE_REBOOT_LOG="/rdklogs/logs/pcie_reboot.txt" + + if [ -f $PCIE_REBOOT_INDICATOR ] + then + echo "Previous Reboot reason:PCIE ENUM failed" > $PCIE_REBOOT_LOG + cat "$PCIE_REBOOT_INDICATOR" >> $PCIE_REBOOT_LOG + rm $PCIE_REBOOT_INDICATOR + if [ -f /nvram/pcie_error_reboot_needed ];then + rm /nvram/pcie_error_reboot_needed + fi + if [ -f /nvram/pcie_error_reboot_counter ];then + rm /nvram/pcie_error_reboot_counter + fi + fi + + #ARRISXB6-2821: + #DOCSIS_TIME_SYNC_NEEDED=yes for devices where DOCSIS and RDKB are in different processors + #and time sync needed before logbackup. + #Checking TimeSync-status before doing backupnvram2logs_on_reboot to ensure uploaded tgz file + #having correct timestamp. + #Will use default time if time not synchronized even after 2 mini of bootup to unblock + #other rdkbLogMonitor.sh functionality + + #TCCBR-4723 To handle all log upload with epoc time cases. Brought this block of code just + #above to the prevoius condition for making all cases to wait for timesync before log upload. + + if [ "`sysevent get wan-status`" != "started" ] || [ "x`sysevent get ntp_time_sync`" != "x1" ];then + loop=1 + retry=1 + while [ "$loop" = "1" ] + do + echo_t "Waiting for time synchronization between processors before logbackup" + WAN_STATUS=`sysevent get wan-status` + NTPD_STATUS=`sysevent get ntp_time_sync` + if [ "$WAN_STATUS" == "started" ] && [ "x$NTPD_STATUS" == "x1" ] + then + echo_t "wan status is $WAN_STATUS, and time sync status $NTPD_STATUS" + echo_t "Time is synced, breaking the loop" + break + elif [ "$retry" -gt "9" ] + then + echo_t "wan status is $WAN_STATUS, and time sync status $NTPD_STATUS" + echo_t "Time is not synced after 3 min retry. Breaking loop and using default time for logbackup" + break + else + echo_t "Time is not synced. Sleeping.. Retry:$retry" + retry=`expr $retry + 1` + sleep 20 + fi + done + fi + + echo_t "RDK_LOGGER: creating $TMP_UPLOAD" + if [ ! -d "$TMP_UPLOAD" ]; then + mkdir -p $TMP_UPLOAD + fi + file_list=`ls $LOG_SYNC_PATH | grep -v tgz` + #TCCBR-4275 to handle factory reset case. + if ( [ "$file_list" != "" ] && [ ! -f "$UPLOAD_ON_REBOOT" ] ) || ( [ "$RebootReason" == "factory-reset" ] ); then + echo_t "RDK_LOGGER: creating tar from nvram2 on reboot" + + #HUB4 uses NTP for syncing time. It doesnt have DOCSIS time sync, Hence waiting for NTP time sync. + if [ "$BOX_TYPE" == "HUB4" ] || [ "$BOX_TYPE" == "SR300" ] || [ "x$BOX_TYPE" == "xSR213" ] || [ "$BOX_TYPE" == "SE501" ] || [ "$BOX_TYPE" == "WNXL11BWL" ] || [ "$BOX_TYPE" == "SCER11BEL" ] || [ "$BOX_TYPE" == "SCXF11BFL" ]; then + loop=1 + retry=1 + while [ "$loop" = "1" ] + do + echo_t "Waiting for time synchronization before logbackup" + TIME_SYNC_STATUS=`timedatectl status | grep "clock synchronized" | cut -d " " -f 4` + if [ "$TIME_SYNC_STATUS" == "yes" ] + then + echo_t "Time synced. Breaking loop" + break + elif [ "$retry" = "12" ] + then + echo_t "Time not synced even after 2 min retry. Breaking loop and using default time for logbackup" + break + else + echo_t "Time not synced yet. Sleeping.. Retry:$retry" + retry=`expr $retry + 1` + sleep 10 + fi + done + fi + backupnvram2logs_on_reboot + #upload_nvram2_logs + + if [ "$LOGBACKUP_ENABLE" == "true" ]; then + #Sync log files immediately after reboot + echo_t "RDK_LOGGER: Sync logs to nvram2 after reboot" + syncLogs_nvram2 + else + BACKUPENABLE=`syscfg get logbackup_enable` + if [ "$BACKUPENABLE" = "true" ]; then + # First time call syncLogs after boot, + # remove existing log files (in $LOG_FILES_NAMES) in $LOG_BACK_UP_REBOOT + curDir=`pwd` + cd $LOG_BACK_UP_REBOOT + for fileName in $LOG_FILES_NAMES + do + rm 2>/dev/null $fileName #avoid error message + done + cd $curDir + syncLogs + fi + fi + elif [ "$file_list" == "" ] && [ ! -f "$UPLOAD_ON_REBOOT" ]; then + if [ "$LOGBACKUP_ENABLE" == "true" ]; then + #Sync log files immediately after reboot + echo_t "RDK_LOGGER: Sync logs to nvram2 after reboot" + syncLogs_nvram2 + fi + fi +fi +} + +service_mode() { + bootup_nvram_upload + + # rdkb-24823 create tar file, then do upload in the background, + bootup_tarlogs + bootup_upload_on_reboot ; tar_bootup_upload & + while [ 1 ]; + do + CRON_MODE=0 + device_state + sleep 60 + regular_upload_state + done + ######################################################## + # RDKB-26588 && TRDKB-355 - Mitigation # + # To Ensure only one instance is running # + ######################################################## + rm -rf $RDKLOG_LOCK_DIR + ######################################################## +} + +if [ "$rdklogger_cron_enable" = "true" ]; then + CRON_MODE=1 + + if [ ! -d "$RDKB_LOGMON_TMP_DIR" ]; then + mkdir -p "$RDKB_LOGMON_TMP_DIR" + fi + + if [ ! -f "$CRON_INITIALIZED_FLAG" ]; then + install_cron_entry + touch "$CRON_INITIALIZED_FLAG" + fi + + if [ ! -f "$BOOT_PROCESSING_DONE" ]; then + if [ ! -f "$ARCHIVE_READY_FLAG" ]; then + bootup_nvram_upload + bootup_tarlogs + touch "$ARCHIVE_READY_FLAG" + fi + + if [ -f "$ARCHIVE_READY_FLAG" ] && [ ! -f "$REBOOT_WORKFLOW_DONE" ]; then + if [ ! -f "$INITIAL_BOOT_MARKER" ]; then + bootup_upload_on_reboot + device_state + regular_upload_state + touch "$INITIAL_BOOT_MARKER" + else + bootup_upload_on_reboot + getLogfileSize "$LOG_PATH" + if [ "$totalSize" -ge "$MAXSIZE" ]; then + device_state + regular_upload_state + fi + fi + exit 0 + fi + + if [ -f "$REBOOT_WORKFLOW_DONE" ] && [ ! -f "$BOOT_ARCHIVE_DONE" ]; then + getLogfileSize "$LOG_PATH" + if [ "$totalSize" -ge "$MAXSIZE" ]; then + device_state + regular_upload_state + fi + + tar_bootup_upload + exit 0 + fi + + if [ -f "$BOOT_ARCHIVE_DONE" ]; then + touch "$BOOT_PROCESSING_DONE" + rm -f "$ARCHIVE_READY_FLAG" \ + "$REBOOT_WORKFLOW_DONE" \ + "$BOOT_ARCHIVE_DONE" + fi + fi + + if [ "$totalSize" -ge "$MAXSIZE" ]; then + device_state + regular_upload_state + fi + + exit 0 +else + CRON_MODE=0 + service_mode +fi diff --git a/update_journal_log.sh b/update_journal_log.sh index 2874dd4..c99c7af 100644 --- a/update_journal_log.sh +++ b/update_journal_log.sh @@ -27,8 +27,41 @@ current_time=0 lastync_time=0 BootupLog_is_updated=0 -while [ 1 ] -do +JOURNAL_RUNTIME_DIR="/run/systemd/journald.conf.d" +JOURNAL_OVERRIDE_FILE="${JOURNAL_RUNTIME_DIR}/override.conf" +LOG_FILE="/rdklogs/logs/Consolelog.txt.0" + +echo_t() +{ + echo "$(date +"%y%m%d-%T.%6N") $*" >> "$LOG_FILE" +} + +JOURNAL_CRON_INSTALLED="/tmp/.journal_log_cron_flag" + +do_journal_iteration() +{ + uptime_in_secs=$(cut -d. -f1 /proc/uptime) + if [ "$uptime_in_secs" -ge 1800 ] && [ ! -f "$JOURNAL_OVERRIDE_FILE" ]; then + echo_t "Applying journald runtime override" + mkdir -p "$JOURNAL_RUNTIME_DIR" + cat > "$JOURNAL_OVERRIDE_FILE" <<'EOF' +[Journal] +RuntimeMaxUse=8M +RuntimeMaxFileSize=4M +RuntimeMaxFiles=2 +EOF + if [ $? -ne 0 ]; then + echo_t "ERROR: Failed to create journald override file" + fi + systemctl restart systemd-journald >/dev/null 2>&1 + rc=$? + + if [ $rc -ne 0 ]; then + echo_t "ERROR: systemd-journald restart failed, rc=$rc" + else + echo_t "journald runtime threshold set to 8MB successfully" + fi + fi current_time=$(date +%s) if [ -f "$lastdmesgsync" ];then lastsync_time=`cat $lastdmesgsync` @@ -43,19 +76,80 @@ do cat ${DMESG_FILE} | grep -i "apparmor" > ${APPARMOR_LOG_FILE} if [ "$BOX_TYPE" = "XB6" ] || [ "$BOX_TYPE" = "XF3" ] || [ "$BOX_TYPE" = "TCCBR" ] || [ "$BOX_TYPE" == "VNTXER5" ] || [ "$BOX_TYPE" == "SCER11BEL" ] || [ "$BOX_TYPE" == "SCXF11BFL" ];then #ARRISXB6-7973: Complete journalctl logs to /rdklogs/logs/journal_logs.txt.0 - uptime_in_secs=$(cut -d. -f1 /proc/uptime) if [ $uptime_in_secs -ge 240 ] && [ $BootupLog_is_updated -eq 0 ]; then nice -n 19 journalctl > ${journal_log} BootupLog_is_updated=1; fi fi - # ARRISXB6-8252 sleep for 60 sec until we populate journalctl - if [ "$BOX_TYPE" = "XB6" -a "$MANUFACTURE" = "Arris" ];then - dmesgsyncinterval=60 - else - dmesgsyncinterval=`syscfg get dmesglogsync_interval` - fi +} + +install_cron_entry() { + if [ "$BOX_TYPE" = "XB6" -a "$MANUFACTURE" = "Arris" ]; then + dmesgsyncinterval_sec=60 + cron="* * * * *" + else + dmesgsyncinterval_sec="$(syscfg get dmesglogsync_interval)" + case "$dmesgsyncinterval_sec" in + "60") cron="* * * * *" ;; + "300") cron="*/5 * * * *" ;; + "600") cron="*/10 * * * *" ;; + "900") cron="*/15 * * * *" ;; + "3600") cron="0 * * * *" ;; + *) cron="*/15 * * * *" ;; + esac + fi + + CRON_LINE="$cron /rdklogger/update_journal_log.sh start" + + if crontab -l 2>/dev/null | grep -q "update_journal_log.sh"; then + echo_t "update_journal_log.sh - Cron entry already present" + return 0 + fi + + (crontab -l 2>/dev/null; echo "$CRON_LINE") | crontab - + rc=$? + + if [ $rc -eq 0 ]; then + echo_t "update_journal_log.sh - Cron installed cleanly: $CRON_LINE" + else + echo_t "update_journal_log.sh - Cron install failed (rc=$rc)" + fi +} + +service_mode() { + + while [ 1 ]; + do + do_journal_iteration + # ARRISXB6-8252 sleep for 60 sec until we populate journalctl + if [ "$BOX_TYPE" = "XB6" -a "$MANUFACTURE" = "Arris" ];then + dmesgsyncinterval=60 + else + dmesgsyncinterval=`syscfg get dmesglogsync_interval` + fi + + sleep $dmesgsyncinterval + + done +} + +rdklogger_cron_enable=`syscfg get RdkbLogCronEnable` - sleep $dmesgsyncinterval - -done +if [ "$rdklogger_cron_enable" = "true" ]; then + + if [ ! -f "$JOURNAL_CRON_INSTALLED" ]; then + install_cron_entry + touch "$JOURNAL_CRON_INSTALLED" + do_journal_iteration + + if [ -f /lib/systemd/system/log_journalmsg.service ]; then + systemctl stop log_journalmsg.service + fi + exit 0 + else + do_journal_iteration + exit 0 + fi +else + service_mode +fi diff --git a/uploadRDKBLogs.sh b/uploadRDKBLogs.sh index e812ee7..1fadac2 100755 --- a/uploadRDKBLogs.sh +++ b/uploadRDKBLogs.sh @@ -239,6 +239,7 @@ CURLPATH="/fss/gw" VERSION="/version.txt" http_code=0 +http_ret=0 OutputFile='/tmp/httpresult.txt' # Function which will upload logs to TFTP server @@ -356,6 +357,9 @@ useDirectRequest() if [ "$http_code" = "200" ] || [ "$http_code" = "302" ] ;then return 0 fi + else + echo_t "Failed: Log Upload: $msg_tls_source Direct Communication - ret:$ret,http_code:$http_code" + http_code=0 fi else http_code=0 @@ -441,6 +445,9 @@ useCodebigRequest() if [ "$http_code" = "200" ] || [ "$http_code" = "302" ] ;then return 0 fi + else + echo_t "Failed:Codebig connection HttpCode received is : $http_code" + http_code=0 fi else http_code=0 @@ -648,6 +655,9 @@ HttpLogUpload() if [ "$http_code" = "200" ];then break fi + else + echo_t "Failed:HttpCode received is : $http_code" + http_code=0 fi else http_code=0 @@ -729,6 +739,9 @@ HttpLogUpload() if [ "$http_code" = "200" ];then break fi + else + echo_t "Failed:HttpCode received is : $http_code" + http_code=0 fi else http_code=0 @@ -778,7 +791,10 @@ HttpLogUpload() if [ "$http_code" = "200" ];then break fi - fi + else + echo_t "http_code:$http_code" + http_code=0 + fi else http_code=0 fi