Feature/24 Ensure input CSVs are expected to have a header row

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -5,27 +5,27 @@ about: Create a report to help us improve
 
 **Environment:**
 
-  **Application version**
-
-  - App version or git commit number + branch
-  - App location (local, dev, test, stage, prod)
-  - Any paticular configuration of the app
+**Application version**
 
-  **Desktop (please complete the following information):**
+- App version or git commit number + branch
+- App location (local, dev, test, stage, prod)
+- Any paticular configuration of the app
 
-  - OS: [e.g. iOS]
-  - Browser [e.g. chrome, safari]
-  - Version [e.g. 22]
+**Desktop (please complete the following information):**
 
-  **Smartphone (please complete the following information):**
+- OS: [e.g. iOS]
+- Browser [e.g. chrome, safari]
+- Version [e.g. 22]
 
-  - Device: [e.g. iPhone6]
-  - OS: [e.g. iOS8.1]
-  - Browser [e.g. stock browser, safari]
-  - Version [e.g. 22]
+**Smartphone (please complete the following information):**
 
-  **Additional context**
-  Add any other context about the problem here.
+- Device: [e.g. iPhone6]
+- OS: [e.g. iOS8.1]
+- Browser [e.g. stock browser, safari]
+- Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.
 
 **To Reproduce**
 Steps to reproduce the behavior:

.gitignore

@@ -7,6 +7,10 @@ node_modules
 # Log files
 *.log
 
+# Allow test log files in test_data directory
+!test_data/find_log_errors/*.octoshift*.log
+!test_data/check_migrations/*/migration-log-*.log
+
 # Filesystem descriptors
 .DS_Store
 

README.md

@@ -1194,7 +1194,7 @@ Fetches secret variables from source repo and sets the corresponding secret vari
 #### Usage
 
 ```
-./migrate_secrets.sh -i [input_csv] [-s [source_token]] [-t [destination_token]] [-z [override_destination_org]] [-y [override_destination_repo_prefix]] [-a [ghes_hostname]] [-l [log_file]]
+npx migrate-secrets -i [input_csv] [-s [source_token]] [-t [destination_token]] [-z [override_destination_org]] [-y [override_destination_repo_prefix]] [-a [ghes_hostname]] [-l [log_file]]
 ```
 
 #### Arguments
@@ -1207,44 +1207,69 @@ Fetches secret variables from source repo and sets the corresponding secret vari
 6. `-a` - ghes_hostname - GHES hostname (not API URL, optional, required for GHES).
 7. `l` - log_file - Log file path (optional, default: migrate_secrets.log).
 
+#### Tests
+
+In order to test the script offline, you can use the following command:
+
+```
+OFFLINE_MODE=true npx migrate-secrets -i test_data/migrate_secrets/input.csv -s "test" -t "test"
+```
+
 ### 5. Check Migration Logs
 
 Checks the downloaded migration log files to see which ones completed successfully and how long the migration took.
 
 #### Usage
 
 ```
-./check_migrations.sh [-d [directory]] [-l [log_file]]
+npx check-migrations [-d [directory]] [-l [log_file]]
 ```
 
 #### Arguments
 
 1. `d` - directory - The directory where the migration log files are located. If no directory is provided, it checks the current working directory for the log files.
 2. `l` - log_file - Log file path (optional, default: check_migrations.log).
 
+#### Tests
+
+In order to test the script offline, you can use the following commands:
+
+```
+npx check-migrations -d test_data/check_migrations/complete/
+npx check-migrations -d test_data/check_migrations/failed/
+```
+
 ### 5. Find Log Errors
 
 Examines a directory containing all the log files after starting migrations with the GEI tool, and finds which ones failed with errors.
 
 #### Usage
 
 ```
-./find_log_errors.sh [-d [directory]] [-l [log_file]]
+npx find-log-errors [-d [directory]] [-l [log_file]]
 ```
 
 #### Arguments
 
 1. `d` - directory - The directory where the migration log files are located. If no directory is provided, it checks the current working directory for the log files.
 2. `l` - log_file - Log file path (optional, default: find_log_errors.log).
 
+#### Tests
+
+In order to test the script offline, you can use the following command:
+
+```
+npx find-log-errors -d ./test_data/find_log_errors
+```
+
 ### 6. Compare Migrations
 
 This script compares migration data between source and destination GitHub organizations based on provided input, downloading necessary data files, and generating a CSV report indicating match status along with signatures of repositories.
 
 #### Usage
 
 ```
-./compare_migrations.sh -i [input_csv] -o [output_csv] -s [source_token] -t [destination_token] -a [source_api_graphql_url] [-p [path_to_analyzer]] [-w [working_directory]] [-z [override_destination_org]] [-y [override_destination_repo_prefix]] [-l [log_file]]
+npx compare-migrations -i [input_csv] -o [output_csv] -s [source_token] -t [destination_token] -a [source_api_graphql_url] [-p [path_to_analyzer]] [-w [working_directory]] [-z [override_destination_org]] [-y [override_destination_repo_prefix]] [-l [log_file]]
 ```
 
 #### Arguments
@@ -1260,6 +1285,42 @@ This script compares migration data between source and destination GitHub organi
 9. `w` - working_directory - Working directory (optional, uses a new temporary directory if not specified).
 10. `l` - log_file - Log file path (optional, default: compare_migrations.log).
 
+#### Tests
+
+In order to test the script offline, you can use the following command:
+
+```
+OFFLINE_MODE=true npx compare-migrations -i test_data/compare_migrations/input.csv -o test_data/compare_migrations/output.csv -a https://test.com -s "test" -t "test" -w "test_data/compare_migrations"
+```
+
+### 7. Change Codeowners
+
+This script automates the process of updating CODEOWNERS files for multiple GitHub repositories within an organization, checking in the root, .github/ and docs/ directories. It reads from an input CSV containing the source organization and repository information, then applies a specified sed script to modify each CODEOWNERS file. The script processes updates in a temporary directory, and commits changes back to the repository if modifications are detected.
+
+#### Usage
+
+```
+npx change-codeowners -s [sed_script] -i [input_csv] -t [temp_dir] -n [commit_username] -e [commit_email] -h [help]
+```
+
+#### Arguments
+
+1. `-s` - sed_script - SED script file for updating CODEOWNERS.
+2. `-i` - input_csv - A CSV with source_org,source_repo.
+3. `-t` - temp_dir - Working directory (optional, uses a new temporary directory if not specified).
+4. `-n` - commit_username - Username for the commit message when CODEOWNERS is updated.
+5. `-e` - commit_email - Email for the commit message when CODEOWNERS is updated.
+6. `-h` - help - Show usage information.
+7. Ensure that a valid `GITHUB_TOKEN` environment variable is set for access to the repositories.
+
+#### Tests
+
+In order to test the script offline, you can use the following command:
+
+```
+OFFLINE_MODE=true npx change-codeowners -s test_data/change_codeowners/codeowners.sed -i test_data/change_codeowners/input.csv -n "test" -e "[email protected]" -t test_data/change_codeowners
+```
+
 ## Prerequisites
 
 **Node V16+**

change_codeowners.js

@@ -0,0 +1,203 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import { execSync } from 'child_process';
+import path from 'path';
+import os from 'os';
+import { parse } from 'fast-csv';
+
+// Function to print usage
+const printUsage = () => {
+	console.log(
+		'Usage: npx change-codeowners -s [sed_script] -i [input_csv] -t [temp_dir] -n [commit_username] -e [commit_email] -h [help]',
+		'\n',
+		'  -s: [sed_script] SED script file for updating CODEOWNERS.',
+		'\n',
+		'  -i: [input_csv] A CSV with source_org,source_repo.',
+		'\n',
+		'  -t: [temp_dir] Working directory (optional, uses a new temporary directory if not specified).',
+		'\n',
+		'  -n: [commit_username] Username for the commit message when CODEOWNERS is updated.',
+		'\n',
+		'  -e: [commit_email] Email for the commit message when CODEOWNERS is updated.',
+		'\n',
+		'  -h: [help] Show usage information.',
+		'\n',
+		'Ensure that a valid GITHUB_TOKEN environment variable is set for access to the repositories.',
+		'\n',
+		'Learn more: https://tinyurl.com/3pzbw4cp',
+		'\n',
+	);
+};
+
+// Default values
+const logFile = 'change_codeowners.log';
+const directories = ['', '.github/', 'docs/'];
+const expectedCSVHeaders = ['source_org', 'source_repo'];
+let tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gh-codeowners-'));
+let sedFile = '';
+let inputFile = '';
+let commitUsername = '';
+let commitEmail = '';
+
+// Function to log messages
+const log = (message) => {
+	const timestamp = new Date().toISOString();
+	const logMessage = `[${timestamp}] ${message}`;
+	fs.appendFileSync(logFile, `${logMessage}\n`);
+	console.log(logMessage);
+};
+
+// Check if a required argument is provided
+const checkRequiredArg = (arg, argName) => {
+	if (!arg) {
+		log(`Error: ${argName} is required.`);
+		printUsage();
+		process.exit(1);
+	}
+};
+
+// Function to validate header row
+const validateHeaders = (headers) => {
+	for (let header of expectedCSVHeaders) {
+		if (!headers.includes(header)) {
+			log(`Error: Missing required header "${header}" in CSV file.`);
+			printUsage();
+			process.exit(1);
+		}
+	}
+};
+
+// Parse command-line options
+const args = process.argv.slice(2);
+
+// Parse arguments
+for (let i = 0; i < args.length; i++) {
+	switch (args[i]) {
+		case '-s':
+			sedFile = args[++i];
+			break;
+		case '-i':
+			inputFile = args[++i];
+			break;
+		case '-t':
+			tempDir = args[++i];
+			break;
+		case '-n':
+			commitUsername = args[++i];
+			break;
+		case '-e':
+			commitEmail = args[++i];
+			break;
+		case '-h':
+			printUsage();
+			process.exit(0);
+		default:
+			printUsage();
+			process.exit(1);
+	}
+}
+
+// Check if required parameters are provided
+checkRequiredArg(sedFile, 'Sed File (-s)');
+checkRequiredArg(inputFile, 'Input CSV (-i)');
+checkRequiredArg(commitUsername, 'Commit Username (-n)');
+checkRequiredArg(commitEmail, 'Commit Email (-e)');
+
+// Helper function to execute shell commands
+const execCommand = (cmd) => {
+	// Possible to run in offline mode, useful for testing
+	if (process.env.OFFLINE_MODE === 'true') {
+		if (cmd.includes('gh api')) {
+			if (cmd.includes('--method PUT')) {
+				return 'Successfully updated CODEOWNERS';
+			}
+
+			return `mocked_sha_value\n${Buffer.from(
+				'Mock CODEOWNERS content\n/docs @old_owner\n/src @old_owner',
+			).toString('base64')}`;
+		}
+	}
+
+	try {
+		return execSync(cmd, { stdio: 'pipe' }).toString();
+	} catch (error) {
+		// If the error comes from `diff` and it's a difference (exit code 1), treat it as expected behavior
+		if (error.status === 1 && cmd.includes('diff')) {
+			return error.stdout.toString(); // Return the diff output as expected
+		}
+
+		log(`Error: Failed to execute command: ${cmd}`);
+		throw error;
+	}
+};
+
+// Parse the CSV file and process each repository
+log('Starting CODEOWNERS update process...');
+fs.createReadStream(inputFile)
+	.pipe(parse({ headers: true }))
+	.on('headers', (headers) => {
+		validateHeaders(headers);
+	})
+	.on('data', (row) => {
+		const { source_org: sourceOrg, source_repo: sourceRepo } = row;
+
+		for (const directory of directories) {
+			const tempFile = path.join(
+				tempDir,
+				`CODEOWNERS_${sourceRepo}_${directory.replace(/\/$/, '')}`,
+			);
+			log(
+				`Processing ${sourceOrg}/${sourceRepo}/${directory}CODEOWNERS (temp file: ${tempFile})`,
+			);
+
+			try {
+				const result = execCommand(
+					`gh api repos/${sourceOrg}/${sourceRepo}/contents/${directory}CODEOWNERS -q ".sha,.content"`,
+				);
+				fs.writeFileSync(tempFile, result);
+
+				const decodedFile = `${tempFile}.decoded`;
+				execCommand(`tail -n +2 ${tempFile} | base64 -d > ${decodedFile}`);
+
+				if (fs.existsSync(decodedFile) && fs.statSync(decodedFile).size > 0) {
+					execCommand(`sed -i.bak -f ${sedFile} ${decodedFile}`);
+					if (fs.existsSync(decodedFile) && fs.statSync(decodedFile).size > 0) {
+						const diffOutput = execCommand(
+							`diff -w ${decodedFile} ${decodedFile}.bak`,
+						);
+						if (diffOutput) {
+							const encodedContent = Buffer.from(
+								fs.readFileSync(tempFile, 'utf8'),
+							).toString('base64');
+
+							const sha = fs.readFileSync(tempFile, 'utf8').split('\n')[0];
+							execCommand(
+								`gh api --method PUT repos/${sourceOrg}/${sourceRepo}/contents/${directory}CODEOWNERS -f "message=update codeowners" -f "committer[name]=${commitUsername}" -f "committer[email]=${commitEmail}" -f "content=${encodedContent}" -f "sha=${sha}"`,
+							);
+							log(
+								`Updated ${directory}CODEOWNERS for ${sourceOrg}/${sourceRepo}`,
+							);
+						}
+					} else {
+						log(
+							`Error: Failed to create valid CODEOWNERS file for ${sourceOrg}/${sourceRepo}`,
+						);
+						process.exit(1);
+					}
+				} else {
+					log(
+						`No ${directory}CODEOWNERS file found for ${sourceOrg}/${sourceRepo}, skipping...`,
+					);
+				}
+			} catch (error) {
+				console.error(error);
+				log(
+					`Failed updating ${directory}CODEOWNERS for ${sourceOrg}/${sourceRepo}`,
+				);
+			}
+		}
+	})
+	.on('end', () => {
+		log('Finished updating CODEOWNERS for all repositories.');
+	});