diff --git a/index.js b/index.js
index e6fd1c8d..9c6707ea 100644
--- a/index.js
+++ b/index.js
@@ -578,9 +578,23 @@ module.exports = (robot, { getRouter }, Settings = require('./lib/settings')) =>
     if (check_suite.before === '0000000000000000000000000000000000000000') {
       check_suite.before = check_suite.pull_requests[0].base.sha
     }
-    params = Object.assign(context.repo(), { basehead: `${check_suite.before}...${check_suite.after}` })
-    const changes = await context.octokit.repos.compareCommitsWithBasehead(params)
-    const files = changes.data.files.map(f => { return f.filename })
+    params = Object.assign(context.repo(), { pull_number: pull_request.number, per_page: 100 })
+
+    const files = await context.octokit.paginate(context.octokit.pulls.listFiles, params, (response) => {
+      const files = new Set()
+
+      for (const file of response.data) {
+        files.add(file.filename)
+
+        if (file.previous_filename) {
+          files.add(file.previous_filename)
+        }
+      }
+
+      return Array.from(files)
+    })
+
+    robot.log.debug('Files changed', { files })
 
     const settingsModified = files.includes(Settings.FILE_PATH)
 
diff --git a/lib/settings.js b/lib/settings.js
index 6c42e439..ec11b517 100644
--- a/lib/settings.js
+++ b/lib/settings.js
@@ -245,8 +245,8 @@ ${this.results.reduce((x, y) => {
         if (y.type === 'ERROR') {
           error = true
           return `${x}
-<tr><td> ❗ ${y.action.msg} </td><td> ${y.plugin} </td><td> ${prettify(y.repo)} </td><td> ${prettify(y.action.additions)} </td><td> ${prettify(y.action.deletions)} </td><td> ${prettify(y.action.modifications)} </td><tr>`
-        } else if (y.action.additions === null && y.action.deletions === null && y.action.modifications === null) {
+<tr><td> ❗ ${y.action?.msg} </td><td> ${y.plugin} </td><td> ${prettify(y.repo)} </td><td> ${prettify(y.action?.additions)} </td><td> ${prettify(y.action?.deletions)} </td><td> ${prettify(y.action?.modifications)} </td><tr>`
+        } else if (y.action?.additions === null && y.action?.deletions === null && y.action?.modifications === null) {
           return `${x}`
         } else {
           if (y.action === undefined) {