diff --git a/db-data/_dl-all-readme.md b/db-data/_dl-all-readme.md new file mode 100644 index 00000000..9460c1b2 --- /dev/null +++ b/db-data/_dl-all-readme.md @@ -0,0 +1,50 @@ +# Firestore Backup Script + +The [db-data/\_dl-all.ts](/db-data/_dl-all.ts) script creates a backup of Firestore collections with these key features: + +1. **File Structure**: + + - Each top-level collection is written to its own `.js` file + - Files use `export default` format for easy importing + - Subcollections are nested within their parent document's data + - An `index.ts` provides typed access to all collections + +2. **Performance**: + + - Documents are processed in parallel chunks (CHUNK_SIZE = 50) + - Collections are processed sequentially to avoid overwhelming Firestore + - Data is written incrementally as it's downloaded + - Memory efficient - doesn't hold entire collections in memory + +3. **Progress Tracking**: + + - Maintains a manifest.json showing status of each collection + - Shows progress within collections (every N documents) + - Records timing for each collection download + +4. **Output Format**: + + ```typescript + // users.js + export default { + "docId1": { + // ... document data ... + __subcollections__: { + "subcollectionName": { + // ... subcollection documents ... + } + } + }, + "docId2": { ... }, + } + ``` + +5. **Usage**: + + ```typescript + // Import specific collections + import { posts, users } from './backup/2024-03-14T12:30/index' + + // Types are automatically inferred + type UserDoc = (typeof users)['docId'] + ``` diff --git a/db-data/_dl-all.ts b/db-data/_dl-all.ts index 8e64f4e2..ba8f89d5 100644 --- a/db-data/_dl-all.ts +++ b/db-data/_dl-all.ts @@ -3,63 +3,127 @@ import './_env' import fs from 'fs' import path from 'path' -import bluebird from 'bluebird' -import { firestore } from 'firebase-admin' - import { firebase } from '../pages/api/_services' -// import elections from './elections.json' - -// Run this to write existing Firebase collections to local JSON files +// Run this to write existing Firebase collections to local JSON file // Execute this file with: -// npx ts-node db-data/_dl-all.ts +// npx tsx db-data/_dl-all.ts + +type NestedDump = Record & { + __subcollections__?: Record +} +const db = firebase.firestore() -// Download all Firebase data to JSON files -type DirectionStr = 'desc' | 'asc' -type Collection = [string, Collection[]?, string?, DirectionStr?] -const collectionsToDownload: Collection[] = [['elections', [['votes', [], 'created_at', 'desc']], 'created_at', 'desc']] +const CHUNK_SIZE = 50 // Process this many documents in parallel -bluebird.mapSeries(collectionsToDownload, async ([collection, subcollections, orderKey, direction]: Collection) => { - console.log(` ⬇️ Downloading ${collection}...`) +async function getNestedDataForCollection(path: string, writeStream?: fs.WriteStream): Promise { + const colRef = db.collection(path) + // TODO: We might want to grab length of collection w/ a much faster count query + // to get the size much faster, to print the expected size *before* doing the big .get() download. + const snapshot = await colRef.get() + console.log(`Found ${path}: ${snapshot.docs.length} documents`) - let query: firestore.Query = firebase.firestore().collection(collection) + const result: NestedDump = {} - // Apply optional orderBy() - if (orderKey) query = query.orderBy(orderKey, direction) + // Top-level collections get writeStreams, subcollections don't. + if (writeStream) writeStream.write('export default {\n') - const data = (await query.get()).docs.map((doc) => ({ id: doc.id, ...doc.data() })) + // Process documents in chunks for parallel processing + for (let i = 0; i < snapshot.docs.length; i += CHUNK_SIZE) { + const chunk = snapshot.docs.slice(i, i + CHUNK_SIZE) - fs.writeFileSync(path.join(__dirname, `/${collection}.json`), JSON.stringify(data)) - console.log(` ✅ Wrote ${data.length} rows to ${collection}.json\n`) + // Process each document in the chunk in parallel + await Promise.all( + chunk.map(async (doc) => { + const docData: NestedDump = { ...doc.data() } - // const data = elections + const subcollections = await doc.ref.listCollections() + if (subcollections.length > 0) { + docData.__subcollections__ = {} + // Process subcollections sequentially to avoid too many concurrent requests + for (const subcol of subcollections) { + docData.__subcollections__[subcol.id] = await getNestedDataForCollection(`${path}/${doc.id}/${subcol.id}`) + } + } - if (subcollections?.length) { - await bluebird.mapSeries(subcollections, async ([sub, , suborder, subdirection]) => { - console.log(` ⬇️ Downloading sub ${sub}...`) + if (writeStream) writeStream.write(` "${doc.id}": ${JSON.stringify(docData, null, 2)},\n`) - let sublength = 0 + result[doc.id] = docData + }), + ) - const subData = await bluebird.reduce( - data, - async (memo, row) => { - let query: firestore.Query = firebase.firestore().collection(collection).doc(row.id).collection(sub) + // Log progress after each chunk + console.log(` Progress: ${Math.min(i + CHUNK_SIZE, snapshot.docs.length)}/${snapshot.docs.length} documents`) + } + + if (writeStream) writeStream.write('}\n') - // Apply optional orderBy() - if (suborder) query = query.orderBy(suborder, subdirection) + return result +} - memo[row.id] = (await query.get()).docs.map((doc) => ({ id: doc.id, ...doc.data() })) - sublength += memo[row.id].length +async function exportNestedFirestore() { + const timestamp = new Date().toISOString().slice(0, 16) + const backupDir = path.join(__dirname, 'local-backup', timestamp) + if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir, { recursive: true }) + console.log(`Writing to ${backupDir}/`) - return memo - }, - {} as Record, - ) + const topLevelCollections = await db.listCollections() + console.log(`Found ${topLevelCollections.length} top-level collections`) - const filename = `${collection}-${sub}.json` - fs.writeFileSync(path.join(__dirname, `/${filename}`), JSON.stringify(subData)) - console.log(` ✅ Wrote ${sublength} rows to ${filename}\n`) - }) + const manifest = { + collections: {} as Record, + timestamp, } -}) + + // Process collections sequentially + for (const col of topLevelCollections) { + const collectionStart = Date.now() + manifest.collections[col.id] = { status: 'pending' } + console.log(`\n🟢 Starting collection: ${col.id}`) + + try { + const collectionFile = path.join(backupDir, `${col.id}.js`) + // TODO: This check for existing data will only work if the timestamp is the same minute as the previous run. + // A much larger range is probably fine, at least for 99% of the data which is years old. + // For now, at least the script should write the data fine. + // We can improve this skipping logic when continuing from a partial download is needed. + if (fs.existsSync(collectionFile)) { + console.log(`Skipping ${col.id} - already downloaded`) + manifest.collections[col.id].status = 'complete' + continue + } + + const writeStream = fs.createWriteStream(collectionFile) + await getNestedDataForCollection(col.id, writeStream) + + manifest.collections[col.id].status = 'complete' + const duration = (Date.now() - collectionStart) / 1000 + console.log(`Finished ${col.id} in ${duration.toFixed(1)}s 🕑`) + } catch (error) { + console.error(`Error processing ${col.id}:`, error) + manifest.collections[col.id].status = 'error' + manifest.collections[col.id].error = error instanceof Error ? error.message : String(error) + } + + // Write manifest after each collection + fs.writeFileSync(path.join(backupDir, 'manifest.json'), JSON.stringify(manifest, null, 2)) + } + + // Create index.ts file with static imports + console.log('\nCreating index file...') + const completedCollections = Object.entries(manifest.collections) + .filter(([, status]) => status.status === 'complete') + .map(([id]) => id) + + const indexContent = `// Auto-generated index file for Firestore backup from ${timestamp} +${completedCollections.map((id) => `export { default as ${id.replaceAll('-', '_')} } from './${id}.js'`).join('\n')} +` + + fs.writeFileSync(path.join(backupDir, 'index.ts'), indexContent) + console.log('Export complete!') + console.log(`Backup files written to: ${backupDir}/`) + console.log('Import from index.ts to access the collections statically') +} + +exportNestedFirestore() diff --git a/db-data/migrations/2024-09-13-merge-db-voters-and-votes-collections.ts b/db-data/migrations/2024-09-13-merge-db-voters-and-votes-collections.ts new file mode 100644 index 00000000..fe529943 --- /dev/null +++ b/db-data/migrations/2024-09-13-merge-db-voters-and-votes-collections.ts @@ -0,0 +1,138 @@ +import '../_env' + +import { firebase } from '../../pages/api/_services' + +/* +Merge old DB `voters` & `votes` collections into `approved-voters` collection + +## Goals of this script: +(context: https://github.com/siv-org/siv/pull/152#issuecomment-2212417637) + +1. Make voters be indexed by auth token, instead of by email. + - Avoids collisions on email (eg after invalidation). Avoid needing emails at all (eg sms auth, qr codes) + - Will want to warn about duplicate emails, but shouldn't strictly be a blocker + +2. And to merge votes collection into voters +3. And to rename that collection to approved-voters + +## How it will work: + +Loop through every election + START DB TRANSACTION + Download all voters + Download all votes + Write existing voters + votes combined to new `approved-voters` collection + + Delete all old 'votes' and 'voters' documents + END DB TRANSACTION +*/ + +const election_ids = ['1726520050751'] + +type Vote = { + auth: string + created_at: { _seconds: number } +} +type Voter = { + auth_token: string +} +type ApprovedVoter = Voter & Partial + +type MigrationResult = { + election_id: string + error?: string + stats?: { approved_voters: number; voters: number; votes: number } + success: boolean +} + +export async function main() { + const db = firebase.firestore() + const results: MigrationResult[] = [] + + for (const election_id of election_ids) { + console.log(`\nProcessing election ${election_id}...`) + const result: MigrationResult = { election_id, success: false } + + try { + const electionDoc = db.collection('elections').doc(election_id) + + // First check document counts to warn about potential transaction limits + const [votersSnapshot, votesSnapshot] = await Promise.all([ + electionDoc.collection('voters').get(), + electionDoc.collection('votes').get(), + ]) + + const totalDocs = votersSnapshot.size + votesSnapshot.size + if (totalDocs > 400) { + // Warning at 80% of limit + console.warn( + `Warning: Election ${election_id} has ${totalDocs} documents, approaching Firestore transaction limit of 500`, + ) + } + + await db.runTransaction(async (transaction) => { + const votersRef = electionDoc.collection('voters') + const votesRef = electionDoc.collection('votes') + + const votersSnapshot = await transaction.get(votersRef) + // Fine to skip if no voters, but should print that it's skipping + const votesSnapshot = await transaction.get(votesRef) + // TODO: Shouldn't fail if no votes + + const approvedVoters: ApprovedVoter[] = [] + + // First we need to create an votesByAuth map + const votesByAuth = votesSnapshot.docs.reduce( + (memo, doc) => ({ ...memo, [doc.data().auth]: doc.data() as Vote }), + {}, + ) + + votersSnapshot.docs.forEach((voterDoc) => { + const voterData = voterDoc.data() as Voter + let voteData: Record | Partial = {} + if (votesByAuth[voterData.auth_token]) { + voteData = votesByAuth[voterData.auth_token] + voteData.voted_at = voteData.created_at + delete voteData.created_at + delete voteData.auth + } + + approvedVoters.push({ ...voterData, ...voteData }) + }) + + const approvedVotersRef = electionDoc.collection('approved-voters') + + // Write combined data to new collection + approvedVoters.forEach((av) => transaction.set(approvedVotersRef.doc(av.auth_token), av)) + + // Delete old collections + votersSnapshot.docs.forEach((voterDoc) => transaction.delete(votersRef.doc(voterDoc.id))) + votesSnapshot.docs.forEach((voteDoc) => transaction.delete(votesRef.doc(voteDoc.id))) + }) + + result.success = true + result.stats = { + approved_voters: votersSnapshot.size, + voters: votersSnapshot.size, + votes: votesSnapshot.size, // Same as voters since we're merging + } + console.log(`✅ Successfully migrated election ${election_id}`) + console.log( + ` Voters: ${result.stats.voters}, Votes: ${result.stats.votes}, Approved Voters: ${result.stats.approved_voters}`, + ) + } catch (error) { + result.error = error instanceof Error ? error.message : String(error) + console.error(`❌ Failed to migrate election ${election_id}:`, result.error) + } + + results.push(result) + } + + // Print failures if any + const failed = results.filter((r) => !r.success) + if (failed.length > 0) { + console.log(`❌ Failed: ${failed.length}`) + failed.forEach((r) => console.log(` - ${r.election_id}: ${r.error}`)) + throw new Error('Some elections failed to migrate') + } +} diff --git a/db-data/migrations/_runner.ts b/db-data/migrations/_runner.ts new file mode 100644 index 00000000..c4f3b026 --- /dev/null +++ b/db-data/migrations/_runner.ts @@ -0,0 +1,103 @@ +import fs from 'fs' +import path from 'path' + +import { firebase } from '../../pages/api/_services' + +// Types and Firestore collection +type MigrationStatus = 'pending' | 'started' | 'completed' | 'failed' + +type MigrationRecord = { + completed_at?: { _seconds: number } + error?: string + id: string + started_at?: { _seconds: number } + status: MigrationStatus +} + +const migrationsCollection = firebase.firestore().collection('migrations') + +// Migration script type and loading +type MigrationScript = { + id: string + path: string + run: () => Promise +} + +async function loadLocalMigrationScripts(): Promise { + const migrationsDir = path.join(__dirname) + const files = fs.readdirSync(migrationsDir) + + return files + .filter((file) => file.endsWith('.ts') && !file.startsWith('_')) + .map((file) => { + const id = file.replace('.ts', '') + const scriptPath = path.join(migrationsDir, file) + return { + id, + path: scriptPath, + run: async () => { + // Import the script dynamically + const script = await import(scriptPath) + if (typeof script.main !== 'function') + throw new Error(`Migration script ${file} does not export a main function`) + + return script.main() + }, + } + }) +} + +async function runMigration(script: MigrationScript): Promise { + // Get migration status + const doc = await migrationsCollection.doc(script.id).get() + const existingMigration = doc.exists ? (doc.data() as MigrationRecord) : null + + if (existingMigration?.status === 'completed') return console.log(`Migration ${script.id} has already been completed`) + if (existingMigration?.status === 'started') return console.log(`Migration ${script.id} is currently running`) + + try { + // Mark migration "started" + await migrationsCollection.doc(script.id).set({ + id: script.id, + started_at: { _seconds: Math.floor(Date.now() / 1000) }, + status: 'started', + }) + console.log(`Starting migration: ${script.id}`) + + await script.run() + + // Mark migration "completed" + await migrationsCollection.doc(script.id).update({ + completed_at: { _seconds: Math.floor(Date.now() / 1000) }, + status: 'completed', + }) + console.log(`Completed migration: ${script.id}`) + } catch (error) { + console.error(`Migration ${script.id} failed:`, error) + // Mark migration "failed" + await migrationsCollection.doc(script.id).update({ + completed_at: { _seconds: Math.floor(Date.now() / 1000) }, + error: error instanceof Error ? error.message : String(error), + status: 'failed', + }) + + throw error // Re-throw to stop further migrations + } +} + +export async function runAllMigrations(): Promise { + const scripts = await loadLocalMigrationScripts() + console.log(`Found ${scripts.length} migration scripts`) + + for (const script of scripts) { + await runMigration(script) + } +} + +// If this file is run directly +if (require.main === module) { + runAllMigrations().catch((error) => { + console.error('Migration runner failed:', error) + process.exit(1) + }) +} diff --git a/db-data/2022-02-11-cipher-unlock-to-lock.ts b/db-data/one-off-scripts/2022-02-11-cipher-unlock-to-lock.ts similarity index 95% rename from db-data/2022-02-11-cipher-unlock-to-lock.ts rename to db-data/one-off-scripts/2022-02-11-cipher-unlock-to-lock.ts index 86a0367d..55aa1c36 100644 --- a/db-data/2022-02-11-cipher-unlock-to-lock.ts +++ b/db-data/one-off-scripts/2022-02-11-cipher-unlock-to-lock.ts @@ -1,11 +1,11 @@ // Execute this file with: // npx ts-node db-data/2022-02-11-cipher-unlock-to-lock.ts -import './_env' +import '../_env' import bluebird from 'bluebird' -import { firebase } from '../pages/api/_services' +import { firebase } from '../../pages/api/_services' import electionVotes from './elections-votes.json' let found = 0 diff --git a/db-data/2022-07-19-copy-david-secureinternetvoting-to-siv.ts b/db-data/one-off-scripts/2022-07-19-copy-david-secureinternetvoting-to-siv.ts similarity index 91% rename from db-data/2022-07-19-copy-david-secureinternetvoting-to-siv.ts rename to db-data/one-off-scripts/2022-07-19-copy-david-secureinternetvoting-to-siv.ts index 5acc4347..50ed6ace 100644 --- a/db-data/2022-07-19-copy-david-secureinternetvoting-to-siv.ts +++ b/db-data/one-off-scripts/2022-07-19-copy-david-secureinternetvoting-to-siv.ts @@ -1,11 +1,11 @@ // Execute this file with: // npx ts-node db-data/2022-07-19-copy-david-secureinternetvoting-to-siv.ts -import './_env' +import '../_env' import bluebird from 'bluebird' -import { firebase } from '../pages/api/_services' +import { firebase } from '../../pages/api/_services' ;(async () => { // Find all elections created by david@secureinternetvoting.org diff --git a/db-data/2023-04-22-find-vote-by-auth.ts b/db-data/one-off-scripts/2023-04-22-find-vote-by-auth.ts similarity index 84% rename from db-data/2023-04-22-find-vote-by-auth.ts rename to db-data/one-off-scripts/2023-04-22-find-vote-by-auth.ts index cd0cc1b4..aa733032 100644 --- a/db-data/2023-04-22-find-vote-by-auth.ts +++ b/db-data/one-off-scripts/2023-04-22-find-vote-by-auth.ts @@ -1,6 +1,6 @@ -import './_env' +import '../_env' -import { firebase } from '../pages/api/_services' +import { firebase } from '../../pages/api/_services' const election_id = '1680323766282' const auth = '' diff --git a/db-data/2023-04-22-get-ballot-design-columns.ts b/db-data/one-off-scripts/2023-04-22-get-ballot-design-columns.ts similarity index 94% rename from db-data/2023-04-22-get-ballot-design-columns.ts rename to db-data/one-off-scripts/2023-04-22-get-ballot-design-columns.ts index 863097f2..e784d7de 100644 --- a/db-data/2023-04-22-get-ballot-design-columns.ts +++ b/db-data/one-off-scripts/2023-04-22-get-ballot-design-columns.ts @@ -1,9 +1,9 @@ // Execute this file with: // npx ts-node db-data/2023-04-22-get-ballot-design-columns.ts -import './_env' +import '../_env' -import { firebase } from '../pages/api/_services' +import { firebase } from '../../pages/api/_services' // CHANGE ME 👇 const admin_email = '' diff --git a/db-data/2023-04-22-simulate-rand-votes.ts b/db-data/one-off-scripts/2023-04-22-simulate-rand-votes.ts similarity index 91% rename from db-data/2023-04-22-simulate-rand-votes.ts rename to db-data/one-off-scripts/2023-04-22-simulate-rand-votes.ts index a5cd69f2..59d58426 100644 --- a/db-data/2023-04-22-simulate-rand-votes.ts +++ b/db-data/one-off-scripts/2023-04-22-simulate-rand-votes.ts @@ -5,15 +5,15 @@ Execute it w/: npx ts-node db-data/2023-04-22-simulate-rand-votes.ts */ -import './_env' +import '../_env' import { mapValues } from 'lodash' -import { firebase } from '../pages/api/_services' -import { RP, random_bigint, stringToPoint } from '../src/crypto/curve' -import encrypt from '../src/crypto/encrypt' -import { CipherStrings } from '../src/crypto/stringify-shuffle' -import { generateTrackingNum } from '../src/vote/tracking-num' +import { firebase } from '../../pages/api/_services' +import { RP, random_bigint, stringToPoint } from '../../src/crypto/curve' +import encrypt from '../../src/crypto/encrypt' +import { CipherStrings } from '../../src/crypto/stringify-shuffle' +import { generateTrackingNum } from '../../src/vote/tracking-num' const election_id = '1682168554981' diff --git a/db-data/2023-04-24-review-rejected-votes.ts b/db-data/one-off-scripts/2023-04-24-review-rejected-votes.ts similarity index 91% rename from db-data/2023-04-24-review-rejected-votes.ts rename to db-data/one-off-scripts/2023-04-24-review-rejected-votes.ts index 1a48d7e4..34f3b27b 100644 --- a/db-data/2023-04-24-review-rejected-votes.ts +++ b/db-data/one-off-scripts/2023-04-24-review-rejected-votes.ts @@ -1,12 +1,12 @@ -import './_env' +import '../_env' import { keyBy, mapValues } from 'lodash' -import { firebase } from '../pages/api/_services' -import { RP, pointToString } from '../src/crypto/curve' -import decrypt from '../src/crypto/decrypt' -import { CipherStrings } from '../src/crypto/stringify-shuffle' -import { tallyVotes } from '../src/status/tally-votes' +import { firebase } from '../../pages/api/_services' +import { RP, pointToString } from '../../src/crypto/curve' +import decrypt from '../../src/crypto/decrypt' +import { CipherStrings } from '../../src/crypto/stringify-shuffle' +import { tallyVotes } from '../../src/status/tally-votes' // CHANGE ME 👇 const election_id = '1680323766282' diff --git a/db-data/2023-04-24-review-submitted-too-late-votes.ts b/db-data/one-off-scripts/2023-04-24-review-submitted-too-late-votes.ts similarity index 90% rename from db-data/2023-04-24-review-submitted-too-late-votes.ts rename to db-data/one-off-scripts/2023-04-24-review-submitted-too-late-votes.ts index 6a14e741..ce5cbab6 100644 --- a/db-data/2023-04-24-review-submitted-too-late-votes.ts +++ b/db-data/one-off-scripts/2023-04-24-review-submitted-too-late-votes.ts @@ -1,12 +1,12 @@ -import './_env' +import '../_env' import { keyBy, mapValues } from 'lodash' -import { firebase } from '../pages/api/_services' -import { RP, pointToString } from '../src/crypto/curve' -import decrypt from '../src/crypto/decrypt' -import { CipherStrings } from '../src/crypto/stringify-shuffle' -import { tallyVotes } from '../src/status/tally-votes' +import { firebase } from '../../pages/api/_services' +import { RP, pointToString } from '../../src/crypto/curve' +import decrypt from '../../src/crypto/decrypt' +import { CipherStrings } from '../../src/crypto/stringify-shuffle' +import { tallyVotes } from '../../src/status/tally-votes' // CHANGE ME 👇 const election_id = '1680323766282' diff --git a/db-data/2023-04-29-investigate-auth.ts b/db-data/one-off-scripts/2023-04-29-investigate-auth.ts similarity index 92% rename from db-data/2023-04-29-investigate-auth.ts rename to db-data/one-off-scripts/2023-04-29-investigate-auth.ts index fc503f2c..c0879815 100644 --- a/db-data/2023-04-29-investigate-auth.ts +++ b/db-data/one-off-scripts/2023-04-29-investigate-auth.ts @@ -1,12 +1,12 @@ -import './_env' +import '../_env' import { mapValues } from 'lodash' import UAParser from 'ua-parser-js' -import { firebase } from '../pages/api/_services' -import { RP, pointToString } from '../src/crypto/curve' -import decrypt from '../src/crypto/decrypt' -import { CipherStrings } from '../src/crypto/stringify-shuffle' +import { firebase } from '../../pages/api/_services' +import { RP, pointToString } from '../../src/crypto/curve' +import decrypt from '../../src/crypto/decrypt' +import { CipherStrings } from '../../src/crypto/stringify-shuffle' // CHANGE ME 👇 const election_id = '1680323766282' diff --git a/db-data/2023-05-02-analyze-encryption-details.ts b/db-data/one-off-scripts/2023-05-02-analyze-encryption-details.ts similarity index 79% rename from db-data/2023-05-02-analyze-encryption-details.ts rename to db-data/one-off-scripts/2023-05-02-analyze-encryption-details.ts index 60ba3cd0..78679667 100644 --- a/db-data/2023-05-02-analyze-encryption-details.ts +++ b/db-data/one-off-scripts/2023-05-02-analyze-encryption-details.ts @@ -1,4 +1,4 @@ -import { RP, pointToString, stringToPoint } from '../src/crypto/curve' +import { RP, pointToString, stringToPoint } from '../../src/crypto/curve' const encodeds = [''] diff --git a/db-data/2023-05-02-count-encryption-holes.ts b/db-data/one-off-scripts/2023-05-02-count-encryption-holes.ts similarity index 93% rename from db-data/2023-05-02-count-encryption-holes.ts rename to db-data/one-off-scripts/2023-05-02-count-encryption-holes.ts index e398566d..6ec9c0f2 100644 --- a/db-data/2023-05-02-count-encryption-holes.ts +++ b/db-data/one-off-scripts/2023-05-02-count-encryption-holes.ts @@ -1,15 +1,15 @@ -import './_env' +import '../_env' import { inspect } from 'util' import { keyBy, mapValues, pick } from 'lodash' import UAParser from 'ua-parser-js' -import { firebase } from '../pages/api/_services' -import { RP, pointToString } from '../src/crypto/curve' -import decrypt from '../src/crypto/decrypt' -import { CipherStrings } from '../src/crypto/stringify-shuffle' -import { tallyVotes } from '../src/status/tally-votes' +import { firebase } from '../../pages/api/_services' +import { RP, pointToString } from '../../src/crypto/curve' +import decrypt from '../../src/crypto/decrypt' +import { CipherStrings } from '../../src/crypto/stringify-shuffle' +import { tallyVotes } from '../../src/status/tally-votes' const election_id = '1680323766282' diff --git a/db-data/2023-05-02-get-holed-verifications.ts b/db-data/one-off-scripts/2023-05-02-get-holed-verifications.ts similarity index 87% rename from db-data/2023-05-02-get-holed-verifications.ts rename to db-data/one-off-scripts/2023-05-02-get-holed-verifications.ts index bfdef40d..59e2da8f 100644 --- a/db-data/2023-05-02-get-holed-verifications.ts +++ b/db-data/one-off-scripts/2023-05-02-get-holed-verifications.ts @@ -1,11 +1,11 @@ -import './_env' +import '../_env' import { mapValues } from 'lodash' -import { firebase } from '../pages/api/_services' -import { RP, pointToString } from '../src/crypto/curve' -import decrypt from '../src/crypto/decrypt' -import { CipherStrings } from '../src/crypto/stringify-shuffle' +import { firebase } from '../../pages/api/_services' +import { RP, pointToString } from '../../src/crypto/curve' +import decrypt from '../../src/crypto/decrypt' +import { CipherStrings } from '../../src/crypto/stringify-shuffle' import { votesWithHoles } from './votes-with-holes' const election_id = '1680323766282' diff --git a/db-data/2023-05-05-calc-max-burst.ts b/db-data/one-off-scripts/2023-05-05-calc-max-burst.ts similarity index 100% rename from db-data/2023-05-05-calc-max-burst.ts rename to db-data/one-off-scripts/2023-05-05-calc-max-burst.ts diff --git a/db-data/2024-07-18-duplicate-election.ts b/db-data/one-off-scripts/2024-07-18-duplicate-election.ts similarity index 97% rename from db-data/2024-07-18-duplicate-election.ts rename to db-data/one-off-scripts/2024-07-18-duplicate-election.ts index 6ee5a8c7..1c8e7978 100644 --- a/db-data/2024-07-18-duplicate-election.ts +++ b/db-data/one-off-scripts/2024-07-18-duplicate-election.ts @@ -6,11 +6,11 @@ // - Other trustees, in their local storage, also need to copy their private keys to the new election_id // See function `cloneTrusteeDetails` at bottom -import './_env' +import '../_env' import { pick } from 'lodash' -import { firebase } from '../pages/api/_services' +import { firebase } from '../../pages/api/_services' const election_id_from = '1721122924218' const election_id_to = '1722034716600' diff --git a/db-data/2024-07-18-manually-approve-pending.ts b/db-data/one-off-scripts/2024-07-18-manually-approve-pending.ts similarity index 89% rename from db-data/2024-07-18-manually-approve-pending.ts rename to db-data/one-off-scripts/2024-07-18-manually-approve-pending.ts index c2cc3160..0ea6e15d 100644 --- a/db-data/2024-07-18-manually-approve-pending.ts +++ b/db-data/one-off-scripts/2024-07-18-manually-approve-pending.ts @@ -1,9 +1,9 @@ // We have 137 "pending" votes (from link-auth). We want admin to begin unlocking them. // So we need to move them from 'votes-pending' to 'votes -import './_env' +import '../_env' -import { firebase } from '../pages/api/_services' +import { firebase } from '../../pages/api/_services' const election_id = '1722034716600' // NDP, Final, All 230 Votes @@ -16,12 +16,13 @@ async function renameSubcollection( const sourceSubcollection = parentDoc.collection(oldSubcollectionName) const targetSubcollection = parentDoc.collection(newSubcollectionName) - const snapshot = await sourceSubcollection.get() - const docs = snapshot.docs + const allSourceDocs = await sourceSubcollection.get() + const docs = allSourceDocs.docs const amount = docs.length let index = 0 const intervalToReport = 10 + // for...of loop for (const doc of docs) { const data = doc.data() const auth = data.link_auth diff --git a/db-data/2024-07-26-duplicate-election-subset.ts b/db-data/one-off-scripts/2024-07-26-duplicate-election-subset.ts similarity index 98% rename from db-data/2024-07-26-duplicate-election-subset.ts rename to db-data/one-off-scripts/2024-07-26-duplicate-election-subset.ts index 58636038..f97a6f42 100644 --- a/db-data/2024-07-26-duplicate-election-subset.ts +++ b/db-data/one-off-scripts/2024-07-26-duplicate-election-subset.ts @@ -6,11 +6,11 @@ // - Other trustees, in their local storage, also need to copy their private keys to the new election_id // See function `cloneTrusteeDetails` at bottom -import './_env' +import '../_env' import { pick } from 'lodash' -import { firebase } from '../pages/api/_services' +import { firebase } from '../../pages/api/_services' const election_id_from = '1721122924218' const election_id_to = '1722029077573' diff --git a/db-data/one-off-scripts/2025-04-20-test-local-backup.ts b/db-data/one-off-scripts/2025-04-20-test-local-backup.ts new file mode 100644 index 00000000..b9445729 --- /dev/null +++ b/db-data/one-off-scripts/2025-04-20-test-local-backup.ts @@ -0,0 +1,3 @@ +import { admins } from '../local-backup/2025-04-21T03:29/index.ts' + +console.log(admins) diff --git a/db-data/one-off-scripts/_readme.md b/db-data/one-off-scripts/_readme.md new file mode 100644 index 00000000..2020ea31 --- /dev/null +++ b/db-data/one-off-scripts/_readme.md @@ -0,0 +1,3 @@ +Any one-off scripts in this folder were useful at one point, and are retained as a possible reference, but are not assumed to still work. + +For example, many of them try to read from `collection('voters')` or `collection('votes')`, but in Sept 2024 these were merged into a new single `collection('approved-voters')`. diff --git a/package.json b/package.json index 50ecc233..4be80818 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "siv-web", - "version": "1.1.0", + "version": "2.0.0", "private": true, "scripts": { "build": "next build", @@ -8,7 +8,7 @@ "dev": "next dev", "e2e": "cypress run --config-file cypress/cypress-config.json", "format": "prettier --write **/*.{js,ts,tsx}", - "postinstall": "patch-package", + "postinstall": "patch-package && ts-node db-data/migrations/_runner.ts", "lint": "eslint src/ pages/", "start": "next start", "test": "bun lint && bun typecheck && bun unit && bun e2e", diff --git a/pages/api/check-auth-token.ts b/pages/api/check-auth-token.ts index 6d810bdc..db76694b 100644 --- a/pages/api/check-auth-token.ts +++ b/pages/api/check-auth-token.ts @@ -23,8 +23,7 @@ export async function validateAuthToken( // Begin preloading these docs const electionDoc = firebase.firestore().collection('elections').doc(election_id) const election = electionDoc.get() - const voters = electionDoc.collection('voters').where('auth_token', '==', auth).get() - const votes = electionDoc.collection('votes').where('auth', '==', auth).get() + const voters = electionDoc.collection('approved-voters').where('auth_token', '==', auth).get() // Did they send us an Auth Token? if (!auth) return fail('Missing Auth Token. Only registered voters are allowed to vote.') @@ -54,9 +53,8 @@ export async function validateAuthToken( } // Has Auth Token already been used? - const [vote] = (await votes).docs - if (vote) { - const previous_at = new Date(vote.data().created_at?._seconds * 1000) + if (voter.data().voted_at) { + const previous_at = new Date(voter.data().voted_at?._seconds * 1000) return fail(`Vote already recorded. (${format(previous_at)})`) } diff --git a/pages/api/create-sample-votes.ts b/pages/api/create-sample-votes.ts deleted file mode 100644 index f5189ae6..00000000 --- a/pages/api/create-sample-votes.ts +++ /dev/null @@ -1,71 +0,0 @@ -// import { mapValues } from 'lodash-es' -import { NextApiRequest, NextApiResponse } from 'next' - -/* -import { encode } from '../../src/crypto/encode' -import encrypt from '../../src/crypto/encrypt' -import pick_random_integer from '../../src/crypto/pick-random-integer' -import { big, bigPubKey } from '../../src/crypto/types' -import { generateTrackingNum } from '../../src/vote/tracking-num' -import { firebase } from './_services' -import { generateAuthToken } from 'src/crypto/generate-auth-tokens' -import { pusher } from './pusher' -*/ - -export default async (req: NextApiRequest, res: NextApiResponse) => { - // Disable this endpoint - return res.status(401).end() - - /* - const election_id = '1612818814403' - const p = '84490233071588324613543045838826431628034872330024413446004719838344478256747' - const threshold_key = '25753591117431663234613254388754657393582184952307187183880669779500171305735' - const pub_key = bigPubKey({ generator: '4', modulo: p, recipient: threshold_key }) - - const num_samples = 100 - - const election = firebase.firestore().collection('elections').doc(election_id) - - await Promise.all( - new Array(num_samples).fill('').map(() => { - const tracking = generateTrackingNum() - const randomVote = (item_index: 0 | 1) => - `${tracking}:${ - ballot_schema[item_index].options[Math.floor(Math.random() * (ballot_schema[0].options.length + 1))]?.name || - 'BLANK' - }` - - const plaintext = { - favorite_apple: randomVote(0), - } - - const encoded = mapValues(plaintext, encode) - const randomizers = mapValues(plaintext, () => pick_random_integer(big(p))) - const encrypted_vote = mapValues(plaintext, (_, key: keyof typeof plaintext) => - mapValues(encrypt(pub_key, randomizers[key], big(encoded[key])), (b) => b.toString()), - ) - - const auth = generateAuthToken() - const created_at = new Date() - - console.log({ auth, created_at, encoded, encrypted_vote, plaintext }) - return election.collection('votes').add({ auth, created_at, encrypted_vote }) - }), - ) - - await pusher.trigger(`status-${election_id}`, 'votes', 'sample-votes') - - return res.status(200).json({ message: `Inserted ${num_samples} sample vote` }) - */ -} - -/* -const ballot_schema = [ - { - id: 'favorite_apple', - options: [{ name: 'Cosmic Crisp' }, { name: 'Honey Crisp' }, { name: 'Golden Delicious' }], - title: 'Which is the best Washington State apple?', - write_in_allowed: true, - }, -] -*/ diff --git a/pages/api/election/[election_id]/accepted-votes.ts b/pages/api/election/[election_id]/accepted-votes.ts index 03d5cf89..cf5e2cff 100644 --- a/pages/api/election/[election_id]/accepted-votes.ts +++ b/pages/api/election/[election_id]/accepted-votes.ts @@ -2,7 +2,6 @@ import { NextApiRequest, NextApiResponse } from 'next' import { getStatus } from '../../../../src/admin/Voters/Signature' import { firebase } from '../../_services' -import { ReviewLog } from './admin/load-admin' export default async (req: NextApiRequest, res: NextApiResponse) => { const { election_id, num_new_accepted_votes, num_new_pending_votes } = req.query @@ -14,7 +13,7 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { .doc(election_id as string) // Begin preloading - let votesQuery = electionDoc.collection('votes').orderBy('created_at') + let votesQuery = electionDoc.collection('approved-voters').where('voted_at', '!=', null).orderBy('voted_at') let pendingVotesQuery = electionDoc.collection('votes-pending').orderBy('created_at') if (num_new_accepted_votes) votesQuery = votesQuery.limitToLast(+num_new_accepted_votes) if (num_new_pending_votes) pendingVotesQuery = pendingVotesQuery.limitToLast(+num_new_pending_votes) @@ -26,31 +25,27 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { // Is election_id in DB? if (!election.exists) return res.status(400).json({ error: 'Unknown Election ID.' }) - let votes = (await loadVotes).docs.map((doc) => { - const { auth, encrypted_vote } = doc.data() - return { auth, ...encrypted_vote } - }) + const votes = (await loadVotes).docs + .map((doc) => { + const { auth_token, encrypted_vote, esignature_review, invalidated_at } = doc.data() + const vote = { auth: auth_token, ...encrypted_vote } + + // Filter out invalidated votes + if (invalidated_at) return null + + // If esignatures enabled, include review status + if (election.data()?.esignature_requested) { + vote.signature_approved = getStatus(esignature_review) === 'approve' + } + + return vote + }) + .filter((v) => v) const pendingVotes = (await loadPendingVotes).docs.map((doc) => { const { encrypted_vote } = doc.data() return { auth: 'pending', ...encrypted_vote } }) - // If we need esignatures, we need to load all voters as well to get their esignature status - if (election.data()?.esignature_requested) { - type VotersByAuth = Record - const voters = await electionDoc.collection('voters').get() - const votersByAuth: VotersByAuth = voters.docs.reduce((acc: VotersByAuth, doc) => { - const data = doc.data() - return { ...acc, [data.auth_token]: data } - }, {}) - - // Add signature status - votes = votes.map((vote) => { - const voter = votersByAuth[vote.auth] - return { ...vote, signature_approved: getStatus(voter?.esignature_review) === 'approve' } - }) - } - return res.status(200).json([...votes, ...pendingVotes]) } diff --git a/pages/api/election/[election_id]/admin/add-voters.ts b/pages/api/election/[election_id]/admin/add-voters.ts index 230722dd..9defb0fc 100644 --- a/pages/api/election/[election_id]/admin/add-voters.ts +++ b/pages/api/election/[election_id]/admin/add-voters.ts @@ -35,7 +35,7 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { export async function addVotersToElection(new_voters: string[], election_id: string) { // Get existing voter from DB const electionDoc = firebase.firestore().collection('elections').doc(election_id) - const loadVoters = electionDoc.collection('voters').get() + const loadVoters = electionDoc.collection('approved-voters').get() const existing_voters = new Set() ;(await loadVoters).docs.map((d) => existing_voters.add(d.data().email)) @@ -58,15 +58,15 @@ export async function addVotersToElection(new_voters: string[], election_id: str console.log('Add-voters:', { already_added, duplicates_in_submission, election_id, unique_new_emails }) const email_to_auth: Record = {} - // Generate and store auths for uniques + // Generate and store auth tokens for uniques await Promise.all( unique_new_emails .map((email: string, index: number) => { const auth_token = generateAuthToken() email_to_auth[email] = auth_token return electionDoc - .collection('voters') - .doc(email) + .collection('approved-voters') + .doc(auth_token) .set({ added_at: new Date(), auth_token, diff --git a/pages/api/election/[election_id]/admin/check-voter-invite-status.ts b/pages/api/election/[election_id]/admin/check-voter-invite-status.ts index 4a458214..1501d9bc 100644 --- a/pages/api/election/[election_id]/admin/check-voter-invite-status.ts +++ b/pages/api/election/[election_id]/admin/check-voter-invite-status.ts @@ -1,11 +1,10 @@ +import { firebase, mailgun, pushover } from 'api/_services' +import { checkJwtOwnsElection } from 'api/validate-admin-jwt' import bluebird from 'bluebird' import { firestore } from 'firebase-admin' import { NextApiRequest, NextApiResponse } from 'next' import { buildSubject } from 'pages/api/invite-voters' -import { firebase, mailgun, pushover } from '../../../_services' -import { checkJwtOwnsElection } from '../../../validate-admin-jwt' - export default async (req: NextApiRequest, res: NextApiResponse) => { const { election_id } = req.query as { election_id: string } if (!election_id) return res.status(401).json({ error: 'Missing election_id' }) @@ -46,15 +45,14 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { // Skip replies to us if (to === 'election@siv.org') return - const voterDoc = electionDoc.collection('voters').doc(to) + const voterDocs = electionDoc.collection('approved-voters').where('email', '==', to) // Confirm voterDoc exists - if (!(await voterDoc.get()).exists) { - return console.log(`No voter doc for ${to}`) - } + const first = (await voterDocs.get()).docs[0] + if (!first) return console.log(`No voter doc for ${to}`) num_events++ // Store new items on voters' docs - return voterDoc.update({ [`mailgun_events.${item.event}`]: firestore.FieldValue.arrayUnion(item) }) + return first.ref.update({ [`mailgun_events.${item.event}`]: firestore.FieldValue.arrayUnion(item) }) }, { concurrency: 3 }, ) diff --git a/pages/api/election/[election_id]/admin/edit-voter-email.ts b/pages/api/election/[election_id]/admin/edit-voter-email.ts index 41bb1f7a..87059806 100644 --- a/pages/api/election/[election_id]/admin/edit-voter-email.ts +++ b/pages/api/election/[election_id]/admin/edit-voter-email.ts @@ -1,12 +1,10 @@ +import { firebase } from 'api/_services' +import { checkJwtOwnsElection } from 'api/validate-admin-jwt' import { validate as validateEmail } from 'email-validator' import { NextApiRequest, NextApiResponse } from 'next' -import { firebase } from '../../../_services' -import { checkJwtOwnsElection } from '../../../validate-admin-jwt' - export default async (req: NextApiRequest, res: NextApiResponse) => { - const { new_email, old_email } = req.body - if (new_email === old_email) return res.status(401).json({ error: 'new_email must be different from old_email' }) + const { auth_token, new_email } = req.body const { election_id } = req.query as { election_id: string } @@ -14,26 +12,25 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { const jwt = await checkJwtOwnsElection(req, res, election_id) if (!jwt.valid) return - const votersCollection = firebase.firestore().collection('elections').doc(election_id).collection('voters') - - // Get old voter data - const old_voter_doc = await votersCollection.doc(old_email).get() - if (!old_voter_doc.exists) return res.status(401).json({ error: `Can't find voter ${old_email}` }) - const old_voter_data = { ...old_voter_doc.data() } - old_voter_data.email = new_email + const votersCollection = firebase.firestore().collection('elections').doc(election_id).collection('approved-voters') // Confirm new voter doesn't already exist - if ((await votersCollection.doc(new_email).get()).exists) + if ((await votersCollection.where('email', '==', new_email).get()).docs.length > 0) return res.status(401).json({ error: `There's already a voter ${new_email}` }) // Validate new_email is a valid email address if (!validateEmail(new_email)) return res.status(401).json({ error: `Invalid email: ${new_email}` }) - // Store record of edit on new voter - old_voter_data.email_edits = [...(old_voter_data.email_edits || []), { edited_at: new Date(), email: old_email }] - - // Delete the old voter and move its data to new_email - await Promise.all([votersCollection.doc(old_email).delete(), votersCollection.doc(new_email).set(old_voter_data)]) + // Get old voter data + const voterDoc = votersCollection.doc(auth_token) + const voter = await voterDoc.get() + if (!voter.exists) return res.status(401).json({ error: `Can't find voter: ${auth_token}` }) + const voter_data = voter.data() + const old_email = voter_data?.email + const email_edits = [...(voter_data?.email_edits || []), { edited_at: new Date(), old_email }] + + // Store record of edit + await voterDoc.update({ email: new_email, email_edits }) return res.status(201).json({ message: 'Changed email' }) } diff --git a/pages/api/election/[election_id]/admin/invalidate-voters.ts b/pages/api/election/[election_id]/admin/invalidate-voters.ts index e614a8cd..b549e79f 100644 --- a/pages/api/election/[election_id]/admin/invalidate-voters.ts +++ b/pages/api/election/[election_id]/admin/invalidate-voters.ts @@ -1,8 +1,8 @@ import { firebase, sendEmail } from 'api/_services' +import { checkJwtOwnsElection } from 'api/validate-admin-jwt' import { firestore } from 'firebase-admin' import { NextApiRequest, NextApiResponse } from 'next' -import { checkJwtOwnsElection } from '../../../validate-admin-jwt' import { Voter } from './load-admin' export default async (req: NextApiRequest, res: NextApiResponse) => { @@ -17,33 +17,24 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { voters_to_invalidate.map(async (voter) => { const db = firebase.firestore() const electionDoc = db.collection('elections').doc(election_id) - const voterRef = electionDoc.collection('voters').doc(voter.email) - const votes = await electionDoc.collection('votes').where('auth', '==', voter.auth_token).get() + const voterRef = electionDoc.collection('approved-voters').doc(voter.auth_token) + const has_voted = !!(await voterRef.get()).data()?.voted_at // Do all in parallel return Promise.all([ // 1. Mark the auth token as invalidated voterRef.update({ invalidated_at: new Date() }), - // 2. If votes were cast with this auth, move them to an 'invalidated-votes' collection - (async function invalidateVotes() { - await Promise.all( - votes.docs.map(async (vote) => { - const invalidatedVote = { ...vote.data(), invalidated_at: new Date() } - await electionDoc.collection('invalidated_votes').doc(vote.id).set(invalidatedVote) - await vote.ref.delete() - await electionDoc.update({ num_invalidated_votes: firestore.FieldValue.increment(1) }) - }), - ) + // 2. If voted, increment election's cached num_invalidated_votes counter + (function invalidateVotes() { + if (!has_voted) return + + return electionDoc.update({ num_invalidated_votes: firestore.FieldValue.increment(1) }) })(), - // 3. Notify the voter over email + // 3. If voted, notify the voter over email (function notifyVoter() { - // Skip if they have not voted - if (!votes.docs.length) return - - // TODO: Skip if voter's email address is unverified, BLOCKED by PR #125 Registration link - // if (voter.status == 'pending') return + if (!has_voted) return return sendEmail({ recipient: voter.email, diff --git a/pages/api/election/[election_id]/admin/invite-voters.ts b/pages/api/election/[election_id]/admin/invite-voters.ts index 33f00278..a66b1c9e 100644 --- a/pages/api/election/[election_id]/admin/invite-voters.ts +++ b/pages/api/election/[election_id]/admin/invite-voters.ts @@ -1,10 +1,9 @@ +import { firebase } from 'api/_services' +import { buildSubject, send_invitation_email } from 'api/invite-voters' +import { checkJwtOwnsElection } from 'api/validate-admin-jwt' import { NextApiRequest, NextApiResponse } from 'next' import throat from 'throat' -import { firebase } from '../../../_services' -import { buildSubject, send_invitation_email } from '../../../invite-voters' -import { checkJwtOwnsElection } from '../../../validate-admin-jwt' - export type QueueLog = { result: string; time: Date } export default async (req: NextApiRequest, res: NextApiResponse) => { @@ -25,12 +24,12 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { // Email each voter their auth token await Promise.all( voters.map( - throat(10, async (email: string) => { + throat(10, async (auth_token: string) => { // Lookup voter info - const voter_doc = electionDoc.collection('voters').doc(email) + const voter_doc = electionDoc.collection('approved-voters').doc(auth_token) const voter = await voter_doc.get() - if (!voter.exists) return { error: `Can't find voter ${email}` } - const { auth_token, invite_queued } = { ...voter.data() } as { auth_token: string; invite_queued?: QueueLog[] } + if (!voter.exists) return { error: `Can't find voter: ${auth_token}` } + const { email, invite_queued } = { ...voter.data() } as { email: string; invite_queued?: QueueLog[] } const link = `${req.headers.origin}/election/${election_id}/vote?auth=${auth_token}` // const link = `https://siv.org/election/${election_id}/vote?auth=${auth_token}` diff --git a/pages/api/election/[election_id]/admin/load-admin.ts b/pages/api/election/[election_id]/admin/load-admin.ts index f0adb1f2..73f1605f 100644 --- a/pages/api/election/[election_id]/admin/load-admin.ts +++ b/pages/api/election/[election_id]/admin/load-admin.ts @@ -1,8 +1,8 @@ +import { firebase } from 'api/_services' +import { checkJwtOwnsElection } from 'api/validate-admin-jwt' import { NextApiRequest, NextApiResponse } from 'next' import UAParser from 'ua-parser-js' -import { firebase } from '../../../_services' -import { checkJwtOwnsElection } from '../../../validate-admin-jwt' import { QueueLog } from './invite-voters' export type AdminData = { @@ -75,10 +75,8 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { // Begin preloading all these docs const loadElection = election.get() const loadTrustees = election.collection('trustees').orderBy('index', 'asc').get() - const loadVoters = election.collection('voters').orderBy('index', 'asc').get() - const loadVotes = election.collection('votes').get() + const loadApprovedVoters = election.collection('approved-voters').orderBy('index', 'asc').get() const loadPendingVotes = election.collection('votes-pending').get() - const loadInvalidatedVotes = election.collection('invalidated_votes').get() // Is election_id in DB? const electionDoc = await loadElection @@ -138,24 +136,12 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { ] }, []) - // Gather who's voted already - const votesByAuth: Record = (await loadVotes).docs.reduce((acc, doc) => { - const data = doc.data() - return { ...acc, [data.auth]: [true, data.esignature] } - }, {}) - - // Gather whose votes were invalidated - const invalidatedVotesByAuth: Record = {} - ;(await loadInvalidatedVotes).docs.forEach((doc) => { - const data = doc.data() - invalidatedVotesByAuth[data.auth] = true - }) - // Build voters objects - const voters: Voter[] = (await loadVoters).docs.reduce((acc: Voter[], doc) => { + const voters: Voter[] = (await loadApprovedVoters).docs.reduce((acc: Voter[], doc) => { const { auth_token, email, + esignature, esignature_review, first_name, index, @@ -164,11 +150,13 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { is_email_verified, last_name, mailgun_events, + voted_at, } = { ...doc.data(), } as { auth_token: string email: string + esignature?: string esignature_review: ReviewLog[] first_name: string index: number @@ -177,16 +165,17 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { is_email_verified?: boolean last_name: string mailgun_events: { accepted: MgEvent[]; delivered: MgEvent[] } + voted_at: { _seconds: number } } return [ ...acc, { auth_token, email, - esignature: (votesByAuth[auth_token] || [])[1], + esignature, esignature_review, first_name, - has_voted: !!votesByAuth[auth_token] || !!invalidatedVotesByAuth[auth_token], + has_voted: !!voted_at, index, invalidated: invalidated_at ? true : undefined, invite_queued, diff --git a/pages/api/election/[election_id]/admin/notify-unlocked.ts b/pages/api/election/[election_id]/admin/notify-unlocked.ts index 692bec7e..af65bc6e 100644 --- a/pages/api/election/[election_id]/admin/notify-unlocked.ts +++ b/pages/api/election/[election_id]/admin/notify-unlocked.ts @@ -1,10 +1,9 @@ +import { firebase, sendEmail } from 'api/_services' +import { checkJwtOwnsElection } from 'api/validate-admin-jwt' import bluebird from 'bluebird' import { validate } from 'email-validator' import { NextApiRequest, NextApiResponse } from 'next' -import { firebase, sendEmail } from '../../../_services' -import { checkJwtOwnsElection } from '../../../validate-admin-jwt' - export default async (req: NextApiRequest, res: NextApiResponse) => { const { election_id } = req.query as { election_id?: string } @@ -18,8 +17,7 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { // Begin preloading all these docs const loadElection = election.get() - const loadVoters = election.collection('voters').orderBy('index', 'asc').get() - const loadVotes = election.collection('votes').get() + const loadVoters = election.collection('approved-voters').where('voted_at', '!=', null).get() // Confirm they're a valid admin that created this election const jwt = await checkJwtOwnsElection(req, res, election_id) @@ -36,24 +34,13 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { election_title?: string } - // Gather who's voted already - const votesByAuth: Record = (await loadVotes).docs.reduce((acc, doc) => { - const data = doc.data() - return { ...acc, [data.auth]: [true, data.esignature] } - }, {}) - // Build voters objects const voters: string[] = [] ;(await loadVoters).docs.forEach((doc) => { - const { auth_token, email, invalidated_at } = { ...doc.data() } as { - auth_token: string - email: string - invalidated_at?: Date - } + const { email, invalidated_at } = { ...doc.data() } as { email: string; invalidated_at?: Date } if (!validate(email)) return if (invalidated_at) return - if (!votesByAuth[auth_token]) return voters.push(email) }) diff --git a/pages/api/election/[election_id]/admin/review-signature.ts b/pages/api/election/[election_id]/admin/review-signature.ts index c9c5781a..913d07cc 100644 --- a/pages/api/election/[election_id]/admin/review-signature.ts +++ b/pages/api/election/[election_id]/admin/review-signature.ts @@ -7,7 +7,7 @@ import { checkJwtOwnsElection } from '../../../validate-admin-jwt' export default async (req: NextApiRequest, res: NextApiResponse) => { const { election_id } = req.query as { election_id: string } - const { emails, review } = req.body + const { auths, review } = req.body // Confirm they're a valid admin that created this election const jwt = await checkJwtOwnsElection(req, res, election_id) @@ -15,13 +15,13 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { // Update voter w/ review await Promise.all( - emails.map((email: string) => + auths.map((auth_token: string) => firebase .firestore() .collection('elections') .doc(election_id) - .collection('voters') - .doc(email) + .collection('approved-voters') + .doc(auth_token) .update({ esignature_review: firestore.FieldValue.arrayUnion({ review, reviewed_at: new Date() }) }), ), ) diff --git a/pages/api/election/[election_id]/admin/unlock.ts b/pages/api/election/[election_id]/admin/unlock.ts index 206d4314..3f468f65 100644 --- a/pages/api/election/[election_id]/admin/unlock.ts +++ b/pages/api/election/[election_id]/admin/unlock.ts @@ -1,3 +1,6 @@ +import { firebase, pushover } from 'api/_services' +import { pusher } from 'api/pusher' +import { checkJwtOwnsElection } from 'api/validate-admin-jwt' import bluebird from 'bluebird' import { mapValues } from 'lodash-es' import { NextApiRequest, NextApiResponse } from 'next' @@ -6,14 +9,11 @@ import { RP } from 'src/crypto/curve' import { fastShuffle, shuffleWithoutProof, shuffleWithProof } from 'src/crypto/shuffle' import { CipherStrings, stringifyShuffle, stringifyShuffleWithoutProof } from 'src/crypto/stringify-shuffle' -import { firebase, pushover } from '../../../_services' -import { pusher } from '../../../pusher' -import { checkJwtOwnsElection } from '../../../validate-admin-jwt' -import { ReviewLog } from './load-admin' - const { ADMIN_EMAIL } = process.env export default async (req: NextApiRequest, res: NextApiResponse) => { + // Initialize profiling code + const start = new Date() const times = [Date.now()] const elapsed = (label: number | string) => { const l = String(label) @@ -24,12 +24,11 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { console.log(`${l.padStart(23, ' ')} ${diff.padStart(5, ' ')}ms`) } - const start = new Date() - if (!ADMIN_EMAIL) return res.status(501).json({ error: 'Missing process.env.ADMIN_EMAIL' }) elapsed('init') + // Grab query params const { election_id } = req.query as { election_id: string } const { options = {} } = req.body const { skip_shuffle_proofs } = options @@ -39,14 +38,14 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { if (!jwt.valid) return elapsed('check jwt') + // Grab election doc const electionDoc = firebase .firestore() .collection('elections') .doc(election_id as string) // Begin preloading these requests - const loadVotes = electionDoc.collection('votes').get() - const loadVoters = electionDoc.collection('voters').get() + const loadVotes = electionDoc.collection('approved-voters').where('voted_at', '!=', null).get() const election = electionDoc.get() const adminDoc = electionDoc.collection('trustees').doc(ADMIN_EMAIL) const admin = adminDoc.get() @@ -56,6 +55,7 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { if (!(await election).exists) return res.status(400).json({ error: `Unknown Election ID: '${election_id}'` }) elapsed('election exists?') + // Grab election params const { esignature_requested, t, threshold_public_key } = { ...(await election).data() } as { esignature_requested: boolean t: number @@ -64,24 +64,17 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { elapsed('election data') if (!threshold_public_key) return res.status(400).json({ error: 'Election missing `threshold_public_key`' }) - // If esignature_requested, filter for only approved - let votes_to_unlock = (await loadVotes).docs - if (esignature_requested) { - type VotersByAuth = Record - const votersByAuth: VotersByAuth = (await loadVoters).docs.reduce((acc: VotersByAuth, doc) => { - const data = doc.data() - return { ...acc, [data.auth_token]: data } - }, {}) - - votes_to_unlock = votes_to_unlock.filter((doc) => { - const { auth } = doc.data() as { auth: string } - return getStatus(votersByAuth[auth].esignature_review) === 'approve' - }) - } + // Filter out invalidated votes + let votes_to_unlock = (await loadVotes).docs.map((doc) => doc.data()).filter((v) => !v.invalidated_at) + + // If esignature_requested, filter out non-approved + if (esignature_requested) + votes_to_unlock = votes_to_unlock.filter((v) => getStatus(v.esignature_review) === 'approve') + elapsed('load votes, filter esig') // Admin removes the auth tokens - const encrypteds_without_auth_tokens = votes_to_unlock.map((doc) => doc.data().encrypted_vote) + const encrypteds_without_auth_tokens = votes_to_unlock.map((v) => v.encrypted_vote) elapsed('remove auth tokens') // Then we split up the votes into individual lists for each item diff --git a/pages/api/election/[election_id]/invalidated-votes.ts b/pages/api/election/[election_id]/invalidated-votes.ts index f2ed69db..fa4c097c 100644 --- a/pages/api/election/[election_id]/invalidated-votes.ts +++ b/pages/api/election/[election_id]/invalidated-votes.ts @@ -1,6 +1,8 @@ import { firebase } from 'api/_services' import { NextApiRequest, NextApiResponse } from 'next' +type InvalidatedVote = { auth: string; encrypted_vote: Record } + export default async (req: NextApiRequest, res: NextApiResponse) => { const { election_id } = req.query @@ -10,19 +12,20 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { .doc(election_id as string) // Begin preloading - const loadInvalidatedVotes = electionDoc.collection('invalidated_votes').get() + const loadInvalidatedVotes = electionDoc.collection('approved-voters').where('invalidated_at', '!=', null).get() // Is election_id in DB? if (!(await electionDoc.get()).exists) return res.status(400).json({ error: 'Unknown Election ID.' }) // Grab public votes fields including encrypted_vote - const votes = (await loadInvalidatedVotes).docs.map((doc) => { - const { auth, encrypted_vote } = doc.data() - return { - auth, - encrypted_vote, - } - }) - - res.status(200).json(votes) + const invalidated_votes = (await loadInvalidatedVotes).docs.reduce((memo, doc) => { + const { auth_token, encrypted_vote } = doc.data() + // Filter for only docs where they voted + // (would be a bit more efficient to do this at the DB query layer, but Firebase only allows one NOT_EQUAL filter per query) + if (encrypted_vote) memo.push({ auth: auth_token, encrypted_vote }) + + return memo + }, [] as InvalidatedVote[]) + + res.status(200).json(invalidated_votes) } diff --git a/pages/api/election/[election_id]/submit-invalidation-response.ts b/pages/api/election/[election_id]/submit-invalidation-response.ts index ee77d78b..0217e0d0 100644 --- a/pages/api/election/[election_id]/submit-invalidation-response.ts +++ b/pages/api/election/[election_id]/submit-invalidation-response.ts @@ -13,34 +13,31 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) // Begin preloading const electionDoc = election.get() - const loadVoters = election.collection('voters').where('auth_token', '==', auth).get() + const voterDoc = election.collection('approved-voters').doc(auth) + const voter = await voterDoc.get() + if (!voter.exists) return res.status(404).json({ error: 'No voter w/ this auth_token: ' + auth }) // Store in database on the invalidated_vote doc - const votes = await election.collection('invalidated_votes').where('auth', '==', auth).get() - await Promise.all( - votes.docs.map((vote) => - vote.ref.update({ responses: firestore.FieldValue.arrayUnion({ message, timestamp: new Date() }) }), - ), - ) + voterDoc.update({ invalidated_vote_reply: firestore.FieldValue.arrayUnion({ message, timestamp: new Date() }) }) // Send admin email - const voter = (await loadVoters).docs[0].data() + const { email, first_name, last_name } = voter.data() || {} const electionData = (await electionDoc).data() sendEmail({ bcc: 'admin@siv.org', recipient: electionData?.creator, - subject: 'Invalidated Vote: Voter Response', + subject: `Invalidated Vote: Voter Response from ${email}`, text: `You have received a message from a voter whose vote you invalidated. Election Title: ${electionData?.election_title} Election ID: ${election_id} Voter details: - - Auth token: ${voter.auth_token} - - Email: ${voter.email} - - First Name: ${voter.first_name || 'Not provided'} - - Last Name: ${voter.last_name || 'Not provided'} + - Auth token: ${auth} + - Email: ${email} + - First Name: ${first_name || 'Not provided'} + - Last Name: ${last_name || 'Not provided'} Their message below: diff --git a/pages/api/election/[election_id]/was-vote-invalidated.ts b/pages/api/election/[election_id]/was-vote-invalidated.ts index c62e853a..fb571494 100644 --- a/pages/api/election/[election_id]/was-vote-invalidated.ts +++ b/pages/api/election/[election_id]/was-vote-invalidated.ts @@ -5,14 +5,16 @@ import { NextApiRequest, NextApiResponse } from 'next' export default async function (req: NextApiRequest, res: NextApiResponse) { const { auth, election_id } = req.query + if (typeof election_id !== 'string') return res.status(401).json({ error: `Missing election_id` }) + if (typeof auth !== 'string') return res.status(401).json({ error: `Missing auth` }) - const voters = await firebase + const voter = await firebase .firestore() .collection('elections') - .doc(election_id as string) - .collection('voters') - .where('auth_token', '==', auth) + .doc(election_id) + .collection('approved-voters') + .doc(auth) .get() - return res.status(200).json(!!voters.docs[0]?.data().invalidated_at) + return res.status(200).json(!!voter?.data()?.invalidated_at) } diff --git a/pages/api/submit-esignature.ts b/pages/api/submit-esignature.ts index 84577e6b..509f06b9 100644 --- a/pages/api/submit-esignature.ts +++ b/pages/api/submit-esignature.ts @@ -8,13 +8,14 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { const electionDoc = firebase.firestore().collection('elections').doc(election_id) - // Is there an encrypted vote w/ this auth token? - const [voteDoc] = (await electionDoc.collection('votes').where('auth', '==', auth).get()).docs - if (!voteDoc.exists) return res.status(404).json({ error: 'No vote w/ this auth_token' }) + // Is there a voter w/ this auth token? + const voterDoc = electionDoc.collection('approved-voters').doc(auth) + const voter = await voterDoc.get() + if (!voter.exists) return res.status(404).json({ error: 'No voter w/ this auth_token' }) // Without an existing esignature? - if (voteDoc.data().esignature) return res.status(400).json({ error: 'Vote already has an esignature' }) + if (voter.data()?.esignature) return res.status(400).json({ error: 'Vote already has an esignature' }) - await electionDoc.collection('votes').doc(voteDoc.id).update({ esignature, esigned_at: new Date() }) + await voterDoc.update({ esignature, esigned_at: new Date() }) await pusher.trigger(`status-${election_id}`, 'votes', auth) diff --git a/pages/api/submit-vote.ts b/pages/api/submit-vote.ts index a03ec9b8..a1de9b8d 100644 --- a/pages/api/submit-vote.ts +++ b/pages/api/submit-vote.ts @@ -72,18 +72,19 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { if (!validated) return // Begin preloading - const voter = electionDoc.collection('voters').where('auth_token', '==', auth).get() const election = electionDoc.get() + const voterDoc = electionDoc.collection('approved-voters').doc(auth) + const loadVoter = voterDoc.get() await Promise.all([ // 2a. Store the encrypted vote in db - electionDoc.collection('votes').add({ auth, created_at: new Date(), encrypted_vote, headers: req.headers }), + voterDoc.update({ encrypted_vote, headers: req.headers, voted_at: new Date() }), // 2b. Update elections cached tally of num_votes electionDoc.update({ num_votes: firestore.FieldValue.increment(1) }), ]) // 3. Email the voter their submission receipt - const { email } = (await voter).docs[0].data() + const { email } = (await loadVoter).data() || {} const promises: Promise[] = [] // Skip if email isn't valid (e.g. used QR invitations) diff --git a/src/admin/Voters/InvalidVotersTable.tsx b/src/admin/Voters/InvalidVotersTable.tsx index 12fa7f98..283496df 100644 --- a/src/admin/Voters/InvalidVotersTable.tsx +++ b/src/admin/Voters/InvalidVotersTable.tsx @@ -53,7 +53,7 @@ export const InvalidVotersTable = ({ hide_approved, hide_voted }: { hide_approve {esignature_requested && - (has_voted ? : )} + (has_voted ? : )} ))} diff --git a/src/admin/Voters/SendInvitationsButton.tsx b/src/admin/Voters/SendInvitationsButton.tsx index cce3d8fb..a30173e8 100644 --- a/src/admin/Voters/SendInvitationsButton.tsx +++ b/src/admin/Voters/SendInvitationsButton.tsx @@ -31,7 +31,7 @@ export const SendInvitationsButton = ({ toggle_sending() const voters_to_invite = checked.reduce((acc: string[], is_checked, index) => { - if (is_checked) acc.push(valid_voters[index].email) + if (is_checked) acc.push(valid_voters[index].auth_token) return acc }, []) diff --git a/src/admin/Voters/Signature.tsx b/src/admin/Voters/Signature.tsx index e137eec6..9d3f9bc1 100644 --- a/src/admin/Voters/Signature.tsx +++ b/src/admin/Voters/Signature.tsx @@ -7,19 +7,19 @@ export const getStatus = (esignature_review?: ReviewLog[]) => esignature_review ? esignature_review[esignature_review.length - 1]?.review : undefined export const Signature = ({ + auth_token, election_id, - email, esignature, esignature_review, }: { + auth_token: string election_id?: string - email: string esignature?: string esignature_review?: ReviewLog[] }) => { const storeReview = (review: 'approve' | 'pending' | 'reject', setIsShown: (setting: boolean) => void) => async () => (await api(`election/${election_id}/admin/review-signature`, { - emails: [email], + auths: [auth_token], review, })) && setIsShown(false) diff --git a/src/admin/Voters/ValidVotersTable.tsx b/src/admin/Voters/ValidVotersTable.tsx index ea389b4e..206c8d78 100644 --- a/src/admin/Voters/ValidVotersTable.tsx +++ b/src/admin/Voters/ValidVotersTable.tsx @@ -79,7 +79,7 @@ export const ValidVotersTable = ({ onClick={() => { if (confirm(`Do you want to approve all ${num_voted} signatures?`)) { api(`election/${election_id}/admin/review-signature`, { - emails: shown_voters.filter(({ has_voted }) => has_voted).map((v) => v.email), + auths: shown_voters.filter(({ has_voted }) => has_voted).map((v) => v.auth_token), review: 'approve', }) } @@ -133,8 +133,8 @@ export const ValidVotersTable = ({ // Store new email in API const response = await api(`election/${election_id}/admin/edit-voter-email`, { + auth_token, new_email, - old_email: email, }) if (response.status === 201) { @@ -159,7 +159,7 @@ export const ValidVotersTable = ({ {has_voted ? '✓' : ''} {esignature_requested && - (has_voted ? : )} + (has_voted ? : )} ), )} diff --git a/src/status/AcceptedVotes.tsx b/src/status/AcceptedVotes.tsx index 1380e90e..d6621189 100644 --- a/src/status/AcceptedVotes.tsx +++ b/src/status/AcceptedVotes.tsx @@ -34,7 +34,7 @@ export const AcceptedVotes = ({ fetcher, 1, ) as { data: NumAcceptedVotes } - const { num_pending_votes = 0, num_votes = 0 } = data || {} + const { num_invalidated_votes = 0, num_pending_votes = 0, num_votes = 0 } = data || {} // Load all the encrypted votes (heavy, so only on first load) useEffect(() => { @@ -52,7 +52,7 @@ export const AcceptedVotes = ({ if (!votes || !ballot_design) return
Loading...
- const newTotalVotes = num_votes - votes.length + const newTotalVotes = num_votes - votes.length - num_invalidated_votes return ( <> @@ -136,7 +136,10 @@ export const AcceptedVotes = ({ `/api/election/${election_id}/accepted-votes?num_new_pending_votes=${num_new_pending_votes}&num_new_accepted_votes=${num_new_accepted_votes}`, ) .then((r) => r.json()) - .then((newVotes) => setVotes(() => [...votes, ...newVotes])) + .then((newVotes) => { + if (!newVotes.length) window.location.reload() + setVotes(() => [...votes, ...newVotes]) + }) }} > + Load {newTotalVotes} new