Last active
May 13, 2024 20:18
-
-
Save twosdai/27910bdc4d4f0392eab3c9ffdb3da227 to your computer and use it in GitHub Desktop.
csvChecker.js
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Read in a CSV file and check for errors | |
| // Usage: node csvChecker.js <filename> | |
| // Example: node csvChecker.js test.csv | |
| const fs = require('fs'); | |
| const csv = require('csv-parser'); | |
| const createCsvWriter = require('csv-writer').createObjectCsvWriter | |
| const results = []; | |
| if (process.argv.length !== 3) { | |
| console.error('Usage: node csvChecker.js <filename>'); | |
| process.exit(1); | |
| } | |
| const filename = process.argv[2]; | |
| try{ | |
| let counter = 0; | |
| fs.createReadStream(filename) | |
| .pipe(csv()) | |
| .on('data', (data) => results.push(data)) | |
| .on('end', () => { | |
| const errors = []; | |
| const goodResults = []; | |
| const pilotCampaign = []; | |
| results.forEach((result, index) => { | |
| const emailPersonalMessage = result['Email (Personal Message)']; | |
| const emailJobRole = result['Email (Job Role)']; | |
| const emailIdeas = result['Email (Ideas)']; | |
| const similarityStep3 = result['Similarity Step 3']; | |
| const simplifiedCompanyName = result['Simplified Company Name']; | |
| const step3StartingLine = result['Step 3 Starting Line']; | |
| if(simplifiedCompanyName.match(/[\.]$/)){ | |
| counter++; | |
| // remove period from end of string | |
| result['Simplified Company Name'] = simplifiedCompanyName.slice(0, -1); | |
| } | |
| console.log(`Row ${index + 2}`); | |
| if (!result['Email (Personal Message)']) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Personal Message) is empty or not defined', | |
| result, | |
| }); | |
| return; | |
| } | |
| if (!result['Email (Social Proof)']) { | |
| errors.push( | |
| { | |
| row: index + 2, | |
| error: 'Email (Social Proof) is empty or not defined', | |
| result, | |
| } | |
| ); | |
| return; | |
| } | |
| if (!result['Email (CTA)']) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (CTA) is empty or not defined', | |
| result, | |
| }); | |
| return; | |
| } | |
| if (!result['Email (Ideas)']) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Ideas) is empty or not defined', | |
| result, | |
| }); | |
| return; | |
| } | |
| if (!result['Similarity Step 3']) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Similarity Step 3 is empty or not defined', | |
| result, | |
| }); | |
| return; | |
| } | |
| if (!result['Simplified Company Name']) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Simplified Company Name is empty or not defined', | |
| result, | |
| }); | |
| return; | |
| } | |
| if (!result['Step 3 Starting Line']) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Step 3 Starting Line is empty or not defined', | |
| result, | |
| }); | |
| return; | |
| } | |
| if (!result['Email (Job Role)']) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Job Role) is empty or not defined', | |
| result, | |
| }); | |
| return; | |
| } | |
| const failureCaseForAIResult = new RegExp(/sorry|apologize|apologies|unable|cannot|do not |don't|trouble|invalid|not able|couldn't|afraid|can not |invalid input|unfortunately|did not |n\/a|wasn't|empty|isn't|is not|are not |aren't|was not |weren't/) | |
| // Check to see personal message ends with proper punctuation, period, question mark, excalamation point, or "" | |
| if (!emailPersonalMessage.match(/[\.\?\!\"]$/)) { | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Personal Message) does not end with a period, question mark, or exclamation point', | |
| result, | |
| }); | |
| return; | |
| } | |
| if(!emailJobRole.startsWith('I noticed that you are') || emailJobRole.toLowerCase().match(failureCaseForAIResult)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Job Role) does not start with "I noticed that you are" or contains a failure case for AI result', | |
| result, | |
| }); | |
| return; | |
| } | |
| /** | |
| * {{8.1 Job Role Summary}}?.response?.split(" ")?.length < 15 && !{{8.1 Job Role Summary}}?.response?.split(' ').some(word => word.length > 40) | |
| */ | |
| if(!(emailJobRole.split(" ").length < 15) || emailJobRole.split(' ').some(word => word.length > 40)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Job Role) does not contain at least 15 words and no word is longer than 40 characters', | |
| result, | |
| }) | |
| return; | |
| }; | |
| /** | |
| * ({{7.2 Ideas}}?.response?.startsWith("-") && !{{7.2 Ideas}}?.response?.toLowerCase()?.match(/sorry|apologize|apologies|unable|cannot|do not |don't|trouble|invalid|not able|couldn't|afraid|can not |invalid input|unfortunately|did not |n\/a|wasn't|empty|isn't|is not|are not |aren't|was not |weren't/) && {{7.2 Ideas}}?.response?.split(" ")?.length < 60 && !{{7.2 Ideas}}?.response?.split(' ').some(word => word.length > 40) && ({{7.2 Ideas}}?.response?.match(/\S/g) || []).length <= 800 | |
| */ | |
| if(!emailIdeas.startsWith("-") || emailIdeas.toLowerCase().match(failureCaseForAIResult)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Ideas) does not start with "-" or contains a failure case for AI result', | |
| result, | |
| }); | |
| return; | |
| } | |
| if(!(emailIdeas.split(" ").length < 60) || emailIdeas.split(' ').some(word => word.length > 40)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Email (Ideas) does not contain at least 60 words and no word is longer than 40 characters', | |
| result, | |
| }) | |
| return; | |
| } | |
| /** | |
| * ({{Step 3 Similarity}}?.response?.startsWith("Just like how") && !{{Step 3 Similarity}}?.response?.toLowerCase()?.match(/sorry|apologize|apologies|unable|cannot|do not |don't|trouble|invalid|not able|couldn't|afraid|can not |invalid input|unfortunately|did not |n\/a|wasn't|empty|isn't|is not|are not |aren't|was not |weren't/) && {{Step 3 Similarity}}?.response?.split(" ")?.length < 38 && !{{Step 3 Similarity}}?.response?.split(' ').some(word => word.length > 40) ) | |
| */ | |
| if(!similarityStep3.startsWith("Just like how") || similarityStep3.toLowerCase().match(failureCaseForAIResult)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Similarity Step 3 does not start with "Just like how" or contains a failure case for AI result', | |
| result, | |
| }); | |
| return; | |
| } | |
| if(!(similarityStep3.split(" ").length < 38) || similarityStep3.split(' ').some(word => word.length > 40)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Similarity Step 3 does not contain at least 38 words and no word is longer than 40 characters', | |
| result, | |
| }); | |
| return; | |
| } | |
| /** | |
| * (!{{8.1 Simplified Company Name}}?.response?.toLowerCase()?.match(/sorry|apologize|apologies|unable|cannot|do not |don't|trouble|invalid|not able|couldn't|afraid|can not |invalid input|unfortunately|did not |n\/a|wasn't|empty|isn't|is not|are not |aren't|was not |weren't/) && {{8.1 Simplified Company Name}}?.response?.split(" ")?.length < 5 && !{{8.1 Simplified Company Name}}?.response?.split(' ').some(word => word.length > 20) ) | |
| */ | |
| if(simplifiedCompanyName.toLowerCase().match(failureCaseForAIResult)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Simplified Company Name contains a failure case for AI result', | |
| result, | |
| }); | |
| return; | |
| } | |
| if(!(simplifiedCompanyName.split(" ").length < 5) || simplifiedCompanyName.split(' ').some(word => word.length > 20)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Simplified Company Name does not contain at least 5 words and no word is longer than 20 characters', | |
| result, | |
| }); | |
| return; | |
| } | |
| /** | |
| * ({{Step 3 Starting line}}?.response?.startsWith("Saw on your website that you") && !{{Step 3 Starting line}}?.response?.toLowerCase()?.match(/sorry|apologize|apologies|unable|cannot|do not |don't|trouble|invalid|not able|couldn't|afraid|can not |invalid input|unfortunately|did not |n\/a|wasn't|empty|isn't|is not|are not |aren't|was not |weren't/) && {{Step 3 Starting line}}?.response?.split(" ")?.length < 40 ) | |
| */ | |
| if(!(step3StartingLine.startsWith("Saw on your website that you") || step3StartingLine.startsWith("I was on your site and noticed")) || step3StartingLine.toLowerCase().match(failureCaseForAIResult)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Step 3 Starting Line does not start with "Saw on your website that you" or contains a failure case for AI result', | |
| result, | |
| }); | |
| return; | |
| } | |
| if(!(step3StartingLine.split(" ").length < 40)){ | |
| errors.push({ | |
| row: index + 2, | |
| error: 'Step 3 Starting Line does not contain at least 40 words', | |
| result, | |
| }); | |
| return; | |
| } | |
| /** | |
| * Personal line starts with “Did you go to” AND | |
| Title includes words such as “architect”, “cto”, “data”, “engineering” (make sure convert and check in lower case) AND | |
| Country in “United States” | |
| */ | |
| if(emailPersonalMessage.startsWith("Did you go to") && result['Title'].toLowerCase().match(/architect|cto|data|engineering/) && result['Country'] === 'United States'){ | |
| pilotCampaign.push(result); | |
| } else{ | |
| goodResults.push(result); | |
| } | |
| }); | |
| if (errors.length === 0) { | |
| console.log('No errors found'); | |
| } else { | |
| // Write errors to a CSV file | |
| const writeableErrors = errors.map((error) => { | |
| return { | |
| Error: error.error, | |
| Row: error.row, | |
| ...error.result, | |
| }; | |
| }); | |
| const csvWriter = createCsvWriter({ | |
| path: 'errors.csv', | |
| header: [ | |
| {id: 'Error', title: 'Error'}, | |
| {id: 'Row', title: 'Row'}, | |
| {id: 'Title Selection', title: 'Title Selection'}, | |
| {id: 'Email (Personal Message)', title: 'Email (Personal Message'}, | |
| {id: 'First Name', title: 'First Name'}, | |
| {id: 'Last Name', title: 'Last Name'}, | |
| {id: 'Title', title: 'Title'}, | |
| {id: 'Company', title: 'Company'}, | |
| {id: 'Email', title: 'Email'}, | |
| {id: 'Person Linkedin Url', title: 'Person Linkedin Url'}, | |
| {id: 'Website', title: 'Website'}, | |
| {id: 'Company Linkedin Url', title: 'Company Linkedin Url'}, | |
| {id: 'City', title: 'City'}, | |
| {id: 'State', title: 'State'}, | |
| {id: 'Country', title: 'Country'}, | |
| {id: 'Email (Job Role)', title: 'Email (Job Role)'}, | |
| {id: 'Email (Social Proof)', title: 'Email (Social Proof)'}, | |
| {id: 'Email (CTA)', title: 'Email (CTA)'}, | |
| {id: 'Company Summary', title: 'Company Summary'}, | |
| {id: 'Email (Ideas)', title: 'Email (Ideas)'}, | |
| {id: 'Similarity Step 3', title: 'Similarity Step 3'}, | |
| {id: 'Simplified Company Name', title: 'Simplified Company Name'}, | |
| {id: 'Step 3 Starting Line', title: 'Step 3 Starting Line'}, | |
| {id: 'Exported', title: 'Exported'}, | |
| ] | |
| }); | |
| csvWriter.writeRecords(writeableErrors).then(() => { | |
| console.log('Errors written to errors.csv'); | |
| }).catch((err) => { | |
| console.error('Error writing errors:', err); | |
| }); | |
| } | |
| const csvWriter = createCsvWriter({ | |
| path: 'generalCampaign.csv', | |
| header: [ | |
| {id: 'Title Selection', title: 'Title Selection'}, | |
| {id: 'Email (Personal Message)', title: 'Email (Personal Message'}, | |
| {id: 'First Name', title: 'First Name'}, | |
| {id: 'Last Name', title: 'Last Name'}, | |
| {id: 'Title', title: 'Title'}, | |
| {id: 'Company', title: 'Company'}, | |
| {id: 'Email', title: 'Email'}, | |
| {id: 'Person Linkedin Url', title: 'Person Linkedin Url'}, | |
| {id: 'Website', title: 'Website'}, | |
| {id: 'Company Linkedin Url', title: 'Company Linkedin Url'}, | |
| {id: 'City', title: 'City'}, | |
| {id: 'State', title: 'State'}, | |
| {id: 'Country', title: 'Country'}, | |
| {id: 'Email (Job Role)', title: 'Email (Job Role)'}, | |
| {id: 'Email (Social Proof)', title: 'Email (Social Proof)'}, | |
| {id: 'Email (CTA)', title: 'Email (CTA)'}, | |
| {id: 'Company Summary', title: 'Company Summary'}, | |
| {id: 'Email (Ideas)', title: 'Email (Ideas)'}, | |
| {id: 'Similarity Step 3', title: 'Similarity Step 3'}, | |
| {id: 'Simplified Company Name', title: 'Simplified Company Name'}, | |
| {id: 'Step 3 Starting Line', title: 'Step 3 Starting Line'}, | |
| {id: 'Exported', title: 'Exported'}, | |
| ] | |
| }); | |
| if(goodResults && goodResults.length){ | |
| csvWriter.writeRecords(goodResults).then(() => { | |
| console.log('Good results written to good.csv'); | |
| }).catch((err) => { | |
| console.error('Error writing good results:', err); | |
| }); | |
| } | |
| if(pilotCampaign && pilotCampaign.length){ | |
| const csvWriter2 = createCsvWriter({ | |
| path: 'pilotCampaign.csv', | |
| header: [ | |
| {id: 'Title Selection', title: 'Title Selection'}, | |
| {id: 'Email (Personal Message)', title: 'Email (Personal Message'}, | |
| {id: 'First Name', title: 'First Name'}, | |
| {id: 'Last Name', title: 'Last Name'}, | |
| {id: 'Title', title: 'Title'}, | |
| {id: 'Company', title: 'Company'}, | |
| {id: 'Email', title: 'Email'}, | |
| {id: 'Person Linkedin Url', title: 'Person Linkedin Url'}, | |
| {id: 'Website', title: 'Website'}, | |
| {id: 'Company Linkedin Url', title: 'Company Linkedin Url'}, | |
| {id: 'City', title: 'City'}, | |
| {id: 'State', title: 'State'}, | |
| {id: 'Country', title: 'Country'}, | |
| {id: 'Email (Job Role)', title: 'Email (Job Role)'}, | |
| {id: 'Email (Social Proof)', title: 'Email (Social Proof)'}, | |
| {id: 'Email (CTA)', title: 'Email (CTA)'}, | |
| {id: 'Company Summary', title: 'Company Summary'}, | |
| {id: 'Email (Ideas)', title: 'Email (Ideas)'}, | |
| {id: 'Similarity Step 3', title: 'Similarity Step 3'}, | |
| {id: 'Simplified Company Name', title: 'Simplified Company Name'}, | |
| {id: 'Step 3 Starting Line', title: 'Step 3 Starting Line'}, | |
| {id: 'Exported', title: 'Exported'}, | |
| ] | |
| }); | |
| csvWriter2.writeRecords(pilotCampaign).then(() => { | |
| console.log('Pilot Campaign written to pilotCampaign.csv'); | |
| }).catch((err) => { | |
| console.error('Error writing pilot campaign:', err); | |
| }); | |
| } | |
| console.log('Good results:', goodResults.length); | |
| console.log('Errors:', errors.length); | |
| console.log('Pilot Campaign:', pilotCampaign.length); | |
| console.log('Total:', results.length); | |
| console.log(`Simplified Company Name ends with period: ${counter}`) | |
| })} catch (err) { | |
| console.error('Error reading file:', err); | |
| process.exit(1); | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment