Compare commits

..

No commits in common. "main" and "75015f8ada1f25d969088299365a72d48aabb7ed" have entirely different histories.

5 changed files with 7 additions and 184 deletions

3
.gitignore vendored
View File

@ -1,3 +1,2 @@
.DS_Store .DS_Store
deduplicatedLeads.json deduplicatedLeads.json
node_modules

View File

@ -1,8 +0,0 @@
Requires Node 12+
```sh
git clone https://git.chuckdries.com/chuckdries/lead-deduplication
cd lead-deduplication
npm i # install dependencies
node index.js # run script
```

View File

@ -1,7 +1,5 @@
#! /usr/bin/env node #! /usr/bin/env node
const fs = require("fs"); const fs = require("fs");
const chalk = require("chalk");
const { leads } = JSON.parse(fs.readFileSync("./leads.json")); const { leads } = JSON.parse(fs.readFileSync("./leads.json"));
// index records by ID for easy lookup // index records by ID for easy lookup
@ -11,7 +9,6 @@ const leadIdssByEmail = {};
const collisions = []; const collisions = [];
// deduplicate leads
for (const currentLead of leads) { for (const currentLead of leads) {
const { _id, email } = currentLead; const { _id, email } = currentLead;
@ -36,7 +33,7 @@ for (const currentLead of leads) {
const discardedValues = [...(collision.right.discardedValues || [])]; const discardedValues = [...(collision.right.discardedValues || [])];
delete collision.right.discardedValues; delete collision.right.discardedValues;
discardedValues.unshift(collision.right); discardedValues.push(collision.right);
const lead = { const lead = {
...collidingLead, ...collidingLead,
discardedValues, discardedValues,
@ -49,7 +46,7 @@ for (const currentLead of leads) {
const discardedValues = [...(collision.left.discardedValues || [])]; const discardedValues = [...(collision.left.discardedValues || [])];
delete collision.left.discardedValues; delete collision.left.discardedValues;
discardedValues.unshift(collision.left); discardedValues.push(collision.left);
const lead = { const lead = {
...currentLead, ...currentLead,
discardedValues, discardedValues,
@ -75,41 +72,6 @@ for (const currentLead of leads) {
} }
} }
const printPropRedIfDiff = (prop, val) => (object) =>
object[prop] !== val && console.log("\t\t", chalk.bgRed(object[prop]));
const prettyPrintItem = ({
_id,
email,
firstName,
lastName,
address,
entryDate,
discardedValues,
}) => {
console.log("------");
console.log("_id:\t\t", _id);
discardedValues && discardedValues.forEach(printPropRedIfDiff("_id", _id));
console.log("email:\t\t", email);
discardedValues &&
discardedValues.forEach(printPropRedIfDiff("email", email));
console.log("firstName:\t", firstName);
discardedValues &&
discardedValues.forEach(printPropRedIfDiff("firstName", firstName));
console.log("lastName:\t", lastName);
discardedValues &&
discardedValues.forEach(printPropRedIfDiff("lastName", lastName));
console.log("address:\t", address);
discardedValues &&
discardedValues.forEach(printPropRedIfDiff("address", address));
console.log("entryDate:\t", entryDate);
discardedValues &&
discardedValues.forEach(printPropRedIfDiff("entryDate", entryDate));
console.log();
};
Object.values(leadsById).forEach(prettyPrintItem);
fs.writeFileSync( fs.writeFileSync(
"./deduplicatedLeads.json", "./deduplicatedLeads.json",
JSON.stringify(Object.values(leadsById), null, 2) JSON.stringify(Object.values(leadsById), null, 2)
@ -118,7 +80,5 @@ fs.writeFileSync(
console.log("records processed:", leads.length); console.log("records processed:", leads.length);
console.log("collisions:", collisions.length); console.log("collisions:", collisions.length);
console.log("output leads:", Object.keys(leadsById).length); console.log("output leads:", Object.keys(leadsById).length);
console.log("leads written to deduplicatedLeads.json");
// // uncomment for more information about deduplication console.log("collisions", collisions);
// console.log("collisions", collisions);

125
package-lock.json generated
View File

@ -1,125 +0,0 @@
{
"name": "lead-deduplication",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "lead-deduplication",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"chalk": "^4.1.2"
}
},
"node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"node_modules/has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"engines": {
"node": ">=8"
}
},
"node_modules/supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dependencies": {
"has-flag": "^4.0.0"
},
"engines": {
"node": ">=8"
}
}
},
"dependencies": {
"ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"requires": {
"color-convert": "^2.0.1"
}
},
"chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"requires": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
},
"supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"requires": {
"has-flag": "^4.0.0"
}
}
}
}

View File

@ -1,14 +1,11 @@
{ {
"name": "lead-deduplication", "name": "adobe-coding-challenge",
"version": "1.0.0", "version": "1.0.0",
"description": "", "description": "",
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"run": "node index.js" "test": "echo \"Error: no test specified\" && exit 1"
}, },
"author": "", "author": "",
"license": "ISC", "license": "ISC"
"dependencies": {
"chalk": "^4.1.2"
}
} }