[offers][feat] add data seeding code to fetch from excel

pull/501/head
Stuart Long Chay Boon 3 years ago
parent 30986ef879
commit 0f349dfc9d

@ -1,15 +1,27 @@
// const xlsxFile = require('read-excel-file/node');
// xlsxFile('/Users/stuartlong/Desktop/tech-interview-handbook/apps/portal/prisma/salaries.xlsx').then((rows) => {
// console.log(rows)
// })
const reader = require("xlsx")
import reader from "xlsx";
import { PrismaClient } from '@prisma/client';
import crypto from 'crypto';
import { baseCurrencyString } from '../src/utils/offers/currency';
import { convert } from '../src/utils/offers/currency/currencyExchange';
const prisma = new PrismaClient();
// Reading our test file
const file = reader.readFile('/Users/stuartlong/Desktop/tech-interview-handbook/apps/portal/prisma/salaries.xlsx')
let data = []
let data: Array<excelData> = []
type excelData = {
Timestamp: Date;
Type: string;
Company: string;
Role: string,
Income?: number | string;
Stocks?: number | string;
SignOn?: number | string;
TC?: number | string;
Bonus?: number | string;
Comments?: string
}
const sheets = file.SheetNames
@ -17,11 +29,206 @@ for(let i = 0; i < sheets.length; i++)
{
const temp = reader.utils.sheet_to_json(
file.Sheets[file.SheetNames[i]])
temp.forEach((res) => {
temp.forEach((res: excelData) => {
data.push(res)
})
}
function xlSerialToJsDate(xlSerial){
return new Date(Date.UTC(0, 0, xlSerial - 1));
}
function generateSpecialization() {
const specializations = ["Frontend", "Backend", "Fullstack"];
return specializations[Math.floor((Math.random() * 300)) % 3];
}
async function seedSalaries() {
console.log('Seeding from salaries sheet...');
const companyIdMappings = {};
(await prisma.company.findMany()).forEach((company) => {
companyIdMappings[company.name] = company.id
});
console.log(companyIdMappings);
const createdProfileIds : Array<string> = [];
//seed here
(await Promise.all([
data.map(async (data: excelData) => {
// only add swe roles
if (data.Role.toUpperCase() === 'SOFTWARE ENGINEER') {
if (data.Income && typeof (data.Income) === "number") {
// check if we have company id
// console.log(data.Income)
// console.log()
if (companyIdMappings[data.Company]) {
const token = crypto.createHash('sha256').update(xlSerialToJsDate(data.Timestamp).toString()).digest('hex')
if (data.Type.toUpperCase() === 'INTERNSHIP') {
// create profile
const dataAdded = await prisma.offersProfile.create({
data: {
profileName: crypto.randomUUID().substring(0, 10),
createdAt: xlSerialToJsDate(data.Timestamp),
editToken: token,
background: {
create: {
totalYoe: 0
}
},
offers: {
create: {
comments: data.Comments ?? "",
company: {
connect: {
id: companyIdMappings[data.Company]
}
},
jobType: "INTERN",
location: "Singapore, Singapore", // TODO: DEFAULT AS SG
monthYearReceived: xlSerialToJsDate(data.Timestamp),
negotiationStrategy: "",
offersIntern: {
create: {
internshipCycle: "Summer",
monthlySalary: {
create: {
baseCurrency: baseCurrencyString,
baseValue: await convert(
data.Income,
'SGD', // assume sgd
baseCurrencyString,
),
currency: 'SGD', // assume sgd
value: data.Income
}
},
specialization: generateSpecialization(), // TODO: check about this
startYear: xlSerialToJsDate(data.Timestamp).getFullYear(),
title: data.Role // TODO: check about this
}
}
}
}
}
})
console.log(dataAdded)
createdProfileIds.push(dataAdded.id)
} else {
// assume rest full time
const dataAdded = await prisma.offersProfile.create({
data: {
profileName: crypto.randomUUID().substring(0, 10),
createdAt: xlSerialToJsDate(data.Timestamp),
editToken: token,
background: {
create: {
totalYoe: 0
}
},
offers: {
create: {
comments: data.Comments ?? "",
company: {
connect: {
id: companyIdMappings[data.Company]
}
},
jobType: "FULLTIME",
location: "Singapore, Singapore", // TODO: DEFAULT AS SG
monthYearReceived: xlSerialToJsDate(data.Timestamp),
negotiationStrategy: "",
offersFullTime: {
create: {
baseSalary: {
create: {
baseCurrency: baseCurrencyString,
baseValue: await convert(
data.Income,
'SGD', // assume sgd
baseCurrencyString,
),
currency: 'SGD', // assume sgd
value: data.Income
}
},
bonus: {
create: {
baseCurrency: baseCurrencyString,
baseValue: await convert(
data.Bonus ? (typeof data.Bonus === 'number' ? data.Bonus : 0) : 0,
'SGD',
baseCurrencyString,
),
currency: 'SGD',
value: data.Bonus ? (typeof data.Bonus === 'number' ? data.Bonus : 0) : 0,
}
},
level: data.Type,
specialization: generateSpecialization(), // TODO: check about this
stocks: {
create: {
baseCurrency: baseCurrencyString,
baseValue: await convert(
data.Stocks ? (typeof data.Stocks === "number" ? data.Stocks : 0) : 0,
'SGD',
baseCurrencyString,
),
currency: 'SGD',
value: data.Stocks ? (typeof data.Stocks === "number" ? data.Stocks : 0) : 0,
}
},
title: data.Role, // TODO: check about this
totalCompensation: {
create: {
baseCurrency: baseCurrencyString,
baseValue: await convert(
data.TC ? (typeof data.TC === "number" ? data.TC : 0) : 0,
'SGD',
baseCurrencyString,
),
currency: 'SGD',
value: data.TC ? (typeof data.TC === "number" ? data.TC : 0) : 0,
}
},
}
}
}
}
}
})
console.log(dataAdded)
createdProfileIds.push(dataAdded.id)
}
} else {
console.log("Invalid Company: " + data.Company)
}
} else {
console.log("Invalid Income not a number: " + data.Income)
}
}
})
]).then((_data) => {
console.log('Seeding from salaries sheet complete')
}));
}
seedSalaries()
.then(async () => {
await prisma.$disconnect();
})
.catch(async (e) => {
console.error(e);
await prisma.$disconnect();
process.exit(1);
});
// Printing data
console.log(data.splice(0,100))
console.table(data.splice(0,100))
// console.log(data.splice(0,100))
// // console.table(data.splice(0,100))
console.log(xlSerialToJsDate(data[0].Timestamp))
export {}

Binary file not shown.

@ -62,4 +62,4 @@ main()
console.error(e);
await prisma.$disconnect();
process.exit(1);
});
});

@ -1,4 +1,5 @@
// API from https://github.com/fawazahmed0/currency-api#readme
import fetch from 'cross-fetch';
export const convert = async (
value: number,

@ -6207,6 +6207,11 @@ damerau-levenshtein@^1.0.8:
resolved "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz"
integrity sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==
data-uri-to-buffer@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz#b5db46aea50f6176428ac05b73be39a57701a64b"
integrity sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==
date-fns@^2.29.1, date-fns@^2.29.3:
version "2.29.3"
resolved "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz"
@ -7790,6 +7795,14 @@ feed@^4.2.2:
dependencies:
xml-js "^1.6.11"
fetch-blob@^3.1.2, fetch-blob@^3.1.4:
version "3.2.0"
resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9"
integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==
dependencies:
node-domexception "^1.0.0"
web-streams-polyfill "^3.0.3"
fetch-retry@^5.0.2:
version "5.0.3"
resolved "https://registry.npmjs.org/fetch-retry/-/fetch-retry-5.0.3.tgz"
@ -8021,6 +8034,13 @@ form-data@^4.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
formdata-polyfill@^4.0.10:
version "4.0.10"
resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423"
integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==
dependencies:
fetch-blob "^3.1.2"
formidable@^2.0.1:
version "2.0.1"
resolved "https://registry.npmjs.org/formidable/-/formidable-2.0.1.tgz"
@ -10658,6 +10678,11 @@ node-dir@^0.1.10:
dependencies:
minimatch "^3.0.2"
node-domexception@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
node-emoji@^1.10.0:
version "1.11.0"
resolved "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz"
@ -10672,6 +10697,15 @@ node-fetch@2.6.7, node-fetch@^2.6.1, node-fetch@^2.6.7:
dependencies:
whatwg-url "^5.0.0"
node-fetch@^3.2.10:
version "3.2.10"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.2.10.tgz#e8347f94b54ae18b57c9c049ef641cef398a85c8"
integrity sha512-MhuzNwdURnZ1Cp4XTazr69K0BTizsBroX7Zx3UgDSVcZYKF/6p0CBe4EUb/hLqmzVhl0UpYfgRljQ4yxE+iCxA==
dependencies:
data-uri-to-buffer "^4.0.0"
fetch-blob "^3.1.4"
formdata-polyfill "^4.0.10"
node-forge@^1:
version "1.3.1"
resolved "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz"
@ -14950,7 +14984,7 @@ web-namespaces@^1.0.0:
resolved "https://registry.npmjs.org/web-namespaces/-/web-namespaces-1.1.4.tgz"
integrity sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw==
web-streams-polyfill@^3.2.1:
web-streams-polyfill@^3.0.3, web-streams-polyfill@^3.2.1:
version "3.2.1"
resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz"
integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==

Loading…
Cancel
Save