From 5cfdf6e87a6d04e5e827b4f371a9a377e70e238c Mon Sep 17 00:00:00 2001 From: Stuart Long Chay Boon Date: Fri, 4 Nov 2022 14:36:06 +0800 Subject: [PATCH] [offers[chore] Generate analysis for seeded data --- apps/portal/prisma/readSheet.ts | 445 ++++++++++-------- apps/portal/prisma/schema.prisma | 2 +- .../pages/offers/test/generateAnalysis.tsx | 2 +- .../src/pages/offers/test/getAnalysis.tsx | 2 +- .../src/utils/offers/analysisGeneration.ts | 386 --------------- 5 files changed, 245 insertions(+), 592 deletions(-) delete mode 100644 apps/portal/src/utils/offers/analysisGeneration.ts diff --git a/apps/portal/prisma/readSheet.ts b/apps/portal/prisma/readSheet.ts index 73ebb562..acb6eaa3 100644 --- a/apps/portal/prisma/readSheet.ts +++ b/apps/portal/prisma/readSheet.ts @@ -1,221 +1,264 @@ -import reader from "xlsx"; +import reader from 'xlsx'; import { PrismaClient } from '@prisma/client'; import crypto from 'crypto'; import { baseCurrencyString } from '../src/utils/offers/currency'; import { convert } from '../src/utils/offers/currency/currencyExchange'; +import { generateAnalysis } from '../src/utils/offers/analysisGeneration'; const prisma = new PrismaClient(); + // Reading our test file -const file = reader.readFile('/Users/stuartlong/Desktop/tech-interview-handbook/apps/portal/prisma/salaries.xlsx') - -let data: Array = [] - -type excelData = { - Timestamp: Date; - Type: string; - Company: string; - Role: string, - Income?: number | string; - Stocks?: number | string; - SignOn?: number | string; - TC?: number | string; - Bonus?: number | string; - Comments?: string -} +const file = reader.readFile('prisma/salaries.xlsx'); + +let data: Array = []; + +type ExcelData = { + Timestamp: Date; + Type: string; + Company: string; + Role: string; + Income?: number | string; + Stocks?: number | string; + SignOn?: number | string; + TC?: number | string; + Bonus?: number | string; + Comments?: string; +}; -const sheets = file.SheetNames +const sheets = file.SheetNames; -for(let i = 0; i < sheets.length; i++) -{ - const temp = reader.utils.sheet_to_json( - file.Sheets[file.SheetNames[i]]) - temp.forEach((res: excelData) => { - data.push(res) - }) +for (let i = 0; i < sheets.length; i++) { + const temp = reader.utils.sheet_to_json(file.Sheets[file.SheetNames[i]]); + temp.forEach((res: ExcelData) => { + data.push(res); + }); } -function xlSerialToJsDate(xlSerial){ +function xlSerialToJsDate(xlSerial) { return new Date(Date.UTC(0, 0, xlSerial - 1)); } function generateSpecialization() { - const specializations = ["Frontend", "Backend", "Fullstack"]; + const specializations = ['Frontend', 'Backend', 'Fullstack']; - return specializations[Math.floor((Math.random() * 300)) % 3]; + return specializations[Math.floor(Math.random() * 300) % 3]; } -async function seedSalaries() { - console.log('Seeding from salaries sheet...'); - - const companyIdMappings = {}; - (await prisma.company.findMany()).forEach((company) => { - companyIdMappings[company.name] = company.id - }); - console.log(companyIdMappings); - - const createdProfileIds : Array = []; - //seed here - (await Promise.all([ - data.map(async (data: excelData) => { - // only add swe roles - if (data.Role.toUpperCase() === 'SOFTWARE ENGINEER') { - if (data.Income && typeof (data.Income) === "number") { - // check if we have company id - // console.log(data.Income) - // console.log() - if (companyIdMappings[data.Company]) { - const token = crypto.createHash('sha256').update(xlSerialToJsDate(data.Timestamp).toString()).digest('hex') - if (data.Type.toUpperCase() === 'INTERNSHIP') { - // create profile - const dataAdded = await prisma.offersProfile.create({ - data: { - profileName: crypto.randomUUID().substring(0, 10), - createdAt: xlSerialToJsDate(data.Timestamp), - editToken: token, - background: { - create: { - totalYoe: 0 - } - }, - offers: { - create: { - comments: data.Comments ?? "", - company: { - connect: { - id: companyIdMappings[data.Company] - } - }, - jobType: "INTERN", - location: "Singapore, Singapore", // TODO: DEFAULT AS SG - monthYearReceived: xlSerialToJsDate(data.Timestamp), - negotiationStrategy: "", - offersIntern: { - create: { - internshipCycle: "Summer", - monthlySalary: { - create: { - baseCurrency: baseCurrencyString, - baseValue: await convert( - data.Income, - 'SGD', // assume sgd - baseCurrencyString, - ), - currency: 'SGD', // assume sgd - value: data.Income - } - }, - specialization: generateSpecialization(), // TODO: check about this - startYear: xlSerialToJsDate(data.Timestamp).getFullYear(), - title: data.Role // TODO: check about this - } - } - } - } - } - }) - - console.log(dataAdded) - createdProfileIds.push(dataAdded.id) - } else { - // assume rest full time - const dataAdded = await prisma.offersProfile.create({ - data: { - profileName: crypto.randomUUID().substring(0, 10), - createdAt: xlSerialToJsDate(data.Timestamp), - editToken: token, - background: { - create: { - totalYoe: 0 - } - }, - offers: { - create: { - comments: data.Comments ?? "", - company: { - connect: { - id: companyIdMappings[data.Company] - } - }, - jobType: "FULLTIME", - location: "Singapore, Singapore", // TODO: DEFAULT AS SG - monthYearReceived: xlSerialToJsDate(data.Timestamp), - negotiationStrategy: "", - offersFullTime: { - create: { - baseSalary: { - create: { - baseCurrency: baseCurrencyString, - baseValue: await convert( - data.Income, - 'SGD', // assume sgd - baseCurrencyString, - ), - currency: 'SGD', // assume sgd - value: data.Income - } - }, - bonus: { - create: { - baseCurrency: baseCurrencyString, - baseValue: await convert( - data.Bonus ? (typeof data.Bonus === 'number' ? data.Bonus : 0) : 0, - 'SGD', - baseCurrencyString, - ), - currency: 'SGD', - value: data.Bonus ? (typeof data.Bonus === 'number' ? data.Bonus : 0) : 0, - } - }, - level: data.Type, - specialization: generateSpecialization(), // TODO: check about this - stocks: { - create: { - baseCurrency: baseCurrencyString, - baseValue: await convert( - data.Stocks ? (typeof data.Stocks === "number" ? data.Stocks : 0) : 0, - 'SGD', - baseCurrencyString, - ), - currency: 'SGD', - value: data.Stocks ? (typeof data.Stocks === "number" ? data.Stocks : 0) : 0, - } - }, - title: data.Role, // TODO: check about this - totalCompensation: { - create: { - baseCurrency: baseCurrencyString, - baseValue: await convert( - data.TC ? (typeof data.TC === "number" ? data.TC : 0) : 0, - 'SGD', - baseCurrencyString, - ), - currency: 'SGD', - value: data.TC ? (typeof data.TC === "number" ? data.TC : 0) : 0, - } - }, - } - } - } - } - } - }) - console.log(dataAdded) - createdProfileIds.push(dataAdded.id) - } - } else { - console.log("Invalid Company: " + data.Company) - } +const createdProfileIds: Array = []; + +const seedSalaries = async () => { + console.log('Seeding from salaries sheet...'); + + const companyIdMappings = {}; + (await prisma.company.findMany()).forEach((company) => { + companyIdMappings[company.name] = company.id; + }); + + //seed here + return await Promise.all( + data.map(async (data: excelData) => { + // only add swe roles + if (data.Role.toUpperCase() === 'SOFTWARE ENGINEER') { + if (data.Income && typeof data.Income === 'number') { + // check if we have company id + // console.log(data.Income) + // console.log() + if (companyIdMappings[data.Company]) { + const token = crypto + .createHash('sha256') + .update(xlSerialToJsDate(data.Timestamp).toString()) + .digest('hex'); + if (data.Type.toUpperCase() === 'INTERNSHIP') { + // create profile + const dataAdded = await prisma.offersProfile.create({ + data: { + profileName: crypto.randomUUID().substring(0, 10), + createdAt: xlSerialToJsDate(data.Timestamp), + editToken: token, + background: { + create: { + totalYoe: 0, + }, + }, + offers: { + create: { + comments: data.Comments ?? '', + company: { + connect: { + id: companyIdMappings[data.Company], + }, + }, + jobType: 'INTERN', + location: 'Singapore, Singapore', // TODO: DEFAULT AS SG + monthYearReceived: xlSerialToJsDate(data.Timestamp), + negotiationStrategy: '', + offersIntern: { + create: { + internshipCycle: 'Summer', + monthlySalary: { + create: { + baseCurrency: baseCurrencyString, + baseValue: await convert( + data.Income, + 'SGD', // assume sgd + baseCurrencyString, + ), + currency: 'SGD', // assume sgd + value: data.Income, + }, + }, + specialization: generateSpecialization(), // TODO: check about this + startYear: xlSerialToJsDate( + data.Timestamp, + ).getFullYear(), + title: data.Role, // TODO: check about this + }, + }, + }, + }, + }, + }); + + console.log('Profile created:', dataAdded.id); + createdProfileIds.push(dataAdded.id); } else { - console.log("Invalid Income not a number: " + data.Income) + // assume rest full time + const dataAdded = await prisma.offersProfile.create({ + data: { + profileName: crypto.randomUUID().substring(0, 10), + createdAt: xlSerialToJsDate(data.Timestamp), + editToken: token, + background: { + create: { + totalYoe: 0, + }, + }, + offers: { + create: { + comments: data.Comments ?? '', + company: { + connect: { + id: companyIdMappings[data.Company], + }, + }, + jobType: 'FULLTIME', + location: 'Singapore, Singapore', // TODO: DEFAULT AS SG + monthYearReceived: xlSerialToJsDate(data.Timestamp), + negotiationStrategy: '', + offersFullTime: { + create: { + baseSalary: { + create: { + baseCurrency: baseCurrencyString, + baseValue: await convert( + data.Income, + 'SGD', // assume sgd + baseCurrencyString, + ), + currency: 'SGD', // assume sgd + value: data.Income, + }, + }, + bonus: { + create: { + baseCurrency: baseCurrencyString, + baseValue: await convert( + data.Bonus + ? typeof data.Bonus === 'number' + ? data.Bonus + : 0 + : 0, + 'SGD', + baseCurrencyString, + ), + currency: 'SGD', + value: data.Bonus + ? typeof data.Bonus === 'number' + ? data.Bonus + : 0 + : 0, + }, + }, + level: data.Type, + specialization: generateSpecialization(), // TODO: check about this + stocks: { + create: { + baseCurrency: baseCurrencyString, + baseValue: await convert( + data.Stocks + ? typeof data.Stocks === 'number' + ? data.Stocks + : 0 + : 0, + 'SGD', + baseCurrencyString, + ), + currency: 'SGD', + value: data.Stocks + ? typeof data.Stocks === 'number' + ? data.Stocks + : 0 + : 0, + }, + }, + title: data.Role, // TODO: check about this + totalCompensation: { + create: { + baseCurrency: baseCurrencyString, + baseValue: await convert( + data.TC + ? typeof data.TC === 'number' + ? data.TC + : 0 + : 0, + 'SGD', + baseCurrencyString, + ), + currency: 'SGD', + value: data.TC + ? typeof data.TC === 'number' + ? data.TC + : 0 + : 0, + }, + }, + }, + }, + }, + }, + }, + }); + console.log('Profile created:', dataAdded.id); + createdProfileIds.push(dataAdded.id); } - } - }) - ]).then((_data) => { - console.log('Seeding from salaries sheet complete') - })); -} + } else { + console.log('Invalid Company: ' + data.Company); + } + } else { + console.log('Invalid Income not a number: ' + data.Income); + } + } + }), + ); +}; -seedSalaries() +const generateAllAnalysis = async () => { + return await Promise.all( + createdProfileIds.map(async (profileId) => { + const analysis = await generateAnalysis({ + ctx: { prisma, session: null }, + input: { profileId }, + }); + console.log('Analysis generated for profile with id:', profileId); + }), + ); +}; + +Promise.all([seedSalaries()]) + .then(() => generateAllAnalysis()) + .then((_data) => { + console.log('Seeding from salaries sheet complete'); + }) .then(async () => { await prisma.$disconnect(); }) @@ -225,10 +268,6 @@ seedSalaries() process.exit(1); }); -// Printing data -// console.log(data.splice(0,100)) -// // console.table(data.splice(0,100)) - -console.log(xlSerialToJsDate(data[0].Timestamp)) +console.log(xlSerialToJsDate(data[0].Timestamp)); -export {} \ No newline at end of file +export {}; diff --git a/apps/portal/prisma/schema.prisma b/apps/portal/prisma/schema.prisma index cbf65808..ca3806c1 100644 --- a/apps/portal/prisma/schema.prisma +++ b/apps/portal/prisma/schema.prisma @@ -235,7 +235,7 @@ model OffersProfile { model OffersBackground { id String @id @default(cuid()) - totalYoe Int + totalYoe Int @default(0) specificYoes OffersSpecificYoe[] experiences OffersExperience[] diff --git a/apps/portal/src/pages/offers/test/generateAnalysis.tsx b/apps/portal/src/pages/offers/test/generateAnalysis.tsx index bf062448..7d6bb6b5 100644 --- a/apps/portal/src/pages/offers/test/generateAnalysis.tsx +++ b/apps/portal/src/pages/offers/test/generateAnalysis.tsx @@ -8,7 +8,7 @@ function GenerateAnalysis() { return (
{JSON.stringify( - analysisMutation.mutate({ profileId: 'cl9lwe9m902k5utskjs52wc0j' }), + analysisMutation.mutate({ profileId: 'cl9luzsqh0005utr2d7jpjabt' }), )}
); diff --git a/apps/portal/src/pages/offers/test/getAnalysis.tsx b/apps/portal/src/pages/offers/test/getAnalysis.tsx index b7faac4c..10490ab4 100644 --- a/apps/portal/src/pages/offers/test/getAnalysis.tsx +++ b/apps/portal/src/pages/offers/test/getAnalysis.tsx @@ -5,7 +5,7 @@ import { trpc } from '~/utils/trpc'; function GetAnalysis() { const analysis = trpc.useQuery([ 'offers.analysis.get', - { profileId: 'cl9jo3e0k004ai9c0zmfzo50j' }, + { profileId: 'cl9luzsqh0005utr2d7jpjabt' }, ]); return
{JSON.stringify(analysis.data)}
; diff --git a/apps/portal/src/utils/offers/analysisGeneration.ts b/apps/portal/src/utils/offers/analysisGeneration.ts deleted file mode 100644 index 57da03da..00000000 --- a/apps/portal/src/utils/offers/analysisGeneration.ts +++ /dev/null @@ -1,386 +0,0 @@ -import type { Session } from 'next-auth'; -import type { - City, - Company, - Country, - OffersBackground, - OffersCurrency, - OffersFullTime, - OffersIntern, - OffersOffer, - OffersProfile, - Prisma, - PrismaClient, - State, -} from '@prisma/client'; -import { TRPCError } from '@trpc/server'; - -import { profileAnalysisDtoMapper } from '../../mappers/offers-mappers'; - -const searchOfferPercentile = ( - offer: OffersOffer & { - company: Company; - offersFullTime: - | (OffersFullTime & { - baseSalary: OffersCurrency; - bonus: OffersCurrency; - stocks: OffersCurrency; - totalCompensation: OffersCurrency; - }) - | null; - offersIntern: (OffersIntern & { monthlySalary: OffersCurrency }) | null; - profile: OffersProfile & { background: OffersBackground | null }; - }, - similarOffers: Array< - OffersOffer & { - company: Company; - offersFullTime: - | (OffersFullTime & { - totalCompensation: OffersCurrency; - }) - | null; - offersIntern: (OffersIntern & { monthlySalary: OffersCurrency }) | null; - profile: OffersProfile & { background: OffersBackground | null }; - } - >, -) => { - for (let i = 0; i < similarOffers.length; i++) { - if (similarOffers[i].id === offer.id) { - return i; - } - } - - return -1; -}; - -export const generateAnalysis = async (params: { - ctx: { - prisma: PrismaClient< - Prisma.PrismaClientOptions, - never, - Prisma.RejectOnNotFound | Prisma.RejectPerOperation | undefined - >; - session: Session | null; - }; - input: { profileId: string }; -}) => { - const { ctx, input } = params; - await ctx.prisma.offersAnalysis.deleteMany({ - where: { - profileId: input.profileId, - }, - }); - - const offers = await ctx.prisma.offersOffer.findMany({ - include: { - company: true, - offersFullTime: { - include: { - baseSalary: true, - bonus: true, - stocks: true, - totalCompensation: true, - }, - }, - offersIntern: { - include: { - monthlySalary: true, - }, - }, - profile: { - include: { - background: true, - }, - }, - }, - orderBy: [ - { - offersFullTime: { - totalCompensation: { - baseValue: 'desc', - }, - }, - }, - { - offersIntern: { - monthlySalary: { - baseValue: 'desc', - }, - }, - }, - ], - where: { - profileId: input.profileId, - }, - }); - - if (!offers || offers.length === 0) { - throw new TRPCError({ - code: 'NOT_FOUND', - message: 'No offers found on this profile', - }); - } - - const overallHighestOffer = offers[0]; - - // TODO: Shift yoe out of background to make it mandatory - if ( - !overallHighestOffer.profile.background || - overallHighestOffer.profile.background.totalYoe == null - ) { - throw new TRPCError({ - code: 'NOT_FOUND', - message: 'YOE not found', - }); - } - - const yoe = overallHighestOffer.profile.background.totalYoe as number; - const monthYearReceived = new Date(overallHighestOffer.monthYearReceived); - monthYearReceived.setFullYear(monthYearReceived.getFullYear() - 1); - - let similarOffers = await ctx.prisma.offersOffer.findMany({ - include: { - company: true, - offersFullTime: { - include: { - totalCompensation: true, - }, - }, - offersIntern: { - include: { - monthlySalary: true, - }, - }, - profile: { - include: { - background: { - include: { - experiences: { - include: { - company: true, - }, - }, - }, - }, - }, - }, - }, - orderBy: [ - { - offersFullTime: { - totalCompensation: { - baseValue: 'desc', - }, - }, - }, - { - offersIntern: { - monthlySalary: { - baseValue: 'desc', - }, - }, - }, - ], - where: { - AND: [ - { - location: overallHighestOffer.location, - }, - { - monthYearReceived: { - gte: monthYearReceived, - }, - }, - { - OR: [ - { - offersFullTime: { - title: overallHighestOffer.offersFullTime?.title, - }, - offersIntern: { - title: overallHighestOffer.offersIntern?.title, - }, - }, - ], - }, - { - profile: { - background: { - AND: [ - { - totalYoe: { - gte: Math.max(yoe - 1, 0), - lte: yoe + 1, - }, - }, - ], - }, - }, - }, - ], - }, - }); - - let similarCompanyOffers = similarOffers.filter( - (offer) => offer.companyId === overallHighestOffer.companyId, - ); - - // CALCULATE PERCENTILES - const overallIndex = searchOfferPercentile( - overallHighestOffer, - similarOffers, - ); - const overallPercentile = - similarOffers.length === 0 - ? 100 - : (100 * overallIndex) / similarOffers.length; - - const companyIndex = searchOfferPercentile( - overallHighestOffer, - similarCompanyOffers, - ); - const companyPercentile = - similarCompanyOffers.length === 0 - ? 100 - : (100 * companyIndex) / similarCompanyOffers.length; - - // FIND TOP >=90 PERCENTILE OFFERS, DOESN'T GIVE 100th PERCENTILE - // e.g. If there only 4 offers, it gives the 2nd and 3rd offer - similarOffers = similarOffers.filter( - (offer) => offer.id !== overallHighestOffer.id, - ); - similarCompanyOffers = similarCompanyOffers.filter( - (offer) => offer.id !== overallHighestOffer.id, - ); - - const noOfSimilarOffers = similarOffers.length; - const similarOffers90PercentileIndex = Math.ceil(noOfSimilarOffers * 0.1); - const topPercentileOffers = - noOfSimilarOffers > 2 - ? similarOffers.slice( - similarOffers90PercentileIndex, - similarOffers90PercentileIndex + 2, - ) - : similarOffers; - - const noOfSimilarCompanyOffers = similarCompanyOffers.length; - const similarCompanyOffers90PercentileIndex = Math.ceil( - noOfSimilarCompanyOffers * 0.1, - ); - const topPercentileCompanyOffers = - noOfSimilarCompanyOffers > 2 - ? similarCompanyOffers.slice( - similarCompanyOffers90PercentileIndex, - similarCompanyOffers90PercentileIndex + 2, - ) - : similarCompanyOffers; - - const analysis = await ctx.prisma.offersAnalysis.create({ - data: { - companyPercentile, - noOfSimilarCompanyOffers, - noOfSimilarOffers, - overallHighestOffer: { - connect: { - id: overallHighestOffer.id, - }, - }, - overallPercentile, - profile: { - connect: { - id: input.profileId, - }, - }, - topCompanyOffers: { - connect: topPercentileCompanyOffers.map((offer) => { - return { id: offer.id }; - }), - }, - topOverallOffers: { - connect: topPercentileOffers.map((offer) => { - return { id: offer.id }; - }), - }, - }, - include: { - overallHighestOffer: { - include: { - company: true, - offersFullTime: { - include: { - totalCompensation: true, - }, - }, - offersIntern: { - include: { - monthlySalary: true, - }, - }, - profile: { - include: { - background: true, - }, - }, - }, - }, - topCompanyOffers: { - include: { - company: true, - offersFullTime: { - include: { - totalCompensation: true, - }, - }, - offersIntern: { - include: { - monthlySalary: true, - }, - }, - profile: { - include: { - background: { - include: { - experiences: { - include: { - company: true, - }, - }, - }, - }, - }, - }, - }, - }, - topOverallOffers: { - include: { - company: true, - offersFullTime: { - include: { - totalCompensation: true, - }, - }, - offersIntern: { - include: { - monthlySalary: true, - }, - }, - profile: { - include: { - background: { - include: { - experiences: { - include: { - company: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }); - - return profileAnalysisDtoMapper(analysis); -};