From 94b7cd05713379e089c02b86b6877cd4f3c5345b Mon Sep 17 00:00:00 2001 From: E Date: Thu, 23 May 2024 10:29:12 +0100 Subject: [PATCH 1/9] Update schema for if scrape b0rks --- .../migration.sql | 14 ++++++++++++++ .../migration.sql | 2 ++ .../migration.sql | 3 +++ prisma/schema.prisma | 8 +++++--- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 prisma/migrations/20240523092504_add_scrape_success_and_info/migration.sql create mode 100644 prisma/migrations/20240523092616_add_scrape_info_can_empty/migration.sql create mode 100644 prisma/migrations/20240523092846_add_price_stock_can_empty/migration.sql diff --git a/prisma/migrations/20240523092504_add_scrape_success_and_info/migration.sql b/prisma/migrations/20240523092504_add_scrape_success_and_info/migration.sql new file mode 100644 index 0000000..b211351 --- /dev/null +++ b/prisma/migrations/20240523092504_add_scrape_success_and_info/migration.sql @@ -0,0 +1,14 @@ +/* + Warnings: + + - You are about to drop the column `imageBase64` on the `ProductScraperHistory` table. All the data in the column will be lost. + - Added the required column `imageUrl` to the `ProductScraperHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `scrapeInfo` to the `ProductScraperHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `successfulScrape` to the `ProductScraperHistory` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "ProductScraperHistory" DROP COLUMN "imageBase64", +ADD COLUMN "imageUrl" TEXT NOT NULL, +ADD COLUMN "scrapeInfo" TEXT NOT NULL, +ADD COLUMN "successfulScrape" BOOLEAN NOT NULL; diff --git a/prisma/migrations/20240523092616_add_scrape_info_can_empty/migration.sql b/prisma/migrations/20240523092616_add_scrape_info_can_empty/migration.sql new file mode 100644 index 0000000..bb76541 --- /dev/null +++ b/prisma/migrations/20240523092616_add_scrape_info_can_empty/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "ProductScraperHistory" ALTER COLUMN "scrapeInfo" DROP NOT NULL; diff --git a/prisma/migrations/20240523092846_add_price_stock_can_empty/migration.sql b/prisma/migrations/20240523092846_add_price_stock_can_empty/migration.sql new file mode 100644 index 0000000..d49bde2 --- /dev/null +++ b/prisma/migrations/20240523092846_add_price_stock_can_empty/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "ProductScraperHistory" ALTER COLUMN "price" DROP NOT NULL, +ALTER COLUMN "inStock" DROP NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6705634..fc93fe1 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -47,10 +47,12 @@ model ScraperLambda { model ProductScraperHistory { id Int @id @default(autoincrement()) - price Float - inStock Boolean + price Float? + inStock Boolean? created DateTime @default(now()) - imageBase64 String + imageUrl String + successfulScrape Boolean + scrapeInfo String? productScraperId Int ProductScraper ProductScraper @relation(fields: [productScraperId], references: [id]) } From f972505a0f00ba3f2acef71353d4ceddd957f85f Mon Sep 17 00:00:00 2001 From: E Date: Thu, 23 May 2024 10:44:52 +0100 Subject: [PATCH 2/9] Move folders and fix errors from schema change --- .../[company_slug]/products/[product_slug]/getProduct.ts | 2 +- .../company/[company_slug]/products/[product_slug]/page.tsx | 2 +- src/app/layout.tsx | 4 ++-- src/{app => }/components/Footer.tsx | 0 src/{app => }/components/Nav.tsx | 0 src/{app => }/components/Product.tsx | 5 +---- src/{app => }/lib/db.ts | 0 7 files changed, 5 insertions(+), 8 deletions(-) rename src/{app => }/components/Footer.tsx (100%) rename src/{app => }/components/Nav.tsx (100%) rename src/{app => }/components/Product.tsx (91%) rename src/{app => }/lib/db.ts (100%) diff --git a/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts b/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts index a66dd50..786535c 100644 --- a/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts +++ b/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts @@ -1,4 +1,4 @@ -import prisma from '@/app/lib/db'; +import prisma from '@/lib/db'; import type { Product } from '@prisma/client'; export async function fetchProductData( diff --git a/src/app/company/[company_slug]/products/[product_slug]/page.tsx b/src/app/company/[company_slug]/products/[product_slug]/page.tsx index a0fc227..560b3f1 100644 --- a/src/app/company/[company_slug]/products/[product_slug]/page.tsx +++ b/src/app/company/[company_slug]/products/[product_slug]/page.tsx @@ -1,7 +1,7 @@ import type { Product as ProductType } from '@prisma/client'; import { notFound } from 'next/navigation'; import { fetchProductData } from './getProduct'; -import Product from '@/app/components/Product'; +import Product from '@/components/Product'; export const metadata = { title: '', diff --git a/src/app/layout.tsx b/src/app/layout.tsx index cf8c9dd..59fdb24 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -1,8 +1,8 @@ import type { Metadata } from 'next'; import { Inter } from 'next/font/google'; import './globals.css'; -import Nav from './components/Nav'; -import Footer from './components/Footer'; +import Nav from '../components/Nav'; +import Footer from '../components/Footer'; const inter = Inter({ subsets: ['latin'] }); diff --git a/src/app/components/Footer.tsx b/src/components/Footer.tsx similarity index 100% rename from src/app/components/Footer.tsx rename to src/components/Footer.tsx diff --git a/src/app/components/Nav.tsx b/src/components/Nav.tsx similarity index 100% rename from src/app/components/Nav.tsx rename to src/components/Nav.tsx diff --git a/src/app/components/Product.tsx b/src/components/Product.tsx similarity index 91% rename from src/app/components/Product.tsx rename to src/components/Product.tsx index 772b5bb..b9ac2ce 100644 --- a/src/app/components/Product.tsx +++ b/src/components/Product.tsx @@ -1,4 +1,3 @@ -import { Product as ProductType } from '@prisma/client'; import { Overpass_Mono } from 'next/font/google'; const overpassMono = Overpass_Mono({ subsets: ['latin'] }); @@ -10,9 +9,7 @@ export default function Product({ product }: any) {
diff --git a/src/app/lib/db.ts b/src/lib/db.ts similarity index 100% rename from src/app/lib/db.ts rename to src/lib/db.ts From a96ce84a185114efe1a8dabbe955e5cc25ae49cc Mon Sep 17 00:00:00 2001 From: E Date: Thu, 23 May 2024 12:46:03 +0100 Subject: [PATCH 3/9] Edit DB to include image xpath --- .../migrations/20240523114429_add_image_xpath/migration.sql | 6 ++++++ .../20240523114540_remove_image_xpath_default/migration.sql | 2 ++ prisma/schema.prisma | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 prisma/migrations/20240523114429_add_image_xpath/migration.sql create mode 100644 prisma/migrations/20240523114540_remove_image_xpath_default/migration.sql diff --git a/prisma/migrations/20240523114429_add_image_xpath/migration.sql b/prisma/migrations/20240523114429_add_image_xpath/migration.sql new file mode 100644 index 0000000..17a4fae --- /dev/null +++ b/prisma/migrations/20240523114429_add_image_xpath/migration.sql @@ -0,0 +1,6 @@ +-- AlterTable +ALTER TABLE "ProductScraper" ADD COLUMN "imageXPATH" TEXT NOT NULL DEFAULT ''; + +-- AlterTable +ALTER TABLE "ProductScraperHistory" ALTER COLUMN "imageUrl" DROP NOT NULL, +ALTER COLUMN "imageUrl" SET DEFAULT 'https://loremflickr.com/1000/1000/cat'; diff --git a/prisma/migrations/20240523114540_remove_image_xpath_default/migration.sql b/prisma/migrations/20240523114540_remove_image_xpath_default/migration.sql new file mode 100644 index 0000000..790878e --- /dev/null +++ b/prisma/migrations/20240523114540_remove_image_xpath_default/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "ProductScraper" ALTER COLUMN "imageXPATH" DROP DEFAULT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index fc93fe1..d8f7b3c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -30,6 +30,7 @@ model ProductScraper { inStockString String outOfStockString String priceXPATH String + imageXPATH String productId Int product Product @relation(fields: [productId], references: [id]) scraperLambdaId Int @@ -49,8 +50,8 @@ model ProductScraperHistory { id Int @id @default(autoincrement()) price Float? inStock Boolean? + imageUrl String? @default(value: "https://loremflickr.com/1000/1000/cat") created DateTime @default(now()) - imageUrl String successfulScrape Boolean scrapeInfo String? productScraperId Int From 6ae13dff80ea294501ac883fabdfe28bfdebe2ab Mon Sep 17 00:00:00 2001 From: E Date: Fri, 24 May 2024 15:45:24 +0100 Subject: [PATCH 4/9] Fix alt text and add ignore for eslint --- src/components/Product.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/components/Product.tsx b/src/components/Product.tsx index b9ac2ce..df74e66 100644 --- a/src/components/Product.tsx +++ b/src/components/Product.tsx @@ -1,3 +1,4 @@ +/* eslint-disable @next/next/no-img-element */ import { Overpass_Mono } from 'next/font/google'; const overpassMono = Overpass_Mono({ subsets: ['latin'] }); @@ -10,6 +11,7 @@ export default function Product({ product }: any) {
{'Image
From acb07f616e4b583dc313d4a354c830ee918fd437 Mon Sep 17 00:00:00 2001 From: E Date: Fri, 24 May 2024 20:43:14 +0100 Subject: [PATCH 5/9] Refactor db design to remove redundant data --- .../20240524194123_refactor/migration.sql | 46 +++++++++++++++++++ prisma/schema.prisma | 30 ++++++++---- 2 files changed, 66 insertions(+), 10 deletions(-) create mode 100644 prisma/migrations/20240524194123_refactor/migration.sql diff --git a/prisma/migrations/20240524194123_refactor/migration.sql b/prisma/migrations/20240524194123_refactor/migration.sql new file mode 100644 index 0000000..68f37ff --- /dev/null +++ b/prisma/migrations/20240524194123_refactor/migration.sql @@ -0,0 +1,46 @@ +/* + Warnings: + + - You are about to drop the column `imageXPATH` on the `ProductScraper` table. All the data in the column will be lost. + - You are about to drop the column `inStockString` on the `ProductScraper` table. All the data in the column will be lost. + - You are about to drop the column `outOfStockString` on the `ProductScraper` table. All the data in the column will be lost. + - You are about to drop the column `priceXPATH` on the `ProductScraper` table. All the data in the column will be lost. + - You are about to drop the column `url` on the `ProductScraper` table. All the data in the column will be lost. + - You are about to drop the column `currencyType` on the `ScraperLambda` table. All the data in the column will be lost. + - Added the required column `url` to the `Product` table without a default value. This is not possible if the table is not empty. + - Added the required column `currency` to the `ProductScraper` table without a default value. This is not possible if the table is not empty. + - Added the required column `scraperPropertiesId` to the `ProductScraper` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "Product" ADD COLUMN "url" TEXT NOT NULL; + +-- AlterTable +ALTER TABLE "ProductScraper" DROP COLUMN "imageXPATH", +DROP COLUMN "inStockString", +DROP COLUMN "outOfStockString", +DROP COLUMN "priceXPATH", +DROP COLUMN "url", +ADD COLUMN "currency" TEXT NOT NULL, +ADD COLUMN "scraperPropertiesId" INTEGER NOT NULL; + +-- AlterTable +ALTER TABLE "ScraperLambda" DROP COLUMN "currencyType"; + +-- CreateTable +CREATE TABLE "ScraperProperties" ( + "id" SERIAL NOT NULL, + "inStockString" TEXT NOT NULL, + "outOfStockString" TEXT NOT NULL, + "priceXPATH" TEXT NOT NULL, + "imageXPATH" TEXT NOT NULL, + "companyId" INTEGER, + + CONSTRAINT "ScraperProperties_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "ScraperProperties" ADD CONSTRAINT "ScraperProperties_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "Company"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "ProductScraper" ADD CONSTRAINT "ProductScraper_scraperPropertiesId_fkey" FOREIGN KEY ("scraperPropertiesId") REFERENCES "ScraperProperties"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d8f7b3c..228dec1 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8,15 +8,28 @@ datasource db { } model Company { - id Int @id @default(autoincrement()) - name String - slug String @unique - Products Product[] + id Int @id @default(autoincrement()) + name String + slug String @unique + Products Product[] + ScraperProperties ScraperProperties[] +} + +model ScraperProperties { + id Int @id @default(autoincrement()) + inStockString String + outOfStockString String + priceXPATH String + imageXPATH String + companyId Int? + Company Company? @relation(fields: [companyId], references: [id]) + ProductScraper ProductScraper[] } model Product { id Int @id @default(autoincrement()) name String + url String created DateTime @default(now()) slug String @unique companyId Int @@ -26,13 +39,11 @@ model Product { model ProductScraper { id Int @id @default(autoincrement()) - url String - inStockString String - outOfStockString String - priceXPATH String - imageXPATH String + currency String productId Int product Product @relation(fields: [productId], references: [id]) + scraperPropertiesId Int + ScraperProperties ScraperProperties @relation(fields: [scraperPropertiesId], references: [id]) scraperLambdaId Int ScraperLambda ScraperLambda @relation(fields: [scraperLambdaId], references: [id]) ProductScraperHistory ProductScraperHistory[] @@ -42,7 +53,6 @@ model ScraperLambda { id Int @id @default(autoincrement()) url String @unique region String - currencyType String ProductScraper ProductScraper[] } From aad522218276567b7f283a133bdff6fdf53868cb Mon Sep 17 00:00:00 2001 From: E Date: Fri, 24 May 2024 20:54:54 +0100 Subject: [PATCH 6/9] Add name fields and rename imageUrl to imageBase64 --- .../migration.sql | 17 +++++++++++++++++ prisma/schema.prisma | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 prisma/migrations/20240524195435_rename_image_add_names/migration.sql diff --git a/prisma/migrations/20240524195435_rename_image_add_names/migration.sql b/prisma/migrations/20240524195435_rename_image_add_names/migration.sql new file mode 100644 index 0000000..314c4b0 --- /dev/null +++ b/prisma/migrations/20240524195435_rename_image_add_names/migration.sql @@ -0,0 +1,17 @@ +/* + Warnings: + + - You are about to drop the column `imageUrl` on the `ProductScraperHistory` table. All the data in the column will be lost. + - Added the required column `name` to the `ProductScraper` table without a default value. This is not possible if the table is not empty. + - Added the required column `name` to the `ScraperProperties` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "ProductScraper" ADD COLUMN "name" TEXT NOT NULL; + +-- AlterTable +ALTER TABLE "ProductScraperHistory" DROP COLUMN "imageUrl", +ADD COLUMN "imageBase64" TEXT; + +-- AlterTable +ALTER TABLE "ScraperProperties" ADD COLUMN "name" TEXT NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 228dec1..3c6051f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -17,6 +17,7 @@ model Company { model ScraperProperties { id Int @id @default(autoincrement()) + name String inStockString String outOfStockString String priceXPATH String @@ -39,6 +40,7 @@ model Product { model ProductScraper { id Int @id @default(autoincrement()) + name String currency String productId Int product Product @relation(fields: [productId], references: [id]) @@ -60,7 +62,7 @@ model ProductScraperHistory { id Int @id @default(autoincrement()) price Float? inStock Boolean? - imageUrl String? @default(value: "https://loremflickr.com/1000/1000/cat") + imageBase64 String? created DateTime @default(now()) successfulScrape Boolean scrapeInfo String? From ac3a69ed06c44bc5a50560dfa3d5d966e8718ca4 Mon Sep 17 00:00:00 2001 From: E Date: Fri, 24 May 2024 20:58:21 +0100 Subject: [PATCH 7/9] Fix product page --- .../[company_slug]/products/[product_slug]/getProduct.ts | 5 ----- src/components/Product.tsx | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts b/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts index 786535c..a61eabc 100644 --- a/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts +++ b/src/app/company/[company_slug]/products/[product_slug]/getProduct.ts @@ -17,11 +17,6 @@ export async function fetchProductData( }, take: 25, }, - ScraperLambda: { - select: { - currencyType: true, - }, - }, }, }, }, diff --git a/src/components/Product.tsx b/src/components/Product.tsx index df74e66..58a14f4 100644 --- a/src/components/Product.tsx +++ b/src/components/Product.tsx @@ -10,7 +10,7 @@ export default function Product({ product }: any) {
{'Image
@@ -36,7 +36,7 @@ export default function Product({ product }: any) { >

Price - {product.ProductScraper[0].ScraperLambda.currencyType} + {product.ProductScraper[0].currency} {product.ProductScraper[0].ProductScraperHistory[0].price}

From 9510268e6b12c2def35b6942bee8f72320a9ac3f Mon Sep 17 00:00:00 2001 From: E Date: Fri, 24 May 2024 21:00:14 +0100 Subject: [PATCH 8/9] Remove old line from .env.example --- .env.example | 1 - 1 file changed, 1 deletion(-) diff --git a/.env.example b/.env.example index 2635f4d..ce81892 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1 @@ DATABASE_URL= -LAMBDA_URL= From 52f59f9888b9df2f2856d891753ce4563bf7e3c5 Mon Sep 17 00:00:00 2001 From: E Date: Fri, 24 May 2024 21:55:27 +0100 Subject: [PATCH 9/9] Run prettier --- src/components/Product.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/Product.tsx b/src/components/Product.tsx index 58a14f4..460ec90 100644 --- a/src/components/Product.tsx +++ b/src/components/Product.tsx @@ -10,7 +10,9 @@ export default function Product({ product }: any) {
{'Image